From e4db511f52d0f39d0ef9609efe7116a470e39732 Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Sat, 30 May 2026 22:37:33 +0800 Subject: [PATCH 01/30] perf(trt): GPU-fuse face-restoration paste_back (39ms->2.4ms) + benchmark harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1 first cut on the TensorRT face-restoration (GFPGAN) stage. Profiling showed paste_back was ~50% of the pipeline: two full-frame cv::warpAffine on the CPU plus per-call cudaMalloc/free and synchronous copies. Rewrote it as a single inverse-mapping CUDA kernel with reused device buffers and pinned/async copies. Measured on RTX 4090 / TRT 10.1 fp32, gfpgan 512: paste_back 39.07ms -> 2.37ms (16.5x); end-to-end 78.2ms -> 30.4ms; 12.8 -> 32.9 FPS. CPU vs GPU output is numerically equivalent (max|diff| = 2/255). Per-file changes: - lite/bench/profiler.h: new header-only, backend-agnostic profiler. CPU chrono + CUDA-event timers, mean/p50/p90/p99 + FPS aggregation, CSV export, scoped-timer macros. - lite/trt/kernel/paste_back.cu, paste_back.cuh: add paste_back_fused_kernel — per-output-pixel inverse mapping (uses the original->crop affine directly, no inversion), bilinear crop/mask sampling with border-0, blend, writes uint8. Old paste_back_kernel kept. - lite/trt/kernel/paste_back_manager.cpp, paste_back_manager.h: add PasteBackGPU, which owns reusable device + pinned buffers (ensure_capacity) and drives the fused kernel. Old CPU launch_paste_back kept for A/B reference. Comments translated to English. - lite/trt/cv/trt_face_restoration.cpp, trt_face_restoration.h: factor detect() into restore() (returns the frame, no disk write, optional per-stage Profiler); detect() now calls restore() + imwrite; restore() uses the GPU PasteBackGPU member instead of CPU paste_back. - examples/lite/cv/test_lite_face_restoration_bench.cpp: new benchmark — CPU-vs-GPU paste_back equivalence check + compute-only per-stage latency/FPS over N iterations. - examples/lite/CMakeLists.txt: register the lite_face_restoration_bench executable. Co-Authored-By: Claude Opus 4.8 --- examples/lite/CMakeLists.txt | 1 + .../cv/test_lite_face_restoration_bench.cpp | 110 +++++++ lite/bench/profiler.h | 273 ++++++++++++++++++ lite/trt/cv/trt_face_restoration.cpp | 150 +++++----- lite/trt/cv/trt_face_restoration.h | 14 +- lite/trt/kernel/paste_back.cu | 59 ++++ lite/trt/kernel/paste_back.cuh | 10 + lite/trt/kernel/paste_back_manager.cpp | 115 +++++++- lite/trt/kernel/paste_back_manager.h | 31 ++ 9 files changed, 665 insertions(+), 98 deletions(-) create mode 100644 examples/lite/cv/test_lite_face_restoration_bench.cpp create mode 100644 lite/bench/profiler.h diff --git a/examples/lite/CMakeLists.txt b/examples/lite/CMakeLists.txt index b8b90c53..ae0a1dbf 100644 --- a/examples/lite/CMakeLists.txt +++ b/examples/lite/CMakeLists.txt @@ -108,6 +108,7 @@ add_lite_executable(lite_face_68landmarks cv) add_lite_executable(lite_face_recognizer cv) add_lite_executable(lite_face_swap cv) add_lite_executable(lite_face_restoration cv) +add_lite_executable(lite_face_restoration_bench cv) add_lite_executable(lite_facefusion_pipeline cv) add_lite_executable(lite_yolov8 cv) add_lite_executable(lite_yolov11 cv) diff --git a/examples/lite/cv/test_lite_face_restoration_bench.cpp b/examples/lite/cv/test_lite_face_restoration_bench.cpp new file mode 100644 index 00000000..644e4521 --- /dev/null +++ b/examples/lite/cv/test_lite_face_restoration_bench.cpp @@ -0,0 +1,110 @@ +// +// End-to-end benchmark for the GFPGAN face-restoration stage (Phase 0). +// Usage: +// lite_face_restoration_bench [engine_path] [test_img] [iters] [warmup] [csv] +// Defaults point at a gfpgan engine + a test image on the remote 4090; override via argv. +// +// It first runs a CPU-vs-GPU paste_back equivalence check, then a compute-only +// latency/throughput benchmark of restore() with per-stage aggregation +// (preprocess / infer / postprocess / paste_back). Disk I/O (imwrite) is kept +// out of the timed loop; one result image is saved afterwards for visual checking. +// +#include "lite/lite.h" +#include "lite/bench/profiler.h" + +#ifdef ENABLE_TENSORRT +#include "lite/trt/cv/trt_face_restoration.h" +#include "lite/ort/cv/face_utils.h" +#include "lite/trt/kernel/paste_back_manager.h" + +// A/B numerical check: per-pixel difference between the GPU fused paste_back and the +// CPU reference, on the same real inputs (real affine + real crop). +static void check_paste_back_equivalence(const cv::Mat &frame, + std::vector &lmk5) { + // Run the real warp to obtain the real affine + real 512 crop (uint8), then to float (as in the pipeline) + cv::Mat crop_u8, affine; + std::tie(crop_u8, affine) = + face_utils::warp_face_by_face_landmark_5(frame, lmk5, face_utils::FFHQ_512); + cv::Mat crop_f; + crop_u8.convertTo(crop_f, CV_32FC3); + cv::Mat mask = face_utils::create_static_box_mask({512, 512}); + + cv::Mat out_cpu = launch_paste_back(frame, crop_f, mask, affine); + PasteBackGPU gpu; + cv::Mat out_gpu = gpu.paste_back(frame, crop_f, mask, affine, nullptr); + + cv::Mat diff; + cv::absdiff(out_cpu, out_gpu, diff); + cv::Scalar mean_diff = cv::mean(diff); + double max_diff = 0.0; + cv::minMaxLoc(diff.reshape(1), nullptr, &max_diff); + std::cout << "[check] paste_back CPU vs GPU max|diff|=" << max_diff + << " mean|diff|(B,G,R)=" << mean_diff[0] << "," << mean_diff[1] + << "," << mean_diff[2] << " (uint8 pixel values, smaller = closer)" << std::endl; +} +#endif + +int main(__unused int argc, __unused char *argv[]) { +#ifdef ENABLE_TENSORRT + std::string engine_path = + argc > 1 ? argv[1] : "/root/autodl-tmp/gfpgan_proj/gfpgan_fp32.engine"; + std::string test_img_path = + argc > 2 ? argv[2] : "../../../examples/lite/resources/test_lite_face_restoration.jpg"; + int iters = argc > 3 ? std::atoi(argv[3]) : 50; + int warmup = argc > 4 ? std::atoi(argv[4]) : 10; + std::string csv_path = argc > 5 ? argv[5] : "bench_face_restoration.csv"; + + // Fixed 5-point landmarks (same as test_lite_face_restoration.cpp); the benchmark only + // cares about timing, so whether they exactly match the image does not affect the numbers. + std::vector face_landmark_5 = { + cv::Point2f(569.092041f, 398.845886f), + cv::Point2f(701.891724f, 399.156677f), + cv::Point2f(634.767212f, 482.927216f), + cv::Point2f(584.270996f, 543.294617f), + cv::Point2f(684.877991f, 543.067078f)}; + + cv::Mat img_bgr = cv::imread(test_img_path); + if (img_bgr.empty()) { + std::cerr << "[bench] cannot read test image: " << test_img_path << std::endl; + return 1; + } + + std::cout << "[bench] engine=" << engine_path << "\n[bench] img=" + << test_img_path << " (" << img_bgr.cols << "x" << img_bgr.rows + << ")\n[bench] warmup=" << warmup << " iters=" << iters << std::endl; + + // Numerical correctness check (CPU vs GPU paste_back) before benchmarking + check_paste_back_equivalence(img_bgr, face_landmark_5); + + trtcv::TRTFaceFusionFaceRestoration restorer(engine_path); + const std::string tmp_out = "/tmp/bench_restoration_out.jpg"; + + // Warmup (first runs include lazy engine/context init and cudnn autotune; excluded from stats) + for (int i = 0; i < warmup; ++i) { + restorer.restore(img_bgr, face_landmark_5, nullptr); + } + + // Timed: restore() does not write to disk; the profiler collects per-stage timings + // (preprocess/infer/postprocess/paste_back) and the end-to-end TOTAL. imwrite is moved + // out of the loop; one image is saved at the end for visual verification. + lite::bench::Profiler prof; + cv::Mat dst; + for (int i = 0; i < iters; ++i) { + lite::bench::CpuTimer t; + t.start(); + dst = restorer.restore(img_bgr, face_landmark_5, &prof); + prof.tick(t.stop_ms()); + } + + prof.report("GFPGAN face restoration (compute-only, no disk I/O)"); + prof.to_csv(csv_path); + + if (!dst.empty()) { + cv::imwrite(tmp_out, dst); + std::cout << "[bench] sample result (saved once, outside the loop): " << tmp_out << std::endl; + } +#else + std::cerr << "This benchmark requires ENABLE_TENSORRT=ON." << std::endl; +#endif + return 0; +} diff --git a/lite/bench/profiler.h b/lite/bench/profiler.h new file mode 100644 index 00000000..78ad6a89 --- /dev/null +++ b/lite/bench/profiler.h @@ -0,0 +1,273 @@ +// +// lite.ai.toolkit unified benchmark / timing utility (header-only, backend-agnostic) +// +// Goals: +// * cross-platform, header-only, no dependency on any specific inference backend; +// * CPU stages timed with std::chrono; GPU stages timed with cudaEvent +// (so asynchronous calls are measured correctly, not via wall-clock); +// * aggregate many samples into mean/p50/p90/p99/min/max and end-to-end FPS, +// with optional CSV export. +// +// Typical usage: +// lite::bench::Profiler prof; +// for (int i = 0; i < N; ++i) { +// lite::bench::CpuTimer total; total.start(); +// { LITE_CPU_SCOPE(prof, "preprocess"); /* ... */ } +// { LITE_GPU_SCOPE(prof, "inference", stream); /* enqueueV3 ... */ } +// { LITE_CPU_SCOPE(prof, "postprocess"); /* ... */ } +// prof.tick(total.stop_ms()); // record one full iteration for FPS +// } +// prof.report("FaceFusion pipeline"); +// prof.to_csv("bench_facefusion.csv"); +// +#ifndef LITE_AI_TOOLKIT_BENCH_PROFILER_H +#define LITE_AI_TOOLKIT_BENCH_PROFILER_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(ENABLE_TENSORRT) || defined(__CUDACC__) +#include +#define LITE_BENCH_WITH_CUDA 1 +#endif + +namespace lite { +namespace bench { + +// Aggregated stats for a single stage (unit: ms) +struct Stat { + std::size_t calls = 0; + double mean = 0, p50 = 0, p90 = 0, p99 = 0, min = 0, max = 0; + + static Stat from(std::vector v) { + Stat s; + if (v.empty()) return s; + std::sort(v.begin(), v.end()); + s.calls = v.size(); + s.min = v.front(); + s.max = v.back(); + s.mean = std::accumulate(v.begin(), v.end(), 0.0) / static_cast(v.size()); + s.p50 = percentile(v, 50); + s.p90 = percentile(v, 90); + s.p99 = percentile(v, 99); + return s; + } + + private: + // v must be sorted ascending; nearest-rank percentile + static double percentile(const std::vector &v, double p) { + if (v.empty()) return 0.0; + double rank = (p / 100.0) * static_cast(v.size() - 1); + auto idx = static_cast(std::llround(rank)); + if (idx >= v.size()) idx = v.size() - 1; + return v[idx]; + } +}; + +class Profiler { + public: + // Record one sample (ms) for a stage + void add(const std::string &stage, double ms) { + auto it = samples_.find(stage); + if (it == samples_.end()) { + order_.push_back(stage); + samples_[stage].push_back(ms); + } else { + it->second.push_back(ms); + } + } + + // Record one full-iteration end-to-end latency (used to compute FPS) + void tick(double ms) { add(kTotal, ms); } + + void clear() { + order_.clear(); + samples_.clear(); + } + + // Print an aligned table; the caller is responsible for excluding warmup samples + void report(const std::string &title, std::ostream &os = std::cout) const { + os << "\n==================== Benchmark: " << title + << " ====================\n"; + os << std::left << std::setw(18) << "stage" << std::right << std::setw(8) + << "calls" << std::setw(11) << "mean(ms)" << std::setw(11) << "p50" + << std::setw(11) << "p90" << std::setw(11) << "p99" << std::setw(11) + << "min" << std::setw(11) << "max" << "\n"; + os << std::string(92, '-') << "\n"; + for (const auto &stage : order_) { + if (stage == kTotal) continue; + print_row(os, stage, Stat::from(samples_.at(stage))); + } + auto it = samples_.find(kTotal); + if (it != samples_.end()) { + os << std::string(92, '-') << "\n"; + Stat t = Stat::from(it->second); + print_row(os, "TOTAL", t); + if (t.mean > 0.0) + os << " -> throughput: " << std::fixed << std::setprecision(2) + << (1000.0 / t.mean) << " FPS (by mean), " << (1000.0 / t.p50) + << " FPS (by p50)\n"; + } + os << std::string(92, '=') << "\n"; + } + + // Export CSV (stage,calls,mean,p50,p90,p99,min,max) + void to_csv(const std::string &path) const { + std::ofstream f(path); + if (!f) { + std::cerr << "[profiler] cannot write CSV: " << path << std::endl; + return; + } + f << "stage,calls,mean_ms,p50_ms,p90_ms,p99_ms,min_ms,max_ms\n"; + for (const auto &stage : order_) { + Stat s = Stat::from(samples_.at(stage)); + const char *name = (stage == kTotal) ? "TOTAL" : stage.c_str(); + f << name << "," << s.calls << "," << s.mean << "," << s.p50 << "," + << s.p90 << "," << s.p99 << "," << s.min << "," << s.max << "\n"; + } + std::cout << "[profiler] CSV written: " << path << std::endl; + } + + Stat stat(const std::string &stage) const { + auto it = samples_.find(stage); + return it == samples_.end() ? Stat{} : Stat::from(it->second); + } + + private: + static constexpr const char *kTotal = "__total__"; + + static void print_row(std::ostream &os, const std::string &name, + const Stat &s) { + os << std::left << std::setw(18) << name << std::right << std::setw(8) + << s.calls << std::fixed << std::setprecision(3) << std::setw(11) + << s.mean << std::setw(11) << s.p50 << std::setw(11) << s.p90 + << std::setw(11) << s.p99 << std::setw(11) << s.min << std::setw(11) + << s.max << "\n"; + } + + std::vector order_; + std::unordered_map> samples_; +}; + +// ---------------- CPU timing (chrono) ---------------- +class CpuTimer { + public: + void start() { t0_ = clock::now(); } + double stop_ms() const { + return std::chrono::duration(clock::now() - t0_).count(); + } + + private: + using clock = std::chrono::high_resolution_clock; + clock::time_point t0_ = clock::now(); +}; + +// RAII: record CPU elapsed time into the profiler on scope exit +class ScopedCpuTimer { + public: + ScopedCpuTimer(Profiler &p, std::string stage) + : prof_(p), stage_(std::move(stage)) { + timer_.start(); + } + ~ScopedCpuTimer() { prof_.add(stage_, timer_.stop_ms()); } + + private: + Profiler &prof_; + std::string stage_; + CpuTimer timer_; +}; + +// Optional variant: when prof is nullptr it does nothing (zero overhead). +// Used to instrument library code that is off by default and only on when benchmarking. +class ScopedCpuTimerOpt { + public: + ScopedCpuTimerOpt(Profiler *p, std::string stage) + : prof_(p), stage_(std::move(stage)) { + if (prof_) timer_.start(); + } + ~ScopedCpuTimerOpt() { + if (prof_) prof_->add(stage_, timer_.stop_ms()); + } + + private: + Profiler *prof_; + std::string stage_; + CpuTimer timer_; +}; + +#ifdef LITE_BENCH_WITH_CUDA +// ---------------- GPU timing (cudaEvent) ---------------- +// Note: stop_ms() calls cudaEventSynchronize, so it serializes the stage; acceptable for benchmarking. +class CudaTimer { + public: + CudaTimer() { + cudaEventCreate(&start_); + cudaEventCreate(&stop_); + } + ~CudaTimer() { + cudaEventDestroy(start_); + cudaEventDestroy(stop_); + } + void start(cudaStream_t stream = nullptr) { + stream_ = stream; + cudaEventRecord(start_, stream_); + } + float stop_ms() { + cudaEventRecord(stop_, stream_); + cudaEventSynchronize(stop_); + float ms = 0.f; + cudaEventElapsedTime(&ms, start_, stop_); + return ms; + } + + private: + cudaEvent_t start_{}, stop_{}; + cudaStream_t stream_ = nullptr; +}; + +// RAII: record GPU elapsed time (on the given stream) into the profiler on scope exit +class ScopedCudaTimer { + public: + ScopedCudaTimer(Profiler &p, std::string stage, cudaStream_t stream = nullptr) + : prof_(p), stage_(std::move(stage)) { + timer_.start(stream); + } + ~ScopedCudaTimer() { prof_.add(stage_, timer_.stop_ms()); } + + private: + Profiler &prof_; + std::string stage_; + CudaTimer timer_; +}; +#endif // LITE_BENCH_WITH_CUDA + +} // namespace bench +} // namespace lite + +// ---------------- convenience macros ---------------- +#define LITE_BENCH_CONCAT_(a, b) a##b +#define LITE_BENCH_CONCAT(a, b) LITE_BENCH_CONCAT_(a, b) + +// Time the current scope on the CPU and record it into the profiler +#define LITE_CPU_SCOPE(prof, name) \ + lite::bench::ScopedCpuTimer LITE_BENCH_CONCAT(_lite_cpu_scope_, __LINE__)((prof), (name)) + +// Optional variant taking a Profiler*; zero overhead when nullptr (for library instrumentation, off by default) +#define LITE_CPU_SCOPE_OPT(profptr, name) \ + lite::bench::ScopedCpuTimerOpt LITE_BENCH_CONCAT(_lite_cpu_scope_opt_, __LINE__)((profptr), (name)) + +#ifdef LITE_BENCH_WITH_CUDA +// Time the current scope on the GPU (given stream) and record it into the profiler +#define LITE_GPU_SCOPE(prof, name, stream) \ + lite::bench::ScopedCudaTimer LITE_BENCH_CONCAT(_lite_gpu_scope_, __LINE__)((prof), (name), (stream)) +#endif + +#endif // LITE_AI_TOOLKIT_BENCH_PROFILER_H diff --git a/lite/trt/cv/trt_face_restoration.cpp b/lite/trt/cv/trt_face_restoration.cpp index 431c622d..770da154 100644 --- a/lite/trt/cv/trt_face_restoration.cpp +++ b/lite/trt/cv/trt_face_restoration.cpp @@ -3,103 +3,85 @@ // #include "trt_face_restoration.h" +#include "lite/bench/profiler.h" using trtcv::TRTFaceFusionFaceRestoration; -void TRTFaceFusionFaceRestoration::detect(cv::Mat &face_swap_image, std::vector &target_landmarks_5, - const std::string &face_enchaner_path) { +// Core compute path: returns the restored full frame, no disk write. +// When prof is non-null, records per-stage timings (preprocess / infer / postprocess, +// with paste_back broken out separately). +cv::Mat TRTFaceFusionFaceRestoration::restore(cv::Mat &face_swap_image, + std::vector &target_landmarks_5, + lite::bench::Profiler *prof) { auto ori_image = face_swap_image.clone(); cv::Mat crop_image; cv::Mat affine_matrix; - // 记录时间 - auto start_warp = std::chrono::high_resolution_clock::now(); - std::tie(crop_image,affine_matrix) = face_utils::warp_face_by_face_landmark_5(face_swap_image,target_landmarks_5, - face_utils::FFHQ_512); - - std::vector crop_size = {512,512}; - cv::Mat box_mask = face_utils::create_static_box_mask(crop_size); - std::vector crop_mask_list; - crop_mask_list.emplace_back(box_mask); - - cv::Mat crop_image_rgb; - launch_bgr2rgb(crop_image,crop_image_rgb); - crop_image_rgb.convertTo(crop_image_rgb,CV_32FC3,1.f / 255.f); - crop_image_rgb.convertTo(crop_image_rgb,CV_32FC3,2.0f,-1.f); - + cv::Mat box_mask; std::vector input_vector; - trtcv::utils::transform::create_tensor(crop_image_rgb,input_vector,input_node_dims,trtcv::utils::transform::CHW); - auto end_warp = std::chrono::high_resolution_clock::now(); - std::chrono::duration fp_ms_warp = end_warp - start_warp; - std::cout << "FaceRestoration preprocess time: " << fp_ms_warp.count() << "ms" << std::endl; + // ---------------- preprocess (CPU): warp + bgr2rgb + normalize + build tensor ---------------- + { + LITE_CPU_SCOPE_OPT(prof, "preprocess"); + std::tie(crop_image, affine_matrix) = face_utils::warp_face_by_face_landmark_5( + face_swap_image, target_landmarks_5, face_utils::FFHQ_512); + std::vector crop_size = {512, 512}; + box_mask = face_utils::create_static_box_mask(crop_size); - // 记录时间 - auto start = std::chrono::high_resolution_clock::now(); - // 先不用拷贝了 处理完成再拷贝出来 类似于整个后处理放在GPU上完成 - cudaMemcpyAsync(buffers[0],input_vector.data(),1 * 3 * 512 * 512 * sizeof(float),cudaMemcpyHostToDevice,stream); - // 同步 - cudaStreamSynchronize(stream); - // 推理 - bool status = trt_context->enqueueV3(stream); + cv::Mat crop_image_rgb; + launch_bgr2rgb(crop_image, crop_image_rgb); + crop_image_rgb.convertTo(crop_image_rgb, CV_32FC3, 1.f / 255.f); + crop_image_rgb.convertTo(crop_image_rgb, CV_32FC3, 2.0f, -1.f); - if (!status) { - std::cerr << "Failed to inference" << std::endl; - return; + trtcv::utils::transform::create_tensor(crop_image_rgb, input_vector, input_node_dims, + trtcv::utils::transform::CHW); } - // 同步 - cudaStreamSynchronize(stream); - auto end = std::chrono::high_resolution_clock::now(); - std::chrono::duration fp_ms = end - start; - std::cout << "FaceRestoration Inference time: " << fp_ms.count() << "ms" << std::endl; - std::vector transposed_data(1 * 3 * 512 * 512); - -// std::vector transposed_data(1 * 3 * 512 * 512); - - // 记录时间 - auto start_postprocess = std::chrono::high_resolution_clock::now(); - // 这里buffer1就是输出了 - launch_face_restoration_postprocess( - static_cast(buffers[1]), - transposed_data.data(), - 3, - 512, - 512 - ); - - std::vector transposed_data_float(transposed_data.begin(), - transposed_data.end()); - - - // 获取输出 - std::vector output_vector(1 * 3 * 512 * 512); -// cudaMemcpyAsync(output_vector.data(),buffers[1],1 * 3 * 512 * 512 * sizeof(float),cudaMemcpyDeviceToHost,stream); - cudaStreamSynchronize(stream); - // 后处理 - int channel = 3; - int height = 512; - int width = 512; - - - cv::Mat mat(height, width, CV_32FC3, transposed_data_float.data()); - cv::cvtColor(mat, mat, cv::COLOR_RGB2BGR); - // 到这里为止基本不耗时 + // ---------------- inference (H2D + GPU + sync) ---------------- + { + LITE_CPU_SCOPE_OPT(prof, "infer(H2D+gpu)"); + cudaMemcpyAsync(buffers[0], input_vector.data(), 1 * 3 * 512 * 512 * sizeof(float), + cudaMemcpyHostToDevice, stream); + cudaStreamSynchronize(stream); + bool status = trt_context->enqueueV3(stream); + if (!status) { + std::cerr << "Failed to inference" << std::endl; + return cv::Mat(); + } + cudaStreamSynchronize(stream); + } - auto crop_mask = crop_mask_list[0]; - // 这里的paste_back 40ms左右 - cv::Mat paste_frame = launch_paste_back(ori_image,mat,crop_mask,affine_matrix); -// cv::Mat paste_frame = face_utils::paste_back(ori_image,mat,crop_mask,affine_matrix); - cv::Mat dst_image = face_utils::blend_frame(ori_image,paste_frame); - auto end_postprocess = std::chrono::high_resolution_clock::now(); - std::chrono::duration fp_ms_postprocess = end_postprocess - start_postprocess; - std::cout << "FaceRestoration postprocess time: " << fp_ms_postprocess.count() << "ms" << std::endl; + // ---------------- postprocess: transpose kernel + cvtColor + paste_back + blend ---------------- + cv::Mat dst_image; + { + LITE_CPU_SCOPE_OPT(prof, "postprocess"); + std::vector transposed_data(1 * 3 * 512 * 512); + launch_face_restoration_postprocess( + static_cast(buffers[1]), transposed_data.data(), 3, 512, 512); + std::vector transposed_data_float(transposed_data.begin(), transposed_data.end()); + cudaStreamSynchronize(stream); + + int height = 512, width = 512; + cv::Mat mat(height, width, CV_32FC3, transposed_data_float.data()); + cv::cvtColor(mat, mat, cv::COLOR_RGB2BGR); + + cv::Mat paste_frame; + { + // GPU fused version: inverse-mapping sampling + blend in one kernel, reused + // device buffers, pinned + async copies (replaces the CPU warpAffine bottleneck). + LITE_CPU_SCOPE_OPT(prof, " paste_back"); + paste_frame = paste_back_gpu_.paste_back(ori_image, mat, box_mask, affine_matrix, stream); + } + dst_image = face_utils::blend_frame(ori_image, paste_frame); + } - // 记录时间 - auto start_save = std::chrono::high_resolution_clock::now(); - cv::imwrite(face_enchaner_path,dst_image); - auto end_save = std::chrono::high_resolution_clock::now(); - std::chrono::duration fp_ms_save = end_save - start_save; - std::cout << "FaceRestoration save time: " << fp_ms_save.count() << "ms" << std::endl; + return dst_image; +} -} \ No newline at end of file +void TRTFaceFusionFaceRestoration::detect(cv::Mat &face_swap_image, + std::vector &target_landmarks_5, + const std::string &face_enchaner_path) { + cv::Mat dst_image = restore(face_swap_image, target_landmarks_5, nullptr); + if (!dst_image.empty()) + cv::imwrite(face_enchaner_path, dst_image); +} diff --git a/lite/trt/cv/trt_face_restoration.h b/lite/trt/cv/trt_face_restoration.h index 928e6b7e..8cdbeb77 100644 --- a/lite/trt/cv/trt_face_restoration.h +++ b/lite/trt/cv/trt_face_restoration.h @@ -11,15 +11,27 @@ #include "lite/trt/kernel/face_restoration_postprocess_manager.h" #include "lite/trt/kernel/bgr2rgb_manager.h" #include "lite/trt/kernel/paste_back_manager.h" + +// Forward declaration for benchmark timing; library passes nullptr by default (zero overhead) +namespace lite { namespace bench { class Profiler; } } + namespace trtcv{ class LITE_EXPORTS TRTFaceFusionFaceRestoration : BasicTRTHandler{ public: explicit TRTFaceFusionFaceRestoration(const std::string& _trt_model_path,unsigned int _num_threads = 1) : BasicTRTHandler(_trt_model_path,_num_threads){};; public: - // 这个是直接保存的 + // writes the restored frame straight to disk void detect(cv::Mat &face_swap_image,std::vector &target_landmarks_5 ,const std::string &face_enchaner_path); + // Core compute: returns the restored full frame without writing to disk; when prof is + // non-null, records per-stage timings (preprocess / infer / postprocess / paste_back). + cv::Mat restore(cv::Mat &face_swap_image, std::vector &target_landmarks_5, + lite::bench::Profiler *prof = nullptr); + + private: + PasteBackGPU paste_back_gpu_; // GPU fused paste_back, reuses device buffers + }; } diff --git a/lite/trt/kernel/paste_back.cu b/lite/trt/kernel/paste_back.cu index d44368db..2dbd63ff 100644 --- a/lite/trt/kernel/paste_back.cu +++ b/lite/trt/kernel/paste_back.cu @@ -21,3 +21,62 @@ __global__ void paste_back_kernel(const float* inverse_vision_frame, } } } + +// ---------------- fused inverse-mapping paste_back ---------------- +// Single-channel bilinear sample; out-of-range taps read 0 (matches cv::BORDER_CONSTANT 0) +__device__ __forceinline__ float bilinear1(const float* img, int W, int H, float u, float v) { + int x0 = floorf(u), y0 = floorf(v); + float fx = u - x0, fy = v - y0; + float a = (x0 >= 0 && x0 < W && y0 >= 0 && y0 < H) ? img[y0 * W + x0] : 0.f; + float b = (x0 + 1 >= 0 && x0 + 1 < W && y0 >= 0 && y0 < H) ? img[y0 * W + (x0 + 1)] : 0.f; + float c = (x0 >= 0 && x0 < W && y0 + 1 >= 0 && y0 + 1 < H) ? img[(y0 + 1) * W + x0] : 0.f; + float d = (x0 + 1 >= 0 && x0 + 1 < W && y0 + 1 >= 0 && y0 + 1 < H) ? img[(y0 + 1) * W + (x0 + 1)] : 0.f; + return (a * (1.f - fx) + b * fx) * (1.f - fy) + (c * (1.f - fx) + d * fx) * fy; +} + +// Three-channel (interleaved BGR) bilinear sample; out-of-range taps read 0 +__device__ __forceinline__ float bilinear3(const float* img, int W, int H, float u, float v, int ch) { + int x0 = floorf(u), y0 = floorf(v); + float fx = u - x0, fy = v - y0; + float a = (x0 >= 0 && x0 < W && y0 >= 0 && y0 < H) ? img[(y0 * W + x0) * 3 + ch] : 0.f; + float b = (x0 + 1 >= 0 && x0 + 1 < W && y0 >= 0 && y0 < H) ? img[(y0 * W + (x0 + 1)) * 3 + ch] : 0.f; + float c = (x0 >= 0 && x0 < W && y0 + 1 >= 0 && y0 + 1 < H) ? img[((y0 + 1) * W + x0) * 3 + ch] : 0.f; + float d = (x0 + 1 >= 0 && x0 + 1 < W && y0 + 1 >= 0 && y0 + 1 < H) ? img[((y0 + 1) * W + (x0 + 1)) * 3 + ch] : 0.f; + return (a * (1.f - fx) + b * fx) * (1.f - fy) + (c * (1.f - fx) + d * fx) * fy; +} + +__global__ void paste_back_fused_kernel(const unsigned char* temp, + const float* crop, + const float* mask, + const float* M, + unsigned char* out, + int W, int H, int Cw, int Ch) { + int x = blockIdx.x * blockDim.x + threadIdx.x; + int y = blockIdx.y * blockDim.y + threadIdx.y; + if (x >= W || y >= H) return; + + int oidx = (y * W + x) * 3; + + // original coords -> crop coords (use M directly, no inversion needed) + float u = M[0] * x + M[1] * y + M[2]; + float v = M[3] * x + M[4] * y + M[5]; + + // pixels outside the crop get mask=0 and just copy temp (matches CPU BORDER_CONSTANT 0) + float m = bilinear1(mask, Cw, Ch, u, v); + m = fminf(fmaxf(m, 0.f), 1.f); + + if (m > 0.f) { + float w = 1.f - m; +#pragma unroll + for (int c = 0; c < 3; ++c) { + float cs = bilinear3(crop, Cw, Ch, u, v, c); + float ts = static_cast(temp[oidx + c]); + float o = m * cs + w * ts; + out[oidx + c] = static_cast(fminf(fmaxf(o + 0.5f, 0.f), 255.f)); + } + } else { + out[oidx + 0] = temp[oidx + 0]; + out[oidx + 1] = temp[oidx + 1]; + out[oidx + 2] = temp[oidx + 2]; + } +} diff --git a/lite/trt/kernel/paste_back.cuh b/lite/trt/kernel/paste_back.cuh index 9b11b995..7ad0d03f 100644 --- a/lite/trt/kernel/paste_back.cuh +++ b/lite/trt/kernel/paste_back.cuh @@ -11,4 +11,14 @@ extern "C" __global__ void paste_back_kernel(const float* inverse_vision_frame, int height, int channels); +// Fused inverse-mapping paste_back: for each full-frame pixel, map to crop space with M, +// bilinearly sample and blend. temp/out are full-frame BGR uint8; crop is BGR float (0..255); +// mask is float (0..1); M is 6 floats (the original->crop 2x3 affine, row-major). +__global__ void paste_back_fused_kernel(const unsigned char* temp, + const float* crop, + const float* mask, + const float* M, + unsigned char* out, + int W, int H, int Cw, int Ch); + #endif // PASTE_BACK_CUH diff --git a/lite/trt/kernel/paste_back_manager.cpp b/lite/trt/kernel/paste_back_manager.cpp index f15f7a20..a6fa361f 100644 --- a/lite/trt/kernel/paste_back_manager.cpp +++ b/lite/trt/kernel/paste_back_manager.cpp @@ -1,33 +1,34 @@ #include "paste_back_manager.h" #include +#include cv::Mat launch_paste_back(const cv::Mat& temp_vision_frame, const cv::Mat& crop_vision_frame, const cv::Mat& crop_mask, const cv::Mat& affine_matrix) { - // 转换为float类型 + // convert to float cv::Mat temp_float, crop_float, mask_float; temp_vision_frame.convertTo(temp_float, CV_32F); crop_vision_frame.convertTo(crop_float, CV_32F); crop_mask.convertTo(mask_float, CV_32F); - // 获取仿射变换的逆矩阵 + // inverse of the affine transform cv::Mat inverse_matrix; cv::invertAffineTransform(affine_matrix, inverse_matrix); - // 获取目标尺寸 + // target (full-frame) size cv::Size temp_size(temp_vision_frame.cols, temp_vision_frame.rows); - // 对mask和crop_frame进行反向仿射变换 + // inverse-warp the mask and crop frame back to the full frame cv::Mat inverse_mask, inverse_vision_frame; cv::warpAffine(mask_float, inverse_mask, inverse_matrix, temp_size); cv::warpAffine(crop_float, inverse_vision_frame, inverse_matrix, temp_size); - // 阈值处理 + // clamp mask to [0, 1] cv::threshold(inverse_mask, inverse_mask, 1.0, 1.0, cv::THRESH_TRUNC); cv::threshold(inverse_mask, inverse_mask, 0.0, 0.0, cv::THRESH_TOZERO); - // 准备CUDA内存 + // allocate CUDA memory int width = temp_vision_frame.cols; int height = temp_vision_frame.rows; int channels = temp_vision_frame.channels(); @@ -40,35 +41,35 @@ cv::Mat launch_paste_back(const cv::Mat& temp_vision_frame, cudaMalloc(&d_inverse_mask, mask_size); cudaMalloc(&d_output, total_size); - // 复制数据到GPU + // copy data to GPU cudaMemcpy(d_inverse_vision_frame, inverse_vision_frame.ptr(), total_size, cudaMemcpyHostToDevice); cudaMemcpy(d_temp_frame, temp_float.ptr(), total_size, cudaMemcpyHostToDevice); cudaMemcpy(d_inverse_mask, inverse_mask.ptr(), mask_size, cudaMemcpyHostToDevice); - // 设置kernel参数 + // kernel launch config dim3 block(16, 16); dim3 grid((width + block.x - 1) / block.x, (height + block.y - 1) / block.y); - // 启动kernel + // launch kernel paste_back_kernel<<>>(d_inverse_vision_frame, d_temp_frame, d_inverse_mask, d_output, width, height, channels); - // 创建输出Mat + // output Mat cv::Mat result(height, width, CV_32FC3); - // 复制结果回主机 + // copy result back to host cudaMemcpy(result.ptr(), d_output, total_size, cudaMemcpyDeviceToHost); - // 清理GPU内存 + // free GPU memory cudaFree(d_inverse_vision_frame); cudaFree(d_temp_frame); cudaFree(d_inverse_mask); cudaFree(d_output); - // 如果需要，转换回原始类型 + // convert back to the original type if needed cv::Mat final_result; if(temp_vision_frame.type() != CV_32F) { result.convertTo(final_result, temp_vision_frame.type()); @@ -78,3 +79,91 @@ cv::Mat launch_paste_back(const cv::Mat& temp_vision_frame, return final_result; } + +// ============================ GPU fused version ============================ +PasteBackGPU::~PasteBackGPU() { + if (d_temp_) cudaFree(d_temp_); + if (d_out_) cudaFree(d_out_); + if (d_crop_) cudaFree(d_crop_); + if (d_mask_) cudaFree(d_mask_); + if (d_affine_) cudaFree(d_affine_); + if (h_temp_pinned_) cudaFreeHost(h_temp_pinned_); + if (h_out_pinned_) cudaFreeHost(h_out_pinned_); +} + +void PasteBackGPU::ensure_capacity(size_t temp_bytes, size_t crop_bytes, + size_t mask_bytes, size_t out_bytes) { + if (temp_bytes > cap_temp_) { + if (d_temp_) cudaFree(d_temp_); + if (h_temp_pinned_) cudaFreeHost(h_temp_pinned_); + cudaMalloc(&d_temp_, temp_bytes); + cudaMallocHost(&h_temp_pinned_, temp_bytes); + cap_temp_ = temp_bytes; + } + if (out_bytes > cap_out_) { + if (d_out_) cudaFree(d_out_); + if (h_out_pinned_) cudaFreeHost(h_out_pinned_); + cudaMalloc(&d_out_, out_bytes); + cudaMallocHost(&h_out_pinned_, out_bytes); + cap_out_ = out_bytes; + } + if (crop_bytes > cap_crop_) { + if (d_crop_) cudaFree(d_crop_); + cudaMalloc(&d_crop_, crop_bytes); + cap_crop_ = crop_bytes; + } + if (mask_bytes > cap_mask_) { + if (d_mask_) cudaFree(d_mask_); + cudaMalloc(&d_mask_, mask_bytes); + cap_mask_ = mask_bytes; + } + if (d_affine_ == nullptr) cudaMalloc(&d_affine_, 6 * sizeof(float)); +} + +cv::Mat PasteBackGPU::paste_back(const cv::Mat& temp_vision_frame, + const cv::Mat& crop_vision_frame, + const cv::Mat& crop_mask, + const cv::Mat& affine_matrix, + cudaStream_t stream) { + // normalize temp to a contiguous BGR uint8 frame + cv::Mat temp = temp_vision_frame; + if (temp.type() != CV_8UC3) temp.convertTo(temp, CV_8UC3); + if (!temp.isContinuous()) temp = temp.clone(); + + cv::Mat crop = crop_vision_frame.isContinuous() ? crop_vision_frame : crop_vision_frame.clone(); + cv::Mat mask = crop_mask.isContinuous() ? crop_mask : crop_mask.clone(); + + const int W = temp.cols, H = temp.rows; + const int Cw = crop.cols, Ch = crop.rows; + const size_t temp_bytes = static_cast(W) * H * 3; + const size_t out_bytes = temp_bytes; + const size_t crop_bytes = static_cast(Cw) * Ch * 3 * sizeof(float); + const size_t mask_bytes = static_cast(Cw) * Ch * sizeof(float); + + ensure_capacity(temp_bytes, crop_bytes, mask_bytes, out_bytes); + + // affine -> float[6] (estimateAffinePartial2D usually returns CV_64F) + cv::Mat M64; + affine_matrix.convertTo(M64, CV_64F); + float h_aff[6]; + for (int i = 0; i < 6; ++i) h_aff[i] = static_cast(M64.at(i / 3, i % 3)); + + // H2D (temp goes through a pinned staging buffer for true async transfer) + std::memcpy(h_temp_pinned_, temp.data, temp_bytes); + cudaMemcpyAsync(d_temp_, h_temp_pinned_, temp_bytes, cudaMemcpyHostToDevice, stream); + cudaMemcpyAsync(d_crop_, crop.ptr(), crop_bytes, cudaMemcpyHostToDevice, stream); + cudaMemcpyAsync(d_mask_, mask.ptr(), mask_bytes, cudaMemcpyHostToDevice, stream); + cudaMemcpyAsync(d_affine_, h_aff, 6 * sizeof(float), cudaMemcpyHostToDevice, stream); + + dim3 block(16, 16); + dim3 grid((W + block.x - 1) / block.x, (H + block.y - 1) / block.y); + paste_back_fused_kernel<<>>( + d_temp_, d_crop_, d_mask_, d_affine_, d_out_, W, H, Cw, Ch); + + cudaMemcpyAsync(h_out_pinned_, d_out_, out_bytes, cudaMemcpyDeviceToHost, stream); + cudaStreamSynchronize(stream); + + cv::Mat result(H, W, CV_8UC3); + std::memcpy(result.data, h_out_pinned_, out_bytes); + return result; +} diff --git a/lite/trt/kernel/paste_back_manager.h b/lite/trt/kernel/paste_back_manager.h index c0381e0a..14ac677f 100644 --- a/lite/trt/kernel/paste_back_manager.h +++ b/lite/trt/kernel/paste_back_manager.h @@ -4,9 +4,40 @@ #include "paste_back.cuh" #include +// Old CPU-heavy version (two full-frame warpAffine + per-call malloc/sync copies); kept for A/B. cv::Mat launch_paste_back(const cv::Mat& temp_vision_frame, const cv::Mat& crop_vision_frame, const cv::Mat& crop_mask, const cv::Mat& affine_matrix); +// GPU fused version: inverse-mapping sampling + blend entirely in the kernel, reused device +// buffers, pinned + async copies. Numerically equivalent to launch_paste_back; returns full-frame BGR uint8. +class PasteBackGPU { +public: + PasteBackGPU() = default; + ~PasteBackGPU(); + + PasteBackGPU(const PasteBackGPU&) = delete; + PasteBackGPU& operator=(const PasteBackGPU&) = delete; + + cv::Mat paste_back(const cv::Mat& temp_vision_frame, // CV_8UC3 (auto-converted otherwise) + const cv::Mat& crop_vision_frame, // CV_32FC3, 0..255 + const cv::Mat& crop_mask, // CV_32FC1, 0..1 + const cv::Mat& affine_matrix, // 2x3, original->crop + cudaStream_t stream = nullptr); + +private: + void ensure_capacity(size_t temp_bytes, size_t crop_bytes, + size_t mask_bytes, size_t out_bytes); + + unsigned char* d_temp_ = nullptr; + unsigned char* d_out_ = nullptr; + float* d_crop_ = nullptr; + float* d_mask_ = nullptr; + float* d_affine_ = nullptr; // 6 floats + unsigned char* h_temp_pinned_ = nullptr; + unsigned char* h_out_pinned_ = nullptr; + size_t cap_temp_ = 0, cap_crop_ = 0, cap_mask_ = 0, cap_out_ = 0; +}; + #endif // PASTE_BACK_MANAGER_H From 421c054c9f1e98f20eaeb278d68851499d35664e Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Sat, 30 May 2026 22:37:33 +0800 Subject: [PATCH 02/30] perf(trt): cache static mask + GPU-fuse face-restoration preprocess (20.9ms->17.7ms) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1 continued on the face-restoration stage. Per-stage profiling of preprocess showed two wins: (1) create_static_box_mask was rebuilt every frame (a large-kernel GaussianBlur, ~10ms) although it only depends on the fixed 512 crop size; (2) bgr2rgb + normalize + HWC->CHW ran on the CPU (~3.2ms) and then a separate H2D copied the tensor to the device. Mask is now built once and cached. bgr2rgb/normalize/CHW are fused into a single CUDA kernel that writes the normalized RGB CHW tensor straight into the inference input buffer, removing the per-frame mask rebuild, the CPU tensor build, and the separate H2D. Measured on RTX 4090 / TRT 10.1 fp32, gfpgan 512 (cumulative from the 78.2ms baseline): preprocess 14.4ms -> 1.3ms; end-to-end 30.8ms -> 17.7ms; 32.4 -> 56.6 FPS. Output unchanged (PSNR 59.5 dB vs the pre-change result). Per-file changes: - lite/trt/kernel/face_restoration_preprocess.cu, face_restoration_preprocess.cuh: new face_restoration_preprocess_kernel — one thread per crop pixel, reads interleaved BGR uint8, writes planar RGB float (CHW) normalized to [-1,1] (v/127.5 - 1). - lite/trt/kernel/face_restoration_preprocess_manager.cpp, face_restoration_preprocess_manager.h: new FaceRestorePreprocessGPU, owns reusable device + pinned staging buffers and launches the kernel writing directly into the inference input buffer. - lite/trt/cv/trt_face_restoration.h: add FaceRestorePreprocessGPU member, a cached box mask member, and the new preprocess-manager include. - lite/trt/cv/trt_face_restoration.cpp: cache the static box mask; replace CPU bgr2rgb/normalize/create_tensor with the fused GPU preprocess into buffers[0]; drop the now redundant H2D in the inference step; add per-substage profiler scopes under preprocess. Co-Authored-By: Claude Opus 4.8 --- lite/trt/cv/trt_face_restoration.cpp | 39 ++++++++++--------- lite/trt/cv/trt_face_restoration.h | 5 ++- .../trt/kernel/face_restoration_preprocess.cu | 20 ++++++++++ .../kernel/face_restoration_preprocess.cuh | 10 +++++ .../face_restoration_preprocess_manager.cpp | 37 ++++++++++++++++++ .../face_restoration_preprocess_manager.h | 29 ++++++++++++++ 6 files changed, 121 insertions(+), 19 deletions(-) create mode 100644 lite/trt/kernel/face_restoration_preprocess.cu create mode 100644 lite/trt/kernel/face_restoration_preprocess.cuh create mode 100644 lite/trt/kernel/face_restoration_preprocess_manager.cpp create mode 100644 lite/trt/kernel/face_restoration_preprocess_manager.h diff --git a/lite/trt/cv/trt_face_restoration.cpp b/lite/trt/cv/trt_face_restoration.cpp index 770da154..214190d8 100644 --- a/lite/trt/cv/trt_face_restoration.cpp +++ b/lite/trt/cv/trt_face_restoration.cpp @@ -17,32 +17,35 @@ cv::Mat TRTFaceFusionFaceRestoration::restore(cv::Mat &face_swap_image, cv::Mat crop_image; cv::Mat affine_matrix; cv::Mat box_mask; - std::vector input_vector; // ---------------- preprocess (CPU): warp + bgr2rgb + normalize + build tensor ---------------- { LITE_CPU_SCOPE_OPT(prof, "preprocess"); - std::tie(crop_image, affine_matrix) = face_utils::warp_face_by_face_landmark_5( - face_swap_image, target_landmarks_5, face_utils::FFHQ_512); - - std::vector crop_size = {512, 512}; - box_mask = face_utils::create_static_box_mask(crop_size); - - cv::Mat crop_image_rgb; - launch_bgr2rgb(crop_image, crop_image_rgb); - crop_image_rgb.convertTo(crop_image_rgb, CV_32FC3, 1.f / 255.f); - crop_image_rgb.convertTo(crop_image_rgb, CV_32FC3, 2.0f, -1.f); + { + LITE_CPU_SCOPE_OPT(prof, " warp"); + std::tie(crop_image, affine_matrix) = face_utils::warp_face_by_face_landmark_5( + face_swap_image, target_landmarks_5, face_utils::FFHQ_512); + } + { + // the static box mask only depends on the (fixed) 512 crop size, so build it + // once and reuse — it used to be rebuilt every frame (a large-kernel GaussianBlur, ~10ms) + LITE_CPU_SCOPE_OPT(prof, " mask"); + if (box_mask_cache_.empty()) + box_mask_cache_ = face_utils::create_static_box_mask({512, 512}); + box_mask = box_mask_cache_; + } - trtcv::utils::transform::create_tensor(crop_image_rgb, input_vector, input_node_dims, - trtcv::utils::transform::CHW); + { + // GPU fused: bgr2rgb + normalize + HWC->CHW written straight into the inference + // input buffer (buffers[0]) — also removes the separate H2D below. + LITE_CPU_SCOPE_OPT(prof, " to_chw(gpu)"); + preprocess_gpu_.run(crop_image, static_cast(buffers[0]), stream); + } } - // ---------------- inference (H2D + GPU + sync) ---------------- + // ---------------- inference (GPU + sync); input already in buffers[0] ---------------- { - LITE_CPU_SCOPE_OPT(prof, "infer(H2D+gpu)"); - cudaMemcpyAsync(buffers[0], input_vector.data(), 1 * 3 * 512 * 512 * sizeof(float), - cudaMemcpyHostToDevice, stream); - cudaStreamSynchronize(stream); + LITE_CPU_SCOPE_OPT(prof, "infer(gpu)"); bool status = trt_context->enqueueV3(stream); if (!status) { std::cerr << "Failed to inference" << std::endl; diff --git a/lite/trt/cv/trt_face_restoration.h b/lite/trt/cv/trt_face_restoration.h index 8cdbeb77..ea46ea55 100644 --- a/lite/trt/cv/trt_face_restoration.h +++ b/lite/trt/cv/trt_face_restoration.h @@ -9,6 +9,7 @@ #include "lite/trt/core/trt_config.h" #include "lite/ort/cv/face_utils.h" #include "lite/trt/kernel/face_restoration_postprocess_manager.h" +#include "lite/trt/kernel/face_restoration_preprocess_manager.h" #include "lite/trt/kernel/bgr2rgb_manager.h" #include "lite/trt/kernel/paste_back_manager.h" @@ -30,7 +31,9 @@ namespace trtcv{ lite::bench::Profiler *prof = nullptr); private: - PasteBackGPU paste_back_gpu_; // GPU fused paste_back, reuses device buffers + PasteBackGPU paste_back_gpu_; // GPU fused paste_back, reuses device buffers + FaceRestorePreprocessGPU preprocess_gpu_; // GPU fused bgr2rgb+normalize+CHW into input buffer + cv::Mat box_mask_cache_; // static box mask is size-only; compute once and reuse }; } diff --git a/lite/trt/kernel/face_restoration_preprocess.cu b/lite/trt/kernel/face_restoration_preprocess.cu new file mode 100644 index 00000000..66d94bb4 --- /dev/null +++ b/lite/trt/kernel/face_restoration_preprocess.cu @@ -0,0 +1,20 @@ +#include "face_restoration_preprocess.cuh" + +// One thread per crop pixel. Reads interleaved BGR uint8, writes planar RGB float (CHW), +// normalized to [-1, 1] (v/127.5 - 1). Channel mapping: R->plane0, G->plane1, B->plane2. +__global__ void face_restoration_preprocess_kernel(const unsigned char* crop, float* out, int H, int W) { + int x = blockIdx.x * blockDim.x + threadIdx.x; + int y = blockIdx.y * blockDim.y + threadIdx.y; + if (x >= W || y >= H) return; + + int i = (y * W + x) * 3; + float b = static_cast(crop[i + 0]); + float g = static_cast(crop[i + 1]); + float r = static_cast(crop[i + 2]); + + int plane = H * W; + int off = y * W + x; + out[0 * plane + off] = r / 127.5f - 1.f; + out[1 * plane + off] = g / 127.5f - 1.f; + out[2 * plane + off] = b / 127.5f - 1.f; +} diff --git a/lite/trt/kernel/face_restoration_preprocess.cuh b/lite/trt/kernel/face_restoration_preprocess.cuh new file mode 100644 index 00000000..794f9fb3 --- /dev/null +++ b/lite/trt/kernel/face_restoration_preprocess.cuh @@ -0,0 +1,10 @@ +#ifndef FACE_RESTORATION_PREPROCESS_CUH +#define FACE_RESTORATION_PREPROCESS_CUH + +#include + +// Fused face-restoration preprocess: takes the HxW interleaved BGR uint8 crop and writes a +// CHW (3,H,W) float tensor that is RGB and normalized by v/127.5 - 1 (i.e. (v/255)*2 - 1). +__global__ void face_restoration_preprocess_kernel(const unsigned char* crop, float* out, int H, int W); + +#endif // FACE_RESTORATION_PREPROCESS_CUH diff --git a/lite/trt/kernel/face_restoration_preprocess_manager.cpp b/lite/trt/kernel/face_restoration_preprocess_manager.cpp new file mode 100644 index 00000000..e0f46af1 --- /dev/null +++ b/lite/trt/kernel/face_restoration_preprocess_manager.cpp @@ -0,0 +1,37 @@ +#include "face_restoration_preprocess_manager.h" +#include +#include + +FaceRestorePreprocessGPU::~FaceRestorePreprocessGPU() { + if (d_crop_) cudaFree(d_crop_); + if (h_pinned_) cudaFreeHost(h_pinned_); +} + +void FaceRestorePreprocessGPU::ensure_capacity(size_t bytes) { + if (bytes > cap_) { + if (d_crop_) cudaFree(d_crop_); + if (h_pinned_) cudaFreeHost(h_pinned_); + cudaMalloc(&d_crop_, bytes); + cudaMallocHost(&h_pinned_, bytes); + cap_ = bytes; + } +} + +void FaceRestorePreprocessGPU::run(const cv::Mat& crop_bgr_u8, float* d_out, cudaStream_t stream) { + cv::Mat c = crop_bgr_u8; + if (c.type() != CV_8UC3) c.convertTo(c, CV_8UC3); + if (!c.isContinuous()) c = c.clone(); + + const int H = c.rows, W = c.cols; + const size_t bytes = static_cast(H) * W * 3; + ensure_capacity(bytes); + + std::memcpy(h_pinned_, c.data, bytes); + cudaMemcpyAsync(d_crop_, h_pinned_, bytes, cudaMemcpyHostToDevice, stream); + + dim3 block(16, 16); + dim3 grid((W + block.x - 1) / block.x, (H + block.y - 1) / block.y); + face_restoration_preprocess_kernel<<>>(d_crop_, d_out, H, W); + + cudaStreamSynchronize(stream); +} diff --git a/lite/trt/kernel/face_restoration_preprocess_manager.h b/lite/trt/kernel/face_restoration_preprocess_manager.h new file mode 100644 index 00000000..9d5a076f --- /dev/null +++ b/lite/trt/kernel/face_restoration_preprocess_manager.h @@ -0,0 +1,29 @@ +#ifndef FACE_RESTORATION_PREPROCESS_MANAGER_H +#define FACE_RESTORATION_PREPROCESS_MANAGER_H + +#include "face_restoration_preprocess.cuh" +#include + +// Fuses bgr2rgb + normalize + HWC->CHW into one kernel and writes the normalized RGB CHW +// tensor straight into the device inference input buffer (no D2H, no separate H2D of the +// float tensor). Reuses device + pinned staging buffers across calls. +class FaceRestorePreprocessGPU { +public: + FaceRestorePreprocessGPU() = default; + ~FaceRestorePreprocessGPU(); + + FaceRestorePreprocessGPU(const FaceRestorePreprocessGPU&) = delete; + FaceRestorePreprocessGPU& operator=(const FaceRestorePreprocessGPU&) = delete; + + // crop_bgr_u8: CV_8UC3 (e.g. 512x512). d_out: device float CHW buffer (the inference input). + void run(const cv::Mat& crop_bgr_u8, float* d_out, cudaStream_t stream = nullptr); + +private: + void ensure_capacity(size_t bytes); + + unsigned char* d_crop_ = nullptr; + unsigned char* h_pinned_ = nullptr; + size_t cap_ = 0; +}; + +#endif // FACE_RESTORATION_PREPROCESS_MANAGER_H From 7c122bbc0b8a0a6dbd5d31d134e19e39a295f15a Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Sat, 30 May 2026 22:38:46 +0800 Subject: [PATCH 03/30] docs(readme): add Benchmark section for the GPU-optimized face-restoration stage Document the FaceFusion face-restoration (GFPGAN 1.4) TensorRT optimization with a stage-by-stage baseline vs optimized table (RTX 4090, TRT 10.1, FP32, 512x512): end-to-end 78.2ms -> 17.7ms (4.4x), 12.8 -> 56.6 FPS. Includes setup/methodology (warmup, iterations, p50, compute-only) and how to reproduce via lite_face_restoration_bench. Co-Authored-By: Claude Opus 4.8 --- README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/README.md b/README.md index 5358611a..44e88ef4 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,28 @@ Run `bash ./build.sh tensorrt` to build lite.ai.toolkit with TensorRT support, a auto *yolov5 = new lite::trt::cv::detection::YOLOV5(engine_path); ``` +## ⚡ Benchmark 🔥 +
+ +End-to-end GPU optimization of the **FaceFusion face-restoration** stage (GFPGAN 1.4) on **TensorRT**. The idea is simple: keep the whole stage on the GPU. Pre/post-processing that used to run on the CPU (affine warp glue, `bgr2rgb`, normalize, HWC→CHW, paste-back) is rewritten as fused CUDA kernels with reused device buffers and pinned + async copies, so the stage spends its time on real inference instead of host glue and `cudaMalloc`/sync round-trips. + +> **Setup**: RTX 4090 · TensorRT 10.1 · CUDA 12.4 · GFPGAN 1.4 (512×512, FP32) · compute-only (no disk I/O) · 10 warmup + 50 iterations, median (p50). Reproduce with [`lite_face_restoration_bench`](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_face_restoration_bench.cpp). + +| Stage | Baseline (ms) | Optimized (ms) | Speedup | +|:--|:--:|:--:|:--:| +| preprocess (warp + bgr2rgb + normalize + tensor) | 14.49 | 1.25 | **11.6×** | +| inference (TensorRT) | 11.30 | 10.79 | 1.05× | +| postprocess (incl. paste-back) | 52.02 | 5.32 | **9.8×** | +|   └ paste-back | 39.07 | 2.34 | **16.7×** | +| **End-to-end** | **78.17** | **17.66** | **4.4×** | +| **Throughput** | **12.8 FPS** | **56.6 FPS** | **4.4×** | + +**What changed** +- **paste-back** — replaced two full-frame CPU `cv::warpAffine` plus per-call `cudaMalloc`/synchronous copies with a single inverse-mapping CUDA kernel (reused buffers, pinned + async). Numerically equivalent to the CPU path (max |diff| = 2/255). +- **preprocess** — the static box mask was rebuilt every frame (a large-kernel Gaussian blur); it only depends on the crop size, so it is now built once and cached. `bgr2rgb + normalize + HWC→CHW` are fused into one kernel that writes straight into the inference input buffer, removing the CPU tensor build and a host→device copy. + +After this, inference itself is ~60% of the stage — FP16 / mixed-precision is the next lever (WIP). + ## Quick Setup 👀 To quickly setup `lite.ai.toolkit`, you can follow the `CMakeLists.txt` listed as belows. 👇👀 From 965e37b4a877c08f46c42cbe5ddce404192c2b5c Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Sat, 30 May 2026 22:48:35 +0800 Subject: [PATCH 04/30] docs(readme): reframe Benchmark as a pipeline-wide GPU-optimization effort Position the work as a repeatable playbook (built-in profiler + moving CPU pre/post into fused CUDA kernels) applied across the whole FaceFusion pipeline, with a per-stage optimization-status table (detect/landmarks/recognize/swap = WIP, restoration done, FP16 next). Keep the GFPGAN face-restoration result as a worked deep-dive rather than the whole story. Co-Authored-By: Claude Opus 4.8 --- README.md | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 44e88ef4..ace3f5d2 100644 --- a/README.md +++ b/README.md @@ -104,9 +104,21 @@ auto *yolov5 = new lite::trt::cv::detection::YOLOV5(engine_path); ## ⚡ Benchmark 🔥
-End-to-end GPU optimization of the **FaceFusion face-restoration** stage (GFPGAN 1.4) on **TensorRT**. The idea is simple: keep the whole stage on the GPU. Pre/post-processing that used to run on the CPU (affine warp glue, `bgr2rgb`, normalize, HWC→CHW, paste-back) is rewritten as fused CUDA kernels with reused device buffers and pinned + async copies, so the stage spends its time on real inference instead of host glue and `cudaMalloc`/sync round-trips. +We are driving the **TensorRT path toward extreme GPU inference** with a repeatable playbook: profile every stage with a built-in, backend-agnostic harness ([`lite/bench/profiler.h`](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/lite/bench/profiler.h)), find where the time actually goes, then move the pre/post-processing that traditionally runs on the CPU (affine warp, color convert, normalize, tensor layout, paste-back, NMS …) into **fused CUDA kernels** with reused device buffers and pinned + async copies. The goal: every stage spends its time on real inference, not host glue or `cudaMalloc`/sync round-trips. All numbers below are reproducible via the `lite_*_bench` binaries (10 warmup + 50 iterations, median p50, compute-only). -> **Setup**: RTX 4090 · TensorRT 10.1 · CUDA 12.4 · GFPGAN 1.4 (512×512, FP32) · compute-only (no disk I/O) · 10 warmup + 50 iterations, median (p50). Reproduce with [`lite_face_restoration_bench`](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_face_restoration_bench.cpp). +### FaceFusion pipeline — GPU optimization status +The flagship target is the full face-swap pipeline. Each stage is profiled and moved fully onto the GPU, one at a time. *RTX 4090 · TensorRT 10.1 · CUDA 12.4.* + +| Stage (model) | GPU-fused pre/post | Result | +|:--|:--:|:--| +| Face detect · YOLOv8-face | 🚧 | — | +| 68 landmarks · 2DFAN4 | 🚧 | — | +| Face recognize · ArcFace | 🚧 | — | +| Face swap · InSwapper | 🚧 | — | +| **Face restoration · GFPGAN 1.4** | ✅ | **78.2 → 17.7 ms · 12.8 → 56.6 FPS (4.4×)** | +| FP16 / mixed-precision (all stages) | 🚧 | — | + +### Deep dive: GFPGAN face restoration (512×512, FP32) | Stage | Baseline (ms) | Optimized (ms) | Speedup | |:--|:--:|:--:|:--:| @@ -117,11 +129,10 @@ End-to-end GPU optimization of the **FaceFusion face-restoration** stage (GFPGAN | **End-to-end** | **78.17** | **17.66** | **4.4×** | | **Throughput** | **12.8 FPS** | **56.6 FPS** | **4.4×** | -**What changed** - **paste-back** — replaced two full-frame CPU `cv::warpAffine` plus per-call `cudaMalloc`/synchronous copies with a single inverse-mapping CUDA kernel (reused buffers, pinned + async). Numerically equivalent to the CPU path (max |diff| = 2/255). -- **preprocess** — the static box mask was rebuilt every frame (a large-kernel Gaussian blur); it only depends on the crop size, so it is now built once and cached. `bgr2rgb + normalize + HWC→CHW` are fused into one kernel that writes straight into the inference input buffer, removing the CPU tensor build and a host→device copy. +- **preprocess** — the static box mask was rebuilt every frame (a large-kernel Gaussian blur), although it only depends on the crop size → now built once and cached. `bgr2rgb + normalize + HWC→CHW` are fused into one kernel that writes straight into the inference input buffer, removing the CPU tensor build and a host→device copy. -After this, inference itself is ~60% of the stage — FP16 / mixed-precision is the next lever (WIP). +With pre/post-processing off the critical path, inference is now ~60% of the stage — so FP16 / mixed-precision is the next lever across the whole pipeline. ## Quick Setup 👀 From deb9f94152b3ba28cad2ad612e6698009b3f1678 Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Sat, 30 May 2026 22:52:24 +0800 Subject: [PATCH 05/30] docs(readme): make Benchmark a flat per-algorithm table One row per algorithm (before / after / speedup / what changed) so the log reads as a growing list of optimized algorithms rather than a single case study; FaceFusion face restoration is the first entry, further algorithms are placeholders. Move the GFPGAN per-stage breakdown into a collapsible details block. Co-Authored-By: Claude Opus 4.8 --- README.md | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index ace3f5d2..47eb938e 100644 --- a/README.md +++ b/README.md @@ -104,21 +104,18 @@ auto *yolov5 = new lite::trt::cv::detection::YOLOV5(engine_path); ## ⚡ Benchmark 🔥
-We are driving the **TensorRT path toward extreme GPU inference** with a repeatable playbook: profile every stage with a built-in, backend-agnostic harness ([`lite/bench/profiler.h`](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/lite/bench/profiler.h)), find where the time actually goes, then move the pre/post-processing that traditionally runs on the CPU (affine warp, color convert, normalize, tensor layout, paste-back, NMS …) into **fused CUDA kernels** with reused device buffers and pinned + async copies. The goal: every stage spends its time on real inference, not host glue or `cudaMalloc`/sync round-trips. All numbers below are reproducible via the `lite_*_bench` binaries (10 warmup + 50 iterations, median p50, compute-only). +GPU-inference optimization log. For each algorithm we profile it with a built-in, backend-agnostic harness ([`lite/bench/profiler.h`](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/lite/bench/profiler.h)), then move the CPU pre/post-processing (affine warp, color convert, normalize, tensor layout, paste-back, NMS …) into **fused CUDA kernels** with reused device buffers and pinned + async copies, so the algorithm spends its time on real inference instead of host glue and `cudaMalloc`/sync round-trips. All numbers are **RTX 4090 · TensorRT 10.1 · CUDA 12.4**, median (p50), compute-only, reproducible via the `lite_*_bench` binaries. -### FaceFusion pipeline — GPU optimization status -The flagship target is the full face-swap pipeline. Each stage is profiled and moved fully onto the GPU, one at a time. *RTX 4090 · TensorRT 10.1 · CUDA 12.4.* +| Algorithm | Before | After | Speedup | What changed | +|:--|:--:|:--:|:--:|:--| +| **FaceFusion · face restoration (GFPGAN 1.4)** | 78.2 ms
(12.8 FPS) | **17.7 ms
(56.6 FPS)** | **4.4×** | inverse-mapping paste-back kernel (replaces 2× CPU `warpAffine` + per-frame `cudaMalloc`); cached static mask; fused `bgr2rgb+normalize+CHW` straight into the input buffer | +| FaceFusion · face detect (YOLOv8-face) | 🚧 | 🚧 | — | bbox decode + NMS → CUDA | +| FaceFusion · 68 landmarks (2DFAN4) | 🚧 | 🚧 | — | warp + preprocess → CUDA | +| FaceFusion · face swap (InSwapper) | 🚧 | 🚧 | — | warp + paste → CUDA | +| FP16 / mixed-precision | 🚧 | 🚧 | — | layer-pinned style convs (keep sensitive layers FP32) | -| Stage (model) | GPU-fused pre/post | Result | -|:--|:--:|:--| -| Face detect · YOLOv8-face | 🚧 | — | -| 68 landmarks · 2DFAN4 | 🚧 | — | -| Face recognize · ArcFace | 🚧 | — | -| Face swap · InSwapper | 🚧 | — | -| **Face restoration · GFPGAN 1.4** | ✅ | **78.2 → 17.7 ms · 12.8 → 56.6 FPS (4.4×)** | -| FP16 / mixed-precision (all stages) | 🚧 | — | - -### Deep dive: GFPGAN face restoration (512×512, FP32) +
+FaceFusion · face restoration — per-stage breakdown | Stage | Baseline (ms) | Optimized (ms) | Speedup | |:--|:--:|:--:|:--:| @@ -127,12 +124,10 @@ The flagship target is the full face-swap pipeline. Each stage is profiled and m | postprocess (incl. paste-back) | 52.02 | 5.32 | **9.8×** | |   └ paste-back | 39.07 | 2.34 | **16.7×** | | **End-to-end** | **78.17** | **17.66** | **4.4×** | -| **Throughput** | **12.8 FPS** | **56.6 FPS** | **4.4×** | -- **paste-back** — replaced two full-frame CPU `cv::warpAffine` plus per-call `cudaMalloc`/synchronous copies with a single inverse-mapping CUDA kernel (reused buffers, pinned + async). Numerically equivalent to the CPU path (max |diff| = 2/255). -- **preprocess** — the static box mask was rebuilt every frame (a large-kernel Gaussian blur), although it only depends on the crop size → now built once and cached. `bgr2rgb + normalize + HWC→CHW` are fused into one kernel that writes straight into the inference input buffer, removing the CPU tensor build and a host→device copy. +paste-back is numerically equivalent to the CPU path (max |diff| = 2/255). The static box mask used to be rebuilt every frame (a large-kernel Gaussian blur) although it only depends on the crop size. With pre/post off the critical path, inference is now ~60% of the stage — FP16 is the next lever. -With pre/post-processing off the critical path, inference is now ~60% of the stage — so FP16 / mixed-precision is the next lever across the whole pipeline. +
## Quick Setup 👀 From 68c2cd559a4344d09a2c10520a11520219444509 Mon Sep 17 00:00:00 2001 From: DefTruth <31974251+DefTruth@users.noreply.github.com> Date: Sun, 31 May 2026 14:49:46 +0800 Subject: [PATCH 06/30] Update README.md --- README.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/README.md b/README.md index 47eb938e..592e0c9b 100644 --- a/README.md +++ b/README.md @@ -35,11 +35,7 @@ ## 📖 News 🔥🔥
-- [2026/03] Cache-DiT **[🎉v1.3.0](https://github.com/vipshop/cache-dit)** release is ready, the major updates including: [Ring](https://cache-dit.readthedocs.io/en/latest/user_guide/CONTEXT_PARALLEL) Attention w/ [batched P2P](https://cache-dit.readthedocs.io/en/latest/user_guide/CONTEXT_PARALLEL), [USP](https://cache-dit.readthedocs.io/en/latest/user_guide/CONTEXT_PARALLEL/) (Hybrid Ring and Ulysses), Hybrid 2D and 3D Parallelism (💥[USP + TP](https://cache-dit.readthedocs.io/en/latest/user_guide/HYBRID_PARALLEL/)), VAE-P Comm overhead reduce. - -![arch](https://github.com/vipshop/cache-dit/raw/main/assets/arch_v2.png) - -- Most of my time now is focused on **LLM/VLM** Inference. Please check 📖[Awesome-LLM-Inference](https://github.com/xlite-dev/Awesome-LLM-Inference) ![](https://img.shields.io/github/stars/xlite-dev/Awesome-LLM-Inference.svg?style=social) and 📖[LeetCUDA](https://github.com/xlite-dev/LeetCUDA) ![](https://img.shields.io/github/stars/xlite-dev/LeetCUDA.svg?style=social) for more details. Now, [lite.ai.toolkit](https://github.com/xlite-dev/lite.ai.toolkit) ![](https://img.shields.io/github/stars/xlite-dev/lite.ai.toolkit.svg?style=social) is mainly maintained by 🎉[@wangzijian1010](https://github.com/wangzijian1010). +- Now, [lite.ai.toolkit](https://github.com/xlite-dev/lite.ai.toolkit) ![](https://img.shields.io/github/stars/xlite-dev/lite.ai.toolkit.svg?style=social) is mainly maintained by 🎉[@wangzijian1010](https://github.com/wangzijian1010). Many thanks ~ 🎉🎉 ## Citations 🎉🎉 ```BibTeX From e07415ccb708ed94b6528a246a35f9541f9e61c5 Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Sun, 31 May 2026 16:43:57 +0800 Subject: [PATCH 07/30] refactor: drop MNN/NCNN/TNN backends, keep ORT (reference) + TRT (product) MNN/NCNN/TNN were thin per-model wrappers with high maintenance surface and no differentiated value. They are frozen on branch/tag `v0.2-all-backends` and removed from the active line. ONNXRuntime is retained deliberately: it is the numerical-reference oracle for CPU/GPU equivalence checks and the only backend that can build the test suite (CMake forces ENABLE_TEST=OFF without it). TensorRT remains the maintained high-performance product line. - delete lite/{mnn,ncnn,tnn}/ (480 files) + cmake/{MNN,ncnn,TNN}.cmake + docs/hub/*.{mnn,ncnn,tnn}.md - CMakeLists.txt: drop ENABLE_{MNN,NCNN,TNN} options; require ORT - cmake/utils.cmake + lite.ai.toolkit.cmake.in: drop backend wiring - lite/config.h.in: drop cmakedefines - lite/models.h: drop include/typedef blocks; alias resolves to ORT Example backend stubs stay guarded by #ifdef and compile out. Not yet build-verified (no GPU host available); pending remote 4090 sync via scripts/remote.sh. Co-Authored-By: Claude Opus 4.8 --- CMakeLists.txt | 16 +- cmake/MNN.cmake | 38 - cmake/TNN.cmake | 27 - cmake/lite.ai.toolkit.cmake.in | 34 +- cmake/ncnn.cmake | 27 - cmake/utils.cmake | 18 - docs/hub/lite.ai.toolkit.hub.mnn.md | 333 ------- docs/hub/lite.ai.toolkit.hub.ncnn.md | 271 ------ docs/hub/lite.ai.toolkit.hub.tnn.md | 311 ------- lite/config.h.in | 3 - lite/mnn/.gitignore | 0 lite/mnn/core/mnn_config.h | 20 - lite/mnn/core/mnn_core.h | 114 --- lite/mnn/core/mnn_defs.h | 24 - lite/mnn/core/mnn_handler.cpp | 90 -- lite/mnn/core/mnn_handler.h | 55 -- lite/mnn/core/mnn_types.h | 15 - lite/mnn/core/mnn_utils.cpp | 5 - lite/mnn/core/mnn_utils.h | 13 - lite/mnn/cv/mnn_age_googlenet.cpp | 62 -- lite/mnn/cv/mnn_age_googlenet.h | 42 - lite/mnn/cv/mnn_backgroundmattingv2.cpp | 217 ----- lite/mnn/cv/mnn_backgroundmattingv2.h | 91 -- lite/mnn/cv/mnn_cava_combined_face.cpp | 57 -- lite/mnn/cv/mnn_cava_combined_face.h | 34 - lite/mnn/cv/mnn_cava_ghost_arcface.cpp | 58 -- lite/mnn/cv/mnn_cava_ghost_arcface.h | 33 - lite/mnn/cv/mnn_center_loss_face.cpp | 58 -- lite/mnn/cv/mnn_center_loss_face.h | 32 - lite/mnn/cv/mnn_colorizer.cpp | 110 --- lite/mnn/cv/mnn_colorizer.h | 29 - lite/mnn/cv/mnn_deeplabv3_resnet101.cpp | 166 ---- lite/mnn/cv/mnn_deeplabv3_resnet101.h | 69 -- lite/mnn/cv/mnn_densenet.cpp | 68 -- lite/mnn/cv/mnn_densenet.h | 409 --------- lite/mnn/cv/mnn_efficient_emotion7.cpp | 60 -- lite/mnn/cv/mnn_efficient_emotion7.h | 36 - lite/mnn/cv/mnn_efficient_emotion8.cpp | 60 -- lite/mnn/cv/mnn_efficient_emotion8.h | 36 - lite/mnn/cv/mnn_efficientnet_lite4.cpp | 78 -- lite/mnn/cv/mnn_efficientnet_lite4.h | 407 --------- lite/mnn/cv/mnn_emotion_ferplus.cpp | 60 -- lite/mnn/cv/mnn_emotion_ferplus.h | 37 - lite/mnn/cv/mnn_face_hair_seg.cpp | 101 --- lite/mnn/cv/mnn_face_hair_seg.h | 37 - lite/mnn/cv/mnn_face_landmarks_1000.cpp | 68 -- lite/mnn/cv/mnn_face_landmarks_1000.h | 32 - lite/mnn/cv/mnn_face_parsing_bisenet.cpp | 182 ---- lite/mnn/cv/mnn_face_parsing_bisenet.h | 37 - lite/mnn/cv/mnn_faceboxes.cpp | 244 ------ lite/mnn/cv/mnn_faceboxes.h | 70 -- lite/mnn/cv/mnn_faceboxesv2.cpp | 208 ----- lite/mnn/cv/mnn_faceboxesv2.h | 71 -- lite/mnn/cv/mnn_facenet.cpp | 58 -- lite/mnn/cv/mnn_facenet.h | 32 - lite/mnn/cv/mnn_fast_portrait_seg.cpp | 141 --- lite/mnn/cv/mnn_fast_portrait_seg.h | 55 -- lite/mnn/cv/mnn_fast_style_transfer.cpp | 69 -- lite/mnn/cv/mnn_fast_style_transfer.h | 33 - lite/mnn/cv/mnn_fcn_resnet101.cpp | 167 ---- lite/mnn/cv/mnn_fcn_resnet101.h | 69 -- lite/mnn/cv/mnn_female_photo2cartoon.cpp | 133 --- lite/mnn/cv/mnn_female_photo2cartoon.h | 35 - lite/mnn/cv/mnn_focal_arcface.cpp | 58 -- lite/mnn/cv/mnn_focal_arcface.h | 33 - lite/mnn/cv/mnn_focal_asia_arcface.cpp | 58 -- lite/mnn/cv/mnn_focal_asia_arcface.h | 33 - lite/mnn/cv/mnn_fsanet.cpp | 66 -- lite/mnn/cv/mnn_fsanet.h | 34 - lite/mnn/cv/mnn_gender_googlenet.cpp | 60 -- lite/mnn/cv/mnn_gender_googlenet.h | 33 - lite/mnn/cv/mnn_ghostnet.cpp | 68 -- lite/mnn/cv/mnn_ghostnet.h | 409 --------- lite/mnn/cv/mnn_glint_arcface.cpp | 58 -- lite/mnn/cv/mnn_glint_arcface.h | 32 - lite/mnn/cv/mnn_glint_cosface.cpp | 58 -- lite/mnn/cv/mnn_glint_cosface.h | 33 - lite/mnn/cv/mnn_glint_partial_fc.cpp | 58 -- lite/mnn/cv/mnn_glint_partial_fc.h | 33 - lite/mnn/cv/mnn_hair_seg.cpp | 85 -- lite/mnn/cv/mnn_hair_seg.h | 38 - lite/mnn/cv/mnn_hdrdnet.cpp | 68 -- lite/mnn/cv/mnn_hdrdnet.h | 409 --------- lite/mnn/cv/mnn_head_seg.cpp | 102 --- lite/mnn/cv/mnn_head_seg.h | 46 - lite/mnn/cv/mnn_ibnnet.cpp | 68 -- lite/mnn/cv/mnn_ibnnet.h | 409 --------- lite/mnn/cv/mnn_insectdet.cpp | 159 ---- lite/mnn/cv/mnn_insectdet.h | 62 -- lite/mnn/cv/mnn_insectid.cpp | 68 -- lite/mnn/cv/mnn_insectid.h | 372 -------- lite/mnn/cv/mnn_mg_matting.cpp | 381 -------- lite/mnn/cv/mnn_mg_matting.h | 90 -- lite/mnn/cv/mnn_mobile_emotion7.cpp | 81 -- lite/mnn/cv/mnn_mobile_emotion7.h | 36 - lite/mnn/cv/mnn_mobile_facenet.cpp | 58 -- lite/mnn/cv/mnn_mobile_facenet.h | 33 - lite/mnn/cv/mnn_mobile_hair_seg.cpp | 85 -- lite/mnn/cv/mnn_mobile_hair_seg.h | 37 - lite/mnn/cv/mnn_mobile_human_matting.cpp | 115 --- lite/mnn/cv/mnn_mobile_human_matting.h | 39 - lite/mnn/cv/mnn_mobilenetv2.cpp | 68 -- lite/mnn/cv/mnn_mobilenetv2.h | 410 --------- lite/mnn/cv/mnn_mobilenetv2_68.cpp | 68 -- lite/mnn/cv/mnn_mobilenetv2_68.h | 32 - lite/mnn/cv/mnn_mobilenetv2_se_68.cpp | 66 -- lite/mnn/cv/mnn_mobilenetv2_se_68.h | 33 - lite/mnn/cv/mnn_mobilese_focal_face.cpp | 57 -- lite/mnn/cv/mnn_mobilese_focal_face.h | 33 - lite/mnn/cv/mnn_modnet.cpp | 116 --- lite/mnn/cv/mnn_modnet.h | 38 - lite/mnn/cv/mnn_nanodet.cpp | 249 ------ lite/mnn/cv/mnn_nanodet.h | 109 --- lite/mnn/cv/mnn_nanodet_efficientnet_lite.cpp | 250 ------ lite/mnn/cv/mnn_nanodet_efficientnet_lite.h | 108 --- lite/mnn/cv/mnn_nanodet_plus.cpp | 224 ----- lite/mnn/cv/mnn_nanodet_plus.h | 99 --- lite/mnn/cv/mnn_pfld.cpp | 83 -- lite/mnn/cv/mnn_pfld.h | 32 - lite/mnn/cv/mnn_pfld68.cpp | 67 -- lite/mnn/cv/mnn_pfld68.h | 33 - lite/mnn/cv/mnn_pfld98.cpp | 66 -- lite/mnn/cv/mnn_pfld98.h | 32 - lite/mnn/cv/mnn_pipnet19.cpp | 206 ----- lite/mnn/cv/mnn_pipnet19.h | 66 -- lite/mnn/cv/mnn_pipnet29.cpp | 206 ----- lite/mnn/cv/mnn_pipnet29.h | 78 -- lite/mnn/cv/mnn_pipnet68.cpp | 206 ----- lite/mnn/cv/mnn_pipnet68.h | 127 --- lite/mnn/cv/mnn_pipnet98.cpp | 206 ----- lite/mnn/cv/mnn_pipnet98.h | 135 --- lite/mnn/cv/mnn_plantid.cpp | 68 -- lite/mnn/cv/mnn_plantid.h | 816 ----------------- lite/mnn/cv/mnn_portrait_seg_extremec3net.cpp | 135 --- lite/mnn/cv/mnn_portrait_seg_extremec3net.h | 55 -- lite/mnn/cv/mnn_portrait_seg_sinet.cpp | 142 --- lite/mnn/cv/mnn_portrait_seg_sinet.h | 55 -- lite/mnn/cv/mnn_resnet.cpp | 68 -- lite/mnn/cv/mnn_resnet.h | 409 --------- lite/mnn/cv/mnn_resnext.cpp | 68 -- lite/mnn/cv/mnn_resnext.h | 409 --------- lite/mnn/cv/mnn_retinaface.cpp | 179 ---- lite/mnn/cv/mnn_retinaface.h | 70 -- lite/mnn/cv/mnn_rexnet_emotion7.cpp | 60 -- lite/mnn/cv/mnn_rexnet_emotion7.h | 35 - lite/mnn/cv/mnn_rvm.cpp | 346 -------- lite/mnn/cv/mnn_rvm.h | 201 ----- lite/mnn/cv/mnn_scrfd.cpp | 415 --------- lite/mnn/cv/mnn_scrfd.h | 106 --- lite/mnn/cv/mnn_shufflenetv2.cpp | 68 -- lite/mnn/cv/mnn_shufflenetv2.h | 410 --------- lite/mnn/cv/mnn_sphere_face.cpp | 58 -- lite/mnn/cv/mnn_sphere_face.h | 33 - lite/mnn/cv/mnn_ssrnet.cpp | 59 -- lite/mnn/cv/mnn_ssrnet.h | 35 - lite/mnn/cv/mnn_subpixel_cnn.cpp | 87 -- lite/mnn/cv/mnn_subpixel_cnn.h | 29 - lite/mnn/cv/mnn_tencent_cifp_face.cpp | 58 -- lite/mnn/cv/mnn_tencent_cifp_face.h | 33 - lite/mnn/cv/mnn_tencent_curricular_face.cpp | 58 -- lite/mnn/cv/mnn_tencent_curricular_face.h | 33 - lite/mnn/cv/mnn_ultraface.cpp | 136 --- lite/mnn/cv/mnn_ultraface.h | 48 - lite/mnn/cv/mnn_yolo5face.cpp | 213 ----- lite/mnn/cv/mnn_yolo5face.h | 64 -- lite/mnn/cv/mnn_yolop.cpp | 272 ------ lite/mnn/cv/mnn_yolop.h | 73 -- lite/mnn/cv/mnn_yolor.cpp | 179 ---- lite/mnn/cv/mnn_yolor.h | 78 -- lite/mnn/cv/mnn_yolov5.cpp | 198 ----- lite/mnn/cv/mnn_yolov5.h | 78 -- lite/mnn/cv/mnn_yolov5_blazeface.cpp | 213 ----- lite/mnn/cv/mnn_yolov5_blazeface.h | 64 -- lite/mnn/cv/mnn_yolov5_v6.0.cpp | 179 ---- lite/mnn/cv/mnn_yolov5_v6.0.h | 78 -- lite/mnn/cv/mnn_yolov5_v6.1.cpp | 179 ---- lite/mnn/cv/mnn_yolov5_v6.1.h | 79 -- lite/mnn/cv/mnn_yolov6.cpp | 221 ----- lite/mnn/cv/mnn_yolov6.h | 91 -- lite/mnn/cv/mnn_yolox.cpp | 218 ----- lite/mnn/cv/mnn_yolox.h | 91 -- lite/mnn/cv/mnn_yolox_v0.1.1.cpp | 218 ----- lite/mnn/cv/mnn_yolox_v0.1.1.h | 91 -- lite/models.h | 723 --------------- lite/ncnn/.gitignore | 0 lite/ncnn/core/ncnn_config.h | 18 - lite/ncnn/core/ncnn_core.h | 102 --- lite/ncnn/core/ncnn_custom.cpp | 47 - lite/ncnn/core/ncnn_custom.h | 24 - lite/ncnn/core/ncnn_defs.h | 25 - lite/ncnn/core/ncnn_handler.cpp | 76 -- lite/ncnn/core/ncnn_handler.h | 56 -- lite/ncnn/core/ncnn_types.h | 15 - lite/ncnn/core/ncnn_utils.cpp | 5 - lite/ncnn/core/ncnn_utils.h | 13 - lite/ncnn/cv/ncnn_age_googlenet.cpp | 57 -- lite/ncnn/cv/ncnn_age_googlenet.h | 46 - lite/ncnn/cv/ncnn_backgroundmattingv2.cpp | 4 - lite/ncnn/cv/ncnn_backgroundmattingv2.h | 8 - lite/ncnn/cv/ncnn_cava_combined_face.cpp | 42 - lite/ncnn/cv/ncnn_cava_combined_face.h | 37 - lite/ncnn/cv/ncnn_cava_ghost_arcface.cpp | 42 - lite/ncnn/cv/ncnn_cava_ghost_arcface.h | 38 - lite/ncnn/cv/ncnn_center_loss_face.cpp | 42 - lite/ncnn/cv/ncnn_center_loss_face.h | 37 - lite/ncnn/cv/ncnn_colorizer.cpp | 106 --- lite/ncnn/cv/ncnn_colorizer.h | 35 - lite/ncnn/cv/ncnn_deeplabv3_resnet101.cpp | 106 --- lite/ncnn/cv/ncnn_deeplabv3_resnet101.h | 46 - lite/ncnn/cv/ncnn_densenet.h | 413 --------- lite/ncnn/cv/ncnn_densent.cpp | 62 -- lite/ncnn/cv/ncnn_efficient_emotion7.cpp | 54 -- lite/ncnn/cv/ncnn_efficient_emotion7.h | 38 - lite/ncnn/cv/ncnn_efficient_emotion8.cpp | 54 -- lite/ncnn/cv/ncnn_efficient_emotion8.h | 39 - lite/ncnn/cv/ncnn_efficientnet_lite4.cpp | 60 -- lite/ncnn/cv/ncnn_efficientnet_lite4.h | 412 --------- lite/ncnn/cv/ncnn_emotion_ferplus.cpp | 54 -- lite/ncnn/cv/ncnn_emotion_ferplus.h | 40 - lite/ncnn/cv/ncnn_face_landmarks_1000.cpp | 60 -- lite/ncnn/cv/ncnn_face_landmarks_1000.h | 36 - lite/ncnn/cv/ncnn_face_parsing_bisenet.cpp | 190 ---- lite/ncnn/cv/ncnn_face_parsing_bisenet.h | 43 - lite/ncnn/cv/ncnn_faceboxes.cpp | 203 ----- lite/ncnn/cv/ncnn_faceboxes.h | 79 -- lite/ncnn/cv/ncnn_faceboxesv2.cpp | 203 ----- lite/ncnn/cv/ncnn_faceboxesv2.h | 79 -- lite/ncnn/cv/ncnn_facenet.cpp | 42 - lite/ncnn/cv/ncnn_facenet.h | 37 - lite/ncnn/cv/ncnn_fast_style_transfer.cpp | 67 -- lite/ncnn/cv/ncnn_fast_style_transfer.h | 39 - lite/ncnn/cv/ncnn_fcn_resnet101.cpp | 107 --- lite/ncnn/cv/ncnn_fcn_resnet101.h | 47 - lite/ncnn/cv/ncnn_female_photo2cartoon.cpp | 99 --- lite/ncnn/cv/ncnn_female_photo2cartoon.h | 41 - lite/ncnn/cv/ncnn_focal_arcface.cpp | 42 - lite/ncnn/cv/ncnn_focal_arcface.h | 37 - lite/ncnn/cv/ncnn_focal_asia_arcface.cpp | 42 - lite/ncnn/cv/ncnn_focal_asia_arcface.h | 36 - lite/ncnn/cv/ncnn_gender_googlenet.cpp | 55 -- lite/ncnn/cv/ncnn_gender_googlenet.h | 37 - lite/ncnn/cv/ncnn_ghostnet.cpp | 62 -- lite/ncnn/cv/ncnn_ghostnet.h | 413 --------- lite/ncnn/cv/ncnn_glint_arcface.cpp | 42 - lite/ncnn/cv/ncnn_glint_arcface.h | 37 - lite/ncnn/cv/ncnn_glint_cosface.cpp | 42 - lite/ncnn/cv/ncnn_glint_cosface.h | 36 - lite/ncnn/cv/ncnn_glint_partial_fc.cpp | 42 - lite/ncnn/cv/ncnn_glint_partial_fc.h | 37 - lite/ncnn/cv/ncnn_hdrdnet.cpp | 62 -- lite/ncnn/cv/ncnn_hdrdnet.h | 413 --------- lite/ncnn/cv/ncnn_ibnnet.cpp | 62 -- lite/ncnn/cv/ncnn_ibnnet.h | 414 --------- lite/ncnn/cv/ncnn_insectid.cpp | 63 -- lite/ncnn/cv/ncnn_insectid.h | 375 -------- lite/ncnn/cv/ncnn_mobile_emotion7.cpp | 64 -- lite/ncnn/cv/ncnn_mobile_emotion7.h | 38 - lite/ncnn/cv/ncnn_mobile_facenet.cpp | 42 - lite/ncnn/cv/ncnn_mobile_facenet.h | 38 - lite/ncnn/cv/ncnn_mobilenetv2.cpp | 62 -- lite/ncnn/cv/ncnn_mobilenetv2.h | 414 --------- lite/ncnn/cv/ncnn_mobilenetv2_68.cpp | 58 -- lite/ncnn/cv/ncnn_mobilenetv2_68.h | 36 - lite/ncnn/cv/ncnn_mobilenetv2_se_68.cpp | 58 -- lite/ncnn/cv/ncnn_mobilenetv2_se_68.h | 35 - lite/ncnn/cv/ncnn_mobilese_focal_face.cpp | 42 - lite/ncnn/cv/ncnn_mobilese_focal_face.h | 38 - lite/ncnn/cv/ncnn_modnet.cpp | 114 --- lite/ncnn/cv/ncnn_modnet.h | 43 - lite/ncnn/cv/ncnn_nanodet.cpp | 243 ------ lite/ncnn/cv/ncnn_nanodet.h | 115 --- lite/ncnn/cv/ncnn_nanodet_depreciated.cpp | 259 ------ lite/ncnn/cv/ncnn_nanodet_depreciated.h | 115 --- ...nn_nanodet_efficientdet_lite_depreciated.h | 115 --- .../cv/ncnn_nanodet_efficientnet_lite.cpp | 244 ------ lite/ncnn/cv/ncnn_nanodet_efficientnet_lite.h | 115 --- ..._nanodet_efficientnet_lite_depreciated.cpp | 264 ------ lite/ncnn/cv/ncnn_nanodet_plus.cpp | 222 ----- lite/ncnn/cv/ncnn_nanodet_plus.h | 104 --- lite/ncnn/cv/ncnn_pfld.cpp | 58 -- lite/ncnn/cv/ncnn_pfld.h | 35 - lite/ncnn/cv/ncnn_pfld68.cpp | 58 -- lite/ncnn/cv/ncnn_pfld68.h | 35 - lite/ncnn/cv/ncnn_pfld98.cpp | 58 -- lite/ncnn/cv/ncnn_pfld98.h | 35 - lite/ncnn/cv/ncnn_pipnet19.cpp | 193 ----- lite/ncnn/cv/ncnn_pipnet19.h | 74 -- lite/ncnn/cv/ncnn_pipnet29.cpp | 193 ----- lite/ncnn/cv/ncnn_pipnet29.h | 86 -- lite/ncnn/cv/ncnn_pipnet68.cpp | 193 ----- lite/ncnn/cv/ncnn_pipnet68.h | 134 --- lite/ncnn/cv/ncnn_pipnet98.cpp | 193 ----- lite/ncnn/cv/ncnn_pipnet98.h | 143 --- lite/ncnn/cv/ncnn_plantid.cpp | 63 -- lite/ncnn/cv/ncnn_plantid.h | 820 ------------------ lite/ncnn/cv/ncnn_resnet.cpp | 63 -- lite/ncnn/cv/ncnn_resnet.h | 413 --------- lite/ncnn/cv/ncnn_resnext.cpp | 62 -- lite/ncnn/cv/ncnn_resnext.h | 413 --------- lite/ncnn/cv/ncnn_retinaface.cpp | 216 ----- lite/ncnn/cv/ncnn_retinaface.h | 78 -- lite/ncnn/cv/ncnn_rvm.cpp | 224 ----- lite/ncnn/cv/ncnn_rvm.h | 158 ---- lite/ncnn/cv/ncnn_scrfd.cpp | 433 --------- lite/ncnn/cv/ncnn_scrfd.h | 112 --- lite/ncnn/cv/ncnn_shufflenetv2.cpp | 62 -- lite/ncnn/cv/ncnn_shufflenetv2.h | 414 --------- lite/ncnn/cv/ncnn_sphere_face.cpp | 42 - lite/ncnn/cv/ncnn_sphere_face.h | 37 - lite/ncnn/cv/ncnn_subpixel_cnn.cpp | 83 -- lite/ncnn/cv/ncnn_subpixel_cnn.h | 35 - lite/ncnn/cv/ncnn_tencent_cifp_face.cpp | 42 - lite/ncnn/cv/ncnn_tencent_cifp_face.h | 37 - lite/ncnn/cv/ncnn_tencent_curricular_face.cpp | 42 - lite/ncnn/cv/ncnn_tencent_curricular_face.h | 37 - lite/ncnn/cv/ncnn_ultraface.cpp | 217 ----- lite/ncnn/cv/ncnn_ultraface.h | 82 -- lite/ncnn/cv/ncnn_yolo5face.cpp | 467 ---------- lite/ncnn/cv/ncnn_yolo5face.h | 114 --- lite/ncnn/cv/ncnn_yolop.cpp | 507 ----------- lite/ncnn/cv/ncnn_yolop.h | 124 --- lite/ncnn/cv/ncnn_yolor.cpp | 477 ---------- lite/ncnn/cv/ncnn_yolor.h | 131 --- lite/ncnn/cv/ncnn_yolor_ssss.cpp | 428 --------- lite/ncnn/cv/ncnn_yolor_ssss.h | 131 --- lite/ncnn/cv/ncnn_yolov5.cpp | 429 --------- lite/ncnn/cv/ncnn_yolov5.h | 131 --- lite/ncnn/cv/ncnn_yolov5_v6.0.cpp | 428 --------- lite/ncnn/cv/ncnn_yolov5_v6.0.h | 131 --- lite/ncnn/cv/ncnn_yolov5_v6.0_p6.cpp | 573 ------------ lite/ncnn/cv/ncnn_yolov5_v6.0_p6.h | 131 --- lite/ncnn/cv/ncnn_yolov6.cpp | 269 ------ lite/ncnn/cv/ncnn_yolov6.h | 113 --- lite/ncnn/cv/ncnn_yolox.cpp | 276 ------ lite/ncnn/cv/ncnn_yolox.h | 115 --- lite/ncnn/cv/ncnn_yolox_v0.1.1.cpp | 275 ------ lite/ncnn/cv/ncnn_yolox_v0.1.1.h | 111 --- lite/tnn/core/tnn_config.h | 22 - lite/tnn/core/tnn_core.h | 104 --- lite/tnn/core/tnn_defs.h | 23 - lite/tnn/core/tnn_handler.cpp | 377 -------- lite/tnn/core/tnn_handler.h | 103 --- lite/tnn/core/tnn_types.h | 15 - lite/tnn/core/tnn_utils.cpp | 5 - lite/tnn/core/tnn_utils.h | 13 - lite/tnn/cv/tnn_age_googlenet.cpp | 90 -- lite/tnn/cv/tnn_age_googlenet.h | 43 - lite/tnn/cv/tnn_backgroundmattingv2.cpp | 296 ------- lite/tnn/cv/tnn_backgroundmattingv2.h | 98 --- lite/tnn/cv/tnn_cava_combined_face.cpp | 92 -- lite/tnn/cv/tnn_cava_combined_face.h | 35 - lite/tnn/cv/tnn_cava_ghost_arcface.cpp | 92 -- lite/tnn/cv/tnn_cava_ghost_arcface.h | 34 - lite/tnn/cv/tnn_center_loss_face.cpp | 88 -- lite/tnn/cv/tnn_center_loss_face.h | 34 - lite/tnn/cv/tnn_colorizer.cpp | 137 --- lite/tnn/cv/tnn_colorizer.h | 29 - lite/tnn/cv/tnn_deeplabv3_resnet101.cpp | 307 ------- lite/tnn/cv/tnn_deeplabv3_resnet101.h | 76 -- lite/tnn/cv/tnn_densenet.cpp | 94 -- lite/tnn/cv/tnn_densenet.h | 414 --------- lite/tnn/cv/tnn_efficient_emotion7.cpp | 86 -- lite/tnn/cv/tnn_efficient_emotion7.h | 41 - lite/tnn/cv/tnn_efficient_emotion8.cpp | 86 -- lite/tnn/cv/tnn_efficient_emotion8.h | 41 - lite/tnn/cv/tnn_efficientnet_lite4.cpp | 94 -- lite/tnn/cv/tnn_efficientnet_lite4.h | 409 --------- lite/tnn/cv/tnn_emotion_ferplus.cpp | 86 -- lite/tnn/cv/tnn_emotion_ferplus.h | 38 - lite/tnn/cv/tnn_face_landmarks_1000.cpp | 93 -- lite/tnn/cv/tnn_face_landmarks_1000.h | 34 - lite/tnn/cv/tnn_face_parsing_bisenet.cpp | 201 ----- lite/tnn/cv/tnn_face_parsing_bisenet.h | 45 - lite/tnn/cv/tnn_faceboxes.cpp | 278 ------ lite/tnn/cv/tnn_faceboxes.h | 75 -- lite/tnn/cv/tnn_faceboxesv2.cpp | 236 ----- lite/tnn/cv/tnn_faceboxesv2.h | 76 -- lite/tnn/cv/tnn_facenet.cpp | 87 -- lite/tnn/cv/tnn_facenet.h | 34 - lite/tnn/cv/tnn_fast_style_transfer.cpp | 95 -- lite/tnn/cv/tnn_fast_style_transfer.h | 34 - lite/tnn/cv/tnn_fcn_resnet101.cpp | 289 ------ lite/tnn/cv/tnn_fcn_resnet101.h | 76 -- lite/tnn/cv/tnn_female_photo2cartoon.cpp | 149 ---- lite/tnn/cv/tnn_female_photo2cartoon.h | 37 - lite/tnn/cv/tnn_focal_arcface.cpp | 88 -- lite/tnn/cv/tnn_focal_arcface.h | 34 - lite/tnn/cv/tnn_focal_asia_arcface.cpp | 87 -- lite/tnn/cv/tnn_focal_asia_arcface.h | 34 - lite/tnn/cv/tnn_fsanet.cpp | 93 -- lite/tnn/cv/tnn_fsanet.h | 34 - lite/tnn/cv/tnn_gender_googlenet.cpp | 86 -- lite/tnn/cv/tnn_gender_googlenet.h | 35 - lite/tnn/cv/tnn_ghostnet.cpp | 94 -- lite/tnn/cv/tnn_ghostnet.h | 413 --------- lite/tnn/cv/tnn_glint_arcface.cpp | 111 --- lite/tnn/cv/tnn_glint_arcface.h | 34 - lite/tnn/cv/tnn_glint_cosface.cpp | 88 -- lite/tnn/cv/tnn_glint_cosface.h | 34 - lite/tnn/cv/tnn_glint_partial_fc.cpp | 86 -- lite/tnn/cv/tnn_glint_partial_fc.h | 34 - lite/tnn/cv/tnn_hdrdnet.cpp | 94 -- lite/tnn/cv/tnn_hdrdnet.h | 414 --------- lite/tnn/cv/tnn_head_seg.cpp | 217 ----- lite/tnn/cv/tnn_head_seg.h | 72 -- lite/tnn/cv/tnn_ibnnet.cpp | 94 -- lite/tnn/cv/tnn_ibnnet.h | 413 --------- lite/tnn/cv/tnn_insectdet.cpp | 189 ---- lite/tnn/cv/tnn_insectdet.h | 62 -- lite/tnn/cv/tnn_insectid.cpp | 94 -- lite/tnn/cv/tnn_insectid.h | 376 -------- lite/tnn/cv/tnn_mg_matting.cpp | 530 ----------- lite/tnn/cv/tnn_mg_matting.h | 98 --- lite/tnn/cv/tnn_mobile_emotion7.cpp | 98 --- lite/tnn/cv/tnn_mobile_emotion7.h | 36 - lite/tnn/cv/tnn_mobile_facenet.cpp | 87 -- lite/tnn/cv/tnn_mobile_facenet.h | 34 - lite/tnn/cv/tnn_mobilenetv2.cpp | 94 -- lite/tnn/cv/tnn_mobilenetv2.h | 414 --------- lite/tnn/cv/tnn_mobilenetv2_68.cpp | 91 -- lite/tnn/cv/tnn_mobilenetv2_68.h | 41 - lite/tnn/cv/tnn_mobilenetv2_se_68.cpp | 91 -- lite/tnn/cv/tnn_mobilenetv2_se_68.h | 42 - lite/tnn/cv/tnn_mobilese_focal_face.cpp | 86 -- lite/tnn/cv/tnn_mobilese_focal_face.h | 35 - lite/tnn/cv/tnn_modnet.cpp | 142 --- lite/tnn/cv/tnn_modnet.h | 40 - lite/tnn/cv/tnn_nanodet.cpp | 293 ------- lite/tnn/cv/tnn_nanodet.h | 110 --- lite/tnn/cv/tnn_nanodet_efficientnet_lite.cpp | 291 ------- lite/tnn/cv/tnn_nanodet_efficientnet_lite.h | 111 --- lite/tnn/cv/tnn_nanodet_plus.cpp | 256 ------ lite/tnn/cv/tnn_nanodet_plus.h | 102 --- lite/tnn/cv/tnn_pfld.cpp | 90 -- lite/tnn/cv/tnn_pfld.h | 33 - lite/tnn/cv/tnn_pfld68.cpp | 91 -- lite/tnn/cv/tnn_pfld68.h | 33 - lite/tnn/cv/tnn_pfld98.cpp | 90 -- lite/tnn/cv/tnn_pfld98.h | 34 - lite/tnn/cv/tnn_pipnet19.cpp | 229 ----- lite/tnn/cv/tnn_pipnet19.h | 69 -- lite/tnn/cv/tnn_pipnet29.cpp | 229 ----- lite/tnn/cv/tnn_pipnet29.h | 81 -- lite/tnn/cv/tnn_pipnet68.cpp | 229 ----- lite/tnn/cv/tnn_pipnet68.h | 130 --- lite/tnn/cv/tnn_pipnet98.cpp | 229 ----- lite/tnn/cv/tnn_pipnet98.h | 140 --- lite/tnn/cv/tnn_plantid.cpp | 94 -- lite/tnn/cv/tnn_plantid.h | 820 ------------------ lite/tnn/cv/tnn_resnet.cpp | 94 -- lite/tnn/cv/tnn_resnet.h | 414 --------- lite/tnn/cv/tnn_resnext.cpp | 94 -- lite/tnn/cv/tnn_resnext.h | 414 --------- lite/tnn/cv/tnn_retinaface.cpp | 245 ------ lite/tnn/cv/tnn_retinaface.h | 75 -- lite/tnn/cv/tnn_rexnet_emotion7.cpp | 86 -- lite/tnn/cv/tnn_rexnet_emotion7.h | 40 - lite/tnn/cv/tnn_rvm.cpp | 492 ----------- lite/tnn/cv/tnn_rvm.h | 145 ---- lite/tnn/cv/tnn_scrfd.cpp | 482 ---------- lite/tnn/cv/tnn_scrfd.h | 161 ---- lite/tnn/cv/tnn_shufflenetv2.cpp | 94 -- lite/tnn/cv/tnn_shufflenetv2.h | 414 --------- lite/tnn/cv/tnn_sphere_face.cpp | 86 -- lite/tnn/cv/tnn_sphere_face.h | 34 - lite/tnn/cv/tnn_ssrnet.cpp | 108 --- lite/tnn/cv/tnn_ssrnet.h | 36 - lite/tnn/cv/tnn_subpixel_cnn.cpp | 110 --- lite/tnn/cv/tnn_subpixel_cnn.h | 28 - lite/tnn/cv/tnn_tencent_cifp_face.cpp | 86 -- lite/tnn/cv/tnn_tencent_cifp_face.h | 34 - lite/tnn/cv/tnn_tencent_curricular_face.cpp | 87 -- lite/tnn/cv/tnn_tencent_curricular_face.h | 34 - lite/tnn/cv/tnn_ultraface.cpp | 174 ---- lite/tnn/cv/tnn_ultraface.h | 53 -- lite/tnn/cv/tnn_yolo5face.cpp | 242 ------ lite/tnn/cv/tnn_yolo5face.h | 63 -- lite/tnn/cv/tnn_yolop.cpp | 300 ------- lite/tnn/cv/tnn_yolop.h | 75 -- lite/tnn/cv/tnn_yolor.cpp | 212 ----- lite/tnn/cv/tnn_yolor.h | 80 -- lite/tnn/cv/tnn_yolov5.cpp | 231 ----- lite/tnn/cv/tnn_yolov5.h | 79 -- lite/tnn/cv/tnn_yolov5_v6.0.cpp | 233 ----- lite/tnn/cv/tnn_yolov5_v6.0.h | 80 -- lite/tnn/cv/tnn_yolov6.cpp | 212 ----- lite/tnn/cv/tnn_yolov6.h | 80 -- lite/tnn/cv/tnn_yolox.cpp | 267 ------ lite/tnn/cv/tnn_yolox.h | 93 -- lite/tnn/cv/tnn_yolox_v0.1.1.cpp | 264 ------ lite/tnn/cv/tnn_yolox_v0.1.1.h | 93 -- 491 files changed, 8 insertions(+), 62511 deletions(-) delete mode 100644 cmake/MNN.cmake delete mode 100644 cmake/TNN.cmake delete mode 100644 cmake/ncnn.cmake delete mode 100644 docs/hub/lite.ai.toolkit.hub.mnn.md delete mode 100644 docs/hub/lite.ai.toolkit.hub.ncnn.md delete mode 100644 docs/hub/lite.ai.toolkit.hub.tnn.md delete mode 100644 lite/mnn/.gitignore delete mode 100644 lite/mnn/core/mnn_config.h delete mode 100644 lite/mnn/core/mnn_core.h delete mode 100644 lite/mnn/core/mnn_defs.h delete mode 100644 lite/mnn/core/mnn_handler.cpp delete mode 100644 lite/mnn/core/mnn_handler.h delete mode 100644 lite/mnn/core/mnn_types.h delete mode 100644 lite/mnn/core/mnn_utils.cpp delete mode 100644 lite/mnn/core/mnn_utils.h delete mode 100644 lite/mnn/cv/mnn_age_googlenet.cpp delete mode 100644 lite/mnn/cv/mnn_age_googlenet.h delete mode 100644 lite/mnn/cv/mnn_backgroundmattingv2.cpp delete mode 100644 lite/mnn/cv/mnn_backgroundmattingv2.h delete mode 100644 lite/mnn/cv/mnn_cava_combined_face.cpp delete mode 100644 lite/mnn/cv/mnn_cava_combined_face.h delete mode 100644 lite/mnn/cv/mnn_cava_ghost_arcface.cpp delete mode 100644 lite/mnn/cv/mnn_cava_ghost_arcface.h delete mode 100644 lite/mnn/cv/mnn_center_loss_face.cpp delete mode 100644 lite/mnn/cv/mnn_center_loss_face.h delete mode 100644 lite/mnn/cv/mnn_colorizer.cpp delete mode 100644 lite/mnn/cv/mnn_colorizer.h delete mode 100644 lite/mnn/cv/mnn_deeplabv3_resnet101.cpp delete mode 100644 lite/mnn/cv/mnn_deeplabv3_resnet101.h delete mode 100644 lite/mnn/cv/mnn_densenet.cpp delete mode 100644 lite/mnn/cv/mnn_densenet.h delete mode 100644 lite/mnn/cv/mnn_efficient_emotion7.cpp delete mode 100644 lite/mnn/cv/mnn_efficient_emotion7.h delete mode 100644 lite/mnn/cv/mnn_efficient_emotion8.cpp delete mode 100644 lite/mnn/cv/mnn_efficient_emotion8.h delete mode 100644 lite/mnn/cv/mnn_efficientnet_lite4.cpp delete mode 100644 lite/mnn/cv/mnn_efficientnet_lite4.h delete mode 100644 lite/mnn/cv/mnn_emotion_ferplus.cpp delete mode 100644 lite/mnn/cv/mnn_emotion_ferplus.h delete mode 100644 lite/mnn/cv/mnn_face_hair_seg.cpp delete mode 100644 lite/mnn/cv/mnn_face_hair_seg.h delete mode 100644 lite/mnn/cv/mnn_face_landmarks_1000.cpp delete mode 100644 lite/mnn/cv/mnn_face_landmarks_1000.h delete mode 100644 lite/mnn/cv/mnn_face_parsing_bisenet.cpp delete mode 100644 lite/mnn/cv/mnn_face_parsing_bisenet.h delete mode 100644 lite/mnn/cv/mnn_faceboxes.cpp delete mode 100644 lite/mnn/cv/mnn_faceboxes.h delete mode 100644 lite/mnn/cv/mnn_faceboxesv2.cpp delete mode 100644 lite/mnn/cv/mnn_faceboxesv2.h delete mode 100644 lite/mnn/cv/mnn_facenet.cpp delete mode 100644 lite/mnn/cv/mnn_facenet.h delete mode 100644 lite/mnn/cv/mnn_fast_portrait_seg.cpp delete mode 100644 lite/mnn/cv/mnn_fast_portrait_seg.h delete mode 100644 lite/mnn/cv/mnn_fast_style_transfer.cpp delete mode 100644 lite/mnn/cv/mnn_fast_style_transfer.h delete mode 100644 lite/mnn/cv/mnn_fcn_resnet101.cpp delete mode 100644 lite/mnn/cv/mnn_fcn_resnet101.h delete mode 100644 lite/mnn/cv/mnn_female_photo2cartoon.cpp delete mode 100644 lite/mnn/cv/mnn_female_photo2cartoon.h delete mode 100644 lite/mnn/cv/mnn_focal_arcface.cpp delete mode 100644 lite/mnn/cv/mnn_focal_arcface.h delete mode 100644 lite/mnn/cv/mnn_focal_asia_arcface.cpp delete mode 100644 lite/mnn/cv/mnn_focal_asia_arcface.h delete mode 100644 lite/mnn/cv/mnn_fsanet.cpp delete mode 100644 lite/mnn/cv/mnn_fsanet.h delete mode 100644 lite/mnn/cv/mnn_gender_googlenet.cpp delete mode 100644 lite/mnn/cv/mnn_gender_googlenet.h delete mode 100644 lite/mnn/cv/mnn_ghostnet.cpp delete mode 100644 lite/mnn/cv/mnn_ghostnet.h delete mode 100644 lite/mnn/cv/mnn_glint_arcface.cpp delete mode 100644 lite/mnn/cv/mnn_glint_arcface.h delete mode 100644 lite/mnn/cv/mnn_glint_cosface.cpp delete mode 100644 lite/mnn/cv/mnn_glint_cosface.h delete mode 100644 lite/mnn/cv/mnn_glint_partial_fc.cpp delete mode 100644 lite/mnn/cv/mnn_glint_partial_fc.h delete mode 100644 lite/mnn/cv/mnn_hair_seg.cpp delete mode 100644 lite/mnn/cv/mnn_hair_seg.h delete mode 100644 lite/mnn/cv/mnn_hdrdnet.cpp delete mode 100644 lite/mnn/cv/mnn_hdrdnet.h delete mode 100644 lite/mnn/cv/mnn_head_seg.cpp delete mode 100644 lite/mnn/cv/mnn_head_seg.h delete mode 100644 lite/mnn/cv/mnn_ibnnet.cpp delete mode 100644 lite/mnn/cv/mnn_ibnnet.h delete mode 100644 lite/mnn/cv/mnn_insectdet.cpp delete mode 100644 lite/mnn/cv/mnn_insectdet.h delete mode 100644 lite/mnn/cv/mnn_insectid.cpp delete mode 100644 lite/mnn/cv/mnn_insectid.h delete mode 100644 lite/mnn/cv/mnn_mg_matting.cpp delete mode 100644 lite/mnn/cv/mnn_mg_matting.h delete mode 100644 lite/mnn/cv/mnn_mobile_emotion7.cpp delete mode 100644 lite/mnn/cv/mnn_mobile_emotion7.h delete mode 100644 lite/mnn/cv/mnn_mobile_facenet.cpp delete mode 100644 lite/mnn/cv/mnn_mobile_facenet.h delete mode 100644 lite/mnn/cv/mnn_mobile_hair_seg.cpp delete mode 100644 lite/mnn/cv/mnn_mobile_hair_seg.h delete mode 100644 lite/mnn/cv/mnn_mobile_human_matting.cpp delete mode 100644 lite/mnn/cv/mnn_mobile_human_matting.h delete mode 100644 lite/mnn/cv/mnn_mobilenetv2.cpp delete mode 100644 lite/mnn/cv/mnn_mobilenetv2.h delete mode 100644 lite/mnn/cv/mnn_mobilenetv2_68.cpp delete mode 100644 lite/mnn/cv/mnn_mobilenetv2_68.h delete mode 100644 lite/mnn/cv/mnn_mobilenetv2_se_68.cpp delete mode 100644 lite/mnn/cv/mnn_mobilenetv2_se_68.h delete mode 100644 lite/mnn/cv/mnn_mobilese_focal_face.cpp delete mode 100644 lite/mnn/cv/mnn_mobilese_focal_face.h delete mode 100644 lite/mnn/cv/mnn_modnet.cpp delete mode 100644 lite/mnn/cv/mnn_modnet.h delete mode 100644 lite/mnn/cv/mnn_nanodet.cpp delete mode 100644 lite/mnn/cv/mnn_nanodet.h delete mode 100644 lite/mnn/cv/mnn_nanodet_efficientnet_lite.cpp delete mode 100644 lite/mnn/cv/mnn_nanodet_efficientnet_lite.h delete mode 100644 lite/mnn/cv/mnn_nanodet_plus.cpp delete mode 100644 lite/mnn/cv/mnn_nanodet_plus.h delete mode 100644 lite/mnn/cv/mnn_pfld.cpp delete mode 100644 lite/mnn/cv/mnn_pfld.h delete mode 100644 lite/mnn/cv/mnn_pfld68.cpp delete mode 100644 lite/mnn/cv/mnn_pfld68.h delete mode 100644 lite/mnn/cv/mnn_pfld98.cpp delete mode 100644 lite/mnn/cv/mnn_pfld98.h delete mode 100644 lite/mnn/cv/mnn_pipnet19.cpp delete mode 100644 lite/mnn/cv/mnn_pipnet19.h delete mode 100644 lite/mnn/cv/mnn_pipnet29.cpp delete mode 100644 lite/mnn/cv/mnn_pipnet29.h delete mode 100644 lite/mnn/cv/mnn_pipnet68.cpp delete mode 100644 lite/mnn/cv/mnn_pipnet68.h delete mode 100644 lite/mnn/cv/mnn_pipnet98.cpp delete mode 100644 lite/mnn/cv/mnn_pipnet98.h delete mode 100644 lite/mnn/cv/mnn_plantid.cpp delete mode 100644 lite/mnn/cv/mnn_plantid.h delete mode 100644 lite/mnn/cv/mnn_portrait_seg_extremec3net.cpp delete mode 100644 lite/mnn/cv/mnn_portrait_seg_extremec3net.h delete mode 100644 lite/mnn/cv/mnn_portrait_seg_sinet.cpp delete mode 100644 lite/mnn/cv/mnn_portrait_seg_sinet.h delete mode 100644 lite/mnn/cv/mnn_resnet.cpp delete mode 100644 lite/mnn/cv/mnn_resnet.h delete mode 100644 lite/mnn/cv/mnn_resnext.cpp delete mode 100644 lite/mnn/cv/mnn_resnext.h delete mode 100644 lite/mnn/cv/mnn_retinaface.cpp delete mode 100644 lite/mnn/cv/mnn_retinaface.h delete mode 100644 lite/mnn/cv/mnn_rexnet_emotion7.cpp delete mode 100644 lite/mnn/cv/mnn_rexnet_emotion7.h delete mode 100644 lite/mnn/cv/mnn_rvm.cpp delete mode 100644 lite/mnn/cv/mnn_rvm.h delete mode 100644 lite/mnn/cv/mnn_scrfd.cpp delete mode 100644 lite/mnn/cv/mnn_scrfd.h delete mode 100644 lite/mnn/cv/mnn_shufflenetv2.cpp delete mode 100644 lite/mnn/cv/mnn_shufflenetv2.h delete mode 100644 lite/mnn/cv/mnn_sphere_face.cpp delete mode 100644 lite/mnn/cv/mnn_sphere_face.h delete mode 100644 lite/mnn/cv/mnn_ssrnet.cpp delete mode 100644 lite/mnn/cv/mnn_ssrnet.h delete mode 100644 lite/mnn/cv/mnn_subpixel_cnn.cpp delete mode 100644 lite/mnn/cv/mnn_subpixel_cnn.h delete mode 100644 lite/mnn/cv/mnn_tencent_cifp_face.cpp delete mode 100644 lite/mnn/cv/mnn_tencent_cifp_face.h delete mode 100644 lite/mnn/cv/mnn_tencent_curricular_face.cpp delete mode 100644 lite/mnn/cv/mnn_tencent_curricular_face.h delete mode 100644 lite/mnn/cv/mnn_ultraface.cpp delete mode 100644 lite/mnn/cv/mnn_ultraface.h delete mode 100644 lite/mnn/cv/mnn_yolo5face.cpp delete mode 100644 lite/mnn/cv/mnn_yolo5face.h delete mode 100644 lite/mnn/cv/mnn_yolop.cpp delete mode 100644 lite/mnn/cv/mnn_yolop.h delete mode 100644 lite/mnn/cv/mnn_yolor.cpp delete mode 100644 lite/mnn/cv/mnn_yolor.h delete mode 100644 lite/mnn/cv/mnn_yolov5.cpp delete mode 100644 lite/mnn/cv/mnn_yolov5.h delete mode 100644 lite/mnn/cv/mnn_yolov5_blazeface.cpp delete mode 100644 lite/mnn/cv/mnn_yolov5_blazeface.h delete mode 100644 lite/mnn/cv/mnn_yolov5_v6.0.cpp delete mode 100644 lite/mnn/cv/mnn_yolov5_v6.0.h delete mode 100644 lite/mnn/cv/mnn_yolov5_v6.1.cpp delete mode 100644 lite/mnn/cv/mnn_yolov5_v6.1.h delete mode 100644 lite/mnn/cv/mnn_yolov6.cpp delete mode 100644 lite/mnn/cv/mnn_yolov6.h delete mode 100644 lite/mnn/cv/mnn_yolox.cpp delete mode 100644 lite/mnn/cv/mnn_yolox.h delete mode 100644 lite/mnn/cv/mnn_yolox_v0.1.1.cpp delete mode 100644 lite/mnn/cv/mnn_yolox_v0.1.1.h delete mode 100644 lite/ncnn/.gitignore delete mode 100644 lite/ncnn/core/ncnn_config.h delete mode 100644 lite/ncnn/core/ncnn_core.h delete mode 100644 lite/ncnn/core/ncnn_custom.cpp delete mode 100644 lite/ncnn/core/ncnn_custom.h delete mode 100644 lite/ncnn/core/ncnn_defs.h delete mode 100644 lite/ncnn/core/ncnn_handler.cpp delete mode 100644 lite/ncnn/core/ncnn_handler.h delete mode 100644 lite/ncnn/core/ncnn_types.h delete mode 100644 lite/ncnn/core/ncnn_utils.cpp delete mode 100644 lite/ncnn/core/ncnn_utils.h delete mode 100644 lite/ncnn/cv/ncnn_age_googlenet.cpp delete mode 100644 lite/ncnn/cv/ncnn_age_googlenet.h delete mode 100644 lite/ncnn/cv/ncnn_backgroundmattingv2.cpp delete mode 100644 lite/ncnn/cv/ncnn_backgroundmattingv2.h delete mode 100644 lite/ncnn/cv/ncnn_cava_combined_face.cpp delete mode 100644 lite/ncnn/cv/ncnn_cava_combined_face.h delete mode 100644 lite/ncnn/cv/ncnn_cava_ghost_arcface.cpp delete mode 100644 lite/ncnn/cv/ncnn_cava_ghost_arcface.h delete mode 100644 lite/ncnn/cv/ncnn_center_loss_face.cpp delete mode 100644 lite/ncnn/cv/ncnn_center_loss_face.h delete mode 100644 lite/ncnn/cv/ncnn_colorizer.cpp delete mode 100644 lite/ncnn/cv/ncnn_colorizer.h delete mode 100644 lite/ncnn/cv/ncnn_deeplabv3_resnet101.cpp delete mode 100644 lite/ncnn/cv/ncnn_deeplabv3_resnet101.h delete mode 100644 lite/ncnn/cv/ncnn_densenet.h delete mode 100644 lite/ncnn/cv/ncnn_densent.cpp delete mode 100644 lite/ncnn/cv/ncnn_efficient_emotion7.cpp delete mode 100644 lite/ncnn/cv/ncnn_efficient_emotion7.h delete mode 100644 lite/ncnn/cv/ncnn_efficient_emotion8.cpp delete mode 100644 lite/ncnn/cv/ncnn_efficient_emotion8.h delete mode 100644 lite/ncnn/cv/ncnn_efficientnet_lite4.cpp delete mode 100644 lite/ncnn/cv/ncnn_efficientnet_lite4.h delete mode 100644 lite/ncnn/cv/ncnn_emotion_ferplus.cpp delete mode 100644 lite/ncnn/cv/ncnn_emotion_ferplus.h delete mode 100644 lite/ncnn/cv/ncnn_face_landmarks_1000.cpp delete mode 100644 lite/ncnn/cv/ncnn_face_landmarks_1000.h delete mode 100644 lite/ncnn/cv/ncnn_face_parsing_bisenet.cpp delete mode 100644 lite/ncnn/cv/ncnn_face_parsing_bisenet.h delete mode 100644 lite/ncnn/cv/ncnn_faceboxes.cpp delete mode 100644 lite/ncnn/cv/ncnn_faceboxes.h delete mode 100644 lite/ncnn/cv/ncnn_faceboxesv2.cpp delete mode 100644 lite/ncnn/cv/ncnn_faceboxesv2.h delete mode 100644 lite/ncnn/cv/ncnn_facenet.cpp delete mode 100644 lite/ncnn/cv/ncnn_facenet.h delete mode 100644 lite/ncnn/cv/ncnn_fast_style_transfer.cpp delete mode 100644 lite/ncnn/cv/ncnn_fast_style_transfer.h delete mode 100644 lite/ncnn/cv/ncnn_fcn_resnet101.cpp delete mode 100644 lite/ncnn/cv/ncnn_fcn_resnet101.h delete mode 100644 lite/ncnn/cv/ncnn_female_photo2cartoon.cpp delete mode 100644 lite/ncnn/cv/ncnn_female_photo2cartoon.h delete mode 100644 lite/ncnn/cv/ncnn_focal_arcface.cpp delete mode 100644 lite/ncnn/cv/ncnn_focal_arcface.h delete mode 100644 lite/ncnn/cv/ncnn_focal_asia_arcface.cpp delete mode 100644 lite/ncnn/cv/ncnn_focal_asia_arcface.h delete mode 100644 lite/ncnn/cv/ncnn_gender_googlenet.cpp delete mode 100644 lite/ncnn/cv/ncnn_gender_googlenet.h delete mode 100644 lite/ncnn/cv/ncnn_ghostnet.cpp delete mode 100644 lite/ncnn/cv/ncnn_ghostnet.h delete mode 100644 lite/ncnn/cv/ncnn_glint_arcface.cpp delete mode 100644 lite/ncnn/cv/ncnn_glint_arcface.h delete mode 100644 lite/ncnn/cv/ncnn_glint_cosface.cpp delete mode 100644 lite/ncnn/cv/ncnn_glint_cosface.h delete mode 100644 lite/ncnn/cv/ncnn_glint_partial_fc.cpp delete mode 100644 lite/ncnn/cv/ncnn_glint_partial_fc.h delete mode 100644 lite/ncnn/cv/ncnn_hdrdnet.cpp delete mode 100644 lite/ncnn/cv/ncnn_hdrdnet.h delete mode 100644 lite/ncnn/cv/ncnn_ibnnet.cpp delete mode 100644 lite/ncnn/cv/ncnn_ibnnet.h delete mode 100644 lite/ncnn/cv/ncnn_insectid.cpp delete mode 100644 lite/ncnn/cv/ncnn_insectid.h delete mode 100644 lite/ncnn/cv/ncnn_mobile_emotion7.cpp delete mode 100644 lite/ncnn/cv/ncnn_mobile_emotion7.h delete mode 100644 lite/ncnn/cv/ncnn_mobile_facenet.cpp delete mode 100644 lite/ncnn/cv/ncnn_mobile_facenet.h delete mode 100644 lite/ncnn/cv/ncnn_mobilenetv2.cpp delete mode 100644 lite/ncnn/cv/ncnn_mobilenetv2.h delete mode 100644 lite/ncnn/cv/ncnn_mobilenetv2_68.cpp delete mode 100644 lite/ncnn/cv/ncnn_mobilenetv2_68.h delete mode 100644 lite/ncnn/cv/ncnn_mobilenetv2_se_68.cpp delete mode 100644 lite/ncnn/cv/ncnn_mobilenetv2_se_68.h delete mode 100644 lite/ncnn/cv/ncnn_mobilese_focal_face.cpp delete mode 100644 lite/ncnn/cv/ncnn_mobilese_focal_face.h delete mode 100644 lite/ncnn/cv/ncnn_modnet.cpp delete mode 100644 lite/ncnn/cv/ncnn_modnet.h delete mode 100644 lite/ncnn/cv/ncnn_nanodet.cpp delete mode 100644 lite/ncnn/cv/ncnn_nanodet.h delete mode 100644 lite/ncnn/cv/ncnn_nanodet_depreciated.cpp delete mode 100644 lite/ncnn/cv/ncnn_nanodet_depreciated.h delete mode 100644 lite/ncnn/cv/ncnn_nanodet_efficientdet_lite_depreciated.h delete mode 100644 lite/ncnn/cv/ncnn_nanodet_efficientnet_lite.cpp delete mode 100644 lite/ncnn/cv/ncnn_nanodet_efficientnet_lite.h delete mode 100644 lite/ncnn/cv/ncnn_nanodet_efficientnet_lite_depreciated.cpp delete mode 100644 lite/ncnn/cv/ncnn_nanodet_plus.cpp delete mode 100644 lite/ncnn/cv/ncnn_nanodet_plus.h delete mode 100644 lite/ncnn/cv/ncnn_pfld.cpp delete mode 100644 lite/ncnn/cv/ncnn_pfld.h delete mode 100644 lite/ncnn/cv/ncnn_pfld68.cpp delete mode 100644 lite/ncnn/cv/ncnn_pfld68.h delete mode 100644 lite/ncnn/cv/ncnn_pfld98.cpp delete mode 100644 lite/ncnn/cv/ncnn_pfld98.h delete mode 100644 lite/ncnn/cv/ncnn_pipnet19.cpp delete mode 100644 lite/ncnn/cv/ncnn_pipnet19.h delete mode 100644 lite/ncnn/cv/ncnn_pipnet29.cpp delete mode 100644 lite/ncnn/cv/ncnn_pipnet29.h delete mode 100644 lite/ncnn/cv/ncnn_pipnet68.cpp delete mode 100644 lite/ncnn/cv/ncnn_pipnet68.h delete mode 100644 lite/ncnn/cv/ncnn_pipnet98.cpp delete mode 100644 lite/ncnn/cv/ncnn_pipnet98.h delete mode 100644 lite/ncnn/cv/ncnn_plantid.cpp delete mode 100644 lite/ncnn/cv/ncnn_plantid.h delete mode 100644 lite/ncnn/cv/ncnn_resnet.cpp delete mode 100644 lite/ncnn/cv/ncnn_resnet.h delete mode 100644 lite/ncnn/cv/ncnn_resnext.cpp delete mode 100644 lite/ncnn/cv/ncnn_resnext.h delete mode 100644 lite/ncnn/cv/ncnn_retinaface.cpp delete mode 100644 lite/ncnn/cv/ncnn_retinaface.h delete mode 100644 lite/ncnn/cv/ncnn_rvm.cpp delete mode 100644 lite/ncnn/cv/ncnn_rvm.h delete mode 100644 lite/ncnn/cv/ncnn_scrfd.cpp delete mode 100644 lite/ncnn/cv/ncnn_scrfd.h delete mode 100644 lite/ncnn/cv/ncnn_shufflenetv2.cpp delete mode 100644 lite/ncnn/cv/ncnn_shufflenetv2.h delete mode 100644 lite/ncnn/cv/ncnn_sphere_face.cpp delete mode 100644 lite/ncnn/cv/ncnn_sphere_face.h delete mode 100644 lite/ncnn/cv/ncnn_subpixel_cnn.cpp delete mode 100644 lite/ncnn/cv/ncnn_subpixel_cnn.h delete mode 100644 lite/ncnn/cv/ncnn_tencent_cifp_face.cpp delete mode 100644 lite/ncnn/cv/ncnn_tencent_cifp_face.h delete mode 100644 lite/ncnn/cv/ncnn_tencent_curricular_face.cpp delete mode 100644 lite/ncnn/cv/ncnn_tencent_curricular_face.h delete mode 100644 lite/ncnn/cv/ncnn_ultraface.cpp delete mode 100644 lite/ncnn/cv/ncnn_ultraface.h delete mode 100644 lite/ncnn/cv/ncnn_yolo5face.cpp delete mode 100644 lite/ncnn/cv/ncnn_yolo5face.h delete mode 100644 lite/ncnn/cv/ncnn_yolop.cpp delete mode 100644 lite/ncnn/cv/ncnn_yolop.h delete mode 100644 lite/ncnn/cv/ncnn_yolor.cpp delete mode 100644 lite/ncnn/cv/ncnn_yolor.h delete mode 100644 lite/ncnn/cv/ncnn_yolor_ssss.cpp delete mode 100644 lite/ncnn/cv/ncnn_yolor_ssss.h delete mode 100644 lite/ncnn/cv/ncnn_yolov5.cpp delete mode 100644 lite/ncnn/cv/ncnn_yolov5.h delete mode 100644 lite/ncnn/cv/ncnn_yolov5_v6.0.cpp delete mode 100644 lite/ncnn/cv/ncnn_yolov5_v6.0.h delete mode 100644 lite/ncnn/cv/ncnn_yolov5_v6.0_p6.cpp delete mode 100644 lite/ncnn/cv/ncnn_yolov5_v6.0_p6.h delete mode 100644 lite/ncnn/cv/ncnn_yolov6.cpp delete mode 100644 lite/ncnn/cv/ncnn_yolov6.h delete mode 100644 lite/ncnn/cv/ncnn_yolox.cpp delete mode 100644 lite/ncnn/cv/ncnn_yolox.h delete mode 100644 lite/ncnn/cv/ncnn_yolox_v0.1.1.cpp delete mode 100644 lite/ncnn/cv/ncnn_yolox_v0.1.1.h delete mode 100644 lite/tnn/core/tnn_config.h delete mode 100644 lite/tnn/core/tnn_core.h delete mode 100644 lite/tnn/core/tnn_defs.h delete mode 100644 lite/tnn/core/tnn_handler.cpp delete mode 100644 lite/tnn/core/tnn_handler.h delete mode 100644 lite/tnn/core/tnn_types.h delete mode 100644 lite/tnn/core/tnn_utils.cpp delete mode 100644 lite/tnn/core/tnn_utils.h delete mode 100644 lite/tnn/cv/tnn_age_googlenet.cpp delete mode 100644 lite/tnn/cv/tnn_age_googlenet.h delete mode 100644 lite/tnn/cv/tnn_backgroundmattingv2.cpp delete mode 100644 lite/tnn/cv/tnn_backgroundmattingv2.h delete mode 100644 lite/tnn/cv/tnn_cava_combined_face.cpp delete mode 100644 lite/tnn/cv/tnn_cava_combined_face.h delete mode 100644 lite/tnn/cv/tnn_cava_ghost_arcface.cpp delete mode 100644 lite/tnn/cv/tnn_cava_ghost_arcface.h delete mode 100644 lite/tnn/cv/tnn_center_loss_face.cpp delete mode 100644 lite/tnn/cv/tnn_center_loss_face.h delete mode 100644 lite/tnn/cv/tnn_colorizer.cpp delete mode 100644 lite/tnn/cv/tnn_colorizer.h delete mode 100644 lite/tnn/cv/tnn_deeplabv3_resnet101.cpp delete mode 100644 lite/tnn/cv/tnn_deeplabv3_resnet101.h delete mode 100644 lite/tnn/cv/tnn_densenet.cpp delete mode 100644 lite/tnn/cv/tnn_densenet.h delete mode 100644 lite/tnn/cv/tnn_efficient_emotion7.cpp delete mode 100644 lite/tnn/cv/tnn_efficient_emotion7.h delete mode 100644 lite/tnn/cv/tnn_efficient_emotion8.cpp delete mode 100644 lite/tnn/cv/tnn_efficient_emotion8.h delete mode 100644 lite/tnn/cv/tnn_efficientnet_lite4.cpp delete mode 100644 lite/tnn/cv/tnn_efficientnet_lite4.h delete mode 100644 lite/tnn/cv/tnn_emotion_ferplus.cpp delete mode 100644 lite/tnn/cv/tnn_emotion_ferplus.h delete mode 100644 lite/tnn/cv/tnn_face_landmarks_1000.cpp delete mode 100644 lite/tnn/cv/tnn_face_landmarks_1000.h delete mode 100644 lite/tnn/cv/tnn_face_parsing_bisenet.cpp delete mode 100644 lite/tnn/cv/tnn_face_parsing_bisenet.h delete mode 100644 lite/tnn/cv/tnn_faceboxes.cpp delete mode 100644 lite/tnn/cv/tnn_faceboxes.h delete mode 100644 lite/tnn/cv/tnn_faceboxesv2.cpp delete mode 100644 lite/tnn/cv/tnn_faceboxesv2.h delete mode 100644 lite/tnn/cv/tnn_facenet.cpp delete mode 100644 lite/tnn/cv/tnn_facenet.h delete mode 100644 lite/tnn/cv/tnn_fast_style_transfer.cpp delete mode 100644 lite/tnn/cv/tnn_fast_style_transfer.h delete mode 100644 lite/tnn/cv/tnn_fcn_resnet101.cpp delete mode 100644 lite/tnn/cv/tnn_fcn_resnet101.h delete mode 100644 lite/tnn/cv/tnn_female_photo2cartoon.cpp delete mode 100644 lite/tnn/cv/tnn_female_photo2cartoon.h delete mode 100644 lite/tnn/cv/tnn_focal_arcface.cpp delete mode 100644 lite/tnn/cv/tnn_focal_arcface.h delete mode 100644 lite/tnn/cv/tnn_focal_asia_arcface.cpp delete mode 100644 lite/tnn/cv/tnn_focal_asia_arcface.h delete mode 100644 lite/tnn/cv/tnn_fsanet.cpp delete mode 100644 lite/tnn/cv/tnn_fsanet.h delete mode 100644 lite/tnn/cv/tnn_gender_googlenet.cpp delete mode 100644 lite/tnn/cv/tnn_gender_googlenet.h delete mode 100644 lite/tnn/cv/tnn_ghostnet.cpp delete mode 100644 lite/tnn/cv/tnn_ghostnet.h delete mode 100644 lite/tnn/cv/tnn_glint_arcface.cpp delete mode 100644 lite/tnn/cv/tnn_glint_arcface.h delete mode 100644 lite/tnn/cv/tnn_glint_cosface.cpp delete mode 100644 lite/tnn/cv/tnn_glint_cosface.h delete mode 100644 lite/tnn/cv/tnn_glint_partial_fc.cpp delete mode 100644 lite/tnn/cv/tnn_glint_partial_fc.h delete mode 100644 lite/tnn/cv/tnn_hdrdnet.cpp delete mode 100644 lite/tnn/cv/tnn_hdrdnet.h delete mode 100644 lite/tnn/cv/tnn_head_seg.cpp delete mode 100644 lite/tnn/cv/tnn_head_seg.h delete mode 100644 lite/tnn/cv/tnn_ibnnet.cpp delete mode 100644 lite/tnn/cv/tnn_ibnnet.h delete mode 100644 lite/tnn/cv/tnn_insectdet.cpp delete mode 100644 lite/tnn/cv/tnn_insectdet.h delete mode 100644 lite/tnn/cv/tnn_insectid.cpp delete mode 100644 lite/tnn/cv/tnn_insectid.h delete mode 100644 lite/tnn/cv/tnn_mg_matting.cpp delete mode 100644 lite/tnn/cv/tnn_mg_matting.h delete mode 100644 lite/tnn/cv/tnn_mobile_emotion7.cpp delete mode 100644 lite/tnn/cv/tnn_mobile_emotion7.h delete mode 100644 lite/tnn/cv/tnn_mobile_facenet.cpp delete mode 100644 lite/tnn/cv/tnn_mobile_facenet.h delete mode 100644 lite/tnn/cv/tnn_mobilenetv2.cpp delete mode 100644 lite/tnn/cv/tnn_mobilenetv2.h delete mode 100644 lite/tnn/cv/tnn_mobilenetv2_68.cpp delete mode 100644 lite/tnn/cv/tnn_mobilenetv2_68.h delete mode 100644 lite/tnn/cv/tnn_mobilenetv2_se_68.cpp delete mode 100644 lite/tnn/cv/tnn_mobilenetv2_se_68.h delete mode 100644 lite/tnn/cv/tnn_mobilese_focal_face.cpp delete mode 100644 lite/tnn/cv/tnn_mobilese_focal_face.h delete mode 100644 lite/tnn/cv/tnn_modnet.cpp delete mode 100644 lite/tnn/cv/tnn_modnet.h delete mode 100644 lite/tnn/cv/tnn_nanodet.cpp delete mode 100644 lite/tnn/cv/tnn_nanodet.h delete mode 100644 lite/tnn/cv/tnn_nanodet_efficientnet_lite.cpp delete mode 100644 lite/tnn/cv/tnn_nanodet_efficientnet_lite.h delete mode 100644 lite/tnn/cv/tnn_nanodet_plus.cpp delete mode 100644 lite/tnn/cv/tnn_nanodet_plus.h delete mode 100644 lite/tnn/cv/tnn_pfld.cpp delete mode 100644 lite/tnn/cv/tnn_pfld.h delete mode 100644 lite/tnn/cv/tnn_pfld68.cpp delete mode 100644 lite/tnn/cv/tnn_pfld68.h delete mode 100644 lite/tnn/cv/tnn_pfld98.cpp delete mode 100644 lite/tnn/cv/tnn_pfld98.h delete mode 100644 lite/tnn/cv/tnn_pipnet19.cpp delete mode 100644 lite/tnn/cv/tnn_pipnet19.h delete mode 100644 lite/tnn/cv/tnn_pipnet29.cpp delete mode 100644 lite/tnn/cv/tnn_pipnet29.h delete mode 100644 lite/tnn/cv/tnn_pipnet68.cpp delete mode 100644 lite/tnn/cv/tnn_pipnet68.h delete mode 100644 lite/tnn/cv/tnn_pipnet98.cpp delete mode 100644 lite/tnn/cv/tnn_pipnet98.h delete mode 100644 lite/tnn/cv/tnn_plantid.cpp delete mode 100644 lite/tnn/cv/tnn_plantid.h delete mode 100644 lite/tnn/cv/tnn_resnet.cpp delete mode 100644 lite/tnn/cv/tnn_resnet.h delete mode 100644 lite/tnn/cv/tnn_resnext.cpp delete mode 100644 lite/tnn/cv/tnn_resnext.h delete mode 100644 lite/tnn/cv/tnn_retinaface.cpp delete mode 100644 lite/tnn/cv/tnn_retinaface.h delete mode 100644 lite/tnn/cv/tnn_rexnet_emotion7.cpp delete mode 100644 lite/tnn/cv/tnn_rexnet_emotion7.h delete mode 100644 lite/tnn/cv/tnn_rvm.cpp delete mode 100644 lite/tnn/cv/tnn_rvm.h delete mode 100644 lite/tnn/cv/tnn_scrfd.cpp delete mode 100644 lite/tnn/cv/tnn_scrfd.h delete mode 100644 lite/tnn/cv/tnn_shufflenetv2.cpp delete mode 100644 lite/tnn/cv/tnn_shufflenetv2.h delete mode 100644 lite/tnn/cv/tnn_sphere_face.cpp delete mode 100644 lite/tnn/cv/tnn_sphere_face.h delete mode 100644 lite/tnn/cv/tnn_ssrnet.cpp delete mode 100644 lite/tnn/cv/tnn_ssrnet.h delete mode 100644 lite/tnn/cv/tnn_subpixel_cnn.cpp delete mode 100644 lite/tnn/cv/tnn_subpixel_cnn.h delete mode 100644 lite/tnn/cv/tnn_tencent_cifp_face.cpp delete mode 100644 lite/tnn/cv/tnn_tencent_cifp_face.h delete mode 100644 lite/tnn/cv/tnn_tencent_curricular_face.cpp delete mode 100644 lite/tnn/cv/tnn_tencent_curricular_face.h delete mode 100644 lite/tnn/cv/tnn_ultraface.cpp delete mode 100644 lite/tnn/cv/tnn_ultraface.h delete mode 100644 lite/tnn/cv/tnn_yolo5face.cpp delete mode 100644 lite/tnn/cv/tnn_yolo5face.h delete mode 100644 lite/tnn/cv/tnn_yolop.cpp delete mode 100644 lite/tnn/cv/tnn_yolop.h delete mode 100644 lite/tnn/cv/tnn_yolor.cpp delete mode 100644 lite/tnn/cv/tnn_yolor.h delete mode 100644 lite/tnn/cv/tnn_yolov5.cpp delete mode 100644 lite/tnn/cv/tnn_yolov5.h delete mode 100644 lite/tnn/cv/tnn_yolov5_v6.0.cpp delete mode 100644 lite/tnn/cv/tnn_yolov5_v6.0.h delete mode 100644 lite/tnn/cv/tnn_yolov6.cpp delete mode 100644 lite/tnn/cv/tnn_yolov6.h delete mode 100644 lite/tnn/cv/tnn_yolox.cpp delete mode 100644 lite/tnn/cv/tnn_yolox.h delete mode 100644 lite/tnn/cv/tnn_yolox_v0.1.1.cpp delete mode 100644 lite/tnn/cv/tnn_yolox_v0.1.1.h diff --git a/CMakeLists.txt b/CMakeLists.txt index a3bcf3fd..2c1b2532 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,16 +31,15 @@ add_definitions(-DSOURCE_PATH="${CMAKE_SOURCE_DIR}") option(ENABLE_TEST "build test examples." OFF) option(ENABLE_DEBUG_STRING "enable DEBUG string or not" ON) -option(ENABLE_ONNXRUNTIME "enable ONNXRuntime engine" ON) -option(ENABLE_TENSORRT "enable TensorRT engine" OFF) -option(ENABLE_MNN "enable MNN engine" OFF) -option(ENABLE_NCNN "enable NCNN engine" OFF) -option(ENABLE_TNN "enable TNN engine" OFF) +option(ENABLE_ONNXRUNTIME "enable ONNXRuntime engine (kept as numerical reference + test host)" ON) +option(ENABLE_TENSORRT "enable TensorRT engine (the maintained high-performance backend)" OFF) option(ENABLE_ONNXRUNTIME_CUDA "enable ONNXRuntime engine with CUDA provider" OFF) # for future use option(ENABLE_OPENCV_VIDEOIO "enable opencv videoio modules for detect_video apis" ON) # now, ON only -if ((NOT ENABLE_ONNXRUNTIME) AND (NOT ENABLE_MNN)) - message(FATAL_ERROR "One of ONNXRuntime/MNN Backend must be enable!") +# As of >=0.3.2 the MNN/NCNN/TNN backends were dropped (frozen on tag `v0.2-all-backends`). +# ONNXRuntime is kept as the numerical-reference oracle and the only backend that can build tests. +if (NOT ENABLE_ONNXRUNTIME) + message(FATAL_ERROR "ONNXRuntime backend must be enabled (it hosts the test suite and numerical reference)!") endif() if ((NOT ENABLE_ONNXRUNTIME) AND ENABLE_TEST) @@ -82,8 +81,5 @@ message(STATUS " Root Path: ${CMAKE_SOURCE_DIR}") message(STATUS " OpenCV: ON Version: ${OpenCV_Version}") message(STATUS " ONNXRUNTIME: ${ENABLE_ONNXRUNTIME} Version: ${OnnxRuntime_Version}") message(STATUS " TENSORRT: ${ENABLE_TENSORRT} Version: ${TensorRT_Version}") -message(STATUS " MNN: ${ENABLE_MNN} Version: ${MNN_Version}") -message(STATUS " NCNN: ${ENABLE_NCNN} Version: ${NCNN_Version}") -message(STATUS " TNN: ${ENABLE_TNN} Version: ${TNN_Version}") message(STATUS " INSTALL: ${CMAKE_INSTALL_PREFIX}") message(STATUS "-------------------------- lite.ai.toolkit Configuration Summary --------------------------") diff --git a/cmake/MNN.cmake b/cmake/MNN.cmake deleted file mode 100644 index 1565dcf0..00000000 --- a/cmake/MNN.cmake +++ /dev/null @@ -1,38 +0,0 @@ -set(MNN_Version "2.8.2" CACHE STRING "MNN version" FORCE) -set(MNN_DIR ${THIRD_PARTY_PATH}/MNN) -# download from github if MNN library is not exists -if (NOT EXISTS ${MNN_DIR}) - set(MNN_Filename "MNN-${MNN_Version}-linux-cpu-x86_64.tgz") - set(MNN_URL https://github.com/DefTruth/lite.ai.toolkit/releases/download/v0.2.0-rc0/${MNN_Filename}) - message("[Lite.AI.Toolkit][I] Downloading MNN library: ${MNN_URL}") - download_and_decompress(${MNN_URL} ${MNN_Filename} ${MNN_DIR}) -else() - message("[Lite.AI.Toolkit][I] Found local MNN library: ${MNN_DIR}") -endif() -if(NOT EXISTS ${MNN_DIR}) - message(FATAL_ERROR "[Lite.AI.Toolkit][E] ${MNN_DIR} is not exists!") -endif() - -include_directories(${MNN_DIR}/include) -link_directories(${MNN_DIR}/lib) - -# 1. glob sources files -file(GLOB MNN_CORE_SRCS ${CMAKE_SOURCE_DIR}/lite/mnn/core/*.cpp) -file(GLOB MNN_CV_SRCS ${CMAKE_SOURCE_DIR}/lite/mnn/cv/*.cpp) -file(GLOB MNN_NLP_SRCS ${CMAKE_SOURCE_DIR}/lite/mnn/nlp/*.cpp) -file(GLOB MNN_ASR_SRCS ${CMAKE_SOURCE_DIR}/lite/mnn/asr/*.cpp) -# 2. glob headers files -file(GLOB MNN_CORE_HEAD ${CMAKE_SOURCE_DIR}/lite/mnn/core/*.h) -file(GLOB MNN_CV_HEAD ${CMAKE_SOURCE_DIR}/lite/mnn/cv/*.h) -file(GLOB MNN_NLP_HEAD ${CMAKE_SOURCE_DIR}/lite/mnn/nlp/*.h) -file(GLOB MNN_ASR_HEAD ${CMAKE_SOURCE_DIR}/lite/mnn/asr/*.h) - -set(MNN_SRCS ${MNN_CV_SRCS} ${MNN_NLP_SRCS} ${MNN_ASR_SRCS} ${MNN_CORE_SRCS}) -# 3. copy -message("[Lite.AI.Toolkit][I] Installing Lite.AI.ToolKit Headers for MNN Backend ...") -# "INSTALL" can copy all files from the list to the specified path. -# "COPY" only copies one file to a specified path -file(INSTALL ${MNN_CORE_HEAD} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/lite/mnn/core) -file(INSTALL ${MNN_CV_HEAD} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/lite/mnn/cv) -file(INSTALL ${MNN_ASR_HEAD} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/lite/mnn/asr) -file(INSTALL ${MNN_NLP_HEAD} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/lite/mnn/nlp) diff --git a/cmake/TNN.cmake b/cmake/TNN.cmake deleted file mode 100644 index 08a246a6..00000000 --- a/cmake/TNN.cmake +++ /dev/null @@ -1,27 +0,0 @@ -set(TNN_DIR ${THIRD_PARTY_PATH}/TNN) -if(NOT EXISTS ${TNN_DIR}) - message(FATAL_ERROR "[Lite.AI.Toolkit][E] ${TNN_DIR} is not exists!") -endif() -include_directories(${TNN_DIR}/include) -link_directories(${TNN_DIR}/lib) - -# 1. glob sources files -file(GLOB TNN_CORE_SRCS ${CMAKE_SOURCE_DIR}/lite/tnn/core/*.cpp) -file(GLOB TNN_CV_SRCS ${CMAKE_SOURCE_DIR}/lite/tnn/cv/*.cpp) -file(GLOB TNN_NLP_SRCS ${CMAKE_SOURCE_DIR}/lite/tnn/nlp/*.cpp) -file(GLOB TNN_ASR_SRCS ${CMAKE_SOURCE_DIR}/lite/tnn/asr/*.cpp) -# 2. glob headers files -file(GLOB TNN_CORE_HEAD ${CMAKE_SOURCE_DIR}/lite/tnn/core/*.h) -file(GLOB TNN_CV_HEAD ${CMAKE_SOURCE_DIR}/lite/tnn/cv/*.h) -file(GLOB TNN_NLP_HEAD ${CMAKE_SOURCE_DIR}/lite/tnn/nlp/*.h) -file(GLOB TNN_ASR_HEAD ${CMAKE_SOURCE_DIR}/lite/tnn/asr/*.h) - -set(TNN_SRCS ${TNN_CV_SRCS} ${TNN_NLP_SRCS} ${TNN_ASR_SRCS} ${TNN_CORE_SRCS}) -# 3. copy -message("[Lite.AI.Toolkit][I] Installing Lite.AI.ToolKit Headers for TNN Backend ...") -# "INSTALL" can copy all files from the list to the specified path. -# "COPY" only copies one file to a specified path -file(INSTALL ${TNN_CORE_HEAD} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/lite/tnn/core) -file(INSTALL ${TNN_CV_HEAD} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/lite/tnn/cv) -file(INSTALL ${TNN_ASR_HEAD} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/lite/tnn/asr) -file(INSTALL ${TNN_NLP_HEAD} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/lite/tnn/nlp) diff --git a/cmake/lite.ai.toolkit.cmake.in b/cmake/lite.ai.toolkit.cmake.in index 38e11060..7bd937ea 100644 --- a/cmake/lite.ai.toolkit.cmake.in +++ b/cmake/lite.ai.toolkit.cmake.in @@ -1,9 +1,6 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.8) set(ENABLE_ONNXRUNTIME @ENABLE_ONNXRUNTIME@) -set(ENABLE_MNN @ENABLE_MNN@) -set(ENABLE_NCNN @ENABLE_NCNN@) -set(ENABLE_TNN @ENABLE_TNN@) set(ENABLE_TENSORRT @ENABLE_TENSORRT@) set(CUDA_DIR @CUDA_DIR@) set(TensorRT_DIR @TensorRT_DIR@) @@ -13,8 +10,8 @@ if (NOT (UNIX AND NOT APPLE)) message(FATAL_ERROR "lite.ai.toolkit>=0.2 not support for windows/mac now!") endif() -if ((NOT ENABLE_ONNXRUNTIME) AND (NOT ENABLE_MNN)) - message(FATAL_ERROR "One of ONNXRuntime/MNN Backend must be enable!") +if (NOT ENABLE_ONNXRUNTIME) + message(FATAL_ERROR "ONNXRuntime backend must be enabled!") endif() # lite.ai.toolkit @@ -46,33 +43,6 @@ if (ENABLE_ONNXRUNTIME) list(APPEND Lite_AI_LIBS onnxruntime) endif() -# MNN -if (ENABLE_MNN) - include_directories(${THIRD_PARTY_PATH}/MNN/include) - link_directories(${THIRD_PARTY_PATH}/MNN/lib) - list(APPEND Lite_AI_INCLUDE_DIRS ${THIRD_PARTY_PATH}/MNN/include) - list(APPEND Lite_AI_LIBS_DIRS ${THIRD_PARTY_PATH}/MNN/lib) - list(APPEND Lite_AI_LIBS MNN) -endif() - -# TNN -if (ENABLE_TNN) - include_directories(${THIRD_PARTY_PATH}/TNN/include) - link_directories(${THIRD_PARTY_PATH}/TNN/lib) - list(APPEND Lite_AI_INCLUDE_DIRS ${THIRD_PARTY_PATH}/TNN/include) - list(APPEND Lite_AI_LIBS_DIRS ${THIRD_PARTY_PATH}/TNN/lib) - list(APPEND Lite_AI_LIBS TNN) -endif() - -# ncnn -if (ENABLE_NCNN) - include_directories(${THIRD_PARTY_PATH}/ncnn/include) - link_directories(${THIRD_PARTY_PATH}/ncnn/lib) - list(APPEND Lite_AI_INCLUDE_DIRS ${THIRD_PARTY_PATH}/ncnn/include) - list(APPEND Lite_AI_LIBS_DIRS ${THIRD_PARTY_PATH}/ncnn/lib) - list(APPEND Lite_AI_LIBS ncnn) -endif() - # tensorrt if (ENABLE_TENSORRT) include_directories(${TensorRT_DIR}/include) diff --git a/cmake/ncnn.cmake b/cmake/ncnn.cmake deleted file mode 100644 index 61800e83..00000000 --- a/cmake/ncnn.cmake +++ /dev/null @@ -1,27 +0,0 @@ -set(NCNN_DIR ${THIRD_PARTY_PATH}/ncnn) -if(NOT EXISTS ${NCNN_DIR}) - message(FATAL_ERROR "[Lite.AI.Toolkit][E] ${NCNN_DIR} is not exists!") -endif() -include_directories(${NCNN_DIR}/include) -link_directories(${NCNN_DIR}/lib) - -# 1. glob sources files -file(GLOB NCNN_CORE_SRCS ${CMAKE_SOURCE_DIR}/lite/ncnn/core/*.cpp) -file(GLOB NCNN_CV_SRCS ${CMAKE_SOURCE_DIR}/lite/ncnn/cv/*.cpp) -file(GLOB NCNN_NLP_SRCS ${CMAKE_SOURCE_DIR}/lite/ncnn/nlp/*.cpp) -file(GLOB NCNN_ASR_SRCS ${CMAKE_SOURCE_DIR}/lite/ncnn/asr/*.cpp) -# 2. glob headers files -file(GLOB NCNN_CORE_HEAD ${CMAKE_SOURCE_DIR}/lite/ncnn/core/*.h) -file(GLOB NCNN_CV_HEAD ${CMAKE_SOURCE_DIR}/lite/ncnn/cv/*.h) -file(GLOB NCNN_NLP_HEAD ${CMAKE_SOURCE_DIR}/lite/ncnn/nlp/*.h) -file(GLOB NCNN_ASR_HEAD ${CMAKE_SOURCE_DIR}/lite/ncnn/asr/*.h) - -set(NCNN_SRCS ${NCNN_CV_SRCS} ${NCNN_NLP_SRCS} ${NCNN_ASR_SRCS} ${NCNN_CORE_SRCS}) -# 3. copy -message("[Lite.AI.Toolkit][I] Installing Lite.AI.ToolKit Headers for NCNN Backend ...") -# "INSTALL" can copy all files from the list to the specified path. -# "COPY" only copies one file to a specified path -file(INSTALL ${NCNN_CORE_HEAD} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/lite/ncnn/core) -file(INSTALL ${NCNN_CV_HEAD} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/lite/ncnn/cv) -file(INSTALL ${NCNN_ASR_HEAD} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/lite/ncnn/asr) -file(INSTALL ${NCNN_NLP_HEAD} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/lite/ncnn/nlp) diff --git a/cmake/utils.cmake b/cmake/utils.cmake index 9073a3c4..63157299 100644 --- a/cmake/utils.cmake +++ b/cmake/utils.cmake @@ -57,24 +57,6 @@ function(add_lite_ai_toolkit_shared_library version soversion) link_directories(${CMAKE_SOURCE_DIR}/lite/bin) endif () - if (ENABLE_MNN) - include(cmake/MNN.cmake) - set(LITE_SRCS ${LITE_SRCS} ${MNN_SRCS}) - set(LITE_DEPENDENCIES ${LITE_DEPENDENCIES} MNN) - endif () - - if (ENABLE_NCNN) - include(cmake/ncnn.cmake) - set(LITE_SRCS ${LITE_SRCS} ${NCNN_SRCS}) - set(LITE_DEPENDENCIES ${LITE_DEPENDENCIES} ncnn) - endif () - - if (ENABLE_TNN) - include(cmake/TNN.cmake) - set(LITE_SRCS ${LITE_SRCS} ${TNN_SRCS}) - set(LITE_DEPENDENCIES ${LITE_DEPENDENCIES} TNN) - endif () - # 4. shared library add_library(lite.ai.toolkit SHARED ${LITE_SRCS}) target_link_libraries(lite.ai.toolkit ${LITE_DEPENDENCIES}) diff --git a/docs/hub/lite.ai.toolkit.hub.mnn.md b/docs/hub/lite.ai.toolkit.hub.mnn.md deleted file mode 100644 index e68524bf..00000000 --- a/docs/hub/lite.ai.toolkit.hub.mnn.md +++ /dev/null @@ -1,333 +0,0 @@ -# Lite.AI.ToolKit.Hub.MNN - -You can download all the pretrained models files of MNN format from ([Baidu Drive](https://pan.baidu.com/s/1KyO-bCYUv6qPq2M8BH_Okg) code: 9v63) - -## Object Detection. - -
- -| Class | Pretrained MNN Files | Rename or Converted From (Repo) | Size | -|:---------------------------------------------------:|:----------------------------------:|:------------------------------------------------------------:|:-----:| -| *lite::mnn::cv::detection::NanoDet* | nanodet_m_0.5x.mnn | [nanodet](https://github.com/RangiLyu/nanodet) | 1.1Mb | -| *lite::mnn::cv::detection::NanoDet* | nanodet_m.mnn | [nanodet](https://github.com/RangiLyu/nanodet) | 3.6Mb | -| *lite::mnn::cv::detection::NanoDet* | nanodet_m_1.5x.mnn | [nanodet](https://github.com/RangiLyu/nanodet) | 7.9Mb | -| *lite::mnn::cv::detection::NanoDet* | nanodet_m_1.5x_416.mnn | [nanodet](https://github.com/RangiLyu/nanodet) | 7.9Mb | -| *lite::mnn::cv::detection::NanoDet* | nanodet_m_416.mnn | [nanodet](https://github.com/RangiLyu/nanodet) | 3.6Mb | -| *lite::mnn::cv::detection::NanoDet* | nanodet_g.mnn | [nanodet](https://github.com/RangiLyu/nanodet) | 14Mb | -| *lite::mnn::cv::detection::NanoDet* | nanodet_t.mnn | [nanodet](https://github.com/RangiLyu/nanodet) | 5.1Mb | -| *lite::mnn::cv::detection::NanoDet* | nanodet-RepVGG-A0_416.mnn | [nanodet](https://github.com/RangiLyu/nanodet) | 26Mb | -| *lite::mnn::cv::detection::NanoDetEfficientNetLite* | nanodet-EfficientNet-Lite0_320.mnn | [nanodet](https://github.com/RangiLyu/nanodet) | 12Mb | -| *lite::mnn::cv::detection::NanoDetEfficientNetLite* | nanodet-EfficientNet-Lite1_416.mnn | [nanodet](https://github.com/RangiLyu/nanodet) | 15Mb | -| *lite::mnn::cv::detection::NanoDetEfficientNetLite* | nanodet-EfficientNet-Lite2_512.mnn | [nanodet](https://github.com/RangiLyu/nanodet) | 18Mb | -| *lite::mnn::cv::detection::YoloX* | yolox_x.mnn | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 378Mb | -| *lite::mnn::cv::detection::YoloX* | yolox_l.mnn | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 207Mb | -| *lite::mnn::cv::detection::YoloX* | yolox_m.mnn | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 97Mb | -| *lite::mnn::cv::detection::YoloX* | yolox_s.mnn | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 34Mb | -| *lite::mnn::cv::detection::YoloX* | yolox_tiny.mnn | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 19Mb | -| *lite::mnn::cv::detection::YoloX* | yolox_nano.mnn | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 3.5Mb | -| *lite::mnn::cv::detection::YOLOP* | yolop-320-320.mnn | [YOLOP](https://github.com/hustvl/YOLOP) | 30Mb | -| *lite::mnn::cv::detection::YOLOP* | yolop-640-640.mnn | [YOLOP](https://github.com/hustvl/YOLOP) | 30Mb | -| *lite::mnn::cv::detection::YOLOP* | yolop-1280-1280.mnn | [YOLOP](https://github.com/hustvl/YOLOP) | 30Mb | -| *lite::mnn::cv::detection::YoloV5* | yolov5l.mnn | [yolov5](https://github.com/ultralytics/yolov5) | 188Mb | -| *lite::mnn::cv::detection::YoloV5* | yolov5m.mnn | [yolov5](https://github.com/ultralytics/yolov5) | 85Mb | -| *lite::mnn::cv::detection::YoloV5* | yolov5s.mnn | [yolov5](https://github.com/ultralytics/yolov5) | 29Mb | -| *lite::mnn::cv::detection::YoloV5* | yolov5x.mnn | [yolov5](https://github.com/ultralytics/yolov5) | 351Mb | -| *lite::mnn::cv::detection::YoloX_V_0_1_1* | yolox_x_v0.1.1.mnn | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 378Mb | -| *lite::mnn::cv::detection::YoloX_V_0_1_1* | yolox_l_v0.1.1.mnn | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 207Mb | -| *lite::mnn::cv::detection::YoloX_V_0_1_1* | yolox_m_v0.1.1.mnn | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 97Mb | -| *lite::mnn::cv::detection::YoloX_V_0_1_1* | yolox_s_v0.1.1.mnn | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 34Mb | -| *lite::mnn::cv::detection::YoloX_V_0_1_1* | yolox_tiny_v0.1.1.mnn | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 19Mb | -| *lite::mnn::cv::detection::YoloX_V_0_1_1* | yolox_nano_v0.1.1.mnn | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 3.5Mb | -| *lite::mnn::cv::detection::YoloR* | yolor-p6-320-320.mnn | [yolor](https://github.com/WongKinYiu/yolor) | 157Mb | -| *lite::mnn::cv::detection::YoloR* | yolor-p6-640-640.mnn | [yolor](https://github.com/WongKinYiu/yolor) | 157Mb | -| *lite::mnn::cv::detection::YoloR* | yolor-ssss-s2d-640-640.mnn | [yolor](https://github.com/WongKinYiu/yolor) | 50Mb | -| *lite::mnn::cv::detection::YoloV5_V_6_0* | yolov5l.640-640.v.6.0.mnn | [yolov5](https://github.com/ultralytics/yolov5) | 178Mb | -| *lite::mnn::cv::detection::YoloV5_V_6_0* | yolov5m.640-640.v.6.0.mnn | [yolov5](https://github.com/ultralytics/yolov5) | 81Mb | -| *lite::mnn::cv::detection::YoloV5_V_6_0* | yolov5s.640-640.v.6.0.mnn | [yolov5](https://github.com/ultralytics/yolov5) | 28Mb | -| *lite::mnn::cv::detection::YoloV5_V_6_0* | yolov5n.640-640.v.6.0.mnn | [yolov5](https://github.com/ultralytics/yolov5) | 7.5Mb | -| *lite::mnn::cv::detection::YoloV5_V_6_0* | yolov5l6.640-640.v.6.0.mnn | [yolov5](https://github.com/ultralytics/yolov5) | 294Mb | -| *lite::mnn::cv::detection::YoloV5_V_6_0* | yolov5m6.640-640.v.6.0.mnn | [yolov5](https://github.com/ultralytics/yolov5) | 128Mb | -| *lite::mnn::cv::detection::YoloV5_V_6_0* | yolov5s6.640-640.v.6.0.mnn | [yolov5](https://github.com/ultralytics/yolov5) | 50Mb | -| *lite::mnn::cv::detection::YoloV5_V_6_0* | yolov5n6.640-640.v.6.0.mnn | [yolov5](https://github.com/ultralytics/yolov5) | 14Mb | -| *lite::mnn::cv::detection::YoloV5_V_6_0* | yolov5l6.1280-1280.v.6.0.mnn | [yolov5](https://github.com/ultralytics/yolov5) | 294Mb | -| *lite::mnn::cv::detection::YoloV5_V_6_0* | yolov5m6.1280-1280.v.6.0.mnn | [yolov5](https://github.com/ultralytics/yolov5) | 128Mb | -| *lite::mnn::cv::detection::YoloV5_V_6_0* | yolov5s6.1280-1280.v.6.0.mnn | [yolov5](https://github.com/ultralytics/yolov5) | 50Mb | -| *lite::mnn::cv::detection::YoloV5_V_6_0* | yolov5n6.1280-1280.v.6.0.mnn | [yolov5](https://github.com/ultralytics/yolov5) | 14Mb | -| *lite::mnn::cv::detection::NanoDetPlus* | nanodet-plus-m_320.mnn | [nanodet](https://github.com/RangiLyu/nanodet) | 4.5Mb | -| *lite::mnn::cv::detection::NanoDetPlus* | nanodet-plus-m_416.mnn | [nanodet](https://github.com/RangiLyu/nanodet) | 4.5Mb | -| *lite::mnn::cv::detection::NanoDetPlus* | nanodet-plus-m-1.5x_320.mnn | [nanodet](https://github.com/RangiLyu/nanodet) | 9.4Mb | -| *lite::mnn::cv::detection::NanoDetPlus* | nanodet-plus-m-1.5x_416.mnn | [nanodet](https://github.com/RangiLyu/nanodet) | 9.4Mb | -| *lite::mnn::cv::detection::InsectDet* | quarrying_insect_detector.mnn | [InsectID](https://github.com/quarrying/quarrying-insect-id) | 22Mb | -| *lite::mnn::cv::detection::YoloV5_V_6_1* | yolov5l.v6.1.640x640.mnn | [yolov5](https://github.com/ultralytics/yolov5) | 178Mb | -| *lite::mnn::cv::detection::YoloV5_V_6_1* | yolov5l.v6.1.1280x1280.mnn | [yolov5](https://github.com/ultralytics/yolov5) | 178Mb | -| *lite::mnn::cv::detection::YoloV5_V_6_1* | yolov5m.v6.1.640x640.mnn | [yolov5](https://github.com/ultralytics/yolov5) | 81Mb | -| *lite::mnn::cv::detection::YoloV5_V_6_1* | yolov5x.v6.1.640x640.mnn | [yolov5](https://github.com/ultralytics/yolov5) | 332Mb | -| *lite::mnn::cv::detection::YoloV5_V_6_1* | yolov5x.v6.1.1280x1280.mnn | [yolov5](https://github.com/ultralytics/yolov5) | 332Mb | -| *lite::mnn::cv::detection::YoloV5_V_6_1* | yolov5s.v6.1.640x640.mnn | [yolov5](https://github.com/ultralytics/yolov5) | 28Mb | -| *lite::mnn::cv::detection::YoloV5_V_6_1* | yolov5s.v6.1.320x320.mnn | [yolov5](https://github.com/ultralytics/yolov5) | 28Mb | -| *lite::mnn::cv::detection::YoloV5_V_6_1* | yolov5n.v6.1.640x640.mnn | [yolov5](https://github.com/ultralytics/yolov5) | 7Mb | -| *lite::mnn::cv::detection::YoloV5_V_6_1* | yolov5n.v6.1.320x320.mnn | [yolov5](https://github.com/ultralytics/yolov5) | 7Mb | -| *lite::mnn::cv::detection::YOLOv6* | yolov6n-320x320.mnn | [YOLOv6](https://github.com/meituan/YOLOv6) | 17Mb | -| *lite::mnn::cv::detection::YOLOv6* | yolov6n-640x640.mnn | [YOLOv6](https://github.com/meituan/YOLOv6) | 17Mb | -| *lite::mnn::cv::detection::YOLOv6* | yolov6s-320x320.mnn | [YOLOv6](https://github.com/meituan/YOLOv6) | 66Mb | -| *lite::mnn::cv::detection::YOLOv6* | yolov6n-640x640.mnn | [YOLOv6](https://github.com/meituan/YOLOv6) | 66Mb | -| *lite::mnn::cv::detection::YOLOv6* | yolov6t-640x640.mnn | [YOLOv6](https://github.com/meituan/YOLOv6) | 57Mb | - - -## Matting. - -
- -| Class | Pretrained MNN Files | Rename or Converted From (Repo) | Size | -|:---------------------------------------------:|:--------------------------------------------------:|:-------------------------------------------------------------------------------------------:|:-----:| -| *lite::mnn::cv::matting::RobustVideoMatting* | rvm_mobilenetv3_fp32.mnn | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 14Mb | -| *lite::mnn::cv::matting::RobustVideoMatting* | rvm_mobilenetv3_fp32-480-480.mnn | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 14Mb | -| *lite::mnn::cv::matting::RobustVideoMatting* | rvm_mobilenetv3_fp32-480-640.mnn | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 14Mb | -| *lite::mnn::cv::matting::RobustVideoMatting* | rvm_mobilenetv3_fp32-640-480.mnn | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 14Mb | -| *lite::mnn::cv::matting::RobustVideoMatting* | rvm_mobilenetv3_fp32-1080-1920.mnn | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 14Mb | -| *lite::mnn::cv::matting::RobustVideoMatting* | rvm_resnet50_fp32.mnn | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 50Mb | -| *lite::mnn::cv::matting::RobustVideoMatting* | rvm_resnet50_fp32-480-480.mnn | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 50Mb | -| *lite::mnn::cv::matting::RobustVideoMatting* | rvm_resnet50_fp32-480-640.mnn | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 50Mb | -| *lite::mnn::cv::matting::RobustVideoMatting* | rvm_resnet50_fp32-640-480.mnn | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 50Mb | -| *lite::mnn::cv::matting::RobustVideoMatting* | rvm_resnet50_fp32-1080-1920.mnn | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 50Mb | -| *lite::mnn::cv::matting::MGMatting* | MGMatting-DIM-100k.mnn | [MGMatting](https://github.com/yucornetto/MGMatting) | 113Mb | -| *lite::mnn::cv::matting::MGMatting* | MGMatting-RWP-100k.mnn | [MGMatting](https://github.com/yucornetto/MGMatting) | 113Mb | -| *lite::mnn::cv::matting::MODNet* | modnet_photographic_portrait_matting-1024x1024.mnn | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::mnn::cv::matting::MODNet* | modnet_photographic_portrait_matting-1024x512.mnn | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::mnn::cv::matting::MODNet* | modnet_photographic_portrait_matting-256x256.mnn | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::mnn::cv::matting::MODNet* | modnet_photographic_portrait_matting-256x512.mnn | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::mnn::cv::matting::MODNet* | modnet_photographic_portrait_matting-512x1024.mnn | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::mnn::cv::matting::MODNet* | modnet_photographic_portrait_matting-512x256.mnn | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::mnn::cv::matting::MODNet* | modnet_photographic_portrait_matting-512x512.mnn | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::mnn::cv::matting::MODNet* | modnet_webcam_portrait_matting-1024x1024.mnn | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::mnn::cv::matting::MODNet* | modnet_webcam_portrait_matting-1024x512.mnn | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::mnn::cv::matting::MODNet* | modnet_webcam_portrait_matting-256x256.mnn | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::mnn::cv::matting::MODNet* | modnet_webcam_portrait_matting-256x512.mnn | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::mnn::cv::matting::MODNet* | modnet_webcam_portrait_matting-512x1024.mnn | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::mnn::cv::matting::MODNet* | modnet_webcam_portrait_matting-512x256.mnn | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::mnn::cv::matting::MODNet* | modnet_webcam_portrait_matting-512x512.mnn | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::mnn::cv::matting::BackgroundMattingV2* | BGMv2_mobilenetv2-256x256-full.mnn | [BackgroundMattingV2](https://github.com/PeterL1n/BackgroundMattingV2) | 20Mb | -| *lite::mnn::cv::matting::BackgroundMattingV2* | BGMv2_mobilenetv2-512x512-full.mnn | [BackgroundMattingV2](https://github.com/PeterL1n/BackgroundMattingV2) | 20Mb | -| *lite::mnn::cv::matting::BackgroundMattingV2* | BGMv2_mobilenetv2-1080x1920-full.mnn | [BackgroundMattingV2](https://github.com/PeterL1n/BackgroundMattingV2) | 20Mb | -| *lite::mnn::cv::matting::BackgroundMattingV2* | BGMv2_mobilenetv2-2160x3840-full.mnn | [BackgroundMattingV2](https://github.com/PeterL1n/BackgroundMattingV2) | 20Mb | -| *lite::mnn::cv::matting::BackgroundMattingV2* | BGMv2_resnet50-1080x1920-full.mnn | [BackgroundMattingV2](https://github.com/PeterL1n/BackgroundMattingV2) | 20Mb | -| *lite::mnn::cv::matting::BackgroundMattingV2* | BGMv2_resnet50-2160x3840-full.mnn | [BackgroundMattingV2](https://github.com/PeterL1n/BackgroundMattingV2) | 20Mb | -| *lite::mnn::cv::matting::BackgroundMattingV2* | BGMv2_resnet101-2160x3840-full.mnn | [BackgroundMattingV2](https://github.com/PeterL1n/BackgroundMattingV2) | 154Mb | -| *lite::mnn::cv::matting::MobileHumanMatting* | mobile_human_matting_256x256.mnn | [mobile_phone_human_matting_](https://github.com/lizhengwei1992/mobile_phone_human_matting) | 3Mb | -| *lite::mnn::cv::matting::MobileHumanMatting* | mobile_human_matting_128x128.mnn | [mobile_phone_human_matting_](https://github.com/lizhengwei1992/mobile_phone_human_matting) | 3Mb | -| *lite::mnn::cv::matting::MobileHumanMatting* | mobile_human_matting_320x320.mnn | [mobile_phone_human_matting_](https://github.com/lizhengwei1992/mobile_phone_human_matting) | 3Mb | -| *lite::mnn::cv::matting::MobileHumanMatting* | mobile_human_matting_512x512.mnn | [mobile_phone_human_matting_](https://github.com/lizhengwei1992/mobile_phone_human_matting) | 3Mb | - -## Face Recognition. - -
- - -| Class | Pretrained MNN Files | Rename or Converted From (Repo) | Size | -|:----------------------------------------------:|:------------------------------------------------------:|:----------------------------------------------------------------------:|:-----:| -| *lite::mnn::cv::faceid::GlintArcFace* | ms1mv3_arcface_r100.mnn | [insightface](https://github.com/deepinsight/insightface) | 248Mb | -| *lite::mnn::cv::faceid::GlintArcFace* | ms1mv3_arcface_r50.mnn | [insightface](https://github.com/deepinsight/insightface) | 166Mb | -| *lite::mnn::cv::faceid::GlintArcFace* | ms1mv3_arcface_r34.mnn | [insightface](https://github.com/deepinsight/insightface) | 130Mb | -| *lite::mnn::cv::faceid::GlintArcFace* | ms1mv3_arcface_r18.mnn | [insightface](https://github.com/deepinsight/insightface) | 91Mb | -| *lite::mnn::cv::faceid::GlintCosFace* | glint360k_cosface_r100.mnn | [insightface](https://github.com/deepinsight/insightface) | 248Mb | -| *lite::mnn::cv::faceid::GlintCosFace* | glint360k_cosface_r50.mnn | [insightface](https://github.com/deepinsight/insightface) | 166Mb | -| *lite::mnn::cv::faceid::GlintCosFace* | glint360k_cosface_r34.mnn | [insightface](https://github.com/deepinsight/insightface) | 130Mb | -| *lite::mnn::cv::faceid::GlintCosFace* | glint360k_cosface_r18.mnn | [insightface](https://github.com/deepinsight/insightface) | 91Mb | -| *lite::mnn::cv::faceid::GlintPartialFC* | partial_fc_glint360k_r100.mnn | [insightface](https://github.com/deepinsight/insightface) | 248Mb | -| *lite::mnn::cv::faceid::GlintPartialFC* | partial_fc_glint360k_r50.mnn | [insightface](https://github.com/deepinsight/insightface) | 91Mb | -| *lite::mnn::cv::faceid::FaceNet* | facenet_vggface2_resnet.mnn | [facenet...](https://github.com/timesler/facenet-pytorch) | 89Mb | -| *lite::mnn::cv::faceid::FaceNet* | facenet_casia-webface_resnet.mnn | [facenet...](https://github.com/timesler/facenet-pytorch) | 89Mb | -| *lite::mnn::cv::faceid::FocalArcFace* | focal-arcface-ms1m-ir152.mnn | [face.evoLVe...](https://github.com/ZhaoJ9014/face.evoLVe.PyTorch) | 269Mb | -| *lite::mnn::cv::faceid::FocalArcFace* | focal-arcface-ms1m-ir50-epoch120.mnn | [face.evoLVe...](https://github.com/ZhaoJ9014/face.evoLVe.PyTorch) | 166Mb | -| *lite::mnn::cv::faceid::FocalArcFace* | focal-arcface-ms1m-ir50-epoch63.mnn | [face.evoLVe...](https://github.com/ZhaoJ9014/face.evoLVe.PyTorch) | 166Mb | -| *lite::mnn::cv::faceid::FocalAsiaArcFace* | focal-arcface-bh-ir50-asia.mnn | [face.evoLVe...](https://github.com/ZhaoJ9014/face.evoLVe.PyTorch) | 166Mb | -| *lite::mnn::cv::faceid::TencentCurricularFace* | Tencent_CurricularFace_Backbone.mnn | [TFace](https://github.com/Tencent/TFace) | 249Mb | -| *lite::mnn::cv::faceid::TencentCifpFace* | Tencent_Cifp_BUPT_Balancedface_IR_34.mnn | [TFace](https://github.com/Tencent/TFace) | 130Mb | -| *lite::mnn::cv::faceid::CenterLossFace* | CenterLossFace_epoch_100.mnn | [center-loss...](https://github.com/louis-she/center-loss.pytorch) | 280Mb | -| *lite::mnn::cv::faceid::SphereFace* | sphere20a_20171020.mnn | [sphere...](https://github.com/clcarwin/sphereface_pytorch) | 86Mb | -| *lite::mnn::cv::faceid:MobileFaceNet* | MobileFaceNet_Pytorch_068.mnn | [MobileFace...](https://github.com/Xiaoccer/MobileFaceNet_Pytorch) | 3.8Mb | -| *lite::mnn::cv::faceid:CavaGhostArcFace* | cavaface_GhostNet_x1.3_Arcface_Epoch_24.mnn | [cavaface...](https://github.com/cavalleria/cavaface.pytorch) | 15Mb | -| *lite::mnn::cv::faceid:CavaCombinedFace* | cavaface_IR_SE_100_Combined_Epoch_24.mnn | [cavaface...](https://github.com/cavalleria/cavaface.pytorch) | 250Mb | -| *lite::mnn::cv::faceid:MobileSEFocalFace* | face_recognition.pytorch_Mobilenet_se_focal_121000.mnn | [face_recog...](https://github.com/grib0ed0v/face_recognition.pytorch) | 4.5Mb | - -## Face Detection. - -
- -| Class | Pretrained MNN Files | Rename or Converted From (Repo) | Size | -|:----------------------------------------------:|:------------------------------------------:|:---------------------------------------------------------------------------------------:|:------:| -| *lite::mnn::cv::face::detect::UltraFace* | ultraface-rfb-320.mnn | [Ultra-Light...](https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB) | 1.5Mb | -| *lite::mnn::cv::face::detect::UltraFace* | ultraface-rfb-640.mnn | [Ultra-Light...](https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB) | 1.5Mb | -| *lite::mnn::cv::face::detect::UltraFace* | ultraface-slim-320.mnn | [Ultra-Light...](https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB) | 1.2Mb | -| *lite::mnn::cv::face::detect::UltraFace* | ultraface-slim-640.mnn | [Ultra-Light...](https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB) | 1.2Mb | -| *lite::mnn::cv::face::detect::RetinaFace* | Pytorch_RetinaFace_mobile0.25.mnn | [...Retinaface](https://github.com/biubug6/Pytorch_Retinaface) | 1.6Mb | -| *lite::mnn::cv::face::detect::RetinaFace* | Pytorch_RetinaFace_mobile0.25-640-640.mnn | [...Retinaface](https://github.com/biubug6/Pytorch_Retinaface) | 1.6Mb | -| *lite::mnn::cv::face::detect::RetinaFace* | Pytorch_RetinaFace_mobile0.25-320-320.mnn | [...Retinaface](https://github.com/biubug6/Pytorch_Retinaface) | 1.6Mb | -| *lite::mnn::cv::face::detect::RetinaFace* | Pytorch_RetinaFace_mobile0.25-720-1080.mnn | [...Retinaface](https://github.com/biubug6/Pytorch_Retinaface) | 1.6Mb | -| *lite::mnn::cv::face::detect::FaceBoxes* | FaceBoxes.mnn | [FaceBoxes](https://github.com/zisianw/FaceBoxes.PyTorch) | 3.8Mb | -| *lite::mnn::cv::face::detect::FaceBoxes* | FaceBoxes-640-640.mnn | [FaceBoxes](https://github.com/zisianw/FaceBoxes.PyTorch) | 3.8Mb | -| *lite::mnn::cv::face::detect::FaceBoxes* | FaceBoxes-320-320.mnn | [FaceBoxes](https://github.com/zisianw/FaceBoxes.PyTorch) | 3.8Mb | -| *lite::mnn::cv::face::detect::FaceBoxes* | FaceBoxes-720-1080.mnn | [FaceBoxes](https://github.com/zisianw/FaceBoxes.PyTorch) | 3.8Mb | -| *lite::mnn::cv::face::detect::SCRFD* | scrfd_500m_shape160x160.mnn | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.5Mb | -| *lite::mnn::cv::face::detect::SCRFD* | scrfd_500m_shape320x320.mnn | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.5Mb | -| *lite::mnn::cv::face::detect::SCRFD* | scrfd_500m_shape640x640.mnn | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.5Mb | -| *lite::mnn::cv::face::detect::SCRFD* | scrfd_500m_bnkps_shape160x160.mnn | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.5Mb | -| *lite::mnn::cv::face::detect::SCRFD* | scrfd_500m_bnkps_shape320x320.mnn | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.5Mb | -| *lite::mnn::cv::face::detect::SCRFD* | scrfd_500m_bnkps_shape640x640.mnn | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.5Mb | -| *lite::mnn::cv::face::detect::SCRFD* | scrfd_1g_shape160x160.mnn | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.7Mb | -| *lite::mnn::cv::face::detect::SCRFD* | scrfd_1g_shape320x320.mnn | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.7Mb | -| *lite::mnn::cv::face::detect::SCRFD* | scrfd_1g_shape640x640.mnn | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.7Mb | -| *lite::mnn::cv::face::detect::SCRFD* | scrfd_2.5g_shape160x160.mnn | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 3.3Mb | -| *lite::mnn::cv::face::detect::SCRFD* | scrfd_2.5g_shape320x320.mnn | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 3.3Mb | -| *lite::mnn::cv::face::detect::SCRFD* | scrfd_2.5g_shape640x640.mnn | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 3.3Mb | -| *lite::mnn::cv::face::detect::SCRFD* | scrfd_2.5g_bnkps_shape160x160.mnn | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 3.3Mb | -| *lite::mnn::cv::face::detect::SCRFD* | scrfd_2.5g_bnkps_shape320x320.mnn | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 3.3Mb | -| *lite::mnn::cv::face::detect::SCRFD* | scrfd_2.5g_bnkps_shape640x640.mnn | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 3.3Mb | -| *lite::mnn::cv::face::detect::SCRFD* | scrfd_10g_shape640x640.mnn | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 16.9Mb | -| *lite::mnn::cv::face::detect::SCRFD* | scrfd_10g_shape1280x1280.mnn | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 16.9Mb | -| *lite::mnn::cv::face::detect::SCRFD* | scrfd_10g_bnkps_shape640x640.mnn | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 16.9Mb | -| *lite::mnn::cv::face::detect::SCRFD* | scrfd_10g_bnkps_shape1280x1280.mnn | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 16.9Mb | -| *lite::mnn::cv::face::detect::YOLO5Face* | yolov5face-blazeface-640x640.mnn | [YOLO5Face](https://github.com/deepcam-cn/yolov5-face) | 3.4Mb | -| *lite::mnn::cv::face::detect::YOLO5Face* | yolov5face-l-640x640.mnn | [YOLO5Face](https://github.com/deepcam-cn/yolov5-face) | 181Mb | -| *lite::mnn::cv::face::detect::YOLO5Face* | yolov5face-m-640x640.mnn | [YOLO5Face](https://github.com/deepcam-cn/yolov5-face) | 83Mb | -| *lite::mnn::cv::face::detect::YOLO5Face* | yolov5face-n-0.5-320x320.mnn | [YOLO5Face](https://github.com/deepcam-cn/yolov5-face) | 2.5Mb | -| *lite::mnn::cv::face::detect::YOLO5Face* | yolov5face-n-0.5-640x640.mnn | [YOLO5Face](https://github.com/deepcam-cn/yolov5-face) | 4.6Mb | -| *lite::mnn::cv::face::detect::YOLO5Face* | yolov5face-n-640x640.mnn | [YOLO5Face](https://github.com/deepcam-cn/yolov5-face) | 9.5Mb | -| *lite::mnn::cv::face::detect::YOLO5Face* | yolov5face-s-640x640.mnn | [YOLO5Face](https://github.com/deepcam-cn/yolov5-face) | 30Mb | -| *lite::mnn::cv::face::detect::FaceBoxesV2* | faceboxesv2-640x640.mnn | [FaceBoxesV2](https://github.com/jhb86253817/FaceBoxesV2) | 4.0Mb | -| *lite::mnn::cv::face::detect::YOLOv5BlazeFace* | yolov5face-blazeface-640x640.mnn | [YOLO5Face](https://github.com/deepcam-cn/yolov5-face) | 3.4Mb | - - -## Face Alignment. - -
- - -| Class | Pretrained MNN Files | Rename or Converted From (Repo) | Size | -|:-----------------------------------------------:|:------------------------------------------------------------:|:------------------------------------------------------------------:|:-------:| -| *lite::mnn::cv::face::align::PFLD* | pfld-106-lite.mnn | [pfld_106_...](https://github.com/Hsintao/pfld_106_face_landmarks) | 1.0Mb | -| *lite::mnn::cv::face::align::PFLD* | pfld-106-v3.mnn | [pfld_106_...](https://github.com/Hsintao/pfld_106_face_landmarks) | 5.5Mb | -| *lite::mnn::cv::face::align::PFLD* | pfld-106-v2.mnn | [pfld_106_...](https://github.com/Hsintao/pfld_106_face_landmarks) | 5.0Mb | -| *lite::mnn::cv::face::align::PFLD98* | PFLD-pytorch-pfld.mnn | [PFLD...](https://github.com/polarisZhao/PFLD-pytorch) | 4.8Mb | -| *lite::mnn::cv::face::align::MobileNetV268* | pytorch_face_landmarks_landmark_detection_56.mnn | [...landmark](https://github.com/cunjian/pytorch_face_landmark) | 9.4Mb | -| *lite::mnn::cv::face::align::MobileNetV2SE68* | pytorch_face_landmarks_landmark_detection_56_se_external.mnn | [...landmark](https://github.com/cunjian/pytorch_face_landmark) | 11Mb | -| *lite::mnn::cv::face::align::PFLD68* | pytorch_face_landmarks_pfld.mnn | [...landmark](https://github.com/cunjian/pytorch_face_landmark) | 2.8Mb | -| *lite::mnn::cv::face::align::FaceLandmarks1000* | FaceLandmark1000.mnn | [FaceLandm...](https://github.com/Single430/FaceLandmark1000) | 2.0Mb | -| *lite::mnn::cv::face::align::PIPNet98* | pipnet_resnet18_10x98x32x256_wflw.mnn | [PIPNet](https://github.com/jhb86253817/PIPNet) | 44.0Mb | -| *lite::mnn::cv::face::align::PIPNet68* | pipnet_resnet18_10x68x32x256_300w.mnn | [PIPNet](https://github.com/jhb86253817/PIPNet) | 44.0Mb | -| *lite::mnn::cv::face::align::PIPNet29* | pipnet_resnet18_10x29x32x256_cofw.mnn | [PIPNet](https://github.com/jhb86253817/PIPNet) | 44.0Mb | -| *lite::mnn::cv::face::align::PIPNet19* | pipnet_resnet18_10x19x32x256_aflw.mnn | [PIPNet](https://github.com/jhb86253817/PIPNet) | 44.0Mb | -| *lite::mnn::cv::face::align::PIPNet98* | pipnet_resnet101_10x98x32x256_wflw.mnn | [PIPNet](https://github.com/jhb86253817/PIPNet) | 150.0Mb | -| *lite::mnn::cv::face::align::PIPNet68* | pipnet_resnet101_10x68x32x256_300w.mnn | [PIPNet](https://github.com/jhb86253817/PIPNet) | 150.0Mb | -| *lite::mnn::cv::face::align::PIPNet29* | pipnet_resnet101_10x29x32x256_cofw.mnn | [PIPNet](https://github.com/jhb86253817/PIPNet) | 150.0Mb | -| *lite::mnn::cv::face::align::PIPNet19* | pipnet_resnet101_10x19x32x256_aflw.mnn | [PIPNet](https://github.com/jhb86253817/PIPNet) | 150.0Mb | - - -## Head Pose Estimation. - -
- - -| Class | Pretrained MNN Files | Rename or Converted From (Repo) | Size | -|:-----------------------------------:|:--------------------:|:------------------------------------------------------------------:|:-----:| -| *lite::mnn::cv::face::pose::FSANet* | fsanet-var.mnn | [...fsanet...](https://github.com/omasaht/headpose-fsanet-pytorch) | 1.2Mb | -| *lite::mnn::cv::face::pose::FSANet* | fsanet-1x1.mnn | [...fsanet...](https://github.com/omasaht/headpose-fsanet-pytorch) | 1.2Mb | - -## Face Attributes. - -
- - -| Class | Pretrained MNN Files | Rename or Converted From (Repo) | Size | -|:----------------------------------------------:|:-----------------------------------------------------------:|:-------------------------------------------------------------------------:|:-----:| -| *lite::mnn::cv::face::attr::AgeGoogleNet* | age_googlenet.mnn | [onnx-models](https://github.com/onnx/models) | 23Mb | -| *lite::mnn::cv::face::attr::GenderGoogleNet* | gender_googlenet.mnn | [onnx-models](https://github.com/onnx/models) | 23Mb | -| *lite::mnn::cv::face::attr::EmotionFerPlus* | emotion-ferplus-7.mnn | [onnx-models](https://github.com/onnx/models) | 33Mb | -| *lite::mnn::cv::face::attr::EmotionFerPlus* | emotion-ferplus-8.mnn | [onnx-models](https://github.com/onnx/models) | 33Mb | -| *lite::mnn::cv::face::attr::SSRNet* | ssrnet.mnn | [SSR_Net...](https://github.com/oukohou/SSR_Net_Pytorch) | 190Kb | -| *lite::mnn::cv::face::attr::EfficientEmotion7* | face-emotion-recognition-enet_b0_7.mnn | [face-emo...](https://github.com/HSE-asavchenko/face-emotion-recognition) | 15Mb | -| *lite::mnn::cv::face::attr::EfficientEmotion8* | face-emotion-recognition-enet_b0_8_best_afew.mnn | [face-emo...](https://github.com/HSE-asavchenko/face-emotion-recognition) | 15Mb | -| *lite::mnn::cv::face::attr::EfficientEmotion8* | face-emotion-recognition-enet_b0_8_best_vgaf.mnn | [face-emo...](https://github.com/HSE-asavchenko/face-emotion-recognition) | 15Mb | -| *lite::mnn::cv::face::attr::MobileEmotion7* | face-emotion-recognition-mobilenet_7.mnn | [face-emo...](https://github.com/HSE-asavchenko/face-emotion-recognition) | 13Mb | -| *lite::mnn::cv::face::attr::ReXNetEmotion7* | face-emotion-recognition-affectnet_7_vggface2_rexnet150.mnn | [face-emo...](https://github.com/HSE-asavchenko/face-emotion-recognition) | 30Mb | - -## Classification. - -
- - -| Class | Pretrained MNN Files | Rename or Converted From (Repo) | Size | -|:-------------------------------------------------:|:-------------------------------:|:------------------------------------------------------------:|:-----:| -| *lite::mnn::cv::classification:EfficientNetLite4* | efficientnet-lite4-11.mnn | [onnx-models](https://github.com/onnx/models) | 49Mb | -| *lite::mnn::cv::classification::ShuffleNetV2* | shufflenet-v2-10.mnn | [onnx-models](https://github.com/onnx/models) | 8.7Mb | -| *lite::mnn::cv::classification::DenseNet121* | densenet121.mnn | [torchvision](https://github.com/pytorch/vision) | 30Mb | -| *lite::mnn::cv::classification::GhostNet* | ghostnet.mnn | [torchvision](https://github.com/pytorch/vision) | 20Mb | -| *lite::mnn::cv::classification::HdrDNet* | hardnet.mnn | [torchvision](https://github.com/pytorch/vision) | 13Mb | -| *lite::mnn::cv::classification::IBNNet* | ibnnet18.mnn | [torchvision](https://github.com/pytorch/vision) | 97Mb | -| *lite::mnn::cv::classification::MobileNetV2* | mobilenetv2.mnn | [torchvision](https://github.com/pytorch/vision) | 13Mb | -| *lite::mnn::cv::classification::ResNet* | resnet18.mnn | [torchvision](https://github.com/pytorch/vision) | 44Mb | -| *lite::mnn::cv::classification::ResNeXt* | resnext.mnn | [torchvision](https://github.com/pytorch/vision) | 95Mb | -| *lite::mnn::cv::classification::InsectID* | quarrying_insect_identifier.mnn | [InsectID](https://github.com/quarrying/quarrying-insect-id) | 27Mb | -| *lite::mnn::cv::classification:PlantID* | quarrying_planted_model.mnn | [PlantID](https://github.com/quarrying/quarrying-plant-id) | 30Mb | - - -## Segmentation. - -
- - -| Class | Pretrained MNN Files | Rename or Converted From (Repo) | Size | -|:------------------------------------------------------:|:------------------------------------------:|:---------------------------------------------------------------------------------:|:-----:| -| *lite::mnn::cv::segmentation::DeepLabV3ResNet101* | deeplabv3_resnet101_coco.mnn | [torchvision](https://github.com/pytorch/vision) | 232Mb | -| *lite::mnn::cv::segmentation::FCNResNet101* | fcn_resnet101.mnn | [torchvision](https://github.com/pytorch/vision) | 207Mb | -| *lite::mnn::cv::segmentation::HeadSeg* | minivision_head_seg.mnn | [photo2cartoon](https://github.com/minivision-ai/photo2cartoon) | 31Mb | -| *lite::mnn::cv::segmentation::FastPortraitSeg* | fast_portrait_seg_SINet_bi_192_128.mnn | [Fast-Portrait...](https://github.com/YexingWan/Fast-Portrait-Segmentation) | 400k | -| *lite::mnn::cv::segmentation::FastPortraitSeg* | fast_portrait_seg_SINet_bi_256_160.mnn | [Fast-Portrait...](https://github.com/YexingWan/Fast-Portrait-Segmentation) | 400k | -| *lite::mnn::cv::segmentation::FastPortraitSeg* | fast_portrait_seg_SINet_bi_320_256.mnn | [Fast-Portrait...](https://github.com/YexingWan/Fast-Portrait-Segmentation) | 400k | -| *lite::mnn::cv::segmentation::PortraitSegSINet* | ext_portrait_seg_SINet_224x224.mnn | [ext_portrait...](https://github.com/clovaai/ext_portrait_segmentation) | 380k | -| *lite::mnn::cv::segmentation::PortraitSegExtremeC3Net* | ext_portrait_seg_ExtremeC3_224x224.mnn | [ext_portrait...](https://github.com/clovaai/ext_portrait_segmentation) | 180k | -| *lite::mnn::cv::segmentation::FaceHairSeg* | face_hair_seg_224x224.mnn | [face-seg](https://github.com/kampta/face-seg) | 18M | -| *lite::mnn::cv::segmentation::HairSeg* | hairseg_224x224.mnn | [mobile-semantic-seg](https://github.com/akirasosa/mobile-semantic-segmentation) | 18M | -| *lite::mnn::cv::segmentation::MobileHairSeg* | mobile_hair_seg_hairmattenetv1_224x224.mnn | [mobile-hair...](https://github.com/wonbeomjang/mobile-hair-segmentation-pytorch) | 14M | -| *lite::mnn::cv::segmentation::MobileHairSeg* | mobile_hair_seg_hairmattenetv2_224x224.mnn | [mobile-hair...](https://github.com/wonbeomjang/mobile-hair-segmentation-pytorch) | 14M | -| *lite::mnn::cv::segmentation::FaceParsingBiSeNet* | face_parsing_512x512.mnn | [face-parsing.PyTorch](https://github.com/zllrunning/face-parsing.PyTorch) | 50M | -| *lite::mnn::cv::segmentation::FaceParsingBiSeNet* | face_parsing_1024x1024.mnn | [face-parsing.PyTorch](https://github.com/zllrunning/face-parsing.PyTorch) | 50M | - - -## Style Transfer. - -
- -| Class | Pretrained MNN Files | Rename or Converted From (Repo) | Size | -|:-------------------------------------------:|:-----------------------------------:|:---------------------------------------------------------------:|:-----:| -| *lite::mnn::cv::style::FastStyleTransfer* | style-mosaic-8.mnn | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::mnn::cv::style::FastStyleTransfer* | style-candy-9.mnn | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::mnn::cv::style::FastStyleTransfer* | style-udnie-8.mnn | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::mnn::cv::style::FastStyleTransfer* | style-udnie-9.mnn | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::mnn::cv::style::FastStyleTransfer* | style-pointilism-8.mnn | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::mnn::cv::style::FastStyleTransfer* | style-pointilism-9.mnn | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::mnn::cv::style::FastStyleTransfer* | style-rain-princess-9.mnn | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::mnn::cv::style::FastStyleTransfer* | style-rain-princess-8.mnn | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::mnn::cv::style::FastStyleTransfer* | style-candy-8.mnn | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::mnn::cv::style::FastStyleTransfer* | style-mosaic-9.mnn | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::mnn::cv::style::FemalePhoto2Cartoon* | minivision_female_photo2cartoon.mnn | [photo2cartoon](https://github.com/minivision-ai/photo2cartoon) | 15Mb | - - -## Colorization. - -
- -| Class | Pretrained MNN Files | Rename or Converted From (Repo) | Size | -|:----------------------------------------:|:------------------------:|:---------------------------------------------------------:|:-----:| -| *lite::mnn::cv::colorization::Colorizer* | eccv16-colorizer.mnn | [colorization](https://github.com/richzhang/colorization) | 123Mb | -| *lite::mnn::cv::colorization::Colorizer* | siggraph17-colorizer.mnn | [colorization](https://github.com/richzhang/colorization) | 129Mb | - - -## Super Resolution. - -
- -| Class | Pretrained MNN Files | Rename or Converted From (Repo) | Size | -|:----------------------------------------:|:--------------------:|:---------------------------------------------------------:|:-----:| -| *lite::mnn::cv::resolution::SubPixelCNN* | subpixel-cnn.mnn | [...PIXEL...](https://github.com/niazwazir/SUB_PIXEL_CNN) | 234Kb | - diff --git a/docs/hub/lite.ai.toolkit.hub.ncnn.md b/docs/hub/lite.ai.toolkit.hub.ncnn.md deleted file mode 100644 index 9f553daa..00000000 --- a/docs/hub/lite.ai.toolkit.hub.ncnn.md +++ /dev/null @@ -1,271 +0,0 @@ -# Lite.AI.ToolKit.Hub.NCNN - -You can download all the pretrained models files of NCNN format from ([Baidu Drive](https://pan.baidu.com/s/1hlnqyNsFbMseGFWscgVhgQ) code: sc7f) - -## Object Detection. - -
- -| Class | Pretrained NCNN Files | Rename or Converted From (Repo) | Size | -|:---------------------------------------------------------------:|:--------------------------------------------------------:|:------------------------------------------------------:|:-----:| -| *lite::ncnn::cv::detection::YoloV5* | yolov5l.opt.param&bin | [yolov5](https://github.com/ultralytics/yolov5) | 188Mb | -| *lite::ncnn::cv::detection::YoloV5* | yolov5m.opt.param&bin | [yolov5](https://github.com/ultralytics/yolov5) | 85Mb | -| *lite::ncnn::cv::detection::YoloV5* | yolov5s.opt.param&bin | [yolov5](https://github.com/ultralytics/yolov5) | 29Mb | -| *lite::ncnn::cv::detection::YoloV5* | yolov5x.opt.param&bin | [yolov5](https://github.com/ultralytics/yolov5) | 351Mb | -| *lite::ncnn::cv::detection::YoloX* | yolox_x.opt.param&bin | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 378Mb | -| *lite::ncnn::cv::detection::YoloX* | yolox_l.opt.param&bin | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 207Mb | -| *lite::ncnn::cv::detection::YoloX* | yolox_m.opt.param&bin | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 97Mb | -| *lite::ncnn::cv::detection::YoloX* | yolox_s.opt.param&bin | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 34Mb | -| *lite::ncnn::cv::detection::YoloX* | yolox_tiny.opt.param&bin | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 19Mb | -| *lite::ncnn::cv::detection::YoloX* | yolox_nano.opt.param&bin | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 3.5Mb | -| *lite::ncnn::cv::detection::YOLOP* | yolop-640-640.opt.param&bin | [YOLOP](https://github.com/hustvl/YOLOP) | 30Mb | -| *lite::ncnn::cv::detection::NanoDet* | nanodet_m_0.5x-opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 1.1Mb | -| *lite::ncnn::cv::detection::NanoDet* | nanodet_m-opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 3.6Mb | -| *lite::ncnn::cv::detection::NanoDet* | nanodet_m_1.5x-opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 7.9Mb | -| *lite::ncnn::cv::detection::NanoDet* | nanodet_m_1.5x_416-opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 7.9Mb | -| *lite::ncnn::cv::detection::NanoDet* | nanodet_m_416-opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 3.6Mb | -| *lite::ncnn::cv::detection::NanoDet* | nanodet_g-opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 14Mb | -| *lite::ncnn::cv::detection::NanoDet* | nanodet_t-opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 5.1Mb | -| *lite::ncnn::cv::detection::NanoDet* | nanodet-RepVGG-A0_416-opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 26Mb | -| *lite::ncnn::cv::detection::NanoDetEfficientNetLite* | nanodet-EfficientNet-Lite0_320-opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 12Mb | -| *lite::ncnn::cv::detection::NanoDetEfficientNetLite* | nanodet-EfficientNet-Lite1_416-opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 15Mb | -| *lite::ncnn::cv::detection::NanoDetEfficientNetLite* | nanodet-EfficientNet-Lite2_512-opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 18Mb | -| *lite::ncnn::cv::detection::NanoDetDepreciated* | nanodet_m_0.5x-depreciated-opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 1.1Mb | -| *lite::ncnn::cv::detection::NanoDetDepreciated* | nanodet_m-depreciated-opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 3.6Mb | -| *lite::ncnn::cv::detection::NanoDetDepreciated* | nanodet_m_1.5x-depreciated-opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 7.9Mb | -| *lite::ncnn::cv::detection::NanoDetDepreciated* | nanodet_m_1.5x_416-depreciated-opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 7.9Mb | -| *lite::ncnn::cv::detection::NanoDetDepreciated* | nanodet_m_416-depreciated-opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 3.6Mb | -| *lite::ncnn::cv::detection::NanoDetDepreciated* | nanodet_g-depreciated-opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 14Mb | -| *lite::ncnn::cv::detection::NanoDetDepreciated* | nanodet_t-depreciated-opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 5.1Mb | -| *lite::ncnn::cv::detection::NanoDetDepreciated* | nanodet-RepVGG-A0_416-depreciated-opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 26Mb | -| *lite::ncnn::cv::detection::NanoDetEfficientNetLiteDepreciated* | nanodet-EfficientNet-Lite0_320-depreciated-opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 12Mb | -| *lite::ncnn::cv::detection::NanoDetEfficientNetLiteDepreciated* | nanodet-EfficientNet-Lite1_416-depreciated-opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 15Mb | -| *lite::ncnn::cv::detection::NanoDetEfficientNetLiteDepreciated* | nanodet-EfficientNet-Lite2_512-depreciated-opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 18Mb | -| *lite::ncnn::cv::detection::YoloX_V_0_1_1* | yolox_x_v0.1.1.opt.param&bin | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 378Mb | -| *lite::ncnn::cv::detection::YoloX_V_0_1_1* | yolox_l_v0.1.1.opt.param&bin | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 207Mb | -| *lite::ncnn::cv::detection::YoloX_V_0_1_1* | yolox_m_v0.1.1.opt.param&bin | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 97Mb | -| *lite::ncnn::cv::detection::YoloX_V_0_1_1* | yolox_s_v0.1.1.opt.param&bin | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 34Mb | -| *lite::ncnn::cv::detection::YoloX_V_0_1_1* | yolox_tiny_v0.1.1.opt.param&bin | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 19Mb | -| *lite::ncnn::cv::detection::YoloX_V_0_1_1* | yolox_nano_v0.1.1.opt.param&bin | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 3.5Mb | -| *lite::ncnn::cv::detection::YoloR* | yolor-p6-320-320.opt.param&bin | [yolor](https://github.com/WongKinYiu/yolor) | 157Mb | -| *lite::ncnn::cv::detection::YoloR* | yolor-p6-640-640.opt.param&bin | [yolor](https://github.com/WongKinYiu/yolor) | 157Mb | -| *lite::ncnn::cv::detection::YoloR* | yolor-ssss-s2d-640-640.opt.param&bin | [yolor](https://github.com/WongKinYiu/yolor) | 50Mb | -| *lite::ncnn::cv::detection::YoloV5_V_6_0* | yolov5m.640-640.v.6.0.opt.param&bin | [yolov5](https://github.com/ultralytics/yolov5) | 81Mb | -| *lite::ncnn::cv::detection::YoloV5_V_6_0* | yolov5s.640-640.v.6.0.opt.param&bin | [yolov5](https://github.com/ultralytics/yolov5) | 28Mb | -| *lite::ncnn::cv::detection::YoloV5_V_6_0* | yolov5n.640-640.v.6.0.opt.param&bin | [yolov5](https://github.com/ultralytics/yolov5) | 7.5Mb | -| *lite::ncnn::cv::detection::YoloV5_V_6_0_P6* | yolov5m6.640-640.v.6.0.opt.param&bin | [yolov5](https://github.com/ultralytics/yolov5) | 128Mb | -| *lite::ncnn::cv::detection::YoloV5_V_6_0_P6* | yolov5s6.640-640.v.6.0.opt.param&bin | [yolov5](https://github.com/ultralytics/yolov5) | 50Mb | -| *lite::ncnn::cv::detection::YoloV5_V_6_0_P6* | yolov5n6.640-640.v.6.0.opt.param&bin | [yolov5](https://github.com/ultralytics/yolov5) | 14Mb | -| *lite::ncnn::cv::detection::YoloV5_V_6_0_P6* | yolov5m6.1280-1280.v.6.0.opt.param&bin | [yolov5](https://github.com/ultralytics/yolov5) | 128Mb | -| *lite::ncnn::cv::detection::YoloV5_V_6_0_P6* | yolov5s6.1280-1280.v.6.0.opt.param&bin | [yolov5](https://github.com/ultralytics/yolov5) | 50Mb | -| *lite::ncnn::cv::detection::YoloV5_V_6_0_P6* | yolov5n6.1280-1280.v.6.0.opt.param&bin | [yolov5](https://github.com/ultralytics/yolov5) | 14Mb | -| *lite::ncnn::cv::detection::NanoDetPlus* | nanodet-plus-m_320.opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 4.5Mb | -| *lite::ncnn::cv::detection::NanoDetPlus* | nanodet-plus-m_416.opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 4.5Mb | -| *lite::ncnn::cv::detection::NanoDetPlus* | nanodet-plus-m-1.5x_320.opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 9.4Mb | -| *lite::ncnn::cv::detection::NanoDetPlus* | nanodet-plus-m-1.5x_416.opt.param&bin | [nanodet](https://github.com/RangiLyu/nanodet) | 9.4Mb | -| *lite::ncnn::cv::detection::YOLOv6* | yolov6n-320x320-for-ncnn.opt.param&bin | [YOLOv6](https://github.com/meituan/YOLOv6) | 17Mb | -| *lite::ncnn::cv::detection::YOLOv6* | yolov6n-640x640-for-ncnn.opt.param&bin | [YOLOv6](https://github.com/meituan/YOLOv6) | 17Mb | -| *lite::ncnn::cv::detection::YOLOv6* | yolov6s-320x320-for-ncnn.opt.param&bin | [YOLOv6](https://github.com/meituan/YOLOv6) | 66Mb | -| *lite::ncnn::cv::detection::YOLOv6* | yolov6n-640x640-for-ncnn.opt.param&bin | [YOLOv6](https://github.com/meituan/YOLOv6) | 66Mb | -| *lite::ncnn::cv::detection::YOLOv6* | yolov6t-640x640-for-ncnn.opt.param&bin | [YOLOv6](https://github.com/meituan/YOLOv6) | 57Mb | - - - -## Matting. - -
- -| Class | Pretrained NCNN Files | Rename or Converted From (Repo) | Size | -|:---------------------------------------------:|:--------------------------------------------:|:--------------------------------------------------------------------:|:----:| -| *lite::ncnn::cv::matting::RobustVideoMatting* | rvm_mobilenetv3_fp32-opt.param&bin | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 14Mb | -| *lite::ncnn::cv::matting::RobustVideoMatting* | rvm_mobilenetv3_fp32-480-480-opt.param&bin | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 14Mb | -| *lite::ncnn::cv::matting::RobustVideoMatting* | rvm_mobilenetv3_fp32-480-640-opt.param&bin | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 14Mb | -| *lite::ncnn::cv::matting::RobustVideoMatting* | rvm_mobilenetv3_fp32-640-480-opt.param&bin | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 14Mb | -| *lite::ncnn::cv::matting::RobustVideoMatting* | rvm_mobilenetv3_fp32-1080-1920-opt.param&bin | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 14Mb | -| *lite::ncnn::cv::matting::RobustVideoMatting* | rvm_resnet50_fp32-opt.param&bin | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 50Mb | -| *lite::ncnn::cv::matting::RobustVideoMatting* | rvm_resnet50_fp32-480-480-opt.param&bin | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 50Mb | -| *lite::ncnn::cv::matting::RobustVideoMatting* | rvm_resnet50_fp32-480-640-opt.param&bin | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 50Mb | -| *lite::ncnn::cv::matting::RobustVideoMatting* | rvm_resnet50_fp32-640-480-opt.param&bin | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 50Mb | -| *lite::ncnn::cv::matting::RobustVideoMatting* | rvm_resnet50_fp32-1080-1920-opt.param&bin | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 50Mb | - -## Face Recognition. - -
- - -| Class | Pretrained NCNN Files | Rename or Converted From (Repo) | Size | -|:-----------------------------------------------:|:----------------------------------------------------------------:|:----------------------------------------------------------------------:|:-----:| -| *lite::ncnn::cv::faceid::GlintArcFace* | ms1mv3_arcface_r100.opt.param&bin | [insightface](https://github.com/deepinsight/insightface) | 248Mb | -| *lite::ncnn::cv::faceid::GlintArcFace* | ms1mv3_arcface_r50.opt.param&bin | [insightface](https://github.com/deepinsight/insightface) | 166Mb | -| *lite::ncnn::cv::faceid::GlintArcFace* | ms1mv3_arcface_r34.opt.param&bin | [insightface](https://github.com/deepinsight/insightface) | 130Mb | -| *lite::ncnn::cv::faceid::GlintArcFace* | ms1mv3_arcface_r18.opt.param&bin | [insightface](https://github.com/deepinsight/insightface) | 91Mb | -| *lite::ncnn::cv::faceid::GlintCosFace* | glint360k_cosface_r100.opt.param&bin | [insightface](https://github.com/deepinsight/insightface) | 248Mb | -| *lite::ncnn::cv::faceid::GlintCosFace* | glint360k_cosface_r50.opt.param&bin | [insightface](https://github.com/deepinsight/insightface) | 166Mb | -| *lite::ncnn::cv::faceid::GlintCosFace* | glint360k_cosface_r34.opt.param&bin | [insightface](https://github.com/deepinsight/insightface) | 130Mb | -| *lite::ncnn::cv::faceid::GlintCosFace* | glint360k_cosface_r18.opt.param&bin | [insightface](https://github.com/deepinsight/insightface) | 91Mb | -| *lite::ncnn::cv::faceid::GlintPartialFC* | partial_fc_glint360k_r100.opt.param&bin | [insightface](https://github.com/deepinsight/insightface) | 248Mb | -| *lite::ncnn::cv::faceid::GlintPartialFC* | partial_fc_glint360k_r50.opt.param&bin | [insightface](https://github.com/deepinsight/insightface) | 91Mb | -| *lite::ncnn::cv::faceid::FaceNet* | facenet_vggface2_resnet.opt.param&bin | [facenet...](https://github.com/timesler/facenet-pytorch) | 89Mb | -| *lite::ncnn::cv::faceid::FaceNet* | facenet_casia-webface_resnet.opt.param&bin | [facenet...](https://github.com/timesler/facenet-pytorch) | 89Mb | -| *lite::ncnn::cv::faceid::FocalArcFace* | focal-arcface-ms1m-ir152.opt.param&bin | [face.evoLVe...](https://github.com/ZhaoJ9014/face.evoLVe.PyTorch) | 269Mb | -| *lite::ncnn::cv::faceid::FocalArcFace* | focal-arcface-ms1m-ir50-epoch120.opt.param&bin | [face.evoLVe...](https://github.com/ZhaoJ9014/face.evoLVe.PyTorch) | 166Mb | -| *lite::ncnn::cv::faceid::FocalArcFace* | focal-arcface-ms1m-ir50-epoch63.opt.param&bin | [face.evoLVe...](https://github.com/ZhaoJ9014/face.evoLVe.PyTorch) | 166Mb | -| *lite::ncnn::cv::faceid::FocalAsiaArcFace* | focal-arcface-bh-ir50-asia.opt.param&bin | [face.evoLVe...](https://github.com/ZhaoJ9014/face.evoLVe.PyTorch) | 166Mb | -| *lite::ncnn::cv::faceid::TencentCurricularFace* | Tencent_CurricularFace_Backbone.opt.param&bin | [TFace](https://github.com/Tencent/TFace) | 249Mb | -| *lite::ncnn::cv::faceid::TencentCifpFace* | Tencent_Cifp_BUPT_Balancedface_IR_34.opt.param&bin | [TFace](https://github.com/Tencent/TFace) | 130Mb | -| *lite::ncnn::cv::faceid::CenterLossFace* | CenterLossFace_epoch_100.opt.param&bin | [center-loss...](https://github.com/louis-she/center-loss.pytorch) | 280Mb | -| *lite::ncnn::cv::faceid::SphereFace* | sphere20a_20171020.opt.param&bin | [sphere...](https://github.com/clcarwin/sphereface_pytorch) | 86Mb | -| *lite::ncnn::cv::faceid:MobileFaceNet* | MobileFaceNet_Pytorch_068.opt.param&bin | [MobileFace...](https://github.com/Xiaoccer/MobileFaceNet_Pytorch) | 3.8Mb | -| *lite::ncnn::cv::faceid:CavaGhostArcFace* | cavaface_GhostNet_x1.3_Arcface_Epoch_24.opt.param&bin | [cavaface...](https://github.com/cavalleria/cavaface.pytorch) | 15Mb | -| *lite::ncnn::cv::faceid:CavaCombinedFace* | cavaface_IR_SE_100_Combined_Epoch_24.opt.param&bin | [cavaface...](https://github.com/cavalleria/cavaface.pytorch) | 250Mb | -| *lite::ncnn::cv::faceid:MobileSEFocalFace* | face_recognition.pytorch_Mobilenet_se_focal_121000.opt.param&bin | [face_recog...](https://github.com/grib0ed0v/face_recognition.pytorch) | 4.5Mb | - - -## Face Detection. - -
- -| Class | Pretrained NCNN Files | Rename or Converted From (Repo) | Size | -|:-------------------------------------------:|:----------------------------------------------------:|:---------------------------------------------------------------------------------------:|:------:| -| *lite::ncnn::cv::face::detect::UltraFace* | ultraface-rfb-320.param&bin | [Ultra-Light...](https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB) | 1.5Mb | -| *lite::ncnn::cv::face::detect::UltraFace* | ultraface-slim-320.param&bin | [Ultra-Light...](https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB) | 1.2Mb | -| *lite::ncnn::cv::face::detect::RetinaFace* | Pytorch_RetinaFace_mobile0.25.opt.param&bin | [...Retinaface](https://github.com/biubug6/Pytorch_Retinaface) | 1.6Mb | -| *lite::ncnn::cv::face::detect::RetinaFace* | Pytorch_RetinaFace_mobile0.25-640-640.opt.param&bin | [...Retinaface](https://github.com/biubug6/Pytorch_Retinaface) | 1.6Mb | -| *lite::ncnn::cv::face::detect::RetinaFace* | Pytorch_RetinaFace_mobile0.25-320-320.opt.param&bin | [...Retinaface](https://github.com/biubug6/Pytorch_Retinaface) | 1.6Mb | -| *lite::ncnn::cv::face::detect::RetinaFace* | Pytorch_RetinaFace_mobile0.25-720-1080.opt.param&bin | [...Retinaface](https://github.com/biubug6/Pytorch_Retinaface) | 1.6Mb | -| *lite::ncnn::cv::face::detect::FaceBoxes* | FaceBoxes.opt.param&bin | [FaceBoxes](https://github.com/zisianw/FaceBoxes.PyTorch) | 3.8Mb | -| *lite::ncnn::cv::face::detect::FaceBoxes* | FaceBoxes-640-640.opt.param&bin | [FaceBoxes](https://github.com/zisianw/FaceBoxes.PyTorch) | 3.8Mb | -| *lite::ncnn::cv::face::detect::FaceBoxes* | FaceBoxes-320-320.opt.param&bin | [FaceBoxes](https://github.com/zisianw/FaceBoxes.PyTorch) | 3.8Mb | -| *lite::ncnn::cv::face::detect::FaceBoxes* | FaceBoxes-720-1080.opt.param&bin | [FaceBoxes](https://github.com/zisianw/FaceBoxes.PyTorch) | 3.8Mb | -| *lite::ncnn::cv::face::detect::SCRFD* | scrfd_500m_shape160x160.opt.param&bin | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.5Mb | -| *lite::ncnn::cv::face::detect::SCRFD* | scrfd_500m_shape320x320.opt.param&bin | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.5Mb | -| *lite::ncnn::cv::face::detect::SCRFD* | scrfd_500m_shape640x640.opt.param&bin | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.5Mb | -| *lite::ncnn::cv::face::detect::SCRFD* | scrfd_500m_bnkps_shape160x160.opt.param&bin | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.5Mb | -| *lite::ncnn::cv::face::detect::SCRFD* | scrfd_500m_bnkps_shape320x320.opt.param&bin | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.5Mb | -| *lite::ncnn::cv::face::detect::SCRFD* | scrfd_500m_bnkps_shape640x640.opt.param&bin | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.5Mb | -| *lite::ncnn::cv::face::detect::SCRFD* | scrfd_1g_shape160x160.opt.param&bin | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.7Mb | -| *lite::ncnn::cv::face::detect::SCRFD* | scrfd_1g_shape320x320.opt.param&bin | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.7Mb | -| *lite::ncnn::cv::face::detect::SCRFD* | scrfd_1g_shape640x640.opt.param&bin | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.7Mb | -| *lite::ncnn::cv::face::detect::SCRFD* | scrfd_2.5g_shape160x160.opt.param&bin | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 3.3Mb | -| *lite::ncnn::cv::face::detect::SCRFD* | scrfd_2.5g_shape320x320.opt.param&bin | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 3.3Mb | -| *lite::ncnn::cv::face::detect::SCRFD* | scrfd_2.5g_shape640x640.opt.param&bin | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 3.3Mb | -| *lite::ncnn::cv::face::detect::SCRFD* | scrfd_2.5g_bnkps_shape160x160.opt.param&bin | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 3.3Mb | -| *lite::ncnn::cv::face::detect::SCRFD* | scrfd_2.5g_bnkps_shape320x320.opt.param&bin | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 3.3Mb | -| *lite::ncnn::cv::face::detect::SCRFD* | scrfd_2.5g_bnkps_shape640x640.opt.param&bin | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 3.3Mb | -| *lite::ncnn::cv::face::detect::SCRFD* | scrfd_10g_shape640x640.opt.param&bin | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 16.9Mb | -| *lite::ncnn::cv::face::detect::SCRFD* | scrfd_10g_shape1280x1280.opt.param&bin | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 16.9Mb | -| *lite::ncnn::cv::face::detect::SCRFD* | scrfd_10g_bnkps_shape640x640.opt.param&bin | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 16.9Mb | -| *lite::ncnn::cv::face::detect::SCRFD* | scrfd_10g_bnkps_shape1280x1280.opt.param&bin | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 16.9Mb | -| *lite::ncnn::cv::face::detect::YOLO5Face* | yolov5face-l-640x640.opt.param&bin | [YOLO5Face](https://github.com/deepcam-cn/yolov5-face) | 181Mb | -| *lite::ncnn::cv::face::detect::YOLO5Face* | yolov5face-m-640x640.opt.param&bin | [YOLO5Face](https://github.com/deepcam-cn/yolov5-face) | 83Mb | -| *lite::ncnn::cv::face::detect::YOLO5Face* | yolov5face-n-0.5-320x320.opt.param&bin | [YOLO5Face](https://github.com/deepcam-cn/yolov5-face) | 2.5Mb | -| *lite::ncnn::cv::face::detect::YOLO5Face* | yolov5face-n-0.5-640x640.opt.param&bin | [YOLO5Face](https://github.com/deepcam-cn/yolov5-face) | 4.6Mb | -| *lite::ncnn::cv::face::detect::YOLO5Face* | yolov5face-n-640x640.opt.param&bin | [YOLO5Face](https://github.com/deepcam-cn/yolov5-face) | 9.5Mb | -| *lite::ncnn::cv::face::detect::YOLO5Face* | yolov5face-s-640x640.opt.param&bin | [YOLO5Face](https://github.com/deepcam-cn/yolov5-face) | 30Mb | -| *lite::ncnn::cv::face::detect::FaceBoxesV2* | faceboxesv2-640x640.opt.param&bin | [FaceBoxesV2](https://github.com/jhb86253817/FaceBoxesV2) | 4.0Mb | - - -## Face Alignment. - -
- - -| Class | Pretrained NCNN Files | Rename or Converted From (Repo) | Size | -|:------------------------------------------------:|:----------------------------------------------------------------------:|:------------------------------------------------------------------:|:-------:| -| *lite::ncnn::cv::face::align::PFLD* | pfld-106-lite.opt.param&bin | [pfld_106_...](https://github.com/Hsintao/pfld_106_face_landmarks) | 1.0Mb | -| *lite::ncnn::cv::face::align::PFLD* | pfld-106-v3.opt.param&bin | [pfld_106_...](https://github.com/Hsintao/pfld_106_face_landmarks) | 5.5Mb | -| *lite::ncnn::cv::face::align::PFLD* | pfld-106-v2.opt.param&bin | [pfld_106_...](https://github.com/Hsintao/pfld_106_face_landmarks) | 5.0Mb | -| *lite::ncnn::cv::face::align::PFLD98* | PFLD-pytorch-pfld.opt.param&bin | [PFLD...](https://github.com/polarisZhao/PFLD-pytorch) | 4.8Mb | -| *lite::ncnn::cv::face::align::MobileNetV268* | pytorch_face_landmarks_landmark_detection_56.opt.param&bin | [...landmark](https://github.com/cunjian/pytorch_face_landmark) | 9.4Mb | -| *lite::ncnn::cv::face::align::MobileNetV2SE68* | pytorch_face_landmarks_landmark_detection_56_se_external.opt.param&bin | [...landmark](https://github.com/cunjian/pytorch_face_landmark) | 11Mb | -| *lite::ncnn::cv::face::align::PFLD68* | pytorch_face_landmarks_pfld.opt.param&bin | [...landmark](https://github.com/cunjian/pytorch_face_landmark) | 2.8Mb | -| *lite::ncnn::cv::face::align::FaceLandmarks1000* | FaceLandmark1000.opt.param&bin | [FaceLandm...](https://github.com/Single430/FaceLandmark1000) | 2.0Mb | -| *lite::ncnn::cv::face::align::PIPNet98* | pipnet_resnet18_10x98x32x256_wflw.opt.param&bin | [PIPNet](https://github.com/jhb86253817/PIPNet) | 44.0Mb | -| *lite::ncnn::cv::face::align::PIPNet68* | pipnet_resnet18_10x68x32x256_300w.opt.param&bin | [PIPNet](https://github.com/jhb86253817/PIPNet) | 44.0Mb | -| *lite::ncnn::cv::face::align::PIPNet29* | pipnet_resnet18_10x29x32x256_cofw.opt.param&bin | [PIPNet](https://github.com/jhb86253817/PIPNet) | 44.0Mb | -| *lite::ncnn::cv::face::align::PIPNet19* | pipnet_resnet18_10x19x32x256_aflw.opt.param&bin | [PIPNet](https://github.com/jhb86253817/PIPNet) | 44.0Mb | -| *lite::ncnn::cv::face::align::PIPNet98* | pipnet_resnet101_10x98x32x256_wflw.opt.param&bin | [PIPNet](https://github.com/jhb86253817/PIPNet) | 150.0Mb | -| *lite::ncnn::cv::face::align::PIPNet68* | pipnet_resnet101_10x68x32x256_300w.opt.param&bin | [PIPNet](https://github.com/jhb86253817/PIPNet) | 150.0Mb | -| *lite::ncnn::cv::face::align::PIPNet29* | pipnet_resnet101_10x29x32x256_cofw.opt.param&bin | [PIPNet](https://github.com/jhb86253817/PIPNet) | 150.0Mb | -| *lite::ncnn::cv::face::align::PIPNet19* | pipnet_resnet101_10x19x32x256_aflw.opt.param&bin | [PIPNet](https://github.com/jhb86253817/PIPNet) | 150.0Mb | - - -## Face Attributes. - -
- - -| Class | Pretrained NCNN Files | Rename or Converted From (Repo) | Size | -|:-----------------------------------------------:|:----------------------------------------------------------:|:-------------------------------------------------------------------------:|:----:| -| *lite::ncnn::cv::face::attr::AgeGoogleNet* | age_googlenet.opt.param&bin | [onnx-models](https://github.com/onnx/models) | 23Mb | -| *lite::ncnn::cv::face::attr::GenderGoogleNet* | gender_googlenet.opt.param&bin | [onnx-models](https://github.com/onnx/models) | 23Mb | -| *lite::ncnn::cv::face::attr::EmotionFerPlus* | emotion-ferplus-7.opt.param&bin | [onnx-models](https://github.com/onnx/models) | 33Mb | -| *lite::ncnn::cv::face::attr::EmotionFerPlus* | emotion-ferplus-8.opt.param&bin | [onnx-models](https://github.com/onnx/models) | 33Mb | -| *lite::ncnn::cv::face::attr::EfficientEmotion7* | face-emotion-recognition-enet_b0_7.opt.param&bin | [face-emo...](https://github.com/HSE-asavchenko/face-emotion-recognition) | 15Mb | -| *lite::ncnn::cv::face::attr::EfficientEmotion8* | face-emotion-recognition-enet_b0_8_best_afew.opt.param&bin | [face-emo...](https://github.com/HSE-asavchenko/face-emotion-recognition) | 15Mb | -| *lite::ncnn::cv::face::attr::EfficientEmotion8* | face-emotion-recognition-enet_b0_8_best_vgaf.opt.param&bin | [face-emo...](https://github.com/HSE-asavchenko/face-emotion-recognition) | 15Mb | -| *lite::ncnn::cv::face::attr::MobileEmotion7* | face-emotion-recognition-mobilenet_7.opt.param&bin | [face-emo...](https://github.com/HSE-asavchenko/face-emotion-recognition) | 13Mb | - - -## Classification. - -
- - -| Class | Pretrained NCNN Files | Rename or Converted From (Repo) | Size | -|:----------------------------------------------:|:-----------------------------------------:|:------------------------------------------------------------:|:-----:| -| *lite::ncnn::cv::classification::ShuffleNetV2* | shufflenet-v2-10.opt.param&bin | [onnx-models](https://github.com/onnx/models) | 8.7Mb | -| *lite::ncnn::cv::classification::DenseNet121* | densenet121.opt.param&bin | [torchvision](https://github.com/pytorch/vision) | 30Mb | -| *lite::ncnn::cv::classification::GhostNet* | ghostnet.opt.param&bin | [torchvision](https://github.com/pytorch/vision) | 20Mb | -| *lite::ncnn::cv::classification::HdrDNet* | hardnet.opt.param&bin | [torchvision](https://github.com/pytorch/vision) | 13Mb | -| *lite::ncnn::cv::classification::IBNNet* | ibnnet18.opt.param&bin | [torchvision](https://github.com/pytorch/vision) | 97Mb | -| *lite::ncnn::cv::classification::MobileNetV2* | mobilenetv2.opt.param&bin | [torchvision](https://github.com/pytorch/vision) | 13Mb | -| *lite::ncnn::cv::classification::ResNet* | resnet18.opt.param&bin | [torchvision](https://github.com/pytorch/vision) | 44Mb | -| *lite::ncnn::cv::classification::ResNeXt* | resnext.opt.param&bin | [torchvision](https://github.com/pytorch/vision) | 95Mb | -| *lite::ncnn::cv::classification::InsectID* | quarrying_insect_identifier.opt.param&bin | [InsectID](https://github.com/quarrying/quarrying-insect-id) | 27Mb | -| *lite::ncnn::cv::classification:PlantID* | quarrying_plantid_model.opt.param&bin | [PlantID](https://github.com/quarrying/quarrying-plant-id) | 30Mb | - -## Segmentation. - -
- - -| Class | Pretrained NCNN Files | Rename or Converted From (Repo) | Size | -|:--------------------------------------------------:|:--------------------------------------:|:--------------------------------------------------------------------------:|:-----:| -| *lite::ncnn::cv::segmentation::DeepLabV3ResNet101* | deeplabv3_resnet101_coco.opt.param&bin | [torchvision](https://github.com/pytorch/vision) | 232Mb | -| *lite::ncnn::cv::segmentation::FCNResNet101* | fcn_resnet101.opt.param&bin | [torchvision](https://github.com/pytorch/vision) | 207Mb | -| *lite::ncnn::cv::segmentation::FaceParsingBiSeNet* | face_parsing_512x512.opt.param&bin | [face-parsing.PyTorch](https://github.com/zllrunning/face-parsing.PyTorch) | 50M | -| *lite::ncnn::cv::segmentation::FaceParsingBiSeNet* | face_parsing_1024x1024.opt.param&bin | [face-parsing.PyTorch](https://github.com/zllrunning/face-parsing.PyTorch) | 50M | - - -## Style Transfer. - -
- -| Class | Pretrained NCNN Files | Rename or Converted From (Repo) | Size | -|:------------------------------------------:|:------------------------------------------------------------:|:---------------------------------------------:|:-----:| -| *lite::ncnn::cv::style::FastStyleTransfer* | style-mosaic-8.opt.param&bin | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::ncnn::cv::style::FastStyleTransfer* | style-candy-9.opt.param&bin | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::ncnn::cv::style::FastStyleTransfer* | style-udnie-8.opt.param&bin | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::ncnn::cv::style::FastStyleTransfer* | style-udnie-9.opt.param&bin | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::ncnn::cv::style::FastStyleTransfer* | style-pointilism-8.opt.param&bin | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::ncnn::cv::style::FastStyleTransfer* | style-pointilism-9.opt.param&bin | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::ncnn::cv::style::FastStyleTransfer* | style-rain-princess-9.opt.param&bin | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::ncnn::cv::style::FastStyleTransfer* | style-rain-princess-8.opt.param&bin | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::ncnn::cv::style::FastStyleTransfer* | style-candy-8.opt.param&bin | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::ncnn::cv::style::FastStyleTransfer* | style-mosaic-9.opt.param&bin | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::ncnn::cv::matting::MODNet* | modnet_photographic_portrait_matting-1024x1024.opt.param&bin | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::ncnn::cv::matting::MODNet* | modnet_photographic_portrait_matting-1024x512.opt.param&bin | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::ncnn::cv::matting::MODNet* | modnet_photographic_portrait_matting-256x256.opt.param&bin | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::ncnn::cv::matting::MODNet* | modnet_photographic_portrait_matting-256x512.opt.param&bin | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::ncnn::cv::matting::MODNet* | modnet_photographic_portrait_matting-512x1024.opt.param&bin | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::ncnn::cv::matting::MODNet* | modnet_photographic_portrait_matting-512x256.opt.param&bin | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::ncnn::cv::matting::MODNet* | modnet_photographic_portrait_matting-512x512.opt.param&bin | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::ncnn::cv::matting::MODNet* | modnet_webcam_portrait_matting-1024x1024.opt.param&bin | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::ncnn::cv::matting::MODNet* | modnet_webcam_portrait_matting-1024x512.opt.param&bin | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::ncnn::cv::matting::MODNet* | modnet_webcam_portrait_matting-256x256.opt.param&bin | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::ncnn::cv::matting::MODNet* | modnet_webcam_portrait_matting-256x512.opt.param&bin | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::ncnn::cv::matting::MODNet* | modnet_webcam_portrait_matting-512x1024.opt.param&bin | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::ncnn::cv::matting::MODNet* | modnet_webcam_portrait_matting-512x256.opt.param&bin | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::ncnn::cv::matting::MODNet* | modnet_webcam_portrait_matting-512x512.opt.param&bin | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | - diff --git a/docs/hub/lite.ai.toolkit.hub.tnn.md b/docs/hub/lite.ai.toolkit.hub.tnn.md deleted file mode 100644 index 85b78b2b..00000000 --- a/docs/hub/lite.ai.toolkit.hub.tnn.md +++ /dev/null @@ -1,311 +0,0 @@ -# Lite.AI.ToolKit.Hub.TNN - -You can download all the pretrained models files of TNN format from ([Baidu Drive](https://pan.baidu.com/s/1lvM2YKyUbEc5HKVtqITpcw) code: 6o6k) - -## Object Detection. - -
- -| Class | Pretrained TNN Files | Rename or Converted From (Repo) | Size | -|:---------------------------------------------------:|:----------------------------------------------------:|:------------------------------------------------------------:|:-----:| -| *lite::tnn::cv::detection::YoloV5* | yolov5l.opt.tnnproto&tnnmodel | [yolov5](https://github.com/ultralytics/yolov5) | 188Mb | -| *lite::tnn::cv::detection::YoloV5* | yolov5m.opt.tnnproto&tnnmodel | [yolov5](https://github.com/ultralytics/yolov5) | 85Mb | -| *lite::tnn::cv::detection::YoloV5* | yolov5s.opt.tnnproto&tnnmodel | [yolov5](https://github.com/ultralytics/yolov5) | 29Mb | -| *lite::tnn::cv::detection::YoloV5* | yolov5x.opt.tnnproto&tnnmodel | [yolov5](https://github.com/ultralytics/yolov5) | 351Mb | -| *lite::tnn::cv::detection::YoloX* | yolox_x.opt.tnnproto&tnnmodel | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 378Mb | -| *lite::tnn::cv::detection::YoloX* | yolox_l.opt.tnnproto&tnnmodel | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 207Mb | -| *lite::tnn::cv::detection::YoloX* | yolox_m.opt.tnnproto&tnnmodel | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 97Mb | -| *lite::tnn::cv::detection::YoloX* | yolox_s.opt.tnnproto&tnnmodel | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 34Mb | -| *lite::tnn::cv::detection::YoloX* | yolox_tiny.opt.tnnproto&tnnmodel | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 19Mb | -| *lite::tnn::cv::detection::YoloX* | yolox_nano.opt.tnnproto&tnnmodel | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 3.5Mb | -| *lite::tnn::cv::detection::YOLOP* | yolop-320-320.opt.tnnproto&tnnmodel | [YOLOP](https://github.com/hustvl/YOLOP) | 30Mb | -| *lite::tnn::cv::detection::YOLOP* | yolop-640-640.opt.tnnproto&tnnmodel | [YOLOP](https://github.com/hustvl/YOLOP) | 30Mb | -| *lite::tnn::cv::detection::YOLOP* | yolop-1280-1280.opt.tnnproto&tnnmodel | [YOLOP](https://github.com/hustvl/YOLOP) | 30Mb | -| *lite::tnn::cv::detection::NanoDet* | nanodet_m_0.5x.opt.tnnproto&tnnmodel | [nanodet](https://github.com/RangiLyu/nanodet) | 1.1Mb | -| *lite::tnn::cv::detection::NanoDet* | nanodet_m.opt.tnnproto&tnnmodel | [nanodet](https://github.com/RangiLyu/nanodet) | 3.6Mb | -| *lite::tnn::cv::detection::NanoDet* | nanodet_m_1.5x.opt.tnnproto&tnnmodel | [nanodet](https://github.com/RangiLyu/nanodet) | 7.9Mb | -| *lite::tnn::cv::detection::NanoDet* | nanodet_m_1.5x_416.opt.tnnproto&tnnmodel | [nanodet](https://github.com/RangiLyu/nanodet) | 7.9Mb | -| *lite::tnn::cv::detection::NanoDet* | nanodet_m_416.opt.tnnproto&tnnmodel | [nanodet](https://github.com/RangiLyu/nanodet) | 3.6Mb | -| *lite::tnn::cv::detection::NanoDet* | nanodet_g.opt.tnnproto&tnnmodel | [nanodet](https://github.com/RangiLyu/nanodet) | 14Mb | -| *lite::tnn::cv::detection::NanoDet* | nanodet_t.opt.tnnproto&tnnmodel | [nanodet](https://github.com/RangiLyu/nanodet) | 5.1Mb | -| *lite::tnn::cv::detection::NanoDet* | nanodet-RepVGG-A0_416.opt.tnnproto&tnnmodel | [nanodet](https://github.com/RangiLyu/nanodet) | 26Mb | -| *lite::tnn::cv::detection::NanoDetEfficientNetLite* | nanodet-EfficientNet-Lite0_320.opt.tnnproto&tnnmodel | [nanodet](https://github.com/RangiLyu/nanodet) | 12Mb | -| *lite::tnn::cv::detection::NanoDetEfficientNetLite* | nanodet-EfficientNet-Lite1_416.opt.tnnproto&tnnmodel | [nanodet](https://github.com/RangiLyu/nanodet) | 15Mb | -| *lite::tnn::cv::detection::NanoDetEfficientNetLite* | nanodet-EfficientNet-Lite2_512.opt.tnnproto&tnnmodel | [nanodet](https://github.com/RangiLyu/nanodet) | 18Mb | -| *lite::tnn::cv::detection::YoloX_V_0_1_1* | yolox_x_v0.1.1.opt.tnnproto&tnnmodel | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 378Mb | -| *lite::tnn::cv::detection::YoloX_V_0_1_1* | yolox_l_v0.1.1.opt.tnnproto&tnnmodel | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 207Mb | -| *lite::tnn::cv::detection::YoloX_V_0_1_1* | yolox_m_v0.1.1.opt.tnnproto&tnnmodel | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 97Mb | -| *lite::tnn::cv::detection::YoloX_V_0_1_1* | yolox_s_v0.1.1.opt.tnnproto&tnnmodel | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 34Mb | -| *lite::tnn::cv::detection::YoloX_V_0_1_1* | yolox_tiny_v0.1.1.opt.tnnproto&tnnmodel | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 19Mb | -| *lite::tnn::cv::detection::YoloX_V_0_1_1* | yolox_nano_v0.1.1.opt.tnnproto&tnnmodel | [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) | 3.5Mb | -| *lite::tnn::cv::detection::YoloR* | yolor-p6-320-320.opt.tnnproto&tnnmodel | [yolor](https://github.com/WongKinYiu/yolor) | 157Mb | -| *lite::tnn::cv::detection::YoloR* | yolor-p6-640-640.opt.tnnproto&tnnmodel | [yolor](https://github.com/WongKinYiu/yolor) | 157Mb | -| *lite::tnn::cv::detection::YoloR* | yolor-ssss-s2d-640-640.opt.tnnproto&tnnmodel | [yolor](https://github.com/WongKinYiu/yolor) | 50Mb | -| *lite::tnn::cv::detection::YoloV5_V_6_0* | yolov5m.640-640.v.6.0.opt.tnnproto&tnnmodel | [yolov5](https://github.com/ultralytics/yolov5) | 81Mb | -| *lite::tnn::cv::detection::YoloV5_V_6_0* | yolov5s.640-640.v.6.0.opt.tnnproto&tnnmodel | [yolov5](https://github.com/ultralytics/yolov5) | 28Mb | -| *lite::tnn::cv::detection::YoloV5_V_6_0* | yolov5n.640-640.v.6.0.opt.tnnproto&tnnmodel | [yolov5](https://github.com/ultralytics/yolov5) | 7.5Mb | -| *lite::tnn::cv::detection::YoloV5_V_6_0* | yolov5m6.640-640.v.6.0.opt.tnnproto&tnnmodel | [yolov5](https://github.com/ultralytics/yolov5) | 128Mb | -| *lite::tnn::cv::detection::YoloV5_V_6_0* | yolov5s6.640-640.v.6.0.opt.tnnproto&tnnmodel | [yolov5](https://github.com/ultralytics/yolov5) | 50Mb | -| *lite::tnn::cv::detection::YoloV5_V_6_0* | yolov5n6.640-640.v.6.0.opt.tnnproto&tnnmodel | [yolov5](https://github.com/ultralytics/yolov5) | 14Mb | -| *lite::tnn::cv::detection::YoloV5_V_6_0* | yolov5m6.1280-1280.v.6.0.opt.tnnproto&tnnmodel | [yolov5](https://github.com/ultralytics/yolov5) | 128Mb | -| *lite::tnn::cv::detection::YoloV5_V_6_0* | yolov5s6.1280-1280.v.6.0.opt.tnnproto&tnnmodel | [yolov5](https://github.com/ultralytics/yolov5) | 50Mb | -| *lite::tnn::cv::detection::YoloV5_V_6_0* | yolov5n6.1280-1280.v.6.0.opt.tnnproto&tnnmodel | [yolov5](https://github.com/ultralytics/yolov5) | 14Mb | -| *lite::tnn::cv::detection::NanoDetPlus* | nanodet-plus-m_320.opt.tnnproto&tnnmodel | [nanodet](https://github.com/RangiLyu/nanodet) | 4.5Mb | -| *lite::tnn::cv::detection::NanoDetPlus* | nanodet-plus-m_416.opt.tnnproto&tnnmodel | [nanodet](https://github.com/RangiLyu/nanodet) | 4.5Mb | -| *lite::tnn::cv::detection::NanoDetPlus* | nanodet-plus-m-1.5x_320.opt.tnnproto&tnnmodel | [nanodet](https://github.com/RangiLyu/nanodet) | 9.4Mb | -| *lite::tnn::cv::detection::NanoDetPlus* | nanodet-plus-m-1.5x_416.opt.tnnproto&tnnmodel | [nanodet](https://github.com/RangiLyu/nanodet) | 9.4Mb | -| *lite::tnn::cv::detection::InsectDet* | quarrying_insect_detector.opt.tnnproto&tnnmodel | [InsectID](https://github.com/quarrying/quarrying-insect-id) | 22Mb | -| *lite::tnn::cv::detection::YOLOv6* | yolov6n-320x320.opt.tnnproto&tnnmodel | [YOLOv6](https://github.com/meituan/YOLOv6) | 17Mb | -| *lite::tnn::cv::detection::YOLOv6* | yolov6n-640x640.opt.tnnproto&tnnmodel | [YOLOv6](https://github.com/meituan/YOLOv6) | 17Mb | -| *lite::tnn::cv::detection::YOLOv6* | yolov6s-320x320.opt.tnnproto&tnnmodel | [YOLOv6](https://github.com/meituan/YOLOv6) | 66Mb | -| *lite::tnn::cv::detection::YOLOv6* | yolov6n-640x640.opt.tnnproto&tnnmodel | [YOLOv6](https://github.com/meituan/YOLOv6) | 66Mb | -| *lite::tnn::cv::detection::YOLOv6* | yolov6t-640x640.opt.tnnproto&tnnmodel | [YOLOv6](https://github.com/meituan/YOLOv6) | 57Mb | - - -## Matting. - -
- -| Class | Pretrained TNN Files | Rename or Converted From (Repo) | Size | -|:---------------------------------------------:|:----------------------------------------------------------------:|:----------------------------------------------------------------------:|:-----:| -| *lite::tnn::cv::matting::RobustVideoMatting* | rvm_mobilenetv3_fp32-480-480-sim.tnnproto&tnnmodel | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 14Mb | -| *lite::tnn::cv::matting::RobustVideoMatting* | rvm_mobilenetv3_fp32-480-480-sim.opt.tnnproto&tnnmodel | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 14Mb | -| *lite::tnn::cv::matting::RobustVideoMatting* | rvm_mobilenetv3_fp32-480-640-sim.opt.tnnproto&tnnmodel | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 14Mb | -| *lite::tnn::cv::matting::RobustVideoMatting* | rvm_mobilenetv3_fp32-640-480-sim.opt.tnnproto&tnnmodel | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 14Mb | -| *lite::tnn::cv::matting::RobustVideoMatting* | rvm_mobilenetv3_fp32-1080-1920-sim.opt.tnnproto&tnnmodel | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 14Mb | -| *lite::tnn::cv::matting::RobustVideoMatting* | rvm_resnet50_fp32-480-480-sim.opt.tnnproto&tnnmodel | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 50Mb | -| *lite::tnn::cv::matting::RobustVideoMatting* | rvm_resnet50_fp32-480-640-sim.opt.tnnproto&tnnmodel | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 50Mb | -| *lite::tnn::cv::matting::RobustVideoMatting* | rvm_resnet50_fp32-640-480-sim.opt.tnnproto&tnnmodel | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 50Mb | -| *lite::tnn::cv::matting::RobustVideoMatting* | rvm_resnet50_fp32-1080-1920-sim.opt.tnnproto&tnnmodel | [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 50Mb | -| *lite::tnn::cv::matting::MGMatting* | MGMatting-DIM-100k.opt.tnnproto&tnnmodel | [MGMatting](https://github.com/yucornetto/MGMatting) | 113Mb | -| *lite::tnn::cv::matting::MGMatting* | MGMatting-RWP-100k.opt.tnnproto&tnnmodel | [MGMatting](https://github.com/yucornetto/MGMatting) | 113Mb | -| *lite::tnn::cv::matting::MODNet* | modnet_photographic_portrait_matting-1024x1024.tnnproto&tnnmodel | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::tnn::cv::matting::MODNet* | modnet_photographic_portrait_matting-1024x512.tnnproto&tnnmodel | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::tnn::cv::matting::MODNet* | modnet_photographic_portrait_matting-256x256.tnnproto&tnnmodel | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::tnn::cv::matting::MODNet* | modnet_photographic_portrait_matting-256x512.tnnproto&tnnmodel | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::tnn::cv::matting::MODNet* | modnet_photographic_portrait_matting-512x1024.tnnproto&tnnmodel | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::tnn::cv::matting::MODNet* | modnet_photographic_portrait_matting-512x256.tnnproto&tnnmodel | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::tnn::cv::matting::MODNet* | modnet_photographic_portrait_matting-512x512.tnnproto&tnnmodel | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::tnn::cv::matting::MODNet* | modnet_webcam_portrait_matting-1024x1024.tnnproto&tnnmodel | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::tnn::cv::matting::MODNet* | modnet_webcam_portrait_matting-1024x512.tnnproto&tnnmodel | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::tnn::cv::matting::MODNet* | modnet_webcam_portrait_matting-256x256.tnnproto&tnnmodel | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::tnn::cv::matting::MODNet* | modnet_webcam_portrait_matting-256x512.tnnproto&tnnmodel | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::tnn::cv::matting::MODNet* | modnet_webcam_portrait_matting-512x1024.tnnproto&tnnmodel | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::tnn::cv::matting::MODNet* | modnet_webcam_portrait_matting-512x256.tnnproto&tnnmodel | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::tnn::cv::matting::MODNet* | modnet_webcam_portrait_matting-512x512.tnnproto&tnnmodel | [MODNet](https://github.com/ZHKKKe/MODNet) | 24Mb | -| *lite::tnn::cv::matting::BackgroundMattingV2* | BGMv2_mobilenetv2-256x256-full.opt.tnnproto&tnnmodel | [BackgroundMattingV2](https://github.com/PeterL1n/BackgroundMattingV2) | 20Mb | -| *lite::tnn::cv::matting::BackgroundMattingV2* | BGMv2_mobilenetv2-512x512-full.opt.tnnproto&tnnmodel | [BackgroundMattingV2](https://github.com/PeterL1n/BackgroundMattingV2) | 20Mb | -| *lite::tnn::cv::matting::BackgroundMattingV2* | BGMv2_mobilenetv2-1080x1920-full.opt.tnnproto&tnnmodel | [BackgroundMattingV2](https://github.com/PeterL1n/BackgroundMattingV2) | 20Mb | -| *lite::tnn::cv::matting::BackgroundMattingV2* | BGMv2_mobilenetv2-2160x3840-full.opt.tnnproto&tnnmodel | [BackgroundMattingV2](https://github.com/PeterL1n/BackgroundMattingV2) | 20Mb | -| *lite::tnn::cv::matting::BackgroundMattingV2* | BGMv2_resnet50-1080x1920-full.opt.tnnproto&tnnmodel | [BackgroundMattingV2](https://github.com/PeterL1n/BackgroundMattingV2) | 20Mb | -| *lite::tnn::cv::matting::BackgroundMattingV2* | BGMv2_resnet50-2160x3840-full.opt.tnnproto&tnnmodel | [BackgroundMattingV2](https://github.com/PeterL1n/BackgroundMattingV2) | 20Mb | -| *lite::tnn::cv::matting::BackgroundMattingV2* | BGMv2_resnet101-2160x3840-full.opt.tnnproto&tnnmodel | [BackgroundMattingV2](https://github.com/PeterL1n/BackgroundMattingV2) | 154Mb | - - -## Face Recognition. - -
- - -| Class | Pretrained TNN Files | Rename or Converted From (Repo) | Size | -|:----------------------------------------------:|:------------------------------------------------------------------------:|:----------------------------------------------------------------------:|:-----:| -| *lite::tnn::cv::faceid::GlintArcFace* | ms1mv3_arcface_r100.opt.tnnproto&tnnmodel | [insightface](https://github.com/deepinsight/insightface) | 248Mb | -| *lite::tnn::cv::faceid::GlintArcFace* | ms1mv3_arcface_r50.opt.tnnproto&tnnmodel | [insightface](https://github.com/deepinsight/insightface) | 166Mb | -| *lite::tnn::cv::faceid::GlintArcFace* | ms1mv3_arcface_r34.opt.tnnproto&tnnmodel | [insightface](https://github.com/deepinsight/insightface) | 130Mb | -| *lite::tnn::cv::faceid::GlintArcFace* | ms1mv3_arcface_r18.opt.tnnproto&tnnmodel | [insightface](https://github.com/deepinsight/insightface) | 91Mb | -| *lite::tnn::cv::faceid::GlintCosFace* | glint360k_cosface_r100.opt.tnnproto&tnnmodel | [insightface](https://github.com/deepinsight/insightface) | 248Mb | -| *lite::tnn::cv::faceid::GlintCosFace* | glint360k_cosface_r50.opt.tnnproto&tnnmodel | [insightface](https://github.com/deepinsight/insightface) | 166Mb | -| *lite::tnn::cv::faceid::GlintCosFace* | glint360k_cosface_r34.opt.tnnproto&tnnmodel | [insightface](https://github.com/deepinsight/insightface) | 130Mb | -| *lite::tnn::cv::faceid::GlintCosFace* | glint360k_cosface_r18.opt.tnnproto&tnnmodel | [insightface](https://github.com/deepinsight/insightface) | 91Mb | -| *lite::tnn::cv::faceid::GlintPartialFC* | partial_fc_glint360k_r100.opt.tnnproto&tnnmodel | [insightface](https://github.com/deepinsight/insightface) | 248Mb | -| *lite::tnn::cv::faceid::GlintPartialFC* | partial_fc_glint360k_r50.opt.tnnproto&tnnmodel | [insightface](https://github.com/deepinsight/insightface) | 91Mb | -| *lite::tnn::cv::faceid::FaceNet* | facenet_vggface2_resnet.opt.tnnproto&tnnmodel | [facenet...](https://github.com/timesler/facenet-pytorch) | 89Mb | -| *lite::tnn::cv::faceid::FaceNet* | facenet_casia-webface_resnet.opt.tnnproto&tnnmodel | [facenet...](https://github.com/timesler/facenet-pytorch) | 89Mb | -| *lite::tnn::cv::faceid::FocalArcFace* | focal-arcface-ms1m-ir152.opt.tnnproto&tnnmodel | [face.evoLVe...](https://github.com/ZhaoJ9014/face.evoLVe.PyTorch) | 269Mb | -| *lite::tnn::cv::faceid::FocalArcFace* | focal-arcface-ms1m-ir50-epoch120.opt.tnnproto&tnnmodel | [face.evoLVe...](https://github.com/ZhaoJ9014/face.evoLVe.PyTorch) | 166Mb | -| *lite::tnn::cv::faceid::FocalArcFace* | focal-arcface-ms1m-ir50-epoch63.opt.tnnproto&tnnmodel | [face.evoLVe...](https://github.com/ZhaoJ9014/face.evoLVe.PyTorch) | 166Mb | -| *lite::tnn::cv::faceid::FocalAsiaArcFace* | focal-arcface-bh-ir50-asia.opt.tnnproto&tnnmodel | [face.evoLVe...](https://github.com/ZhaoJ9014/face.evoLVe.PyTorch) | 166Mb | -| *lite::tnn::cv::faceid::TencentCurricularFace* | Tencent_CurricularFace_Backbone.opt.tnnproto&tnnmodel | [TFace](https://github.com/Tencent/TFace) | 249Mb | -| *lite::tnn::cv::faceid::TencentCifpFace* | Tencent_Cifp_BUPT_Balancedface_IR_34.opt.tnnproto&tnnmodel | [TFace](https://github.com/Tencent/TFace) | 130Mb | -| *lite::tnn::cv::faceid::CenterLossFace* | CenterLossFace_epoch_100.opt.tnnproto&tnnmodel | [center-loss...](https://github.com/louis-she/center-loss.pytorch) | 280Mb | -| *lite::tnn::cv::faceid::SphereFace* | sphere20a_20171020.opt.tnnproto&tnnmodel | [sphere...](https://github.com/clcarwin/sphereface_pytorch) | 86Mb | -| *lite::tnn::cv::faceid:MobileFaceNet* | MobileFaceNet_Pytorch_068.opt.tnnproto&tnnmodel | [MobileFace...](https://github.com/Xiaoccer/MobileFaceNet_Pytorch) | 3.8Mb | -| *lite::tnn::cv::faceid:CavaGhostArcFace* | cavaface_GhostNet_x1.3_Arcface_Epoch_24.opt.tnnproto&tnnmodel | [cavaface...](https://github.com/cavalleria/cavaface.pytorch) | 15Mb | -| *lite::tnn::cv::faceid:CavaCombinedFace* | cavaface_IR_SE_100_Combined_Epoch_24.opt.tnnproto&tnnmodel | [cavaface...](https://github.com/cavalleria/cavaface.pytorch) | 250Mb | -| *lite::tnn::cv::faceid:MobileSEFocalFace* | face_recognition.pytorch_Mobilenet_se_focal_121000.opt.tnnproto&tnnmodel | [face_recog...](https://github.com/grib0ed0v/face_recognition.pytorch) | 4.5Mb | - - -## Face Detection. - -
- -| Class | Pretrained TNN Files | Rename or Converted From (Repo) | Size | -|:------------------------------------------:|:------------------------------------------------------------:|:---------------------------------------------------------------------------------------:|:------:| -| *lite::tnn::cv::face::detect::UltraFace* | ultraface-rfb-320.opt.tnnproto&tnnmodel | [Ultra-Light...](https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB) | 1.5Mb | -| *lite::tnn::cv::face::detect::UltraFace* | ultraface-rfb-640.opt.tnnproto&tnnmodel | [Ultra-Light...](https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB) | 1.5Mb | -| *lite::tnn::cv::face::detect::UltraFace* | ultraface-slim-320.opt.tnnproto&tnnmodel | [Ultra-Light...](https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB) | 1.2Mb | -| *lite::tnn::cv::face::detect::UltraFace* | ultraface-slim-640.opt.tnnproto&tnnmodel | [Ultra-Light...](https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB) | 1.2Mb | -| *lite::tnn::cv::face::detect::RetinaFace* | Pytorch_RetinaFace_mobile0.25.opt.tnnproto&tnnmodel | [...Retinaface](https://github.com/biubug6/Pytorch_Retinaface) | 1.6Mb | -| *lite::tnn::cv::face::detect::RetinaFace* | Pytorch_RetinaFace_mobile0.25-640-640.opt.tnnproto&tnnmodel | [...Retinaface](https://github.com/biubug6/Pytorch_Retinaface) | 1.6Mb | -| *lite::tnn::cv::face::detect::RetinaFace* | Pytorch_RetinaFace_mobile0.25-320-320.opt.tnnproto&tnnmodel | [...Retinaface](https://github.com/biubug6/Pytorch_Retinaface) | 1.6Mb | -| *lite::tnn::cv::face::detect::RetinaFace* | Pytorch_RetinaFace_mobile0.25-720-1080.opt.tnnproto&tnnmodel | [...Retinaface](https://github.com/biubug6/Pytorch_Retinaface) | 1.6Mb | -| *lite::tnn::cv::face::detect::FaceBoxes* | FaceBoxes.opt.tnnproto&tnnmodel | [FaceBoxes](https://github.com/zisianw/FaceBoxes.PyTorch) | 3.8Mb | -| *lite::tnn::cv::face::detect::FaceBoxes* | FaceBoxes-640-640.opt.tnnproto&tnnmodel | [FaceBoxes](https://github.com/zisianw/FaceBoxes.PyTorch) | 3.8Mb | -| *lite::tnn::cv::face::detect::FaceBoxes* | FaceBoxes-320-320.opt.tnnproto&tnnmodel | [FaceBoxes](https://github.com/zisianw/FaceBoxes.PyTorch) | 3.8Mb | -| *lite::tnn::cv::face::detect::FaceBoxes* | FaceBoxes-720-1080.opt.tnnproto&tnnmodel | [FaceBoxes](https://github.com/zisianw/FaceBoxes.PyTorch) | 3.8Mb | -| *lite::tnn::cv::face::detect::SCRFD* | scrfd_500m_shape160x160.opt.tnnproto&tnnmodel | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.5Mb | -| *lite::tnn::cv::face::detect::SCRFD* | scrfd_500m_shape320x320.opt.tnnproto&tnnmodel | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.5Mb | -| *lite::tnn::cv::face::detect::SCRFD* | scrfd_500m_shape640x640.opt.tnnproto&tnnmodel | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.5Mb | -| *lite::tnn::cv::face::detect::SCRFD* | scrfd_500m_bnkps_shape160x160.opt.tnnproto&tnnmodel | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.5Mb | -| *lite::tnn::cv::face::detect::SCRFD* | scrfd_500m_bnkps_shape320x320.opt.tnnproto&tnnmodel | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.5Mb | -| *lite::tnn::cv::face::detect::SCRFD* | scrfd_500m_bnkps_shape640x640.opt.tnnproto&tnnmodel | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.5Mb | -| *lite::tnn::cv::face::detect::SCRFD* | scrfd_1g_shape160x160.opt.tnnproto&tnnmodel | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.7Mb | -| *lite::tnn::cv::face::detect::SCRFD* | scrfd_1g_shape320x320.opt.tnnproto&tnnmodel | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.7Mb | -| *lite::tnn::cv::face::detect::SCRFD* | scrfd_1g_shape640x640.opt.tnnproto&tnnmodel | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 2.7Mb | -| *lite::tnn::cv::face::detect::SCRFD* | scrfd_2.5g_shape160x160.opt.tnnproto&tnnmodel | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 3.3Mb | -| *lite::tnn::cv::face::detect::SCRFD* | scrfd_2.5g_shape320x320.opt.tnnproto&tnnmodel | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 3.3Mb | -| *lite::tnn::cv::face::detect::SCRFD* | scrfd_2.5g_shape640x640.opt.tnnproto&tnnmodel | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 3.3Mb | -| *lite::tnn::cv::face::detect::SCRFD* | scrfd_2.5g_bnkps_shape160x160.opt.tnnproto&tnnmodel | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 3.3Mb | -| *lite::tnn::cv::face::detect::SCRFD* | scrfd_2.5g_bnkps_shape320x320.opt.tnnproto&tnnmodel | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 3.3Mb | -| *lite::tnn::cv::face::detect::SCRFD* | scrfd_2.5g_bnkps_shape640x640.opt.tnnproto&tnnmodel | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 3.3Mb | -| *lite::tnn::cv::face::detect::SCRFD* | scrfd_10g_shape640x640.opt.tnnproto&tnnmodel | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 16.9Mb | -| *lite::tnn::cv::face::detect::SCRFD* | scrfd_10g_shape1280x1280.opt.tnnproto&tnnmodel | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 16.9Mb | -| *lite::tnn::cv::face::detect::SCRFD* | scrfd_10g_bnkps_shape640x640.opt.tnnproto&tnnmodel | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 16.9Mb | -| *lite::tnn::cv::face::detect::SCRFD* | scrfd_10g_bnkps_shape1280x1280.opt.tnnproto&tnnmodel | [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd) | 16.9Mb | -| *lite::tnn::cv::face::detect::YOLO5Face* | yolov5face-blazeface-640x640.opt.tnnproto&tnnmodel | [YOLO5Face](https://github.com/deepcam-cn/yolov5-face) | 3.4Mb | -| *lite::tnn::cv::face::detect::YOLO5Face* | yolov5face-l-640x640.opt.tnnproto&tnnmodel | [YOLO5Face](https://github.com/deepcam-cn/yolov5-face) | 181Mb | -| *lite::tnn::cv::face::detect::YOLO5Face* | yolov5face-m-640x640.opt.tnnproto&tnnmodel | [YOLO5Face](https://github.com/deepcam-cn/yolov5-face) | 83Mb | -| *lite::tnn::cv::face::detect::YOLO5Face* | yolov5face-n-0.5-320x320.opt.tnnproto&tnnmodel | [YOLO5Face](https://github.com/deepcam-cn/yolov5-face) | 2.5Mb | -| *lite::tnn::cv::face::detect::YOLO5Face* | yolov5face-n-0.5-640x640.opt.tnnproto&tnnmodel | [YOLO5Face](https://github.com/deepcam-cn/yolov5-face) | 4.6Mb | -| *lite::tnn::cv::face::detect::YOLO5Face* | yolov5face-n-640x640.opt.tnnproto&tnnmodel | [YOLO5Face](https://github.com/deepcam-cn/yolov5-face) | 9.5Mb | -| *lite::tnn::cv::face::detect::YOLO5Face* | yolov5face-s-640x640.opt.tnnproto&tnnmodel | [YOLO5Face](https://github.com/deepcam-cn/yolov5-face) | 30Mb | -| *lite::tnn::cv::face::detect::FaceBoxesV2* | faceboxesv2-640x640.opt.tnnproto&tnnmodel | [FaceBoxesV2](https://github.com/jhb86253817/FaceBoxesV2) | 4.0Mb | - - -## Face Alignment. - -
- - -| Class | Pretrained TNN Files | Rename or Converted From (Repo) | Size | -|:-----------------------------------------------:|:------------------------------------------------------------------------------:|:------------------------------------------------------------------:|:-------:| -| *lite::tnn::cv::face::align::PFLD* | pfld-106-lite.opt.tnnproto&tnnmodel | [pfld_106_...](https://github.com/Hsintao/pfld_106_face_landmarks) | 1.0Mb | -| *lite::tnn::cv::face::align::PFLD* | pfld-106-v3.opt.tnnproto&tnnmodel | [pfld_106_...](https://github.com/Hsintao/pfld_106_face_landmarks) | 5.5Mb | -| *lite::tnn::cv::face::align::PFLD* | pfld-106-v2.opt.tnnproto&tnnmodel | [pfld_106_...](https://github.com/Hsintao/pfld_106_face_landmarks) | 5.0Mb | -| *lite::tnn::cv::face::align::PFLD98* | PFLD-pytorch-pfld.opt.tnnproto&tnnmodel | [PFLD...](https://github.com/polarisZhao/PFLD-pytorch) | 4.8Mb | -| *lite::tnn::cv::face::align::MobileNetV268* | pytorch_face_landmarks_landmark_detection_56.opt.tnnproto&tnnmodel | [...landmark](https://github.com/cunjian/pytorch_face_landmark) | 9.4Mb | -| *lite::tnn::cv::face::align::MobileNetV2SE68* | pytorch_face_landmarks_landmark_detection_56_se_external.opt.tnnproto&tnnmodel | [...landmark](https://github.com/cunjian/pytorch_face_landmark) | 11Mb | -| *lite::tnn::cv::face::align::PFLD68* | pytorch_face_landmarks_pfld.opt.tnnproto&tnnmodel | [...landmark](https://github.com/cunjian/pytorch_face_landmark) | 2.8Mb | -| *lite::tnn::cv::face::align::FaceLandmarks1000* | FaceLandmark1000.opt.tnnproto&tnnmodel | [FaceLandm...](https://github.com/Single430/FaceLandmark1000) | 2.0Mb | -| *lite::tnn::cv::face::align::PIPNet98* | pipnet_resnet18_10x98x32x256_wflw.opt.tnnproto&tnnmodel | [PIPNet](https://github.com/jhb86253817/PIPNet) | 44.0Mb | -| *lite::tnn::cv::face::align::PIPNet68* | pipnet_resnet18_10x68x32x256_300w.opt.tnnproto&tnnmodel | [PIPNet](https://github.com/jhb86253817/PIPNet) | 44.0Mb | -| *lite::tnn::cv::face::align::PIPNet29* | pipnet_resnet18_10x29x32x256_cofw.opt.tnnproto&tnnmodel | [PIPNet](https://github.com/jhb86253817/PIPNet) | 44.0Mb | -| *lite::tnn::cv::face::align::PIPNet19* | pipnet_resnet18_10x19x32x256_aflw.opt.tnnproto&tnnmodel | [PIPNet](https://github.com/jhb86253817/PIPNet) | 44.0Mb | -| *lite::tnn::cv::face::align::PIPNet98* | pipnet_resnet101_10x98x32x256_wflw.opt.tnnproto&tnnmodel | [PIPNet](https://github.com/jhb86253817/PIPNet) | 150.0Mb | -| *lite::tnn::cv::face::align::PIPNet68* | pipnet_resnet101_10x68x32x256_300w.opt.tnnproto&tnnmodel | [PIPNet](https://github.com/jhb86253817/PIPNet) | 150.0Mb | -| *lite::tnn::cv::face::align::PIPNet29* | pipnet_resnet101_10x29x32x256_cofw.opt.tnnproto&tnnmodel | [PIPNet](https://github.com/jhb86253817/PIPNet) | 150.0Mb | -| *lite::tnn::cv::face::align::PIPNet19* | pipnet_resnet101_10x19x32x256_aflw.opt.tnnproto&tnnmodel | [PIPNet](https://github.com/jhb86253817/PIPNet) | 150.0Mb | - - -## Head Pose Estimation. - -
- - -| Class | Pretrained TNN Files | Rename or Converted From (Repo) | Size | -|:-----------------------------------:|:--------------------------------:|:------------------------------------------------------------------:|:-----:| -| *lite::tnn::cv::face::pose::FSANet* | fsanet-var.opt.tnnproto&tnnmodel | [...fsanet...](https://github.com/omasaht/headpose-fsanet-pytorch) | 1.2Mb | -| *lite::tnn::cv::face::pose::FSANet* | fsanet-1x1.opt.tnnproto&tnnmodel | [...fsanet...](https://github.com/omasaht/headpose-fsanet-pytorch) | 1.2Mb | - -## Face Attributes. - -
- - -| Class | Pretrained TNN Files | Rename or Converted From (Repo) | Size | -|:----------------------------------------------:|:-----------------------------------------------------------------------------:|:-------------------------------------------------------------------------:|:-----:| -| *lite::tnn::cv::face::attr::AgeGoogleNet* | age_googlenet.opt.tnnproto&tnnmodel | [onnx-models](https://github.com/onnx/models) | 23Mb | -| *lite::tnn::cv::face::attr::GenderGoogleNet* | gender_googlenet.opt.tnnproto&tnnmodel | [onnx-models](https://github.com/onnx/models) | 23Mb | -| *lite::tnn::cv::face::attr::EmotionFerPlus* | emotion-ferplus-7.opt.tnnproto&tnnmodel | [onnx-models](https://github.com/onnx/models) | 33Mb | -| *lite::tnn::cv::face::attr::EmotionFerPlus* | emotion-ferplus-8.opt.tnnproto&tnnmodel | [onnx-models](https://github.com/onnx/models) | 33Mb | -| *lite::tnn::cv::face::attr::SSRNet* | ssrnet.opt.tnnproto&tnnmodel | [SSR_Net...](https://github.com/oukohou/SSR_Net_Pytorch) | 190Kb | -| *lite::tnn::cv::face::attr::EfficientEmotion7* | face-emotion-recognition-enet_b0_7.opt.tnnproto&tnnmodel | [face-emo...](https://github.com/HSE-asavchenko/face-emotion-recognition) | 15Mb | -| *lite::tnn::cv::face::attr::EfficientEmotion8* | face-emotion-recognition-enet_b0_8_best_afew.opt.tnnproto&tnnmodel | [face-emo...](https://github.com/HSE-asavchenko/face-emotion-recognition) | 15Mb | -| *lite::tnn::cv::face::attr::EfficientEmotion8* | face-emotion-recognition-enet_b0_8_best_vgaf.opt.tnnproto&tnnmodel | [face-emo...](https://github.com/HSE-asavchenko/face-emotion-recognition) | 15Mb | -| *lite::tnn::cv::face::attr::MobileEmotion7* | face-emotion-recognition-mobilenet_7.opt.tnnproto&tnnmodel | [face-emo...](https://github.com/HSE-asavchenko/face-emotion-recognition) | 13Mb | -| *lite::tnn::cv::face::attr::ReXNetEmotion7* | face-emotion-recognition-affectnet_7_vggface2_rexnet150.opt.tnnproto&tnnmodel | [face-emo...](https://github.com/HSE-asavchenko/face-emotion-recognition) | 30Mb | - -## Classification. - -
- - -| Class | Pretrained TNN Files | Rename or Converted From (Repo) | Size | -|:-------------------------------------------------:|:-------------------------------------------------:|:------------------------------------------------------------:|:-----:| -| *lite::tnn::cv::classification:EfficientNetLite4* | efficientnet-lite4-11.opt.tnnproto&tnnmodel | [onnx-models](https://github.com/onnx/models) | 49Mb | -| *lite::tnn::cv::classification::ShuffleNetV2* | shufflenet-v2-10.opt.tnnproto&tnnmodel | [onnx-models](https://github.com/onnx/models) | 8.7Mb | -| *lite::tnn::cv::classification::DenseNet121* | densenet121.opt.tnnproto&tnnmodel | [torchvision](https://github.com/pytorch/vision) | 30Mb | -| *lite::tnn::cv::classification::GhostNet* | ghostnet.opt.tnnproto&tnnmodel | [torchvision](https://github.com/pytorch/vision) | 20Mb | -| *lite::tnn::cv::classification::HdrDNet* | hardnet.opt.tnnproto&tnnmodel | [torchvision](https://github.com/pytorch/vision) | 13Mb | -| *lite::tnn::cv::classification::IBNNet* | ibnnet18.opt.tnnproto&tnnmodel | [torchvision](https://github.com/pytorch/vision) | 97Mb | -| *lite::tnn::cv::classification::MobileNetV2* | mobilenetv2.opt.tnnproto&tnnmodel | [torchvision](https://github.com/pytorch/vision) | 13Mb | -| *lite::tnn::cv::classification::ResNet* | resnet18.opt.tnnproto&tnnmodel | [torchvision](https://github.com/pytorch/vision) | 44Mb | -| *lite::tnn::cv::classification::ResNeXt* | resnext.opt.tnnproto&tnnmodel | [torchvision](https://github.com/pytorch/vision) | 95Mb | -| *lite::tnn::cv::classification::InsectID* | quarrying_insect_identifier.opt.tnnproto&tnnmodel | [InsectID](https://github.com/quarrying/quarrying-insect-id) | 27Mb | -| *lite::tnn::cv::classification:PlantID* | quarrying_planted_model.opt.tnnproto&tnnmodel | [PlantID](https://github.com/quarrying/quarrying-plant-id) | 30Mb | - - -## Segmentation. - -
- - -| Class | Pretrained TNN Files | Rename or Converted From (Repo) | Size | -|:-------------------------------------------------:|:----------------------------------------------:|:--------------------------------------------------------------------------:|:-----:| -| *lite::tnn::cv::segmentation::DeepLabV3ResNet101* | deeplabv3_resnet101_coco.opt.tnnproto&tnnmodel | [torchvision](https://github.com/pytorch/vision) | 232Mb | -| *lite::tnn::cv::segmentation::FCNResNet101* | fcn_resnet101.opt.tnnproto&tnnmodel | [torchvision](https://github.com/pytorch/vision) | 207Mb | -| *lite::tnn::cv::segmentation::HeadSeg* | minivision_head_seg.opt.tnnproto&cmodel | [photo2cartoon](https://github.com/minivision-ai/photo2cartoon) | 31Mb | -| *lite::tnn::cv::segmentation::FaceParsingBiSeNet* | face_parsing_512x512.opt.tnnproto&cmodel | [face-parsing.PyTorch](https://github.com/zllrunning/face-parsing.PyTorch) | 50M | -| *lite::tnn::cv::segmentation::FaceParsingBiSeNet* | face_parsing_1024x1024.opt.tnnproto&cmodel | [face-parsing.PyTorch](https://github.com/zllrunning/face-parsing.PyTorch) | 50M | - - - -## Style Transfer. - -
- -| Class | Pretrained TNN Files | Rename or Converted From (Repo) | Size | -|:-------------------------------------------:|:-----------------------------------------------------:|:---------------------------------------------------------------:|:-----:| -| *lite::tnn::cv::style::FastStyleTransfer* | style-mosaic-8.opt.tnnproto&tnnmodel | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::tnn::cv::style::FastStyleTransfer* | style-candy-9.opt.tnnproto&tnnmodel | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::tnn::cv::style::FastStyleTransfer* | style-udnie-8.opt.tnnproto&tnnmodel | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::tnn::cv::style::FastStyleTransfer* | style-udnie-9.opt.tnnproto&tnnmodel | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::tnn::cv::style::FastStyleTransfer* | style-pointilism-8.opt.tnnproto&tnnmodel | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::tnn::cv::style::FastStyleTransfer* | style-pointilism-9.opt.tnnproto&tnnmodel | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::tnn::cv::style::FastStyleTransfer* | style-rain-princess-9.opt.tnnproto&tnnmodel | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::tnn::cv::style::FastStyleTransfer* | style-rain-princess-8.opt.tnnproto&tnnmodel | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::tnn::cv::style::FastStyleTransfer* | style-candy-8.opt.tnnproto&tnnmodel | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::tnn::cv::style::FastStyleTransfer* | style-mosaic-9.opt.tnnproto&tnnmodel | [onnx-models](https://github.com/onnx/models) | 6.4Mb | -| *lite::tnn::cv::style::FemalePhoto2Cartoon* | minivision_female_photo2cartoon.opt.tnnproto&tnnmodel | [photo2cartoon](https://github.com/minivision-ai/photo2cartoon) | 15Mb | - - -## Colorization. - -
- -| Class | Pretrained TNN Files | Rename or Converted From (Repo) | Size | -|:----------------------------------------:|:------------------------------------------:|:---------------------------------------------------------:|:-----:| -| *lite::tnn::cv::colorization::Colorizer* | eccv16-colorizer.opt.tnnproto&tnnmodel | [colorization](https://github.com/richzhang/colorization) | 123Mb | -| *lite::tnn::cv::colorization::Colorizer* | siggraph17-colorizer.opt.tnnproto&tnnmodel | [colorization](https://github.com/richzhang/colorization) | 129Mb | - - -## Super Resolution. - -
- -| Class | Pretrained TNN Files | Rename or Converted From (Repo) | Size | -|:----------------------------------------:|:----------------------------------:|:---------------------------------------------------------:|:-----:| -| *lite::tnn::cv::resolution::SubPixelCNN* | subpixel-cnn.opt.tnnproto&tnnmodel | [...PIXEL...](https://github.com/niazwazir/SUB_PIXEL_CNN) | 234Kb | - - -# \ No newline at end of file diff --git a/lite/config.h.in b/lite/config.h.in index 6bed6b4a..51083521 100644 --- a/lite/config.h.in +++ b/lite/config.h.in @@ -3,9 +3,6 @@ #cmakedefine ENABLE_ONNXRUNTIME #cmakedefine ENABLE_TENSORRT -#cmakedefine ENABLE_MNN -#cmakedefine ENABLE_NCNN -#cmakedefine ENABLE_TNN #cmakedefine ENABLE_ONNXRUNTIME_CUDA #cmakedefine ENABLE_OPENCV_VIDEOIO #cmakedefine ENABLE_DEBUG_STRING diff --git a/lite/mnn/.gitignore b/lite/mnn/.gitignore deleted file mode 100644 index e69de29b..00000000 diff --git a/lite/mnn/core/mnn_config.h b/lite/mnn/core/mnn_config.h deleted file mode 100644 index 52c9c9b0..00000000 --- a/lite/mnn/core/mnn_config.h +++ /dev/null @@ -1,20 +0,0 @@ -// -// Created by DefTruth on 2021/10/6. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CORE_MNN_CONFIG_H -#define LITE_AI_TOOLKIT_MNN_CORE_MNN_CONFIG_H - -#include "mnn_defs.h" -#include "lite/lite.ai.headers.h" - -#ifdef ENABLE_MNN -#include "MNN/Interpreter.hpp" -#include "MNN/MNNDefine.h" -#include "MNN/Tensor.hpp" -#include "MNN/ImageProcess.hpp" -#endif - -namespace mnncore {} - -#endif //LITE_AI_TOOLKIT_MNN_CORE_MNN_CONFIG_H diff --git a/lite/mnn/core/mnn_core.h b/lite/mnn/core/mnn_core.h deleted file mode 100644 index 1d028023..00000000 --- a/lite/mnn/core/mnn_core.h +++ /dev/null @@ -1,114 +0,0 @@ -// -// Created by DefTruth on 2021/10/6. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CORE_MNN_CORE_H -#define LITE_AI_TOOLKIT_MNN_CORE_MNN_CORE_H - -#include "mnn_config.h" -#include "mnn_handler.h" -#include "mnn_types.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNNanoDet; // [0] * reference: https://github.com/RangiLyu/nanodet - class LITE_EXPORTS MNNNanoDetEfficientNetLite; // [1] * reference: https://github.com/RangiLyu/nanodet - class LITE_EXPORTS MNNRobustVideoMatting; // [2] * reference: https://github.com/PeterL1n/RobustVideoMatting - class LITE_EXPORTS MNNYoloX; // [3] * reference: https://github.com/Megvii-BaseDetection/YOLOX - class LITE_EXPORTS MNNYOLOP; // [4] * reference: https://github.com/hustvl/YOLOP - class LITE_EXPORTS MNNYoloV5; // [5] * reference: https://github.com/ultralytics/yolov5 - class LITE_EXPORTS MNNYoloX_V_0_1_1; // [6] * reference: https://github.com/Megvii-BaseDetection/YOLOX - class LITE_EXPORTS MNNYoloR; // [7] * reference: https://github.com/WongKinYiu/yolor - class LITE_EXPORTS MNNYoloV5_V_6_0; // [8] * reference: https://github.com/ultralytics/yolov5 - class LITE_EXPORTS MNNGlintArcFace; // [9] * reference: https://github.com/deepinsight/insightface/tree/master/recognition/arcface_torch - class LITE_EXPORTS MNNGlintCosFace; // [10] * reference: https://github.com/deepinsight/insightface/tree/master/recognition/arcface_torch - class LITE_EXPORTS MNNGlintPartialFC; // [11] * reference: https://github.com/deepinsight/insightface/tree/master/recognition/partial_fc - class LITE_EXPORTS MNNFaceNet; // [12] * reference: https://github.com/timesler/facenet-pytorch - class LITE_EXPORTS MNNFocalArcFace; // [13] * reference: https://github.com/ZhaoJ9014/face.evoLVe.PyTorch - class LITE_EXPORTS MNNFocalAsiaArcFace; // [14] * reference: https://github.com/ZhaoJ9014/face.evoLVe.PyTorch - class LITE_EXPORTS MNNTencentCurricularFace; // [15] * reference: https://github.com/Tencent/TFace/tree/master/tasks/distfc - class LITE_EXPORTS MNNTencentCifpFace; // [16] * reference: https://github.com/Tencent/TFace/tree/master/tasks/cifp - class LITE_EXPORTS MNNCenterLossFace; // [17] * reference: https://github.com/louis-she/center-loss.pytorch - class LITE_EXPORTS MNNSphereFace; // [18] * reference: https://github.com/clcarwin/sphereface_pytorch - class LITE_EXPORTS MNNMobileFaceNet; // [19] * reference: https://github.com/Xiaoccer/MobileFaceNet_Pytorch - class LITE_EXPORTS MNNCavaGhostArcFace; // [20] * reference: https://github.com/cavalleria/cavaface.pytorch - class LITE_EXPORTS MNNCavaCombinedFace; // [21] * reference: https://github.com/cavalleria/cavaface.pytorch - class LITE_EXPORTS MNNMobileSEFocalFace; // [22] * reference: https://github.com/grib0ed0v/face_recognition.pytorch - class LITE_EXPORTS MNNUltraFace; // [23] * reference: https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB - class LITE_EXPORTS MNNRetinaFace; // [24] * reference: https://github.com/biubug6/Pytorch_Retinaface - class LITE_EXPORTS MNNFaceBoxes; // [25] * reference: https://github.com/zisianw/FaceBoxes.PyTorch - class LITE_EXPORTS MNNPFLD; // [26] * reference: https://github.com/Hsintao/pfld_106_face_landmarks - class LITE_EXPORTS MNNPFLD98; // [27] * reference: https://github.com/polarisZhao/PFLD-pytorch - class LITE_EXPORTS MNNMobileNetV268; // [28] * reference: https://github.com/cunjian/pytorch_face_landmark - class LITE_EXPORTS MNNMobileNetV2SE68; // [29] * reference: https://github.com/cunjian/pytorch_face_landmark - class LITE_EXPORTS MNNPFLD68; // [30] * reference: https://github.com/cunjian/pytorch_face_landmark - class LITE_EXPORTS MNNFaceLandmark1000; // [31] * reference: https://github.com/Single430/FaceLandmark1000 - class LITE_EXPORTS MNNFSANet; // [32] * reference: https://github.com/omasaht/headpose-fsanet-pytorch - class LITE_EXPORTS MNNAgeGoogleNet; // [33] * reference: https://github.com/onnx/models/tree/master/vision/body_analysis/age_gender - class LITE_EXPORTS MNNGenderGoogleNet; // [34] * reference: https://github.com/onnx/models/tree/master/vision/body_analysis/age_gender - class LITE_EXPORTS MNNEmotionFerPlus; // [35] * reference: https://github.com/onnx/models/blob/master/vision/body_analysis/emotion_ferplus - class LITE_EXPORTS MNNSSRNet; // [36] * reference: https://github.com/oukohou/SSR_Net_Pytorch - class LITE_EXPORTS MNNEfficientEmotion7; // [37] * reference: https://github.com/HSE-asavchenko/face-emotion-recognition - class LITE_EXPORTS MNNEfficientEmotion8; // [38] * reference: https://github.com/HSE-asavchenko/face-emotion-recognition - class LITE_EXPORTS MNNMobileEmotion7; // [39] * reference: https://github.com/HSE-asavchenko/face-emotion-recognition - class LITE_EXPORTS MNNReXNetEmotion7; // [40] * reference: https://github.com/HSE-asavchenko/face-emotion-recognition - class LITE_EXPORTS MNNEfficientNetLite4; // [41] * reference: https://github.com/onnx/models/blob/master/vision/classification/efficientnet-lite4 - class LITE_EXPORTS MNNShuffleNetV2; // [42] * reference: https://github.com/onnx/models/blob/master/vision/classification/shufflenet - class LITE_EXPORTS MNNDenseNet; // [43] * reference: https://pytorch.org/hub/pytorch_vision_densenet/ - class LITE_EXPORTS MNNGhostNet; // [44] * reference：https://pytorch.org/hub/pytorch_vision_ghostnet/ - class LITE_EXPORTS MNNHdrDNet; // [45] * reference: https://pytorch.org/hub/pytorch_vision_hardnet/ - class LITE_EXPORTS MNNIBNNet; // [46] * reference: https://pytorch.org/hub/pytorch_vision_ibnnet/ - class LITE_EXPORTS MNNMobileNetV2; // [47] * reference: https://pytorch.org/hub/pytorch_vision_mobilenet_v2/ - class LITE_EXPORTS MNNResNet; // [48] * reference: https://pytorch.org/hub/pytorch_vision_resnet/ - class LITE_EXPORTS MNNResNeXt; // [49] * reference: https://pytorch.org/hub/pytorch_vision_resnext/ - class LITE_EXPORTS MNNFastStyleTransfer; // [50] * reference: https://github.com/onnx/models/blob/master/vision/style_transfer/fast_neural_style - class LITE_EXPORTS MNNColorizer; // [51] * reference: https://github.com/richzhang/colorization - class LITE_EXPORTS MNNSubPixelCNN; // [52] * reference: https://github.com/niazwazir/SUB_PIXEL_CNN - class LITE_EXPORTS MNNDeepLabV3ResNet101; // [53] * reference: https://pytorch.org/hub/pytorch_vision_deeplabv3_resnet101/ - class LITE_EXPORTS MNNFCNResNet101; // [54] * reference: https://pytorch.org/hub/pytorch_vision_fcn_resnet101/ - class LITE_EXPORTS MNNMGMatting; // [55] * reference: https://github.com/yucornetto/MGMatting - class LITE_EXPORTS MNNNanoDetPlus; // [56] * reference: https://github.com/RangiLyu/nanodet - class LITE_EXPORTS MNNSCRFD; // [57] * reference: https://github.com/deepinsight/insightface/tree/master/detection/scrfd - class LITE_EXPORTS MNNYOLO5Face; // [58] * reference: https://github.com/deepcam-cn/yolov5-face - class LITE_EXPORTS MNNFaceBoxesV2; // [59] * reference: https://github.com/jhb86253817/FaceBoxesV2 - class LITE_EXPORTS MNNPIPNet19; // [60] * reference: https://github.com/jhb86253817/PIPNet - class LITE_EXPORTS MNNPIPNet29; // [61] * reference: https://github.com/jhb86253817/PIPNet - class LITE_EXPORTS MNNPIPNet68; // [62] * reference: https://github.com/jhb86253817/PIPNet - class LITE_EXPORTS MNNPIPNet98; // [63] * reference: https://github.com/jhb86253817/PIPNet - class LITE_EXPORTS MNNInsectDet; // [64] * reference: https://github.com/quarrying/quarrying-insect-id - class LITE_EXPORTS MNNInsectID; // [65] * reference: https://github.com/quarrying/quarrying-insect-id - class LITE_EXPORTS MNNPlantID; // [66] * reference: https://github.com/quarrying/quarrying-plant-id - class LITE_EXPORTS MNNMODNet; // [67] * reference: https://github.com/ZHKKKe/MODNet - class LITE_EXPORTS MNNBackgroundMattingV2; // [68] * reference: https://github.com/PeterL1n/BackgroundMattingV2 - class LITE_EXPORTS MNNYOLOv5BlazeFace; // [69] * reference: https://github.com/deepcam-cn/yolov5-face - class LITE_EXPORTS MNNYoloV5_V_6_1; // [70] * reference: https://github.com/ultralytics/yolov5/releases/tag/v6.1 - class LITE_EXPORTS MNNHeadSeg; // [71] * reference: https://github.com/minivision-ai/photo2cartoon - class LITE_EXPORTS MNNFemalePhoto2Cartoon; // [72] * reference: https://github.com/minivision-ai/photo2cartoon - class LITE_EXPORTS MNNFastPortraitSeg; // [73] * reference: https://github.com/YexingWan/Fast-Portrait-Segmentation - class LITE_EXPORTS MNNPortraitSegExtremeC3Net; // [74] * reference: https://github.com/clovaai/ext_portrait_segmentation - class LITE_EXPORTS MNNPortraitSegSINet; // [75] * reference: https://github.com/clovaai/ext_portrait_segmentation - class LITE_EXPORTS MNNFaceHairSeg; // [76] * reference: https://github.com/kampta/face-seg - class LITE_EXPORTS MNNHairSeg; // [77] * reference: https://github.com/akirasosa/mobile-semantic-segmentation - class LITE_EXPORTS MNNMobileHumanMatting; // [78] * reference: https://github.com/lizhengwei1992/mobile_phone_human_matting - class LITE_EXPORTS MNNYOLOv6; // [78] * reference: https://github.com/meituan/YOLOv6 - class LITE_EXPORTS MNNFaceParsingBiSeNet; // [79] * reference: https://github.com/zllrunning/face-parsing.PyTorch - class LITE_EXPORTS MNNFaceMesh; // [80] * reference: https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_mesh - class LITE_EXPORTS MNNIrisLandmarks; // [81] * reference: https://github.com/google/mediapipe/tree/master/mediapipe/graphs/iris_tracking -} - -namespace mnncv -{ - using mnncore::BasicMNNHandler; -} - -namespace mnnnlp -{ - using mnncore::BasicMNNHandler; -} - -namespace mnnasr -{ - using mnncore::BasicMNNHandler; -} - -#endif //LITE_AI_TOOLKIT_MNN_CORE_MNN_CORE_H diff --git a/lite/mnn/core/mnn_defs.h b/lite/mnn/core/mnn_defs.h deleted file mode 100644 index be2f5abb..00000000 --- a/lite/mnn/core/mnn_defs.h +++ /dev/null @@ -1,24 +0,0 @@ -// -// Created by DefTruth on 2021/10/6. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CORE_MNN_DEFS_H -#define LITE_AI_TOOLKIT_MNN_CORE_MNN_DEFS_H - -#include "lite/config.h" -#include "lite/lite.ai.defs.h" - -#ifdef ENABLE_DEBUG_STRING -# define LITEMNN_DEBUG 1 -#else -# define LITEMNN_DEBUG 0 -#endif - - -#ifdef LITE_WIN32 -# ifndef NOMINMAX -# define NOMINMAX -# endif -#endif - -#endif //LITE_AI_TOOLKIT_MNN_CORE_MNN_DEFS_H diff --git a/lite/mnn/core/mnn_handler.cpp b/lite/mnn/core/mnn_handler.cpp deleted file mode 100644 index d1d75f2c..00000000 --- a/lite/mnn/core/mnn_handler.cpp +++ /dev/null @@ -1,90 +0,0 @@ -// -// Created by DefTruth on 2021/10/6. -// - -#include "mnn_handler.h" - -using mnncore::BasicMNNHandler; - -BasicMNNHandler::BasicMNNHandler( - const std::string &_mnn_path, unsigned int _num_threads) : - log_id(_mnn_path.data()), mnn_path(_mnn_path.data()), - num_threads(_num_threads) -{ - initialize_handler(); -} - -void BasicMNNHandler::initialize_handler() -{ - // 1. init interpreter - mnn_interpreter = std::shared_ptr(MNN::Interpreter::createFromFile(mnn_path)); - // 2. init schedule_config - schedule_config.numThread = (int) num_threads; - MNN::BackendConfig backend_config; - backend_config.precision = MNN::BackendConfig::Precision_High; // default Precision_High - schedule_config.backendConfig = &backend_config; - // 3. create session - mnn_session = mnn_interpreter->createSession(schedule_config); - // 4. init input tensor - input_tensor = mnn_interpreter->getSessionInput(mnn_session, nullptr); - // 5. init input dims - input_batch = input_tensor->batch(); - input_channel = input_tensor->channel(); - input_height = input_tensor->height(); - input_width = input_tensor->width(); - dimension_type = input_tensor->getDimensionType(); - // 6. resize tensor & session needed ??? - if (dimension_type == MNN::Tensor::CAFFE) - { - // NCHW - mnn_interpreter->resizeTensor( - input_tensor, {input_batch, input_channel, input_height, input_width}); - mnn_interpreter->resizeSession(mnn_session); - } // NHWC - else if (dimension_type == MNN::Tensor::TENSORFLOW) - { - mnn_interpreter->resizeTensor( - input_tensor, {input_batch, input_height, input_width, input_channel}); - mnn_interpreter->resizeSession(mnn_session); - } // NC4HW4 - else if (dimension_type == MNN::Tensor::CAFFE_C4) - { -#ifdef LITEMNN_DEBUG - std::cout << "Dimension Type is CAFFE_C4, skip resizeTensor & resizeSession!\n"; -#endif - } - // output count - num_outputs = mnn_interpreter->getSessionOutputAll(mnn_session).size(); -#ifdef LITEMNN_DEBUG - this->print_debug_string(); -#endif -} - -BasicMNNHandler::~BasicMNNHandler() -{ - mnn_interpreter->releaseModel(); - if (mnn_session) - mnn_interpreter->releaseSession(mnn_session); -} - -void BasicMNNHandler::print_debug_string() -{ - std::cout << "LITEMNN_DEBUG LogId: " << log_id << "\n"; - std::cout << "=============== Input-Dims ==============\n"; - if (input_tensor) input_tensor->printShape(); - if (dimension_type == MNN::Tensor::CAFFE) - std::cout << "Dimension Type: (CAFFE/PyTorch/ONNX)NCHW" << "\n"; - else if (dimension_type == MNN::Tensor::TENSORFLOW) - std::cout << "Dimension Type: (TENSORFLOW)NHWC" << "\n"; - else if (dimension_type == MNN::Tensor::CAFFE_C4) - std::cout << "Dimension Type: (CAFFE_C4)NC4HW4" << "\n"; - std::cout << "=============== Output-Dims ==============\n"; - auto tmp_output_map = mnn_interpreter->getSessionOutputAll(mnn_session); - std::cout << "getSessionOutputAll done!\n"; - for (auto it = tmp_output_map.cbegin(); it != tmp_output_map.cend(); ++it) - { - std::cout << "Output: " << it->first << ": "; - it->second->printShape(); - } - std::cout << "========================================\n"; -} \ No newline at end of file diff --git a/lite/mnn/core/mnn_handler.h b/lite/mnn/core/mnn_handler.h deleted file mode 100644 index ced21152..00000000 --- a/lite/mnn/core/mnn_handler.h +++ /dev/null @@ -1,55 +0,0 @@ -// -// Created by DefTruth on 2021/10/6. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CORE_MNN_HANDLER_H -#define LITE_AI_TOOLKIT_MNN_CORE_MNN_HANDLER_H - -#include "mnn_config.h" - -namespace mnncore -{ - class LITE_EXPORTS BasicMNNHandler - { - protected: - std::shared_ptr mnn_interpreter; - MNN::Session *mnn_session = nullptr; - MNN::Tensor *input_tensor = nullptr; // assume single input. - MNN::ScheduleConfig schedule_config; - std::shared_ptr pretreat; // init at subclass - const char *log_id = nullptr; - const char *mnn_path = nullptr; - - protected: - const unsigned int num_threads; // initialize at runtime. - int input_batch; - int input_channel; - int input_height; - int input_width; - int dimension_type; - int num_outputs = 1; - - protected: - explicit BasicMNNHandler(const std::string &_mnn_path, unsigned int _num_threads = 1); - - virtual ~BasicMNNHandler(); - - // un-copyable - protected: - BasicMNNHandler(const BasicMNNHandler &) = delete; // - BasicMNNHandler(BasicMNNHandler &&) = delete; // - BasicMNNHandler &operator=(const BasicMNNHandler &) = delete; // - BasicMNNHandler &operator=(BasicMNNHandler &&) = delete; // - - protected: - virtual void transform(const cv::Mat &mat) = 0; // ? needed ? - - private: - void initialize_handler(); - - void print_debug_string(); - - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CORE_MNN_HANDLER_H diff --git a/lite/mnn/core/mnn_types.h b/lite/mnn/core/mnn_types.h deleted file mode 100644 index 2b88a3b8..00000000 --- a/lite/mnn/core/mnn_types.h +++ /dev/null @@ -1,15 +0,0 @@ -// -// Created by DefTruth on 2021/10/6. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CORE_MNN_TYPES_H -#define LITE_AI_TOOLKIT_MNN_CORE_MNN_TYPES_H - -#include "lite/types.h" - -namespace mnncv -{ - namespace types = lite::types; -} - -#endif //LITE_AI_TOOLKIT_MNN_CORE_MNN_TYPES_H diff --git a/lite/mnn/core/mnn_utils.cpp b/lite/mnn/core/mnn_utils.cpp deleted file mode 100644 index baf9570c..00000000 --- a/lite/mnn/core/mnn_utils.cpp +++ /dev/null @@ -1,5 +0,0 @@ -// -// Created by DefTruth on 2021/10/6. -// - -#include "mnn_utils.h" diff --git a/lite/mnn/core/mnn_utils.h b/lite/mnn/core/mnn_utils.h deleted file mode 100644 index 1be56fc3..00000000 --- a/lite/mnn/core/mnn_utils.h +++ /dev/null @@ -1,13 +0,0 @@ -// -// Created by DefTruth on 2021/10/6. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CORE_MNN_UTILS_H -#define LITE_AI_TOOLKIT_MNN_CORE_MNN_UTILS_H - -namespace mnncv -{ - // no specific utils for MNN now. -} - -#endif //LITE_AI_TOOLKIT_MNN_CORE_MNN_UTILS_H diff --git a/lite/mnn/cv/mnn_age_googlenet.cpp b/lite/mnn/cv/mnn_age_googlenet.cpp deleted file mode 100644 index 716ceb33..00000000 --- a/lite/mnn/cv/mnn_age_googlenet.cpp +++ /dev/null @@ -1,62 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#include "mnn_age_googlenet.h" -#include "lite/utils.h" - -using mnncv::MNNAgeGoogleNet; - -MNNAgeGoogleNet::MNNAgeGoogleNet(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNAgeGoogleNet::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNAgeGoogleNet::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - // (1,3,224,224) - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNAgeGoogleNet::detect(const cv::Mat &mat, types::Age &age) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch age. - auto device_age_logits_ptr = output_tensors.at("loss3/loss3_Y"); // (1,8) - MNN::Tensor host_age_logits_tensor(device_age_logits_ptr, device_age_logits_ptr->getDimensionType()); - device_age_logits_ptr->copyToHostTensor(&host_age_logits_tensor); - - auto age_dims = host_age_logits_tensor.shape(); - unsigned int interval = 0; - const unsigned int num_intervals = age_dims.at(1); // 8 - const float *pred_logits_ptr = host_age_logits_tensor.host(); - - auto softmax_probs = lite::utils::math::softmax(pred_logits_ptr, num_intervals, interval); - const float pred_age = static_cast(age_intervals[interval][0] + age_intervals[interval][1]) / 2.0f; - - age.age = pred_age; - age.age_interval[0] = age_intervals[interval][0]; - age.age_interval[1] = age_intervals[interval][1]; - age.interval_prob = softmax_probs[interval]; - age.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_age_googlenet.h b/lite/mnn/cv/mnn_age_googlenet.h deleted file mode 100644 index f1b4221d..00000000 --- a/lite/mnn/cv/mnn_age_googlenet.h +++ /dev/null @@ -1,42 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_AGE_GOOGLENET_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_AGE_GOOGLENET_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNAgeGoogleNet : public BasicMNNHandler - { - public: - explicit MNNAgeGoogleNet(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNAgeGoogleNet() override = default; - - private: - const float mean_vals[3] = {104.0f, 117.0f, 123.0f}; - const float norm_vals[3] = {1.0f, 1.0f, 1.0f}; - const unsigned int age_intervals[8][2] = { - {0, 2}, - {4, 6}, - {8, 12}, - {15, 20}, - {25, 32}, - {38, 43}, - {48, 53}, - {60, 100} - }; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // padding & resize & normalize. - - public: - void detect(const cv::Mat &mat, types::Age &age); - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_AGE_GOOGLENET_H diff --git a/lite/mnn/cv/mnn_backgroundmattingv2.cpp b/lite/mnn/cv/mnn_backgroundmattingv2.cpp deleted file mode 100644 index 08a110d1..00000000 --- a/lite/mnn/cv/mnn_backgroundmattingv2.cpp +++ /dev/null @@ -1,217 +0,0 @@ -// -// Created by DefTruth on 2022/4/9. -// - -#include "mnn_backgroundmattingv2.h" -#include "lite/utils.h" - -using mnncv::MNNBackgroundMattingV2; - -MNNBackgroundMattingV2::MNNBackgroundMattingV2( - const std::string &_mnn_path, - unsigned int _num_threads -) : log_id(_mnn_path.data()), - mnn_path(_mnn_path.data()), - num_threads(_num_threads) -{ - initialize_interpreter(); - initialize_pretreat(); -} - -MNNBackgroundMattingV2::~MNNBackgroundMattingV2() -{ - mnn_interpreter->releaseModel(); - if (mnn_session) - mnn_interpreter->releaseSession(mnn_session); -} - -void MNNBackgroundMattingV2::initialize_interpreter() -{ - // 1. init interpreter - mnn_interpreter = std::shared_ptr(MNN::Interpreter::createFromFile(mnn_path)); - // 2. init schedule_config - schedule_config.numThread = (int) num_threads; - MNN::BackendConfig backend_config; - backend_config.precision = MNN::BackendConfig::Precision_High; // default Precision_High - schedule_config.backendConfig = &backend_config; - // 3. create session - mnn_session = mnn_interpreter->createSession(schedule_config); - // 4. init input tensor - src_tensor = mnn_interpreter->getSessionInput(mnn_session, "src"); - bgr_tensor = mnn_interpreter->getSessionInput(mnn_session, "bgr"); - // 5. init input dims - input_height = src_tensor->height(); - input_width = src_tensor->width(); - dimension_type = src_tensor->getDimensionType(); // CAFFE - mnn_interpreter->resizeTensor(src_tensor, src_tensor->shape()); - mnn_interpreter->resizeTensor(bgr_tensor, bgr_tensor->shape()); - mnn_interpreter->resizeSession(mnn_session); -#ifdef LITEMNN_DEBUG - this->print_debug_string(); -#endif -} - -void MNNBackgroundMattingV2::print_debug_string() -{ - std::cout << "LITEMNN_DEBUG LogId: " << log_id << "\n"; - std::cout << "=============== Input-Dims ==============\n"; - if (src_tensor) src_tensor->printShape(); - if (bgr_tensor) bgr_tensor->printShape(); - if (dimension_type == MNN::Tensor::CAFFE) - std::cout << "Dimension Type: (CAFFE/PyTorch/ONNX)NCHW" << "\n"; - else if (dimension_type == MNN::Tensor::TENSORFLOW) - std::cout << "Dimension Type: (TENSORFLOW)NHWC" << "\n"; - else if (dimension_type == MNN::Tensor::CAFFE_C4) - std::cout << "Dimension Type: (CAFFE_C4)NC4HW4" << "\n"; - std::cout << "=============== Output-Dims ==============\n"; - auto tmp_output_map = mnn_interpreter->getSessionOutputAll(mnn_session); - std::cout << "getSessionOutputAll done!\n"; - for (auto it = tmp_output_map.cbegin(); it != tmp_output_map.cend(); ++it) - { - std::cout << "Output: " << it->first << ": "; - it->second->printShape(); - } - std::cout << "========================================\n"; -} - -void MNNBackgroundMattingV2::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -inline void MNNBackgroundMattingV2::transform(const cv::Mat &mat, const cv::Mat &bgr) -{ - cv::Mat mat_rs, bgr_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::resize(bgr, bgr_rs, cv::Size(input_width, input_height)); - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], src_tensor); - pretreat->convert(bgr_rs.data, input_width, input_height, bgr_rs.step[0], bgr_tensor); -} - -void MNNBackgroundMattingV2::detect(const cv::Mat &mat, const cv::Mat &bgr, - types::MattingContent &content, bool remove_noise, - bool minimum_post_process) -{ - if (mat.empty() || bgr.empty()) return; - // 1. make input tensor - this->transform(mat, bgr); - // 2. inference & run session - mnn_interpreter->runSession(mnn_session); - - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. generate matting - this->generate_matting(output_tensors, mat, content, remove_noise, minimum_post_process); -} - -void MNNBackgroundMattingV2::generate_matting( - const std::map &output_tensors, const cv::Mat &mat, - types::MattingContent &content, bool remove_noise, - bool minimum_post_process) -{ - auto device_fgr_ptr = output_tensors.at("fgr"); - auto device_pha_ptr = output_tensors.at("pha"); - MNN::Tensor host_fgr_tensor(device_fgr_ptr, device_fgr_ptr->getDimensionType()); // NCHW - MNN::Tensor host_pha_tensor(device_pha_ptr, device_pha_ptr->getDimensionType()); // NCHW - device_fgr_ptr->copyToHostTensor(&host_fgr_tensor); - device_pha_ptr->copyToHostTensor(&host_pha_tensor); - const unsigned int h = mat.rows; - const unsigned int w = mat.cols; - const unsigned int out_h = input_height; - const unsigned int out_w = input_width; - - float *fgr_ptr = host_fgr_tensor.host(); - float *pha_ptr = host_pha_tensor.host(); - const unsigned int channel_step = out_h * out_w; - - // fast assign & channel transpose(CHW->HWC). - cv::Mat pmat(out_h, out_w, CV_32FC1, pha_ptr); - if (remove_noise) lite::utils::remove_small_connected_area(pmat, 0.05f); - - std::vector fgr_channel_mats; - cv::Mat rmat(out_h, out_w, CV_32FC1, fgr_ptr); - cv::Mat gmat(out_h, out_w, CV_32FC1, fgr_ptr + channel_step); - cv::Mat bmat(out_h, out_w, CV_32FC1, fgr_ptr + 2 * channel_step); - rmat *= 255.; - bmat *= 255.; - gmat *= 255.; - fgr_channel_mats.push_back(bmat); - fgr_channel_mats.push_back(gmat); - fgr_channel_mats.push_back(rmat); - - content.pha_mat = pmat; - cv::merge(fgr_channel_mats, content.fgr_mat); - content.fgr_mat.convertTo(content.fgr_mat, CV_8UC3); - - if (!minimum_post_process) - { - std::vector merge_channel_mats; - cv::Mat rest = 1. - pmat; - cv::Mat mbmat = bmat.mul(pmat) + rest * 153.; - cv::Mat mgmat = gmat.mul(pmat) + rest * 255.; - cv::Mat mrmat = rmat.mul(pmat) + rest * 120.; - merge_channel_mats.push_back(mbmat); - merge_channel_mats.push_back(mgmat); - merge_channel_mats.push_back(mrmat); - cv::merge(merge_channel_mats, content.merge_mat); - content.merge_mat.convertTo(content.merge_mat, CV_8UC3); - } - - // resize alpha - if (out_h != h || out_w != w) - { - // already allocated a new continuous memory after resize (pha_mat) - cv::resize(content.pha_mat, content.pha_mat, cv::Size(w, h)); - cv::resize(content.fgr_mat, content.fgr_mat, cv::Size(w, h)); - if (!minimum_post_process) - cv::resize(content.merge_mat, content.merge_mat, cv::Size(w, h)); - } // - else - { - // need clone to allocate a new continuous memory if not performed resize. - // The memory elements point to will release after return. - content.pha_mat = content.pha_mat.clone(); - } - - content.flag = true; -} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lite/mnn/cv/mnn_backgroundmattingv2.h b/lite/mnn/cv/mnn_backgroundmattingv2.h deleted file mode 100644 index f25ce219..00000000 --- a/lite/mnn/cv/mnn_backgroundmattingv2.h +++ /dev/null @@ -1,91 +0,0 @@ -// -// Created by DefTruth on 2022/4/9. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_BACKGROUNDMATTINGV2_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_BACKGROUNDMATTINGV2_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNBackgroundMattingV2 - { - public: - explicit MNNBackgroundMattingV2(const std::string &_mnn_path, - unsigned int _num_threads = 1); // - ~MNNBackgroundMattingV2(); - - private: - std::shared_ptr mnn_interpreter; - MNN::Session *mnn_session = nullptr; - MNN::ScheduleConfig schedule_config; - std::shared_ptr pretreat; // init at runtime - const char *log_id = nullptr; - const char *mnn_path = nullptr; - - private: - const float mean_vals[3] = {0.f, 0.f, 0.f}; // RGB - const float norm_vals[3] = {1.f / 255.f, 1.f / 255.f, 1.f / 255.f}; - // hardcode input node names, hint only. - // downsample_ratio has been freeze while onnx exported - // and, the input size of each input has been freeze, also. - std::vector input_node_names = { - "src", - "bgr" - }; - // hardcode output node names, hint only. - std::vector output_node_names = { - "pha", - "fgr", - "pha_sm", - "fgr_sm", - "err_sm", - "ref_sm" - }; - - private: - const unsigned int num_threads; // initialize at runtime. - // multi inputs. - MNN::Tensor *src_tensor = nullptr; - MNN::Tensor *bgr_tensor = nullptr; - // input size, initialize at runtime. - int input_height; - int input_width; - int dimension_type; // hint only - - // un-copyable - protected: - MNNBackgroundMattingV2(const MNNBackgroundMattingV2 &) = delete; // - MNNBackgroundMattingV2(MNNBackgroundMattingV2 &&) = delete; // - MNNBackgroundMattingV2 &operator=(const MNNBackgroundMattingV2 &) = delete; // - MNNBackgroundMattingV2 &operator=(MNNBackgroundMattingV2 &&) = delete; // - - private: - void print_debug_string(); - - private: - void transform(const cv::Mat &mat, const cv::Mat &bgr); - - void initialize_pretreat(); // - - void initialize_interpreter(); - - void generate_matting(const std::map &output_tensors, - const cv::Mat &mat, types::MattingContent &content, - bool remove_noise = false, bool minimum_post_process = false); - - public: - /** - * @param mat cv::Mat input image with BGR format. - * @param bgr cv::Mat input background image with BGR format. - * @param content MattingContent output fgr, pha and merge_mat (if minimum_post_process is false) - * @param remove_noise bool, whether to remove small connected areas. - * @param minimum_post_process bool, will not return demo merge mat if True. - */ - void detect(const cv::Mat &mat, const cv::Mat &bgr, types::MattingContent &content, - bool remove_noise = false, bool minimum_post_process = false); - - }; -} -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_BACKGROUNDMATTINGV2_H diff --git a/lite/mnn/cv/mnn_cava_combined_face.cpp b/lite/mnn/cv/mnn_cava_combined_face.cpp deleted file mode 100644 index c10674c3..00000000 --- a/lite/mnn/cv/mnn_cava_combined_face.cpp +++ /dev/null @@ -1,57 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "mnn_cava_combined_face.h" - -using mnncv::MNNCavaCombinedFace; - -MNNCavaCombinedFace::MNNCavaCombinedFace(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNCavaCombinedFace::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNCavaCombinedFace::transform(const cv::Mat &mat) -{ - // normalize & HWC -> CHW & BGR -> RGB - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNCavaCombinedFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - - auto device_embedding_ptr = output_tensors.at("embedding"); - MNN::Tensor host_embedding_tensor(device_embedding_ptr, device_embedding_ptr->getDimensionType()); // NCHW - device_embedding_ptr->copyToHostTensor(&host_embedding_tensor); - - auto embedding_dims = host_embedding_tensor.shape(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = host_embedding_tensor.host(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_cava_combined_face.h b/lite/mnn/cv/mnn_cava_combined_face.h deleted file mode 100644 index 9dffb8bc..00000000 --- a/lite/mnn/cv/mnn_cava_combined_face.h +++ /dev/null @@ -1,34 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_CAVA_COMBINED_FACE_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_CAVA_COMBINED_FACE_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNCavaCombinedFace : public BasicMNNHandler - { - public: - explicit MNNCavaCombinedFace(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNCavaCombinedFace() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 128.0f, 1.f / 128.0f, 1.f / 128.0f}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_CAVA_COMBINED_FACE_H diff --git a/lite/mnn/cv/mnn_cava_ghost_arcface.cpp b/lite/mnn/cv/mnn_cava_ghost_arcface.cpp deleted file mode 100644 index 86d45ee2..00000000 --- a/lite/mnn/cv/mnn_cava_ghost_arcface.cpp +++ /dev/null @@ -1,58 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "mnn_cava_ghost_arcface.h" - -using mnncv::MNNCavaGhostArcFace; - -MNNCavaGhostArcFace::MNNCavaGhostArcFace(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - - -inline void MNNCavaGhostArcFace::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNCavaGhostArcFace::transform(const cv::Mat &mat) -{ - // normalize & HWC -> CHW & BGR -> RGB - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNCavaGhostArcFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - - auto device_embedding_ptr = output_tensors.at("embedding"); - MNN::Tensor host_embedding_tensor(device_embedding_ptr, device_embedding_ptr->getDimensionType()); // NCHW - device_embedding_ptr->copyToHostTensor(&host_embedding_tensor); - - auto embedding_dims = host_embedding_tensor.shape(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = host_embedding_tensor.host(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_cava_ghost_arcface.h b/lite/mnn/cv/mnn_cava_ghost_arcface.h deleted file mode 100644 index a61850fb..00000000 --- a/lite/mnn/cv/mnn_cava_ghost_arcface.h +++ /dev/null @@ -1,33 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_CAVA_GHOST_ARCFACE_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_CAVA_GHOST_ARCFACE_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNCavaGhostArcFace : public BasicMNNHandler - { - public: - explicit MNNCavaGhostArcFace(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNCavaGhostArcFace() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 128.0f, 1.f / 128.0f, 1.f / 128.0f}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_CAVA_GHOST_ARCFACE_H diff --git a/lite/mnn/cv/mnn_center_loss_face.cpp b/lite/mnn/cv/mnn_center_loss_face.cpp deleted file mode 100644 index 268b973c..00000000 --- a/lite/mnn/cv/mnn_center_loss_face.cpp +++ /dev/null @@ -1,58 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "mnn_center_loss_face.h" - -using mnncv::MNNCenterLossFace; - -MNNCenterLossFace::MNNCenterLossFace(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - - -inline void MNNCenterLossFace::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNCenterLossFace::transform(const cv::Mat &mat) -{ - // normalize & HWC -> CHW & BGR -> RGB - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNCenterLossFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - - auto device_embedding_ptr = output_tensors.at("embedding"); - MNN::Tensor host_embedding_tensor(device_embedding_ptr, device_embedding_ptr->getDimensionType()); // NCHW - device_embedding_ptr->copyToHostTensor(&host_embedding_tensor); - - auto embedding_dims = host_embedding_tensor.shape(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = host_embedding_tensor.host(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_center_loss_face.h b/lite/mnn/cv/mnn_center_loss_face.h deleted file mode 100644 index 1d8dd4ad..00000000 --- a/lite/mnn/cv/mnn_center_loss_face.h +++ /dev/null @@ -1,32 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_CENTER_LOSS_FACE_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_CENTER_LOSS_FACE_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNCenterLossFace : public BasicMNNHandler - { - public: - explicit MNNCenterLossFace(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNCenterLossFace() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_CENTER_LOSS_FACE_H diff --git a/lite/mnn/cv/mnn_colorizer.cpp b/lite/mnn/cv/mnn_colorizer.cpp deleted file mode 100644 index a7405ae7..00000000 --- a/lite/mnn/cv/mnn_colorizer.cpp +++ /dev/null @@ -1,110 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "mnn_colorizer.h" - -using mnncv::MNNColorizer; - -MNNColorizer::MNNColorizer(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNColorizer::initialize_pretreat() -{ - pretreat = nullptr; // no use -} - -void MNNColorizer::transform(const cv::Mat &mat) -{ - cv::Mat mat_l; // assume that input mat is L of Lab - mat.convertTo(mat_l, CV_32FC1, 1.0f, 0.f); // (256,256,1) range (0.,100.) - - auto tmp_host_nchw_tensor = new MNN::Tensor(input_tensor, MNN::Tensor::CAFFE); // tmp - std::memcpy(tmp_host_nchw_tensor->host(), mat_l.data, - input_height * input_width * sizeof(float)); - input_tensor->copyFromHostTensor(tmp_host_nchw_tensor); - - delete tmp_host_nchw_tensor; -} - -void MNNColorizer::detect(const cv::Mat &mat, types::ColorizeContent &colorize_content) -{ - if (mat.empty()) return; - const unsigned int height = mat.rows; - const unsigned int width = mat.cols; - - cv::Mat mat_rs = mat.clone(); - cv::resize(mat_rs, mat_rs, cv::Size(input_width, input_height)); // (256,256,3) - cv::Mat mat_rs_norm, mat_orig_norm; - mat_rs.convertTo(mat_rs_norm, CV_32FC3, 1.0f / 255.0f, 0.f); // (0.,1.) BGR - mat.convertTo(mat_orig_norm, CV_32FC3, 1.0f / 255.0f, 0.f); // (0.,1.) BGR - if (mat_rs_norm.empty() || mat_orig_norm.empty()) return; - - cv::Mat mat_lab_orig, mat_lab_rs; - cv::cvtColor(mat_rs_norm, mat_lab_rs, cv::COLOR_BGR2Lab); - cv::cvtColor(mat_orig_norm, mat_lab_orig, cv::COLOR_BGR2Lab); - - cv::Mat mat_rs_l, mat_orig_l; - std::vector mats_rs_lab, mats_orig_lab; - cv::split(mat_lab_rs, mats_rs_lab); - cv::split(mat_lab_orig, mats_orig_lab); - - mat_rs_l = mats_rs_lab.at(0); - mat_orig_l = mats_orig_lab.at(0); - - // 1. make input tensor - this->transform(mat_rs_l); // (1,1,256,256) - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch. - auto device_pred_ab_ptr = output_tensors.at("out_ab"); - MNN::Tensor host_pred_ab_tensor(device_pred_ab_ptr, device_pred_ab_ptr->getDimensionType()); - device_pred_ab_ptr->copyToHostTensor(&host_pred_ab_tensor); - - auto pred_dims = host_pred_ab_tensor.shape(); // (1,2,256,256) - const unsigned int rows = pred_dims.at(2); // H 256 - const unsigned int cols = pred_dims.at(3); // W 256 - const unsigned int pred_step = rows * cols; - - float *pred_ab_ptr = host_pred_ab_tensor.host(); - - cv::Mat out_a_orig(rows, cols, CV_32FC1); - cv::Mat out_b_orig(rows, cols, CV_32FC1); - - for (unsigned int i = 0; i < rows; ++i) - { - float *pa = out_a_orig.ptr(i); - float *pb = out_b_orig.ptr(i); - for (unsigned int j = 0; j < cols; ++j) - { - pa[j] = pred_ab_ptr[0 * pred_step + i * cols + j]; - pb[j] = pred_ab_ptr[1 * pred_step + i * cols + j]; - } // CHW->HWC - } - - if (rows != height || cols != width) - { - cv::resize(out_a_orig, out_a_orig, cv::Size(width, height)); - cv::resize(out_b_orig, out_b_orig, cv::Size(width, height)); - } - - std::vector out_mats_lab; - out_mats_lab.push_back(mat_orig_l); - out_mats_lab.push_back(out_a_orig); - out_mats_lab.push_back(out_b_orig); - - cv::Mat merge_mat_lab, mat_bgr_norm; - cv::merge(out_mats_lab, merge_mat_lab); - if (merge_mat_lab.empty()) return; - cv::cvtColor(merge_mat_lab, mat_bgr_norm, cv::COLOR_Lab2BGR); // CV_32FC3 - mat_bgr_norm *= 255.0f; - - mat_bgr_norm.convertTo(colorize_content.mat, CV_8UC3); // uint8 - - colorize_content.flag = true; - -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_colorizer.h b/lite/mnn/cv/mnn_colorizer.h deleted file mode 100644 index 0eb6e606..00000000 --- a/lite/mnn/cv/mnn_colorizer.h +++ /dev/null @@ -1,29 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_COLORIZER_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_COLORIZER_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNColorizer : public BasicMNNHandler - { - public: - explicit MNNColorizer(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNColorizer() override = default; - - private: - void initialize_pretreat(); // no use - - void transform(const cv::Mat &mat) override; // resize & normalize. - - public: - void detect(const cv::Mat &mat, types::ColorizeContent &colorize_content); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_COLORIZER_H diff --git a/lite/mnn/cv/mnn_deeplabv3_resnet101.cpp b/lite/mnn/cv/mnn_deeplabv3_resnet101.cpp deleted file mode 100644 index eeb0e74d..00000000 --- a/lite/mnn/cv/mnn_deeplabv3_resnet101.cpp +++ /dev/null @@ -1,166 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "mnn_deeplabv3_resnet101.h" - -using mnncv::MNNDeepLabV3ResNet101; - -MNNDeepLabV3ResNet101::MNNDeepLabV3ResNet101( - const std::string &_mnn_path, unsigned int _num_threads -) : log_id(_mnn_path.data()), - mnn_path(_mnn_path.data()), - num_threads(_num_threads) -{ - initialize_interpreter(); - initialize_pretreat(); -} - -MNNDeepLabV3ResNet101::~MNNDeepLabV3ResNet101() -{ - mnn_interpreter->releaseModel(); - if (mnn_session) - mnn_interpreter->releaseSession(mnn_session); -} - -void MNNDeepLabV3ResNet101::initialize_interpreter() -{ - mnn_interpreter = std::shared_ptr(MNN::Interpreter::createFromFile(mnn_path)); - // 2. init schedule_config - schedule_config.numThread = (int) num_threads; - MNN::BackendConfig backend_config; - backend_config.precision = MNN::BackendConfig::Precision_High; // default Precision_High - schedule_config.backendConfig = &backend_config; - // 3. create session - mnn_session = mnn_interpreter->createSession(schedule_config); - // 4. init input tensor - input_tensor = mnn_interpreter->getSessionInput(mnn_session, nullptr); - // 5. init input dims - dynamic_input_height = input_tensor->height(); - dynamic_input_width = input_tensor->width(); - dimension_type = input_tensor->getDimensionType(); // CAFFE(NCHW) - mnn_interpreter->resizeTensor(input_tensor, {1, 3, dynamic_input_height, dynamic_input_width}); - mnn_interpreter->resizeSession(mnn_session); -#ifdef LITEMNN_DEBUG - this->print_debug_string(); -#endif -} - -void MNNDeepLabV3ResNet101::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNDeepLabV3ResNet101::transform(const cv::Mat &mat) -{ - const int img_width = mat.cols; - const int img_height = mat.rows; - // update dynamic input dims - dynamic_input_height = img_height; - dynamic_input_width = img_width; - - // update input tensor and resize Session - mnn_interpreter->resizeTensor(input_tensor, {1, 3, dynamic_input_height, dynamic_input_width}); - mnn_interpreter->resizeSession(mnn_session); - - // push data into input tensor - pretreat->convert(mat.data, dynamic_input_width, dynamic_input_height, mat.step[0], input_tensor); -} - -void MNNDeepLabV3ResNet101::detect(const cv::Mat &mat, types::SegmentContent &content) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference & run session - mnn_interpreter->runSession(mnn_session); - - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch - auto device_scores_ptr = output_tensors.at("out"); // (1,21,h,w) - MNN::Tensor host_scores_tensor(device_scores_ptr, device_scores_ptr->getDimensionType()); - device_scores_ptr->copyToHostTensor(&host_scores_tensor); -#ifdef LITEMNN_DEBUG - host_scores_tensor.printShape(); -#endif - - auto scores_dims = host_scores_tensor.shape(); - const unsigned int output_classes = scores_dims.at(1); - const unsigned int output_height = scores_dims.at(2); - const unsigned int output_width = scores_dims.at(3); - - const float *scores_ptr = host_scores_tensor.host(); - // time cost! - content.names_map.clear(); - content.class_mat = cv::Mat(output_height, output_width, CV_8UC1, cv::Scalar(0)); - content.color_mat = mat.clone(); - - const unsigned int scores_step = output_height * output_width; // h x w - - for (unsigned int i = 0; i < output_height; ++i) - { - - uchar *p_class = content.class_mat.ptr(i); - cv::Vec3b *p_color = content.color_mat.ptr(i); - - for (unsigned int j = 0; j < output_width; ++j) - { - // argmax - unsigned int max_label = 0; - float max_conf = scores_ptr[0 * scores_step + i * output_width + j]; - - for (unsigned int l = 0; l < output_classes; ++l) - { - float conf = scores_ptr[l * scores_step + i * output_width + j]; - if (conf > max_conf) - { - max_conf = conf; - max_label = l; - } - } - - if (max_label == 0) continue; - - // assign label for pixel(i,j) - p_class[j] = cv::saturate_cast(max_label); - // assign color for detected class at pixel(i,j). - p_color[j][0] = cv::saturate_cast((max_label % 10) * 20); - p_color[j][1] = cv::saturate_cast((max_label % 5) * 40); - p_color[j][2] = cv::saturate_cast((max_label % 10) * 20); - // assign names map - content.names_map[max_label] = class_names[max_label - 1]; // max_label >= 1 - } - - } - - content.flag = true; -} - -void MNNDeepLabV3ResNet101::print_debug_string() -{ - std::cout << "LITEMNN_DEBUG LogId: " << log_id << "\n"; - std::cout << "=============== Input-Dims ==============\n"; - if (input_tensor) input_tensor->printShape(); - if (dimension_type == MNN::Tensor::CAFFE) - std::cout << "Dimension Type: (CAFFE/PyTorch/ONNX)NCHW" << "\n"; - else if (dimension_type == MNN::Tensor::TENSORFLOW) - std::cout << "Dimension Type: (TENSORFLOW)NHWC" << "\n"; - else if (dimension_type == MNN::Tensor::CAFFE_C4) - std::cout << "Dimension Type: (CAFFE_C4)NC4HW4" << "\n"; - std::cout << "=============== Output-Dims ==============\n"; - auto tmp_output_map = mnn_interpreter->getSessionOutputAll(mnn_session); - std::cout << "getSessionOutputAll done!\n"; - for (auto it = tmp_output_map.cbegin(); it != tmp_output_map.cend(); ++it) - { - std::cout << "Output: " << it->first << ": "; - it->second->printShape(); - } - std::cout << "========================================\n"; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_deeplabv3_resnet101.h b/lite/mnn/cv/mnn_deeplabv3_resnet101.h deleted file mode 100644 index da6924b3..00000000 --- a/lite/mnn/cv/mnn_deeplabv3_resnet101.h +++ /dev/null @@ -1,69 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_DEEPLABV3_RESNET101_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_DEEPLABV3_RESNET101_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNDeepLabV3ResNet101 - { - public: - explicit MNNDeepLabV3ResNet101(const std::string &_mnn_path, - unsigned int _num_threads = 8); // - ~MNNDeepLabV3ResNet101(); - - private: - std::shared_ptr mnn_interpreter; - MNN::Session *mnn_session = nullptr; - MNN::ScheduleConfig schedule_config; - std::shared_ptr pretreat; // init at runtime - const char *log_id = nullptr; - const char *mnn_path = nullptr; - MNN::Tensor *input_tensor = nullptr; - - private: - const float norm_vals[3] = {(1.f / 0.229f) * (1.f / 255.f), - (1.f / 0.224f) * (1.f / 255.f), - (1.f / 0.225f) * (1.f / 255.f)}; - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; // RGB - - private: - const unsigned int num_threads; // initialize at runtime. - int dynamic_input_height = 512; // init only, will change according to input mat. - int dynamic_input_width = 512; // init only, will change according to input mat. - int dimension_type; // hint only - - const char *class_names[20] = { - "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", - "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", - "train", "tvmonitor" - }; // 20 classes - - // un-copyable - protected: - MNNDeepLabV3ResNet101(const MNNDeepLabV3ResNet101 &) = delete; // - MNNDeepLabV3ResNet101(MNNDeepLabV3ResNet101 &&) = delete; // - MNNDeepLabV3ResNet101 &operator=(const MNNDeepLabV3ResNet101 &) = delete; // - MNNDeepLabV3ResNet101 &operator=(MNNDeepLabV3ResNet101 &&) = delete; // - - private: - void print_debug_string(); - - private: - void transform(const cv::Mat &mat); - - void initialize_pretreat(); // - - void initialize_interpreter(); - - public: - void detect(const cv::Mat &mat, types::SegmentContent &content); - - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_DEEPLABV3_RESNET101_H diff --git a/lite/mnn/cv/mnn_densenet.cpp b/lite/mnn/cv/mnn_densenet.cpp deleted file mode 100644 index 292f3e06..00000000 --- a/lite/mnn/cv/mnn_densenet.cpp +++ /dev/null @@ -1,68 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "mnn_densenet.h" -#include "lite/utils.h" - -using mnncv::MNNDenseNet; - -MNNDenseNet::MNNDenseNet(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNDenseNet::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNDenseNet::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,3,224,224) - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNDenseNet::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch. - auto device_logits_ptr = output_tensors.at("logits"); - MNN::Tensor host_logits_tensor(device_logits_ptr, device_logits_ptr->getDimensionType()); - device_logits_ptr->copyToHostTensor(&host_logits_tensor); - - auto logits_dims = host_logits_tensor.shape(); - const unsigned int num_classes = logits_dims.at(1); // 1000 - const float *logits = host_logits_tensor.host(); - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_densenet.h b/lite/mnn/cv/mnn_densenet.h deleted file mode 100644 index a21c5255..00000000 --- a/lite/mnn/cv/mnn_densenet.h +++ /dev/null @@ -1,409 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_DENSENET_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_DENSENET_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNDenseNet : public BasicMNNHandler - { - public: - explicit MNNDenseNet(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNDenseNet() override = default; - - private: - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // resize & normalize. - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_DENSENET_H diff --git a/lite/mnn/cv/mnn_efficient_emotion7.cpp b/lite/mnn/cv/mnn_efficient_emotion7.cpp deleted file mode 100644 index 1d80abec..00000000 --- a/lite/mnn/cv/mnn_efficient_emotion7.cpp +++ /dev/null @@ -1,60 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#include "mnn_efficient_emotion7.h" -#include "lite/utils.h" - -using mnncv::MNNEfficientEmotion7; - -MNNEfficientEmotion7::MNNEfficientEmotion7(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNEfficientEmotion7::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNEfficientEmotion7::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,3,224,224) - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNEfficientEmotion7::detect(const cv::Mat &mat, types::Emotions &emotions) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch. - auto device_emotion_logits_ptr = output_tensors.at("logits"); // (1,7) - MNN::Tensor host_emotion_logits_tensor(device_emotion_logits_ptr, device_emotion_logits_ptr->getDimensionType()); - device_emotion_logits_ptr->copyToHostTensor(&host_emotion_logits_tensor); - - auto emotion_dims = host_emotion_logits_tensor.shape(); - const unsigned int num_emotions = emotion_dims.at(1); // 7 - - unsigned int pred_label = 0; - const float *pred_logits_ptr = host_emotion_logits_tensor.host(); - - auto softmax_probs = lite::utils::math::softmax(pred_logits_ptr, num_emotions, pred_label); - emotions.label = pred_label; - emotions.score = softmax_probs[pred_label]; - emotions.text = emotion_texts[pred_label]; - emotions.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_efficient_emotion7.h b/lite/mnn/cv/mnn_efficient_emotion7.h deleted file mode 100644 index c0259511..00000000 --- a/lite/mnn/cv/mnn_efficient_emotion7.h +++ /dev/null @@ -1,36 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_EFFICIENT_EMOTION7_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_EFFICIENT_EMOTION7_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNEfficientEmotion7 : public BasicMNNHandler - { - public: - explicit MNNEfficientEmotion7(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNEfficientEmotion7() override = default; - - private: - const float mean_vals[3] = {255.f * 0.485f, 255.f * 0.456, 255.f * 0.406f}; - const float norm_vals[3] = {1.f / (255.f * 0.229f), 1.f / (255.f * 0.224f), 1.f / (255.f * 0.225f)}; - const char *emotion_texts[7] = { - "angry", "disgust", "fear", "happiness", "neutral", "sadness", "surprise" - }; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // padding & resize & normalize. - - public: - void detect(const cv::Mat &mat, types::Emotions &emotions); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_EFFICIENT_EMOTION7_H diff --git a/lite/mnn/cv/mnn_efficient_emotion8.cpp b/lite/mnn/cv/mnn_efficient_emotion8.cpp deleted file mode 100644 index fbf9036c..00000000 --- a/lite/mnn/cv/mnn_efficient_emotion8.cpp +++ /dev/null @@ -1,60 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#include "mnn_efficient_emotion8.h" -#include "lite/utils.h" - -using mnncv::MNNEfficientEmotion8; - -MNNEfficientEmotion8::MNNEfficientEmotion8(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNEfficientEmotion8::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNEfficientEmotion8::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,3,224,224) - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNEfficientEmotion8::detect(const cv::Mat &mat, types::Emotions &emotions) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch. - auto device_emotion_logits_ptr = output_tensors.at("logits"); // (1,8) - MNN::Tensor host_emotion_logits_tensor(device_emotion_logits_ptr, device_emotion_logits_ptr->getDimensionType()); - device_emotion_logits_ptr->copyToHostTensor(&host_emotion_logits_tensor); - - auto emotion_dims = host_emotion_logits_tensor.shape(); - const unsigned int num_emotions = emotion_dims.at(1); // 8 - - unsigned int pred_label = 0; - const float *pred_logits_ptr = host_emotion_logits_tensor.host(); - - auto softmax_probs = lite::utils::math::softmax(pred_logits_ptr, num_emotions, pred_label); - emotions.label = pred_label; - emotions.score = softmax_probs[pred_label]; - emotions.text = emotion_texts[pred_label]; - emotions.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_efficient_emotion8.h b/lite/mnn/cv/mnn_efficient_emotion8.h deleted file mode 100644 index 7cccb15d..00000000 --- a/lite/mnn/cv/mnn_efficient_emotion8.h +++ /dev/null @@ -1,36 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_EFFICIENT_EMOTION8_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_EFFICIENT_EMOTION8_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNEfficientEmotion8 : public BasicMNNHandler - { - public: - explicit MNNEfficientEmotion8(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNEfficientEmotion8() override = default; - - private: - const float mean_vals[3] = {255.f * 0.485f, 255.f * 0.456, 255.f * 0.406f}; - const float norm_vals[3] = {1 / (255.f * 0.229f), 1 / (255.f * 0.224f), 1 / (255.f * 0.225f)}; - const char *emotion_texts[8] = { - "angry", "contempt", "disgust", "fear", "happiness", "neutral", "sadness", "surprise" - }; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // padding & resize & normalize. - - public: - void detect(const cv::Mat &mat, types::Emotions &emotions); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_EFFICIENT_EMOTION8_H diff --git a/lite/mnn/cv/mnn_efficientnet_lite4.cpp b/lite/mnn/cv/mnn_efficientnet_lite4.cpp deleted file mode 100644 index e96bdee2..00000000 --- a/lite/mnn/cv/mnn_efficientnet_lite4.cpp +++ /dev/null @@ -1,78 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "mnn_efficientnet_lite4.h" -#include "lite/utils.h" - -using mnncv::MNNEfficientNetLite4; - -MNNEfficientNetLite4::MNNEfficientNetLite4(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - // re-init with fixed input shape, due to the error of input - // shape auto-detection while using MNN with NHWC input. - // TODO: pre-process bug fix - dimension_type = MNN::Tensor::TENSORFLOW; - input_batch = 1; - input_channel = 3; - input_width = 224; - input_height = 224; - mnn_interpreter->resizeTensor( - input_tensor, {input_batch, input_height, input_width, input_channel}); - mnn_interpreter->resizeSession(mnn_session); - - initialize_pretreat(); -} - -inline void MNNEfficientNetLite4::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNEfficientNetLite4::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,224,224,3) - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNEfficientNetLite4::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch. - auto device_scores_ptr = output_tensors.at("Softmax:0"); - MNN::Tensor host_scores_tensor(device_scores_ptr, device_scores_ptr->getDimensionType()); - device_scores_ptr->copyToHostTensor(&host_scores_tensor); - - auto scores_dims = host_scores_tensor.shape(); - const unsigned int num_classes = scores_dims.at(1); // 1000 - const float *scores = host_scores_tensor.host(); - - std::vector sorted_indices = lite::utils::math::argsort(scores, num_classes); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_efficientnet_lite4.h b/lite/mnn/cv/mnn_efficientnet_lite4.h deleted file mode 100644 index 296e7c9e..00000000 --- a/lite/mnn/cv/mnn_efficientnet_lite4.h +++ /dev/null @@ -1,407 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_EFFICIENTNET_LITE4_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_EFFICIENTNET_LITE4_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNEfficientNetLite4 : public BasicMNNHandler - { - public: - explicit MNNEfficientNetLite4(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNEfficientNetLite4() override = default; - - private: - const float mean_vals[3] = {127.f, 127.f, 127.f}; - const float norm_vals[3] = {1.f / 128.f, 1.f / 128.f, 1.f / 128.f}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // resize & normalize. - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_EFFICIENTNET_LITE4_H diff --git a/lite/mnn/cv/mnn_emotion_ferplus.cpp b/lite/mnn/cv/mnn_emotion_ferplus.cpp deleted file mode 100644 index 43f00270..00000000 --- a/lite/mnn/cv/mnn_emotion_ferplus.cpp +++ /dev/null @@ -1,60 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#include "mnn_emotion_ferplus.h" -#include "lite/utils.h" - -using mnncv::MNNEmotionFerPlus; - -MNNEmotionFerPlus::MNNEmotionFerPlus(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNEmotionFerPlus::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::GRAY, - mean_vals, 1, - norm_vals, 1 - ) - ); -} - -void MNNEmotionFerPlus::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,1,64,64) - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNEmotionFerPlus::detect(const cv::Mat &mat, types::Emotions &emotions) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch. - auto device_emotion_logits_ptr = output_tensors.at("Plus692_Output_0"); // (1,8) - MNN::Tensor host_emotion_logits_tensor(device_emotion_logits_ptr, device_emotion_logits_ptr->getDimensionType()); - device_emotion_logits_ptr->copyToHostTensor(&host_emotion_logits_tensor); - - auto emotion_dims = host_emotion_logits_tensor.shape(); - const unsigned int num_emotions = emotion_dims.at(1); // 8 - - unsigned int pred_label = 0; - const float *pred_logits_ptr = host_emotion_logits_tensor.host(); - - auto softmax_probs = lite::utils::math::softmax(pred_logits_ptr, num_emotions, pred_label); - emotions.label = pred_label; - emotions.score = softmax_probs[pred_label]; - emotions.text = emotion_texts[pred_label]; - emotions.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_emotion_ferplus.h b/lite/mnn/cv/mnn_emotion_ferplus.h deleted file mode 100644 index 5325c6b9..00000000 --- a/lite/mnn/cv/mnn_emotion_ferplus.h +++ /dev/null @@ -1,37 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_EMOTION_FERPLUS_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_EMOTION_FERPLUS_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNEmotionFerPlus : public BasicMNNHandler - { - public: - explicit MNNEmotionFerPlus(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNEmotionFerPlus() override = default; - - private: - const float mean_vals[3] = {0.0f}; - const float norm_vals[3] = {1.0f}; - const char *emotion_texts[8] = { - "neutral", "happiness", "surprise", "sadness", "anger", - "disgust", "fear", "contempt" - }; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // padding & resize & normalize. - - public: - void detect(const cv::Mat &mat, types::Emotions &emotions); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_EMOTION_FERPLUS_H diff --git a/lite/mnn/cv/mnn_face_hair_seg.cpp b/lite/mnn/cv/mnn_face_hair_seg.cpp deleted file mode 100644 index 6e0cb29d..00000000 --- a/lite/mnn/cv/mnn_face_hair_seg.cpp +++ /dev/null @@ -1,101 +0,0 @@ -// -// Created by DefTruth on 2022/6/19. -// - -#include "mnn_face_hair_seg.h" -#include "lite/utils.h" - -using mnncv::MNNFaceHairSeg; - -MNNFaceHairSeg::MNNFaceHairSeg(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -void MNNFaceHairSeg::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNFaceHairSeg::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,3,224,224) deepcopy inside - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNFaceHairSeg::detect(const cv::Mat &mat, types::FaceHairSegContent &content, - bool remove_noise) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. generate mask - this->generate_mask(output_tensors, mat, content, remove_noise); -} - -static inline float argmax(float *mutable_ptr, const unsigned int &step) -{ - std::vector logits(3, 0.f); - logits[0] = *mutable_ptr; // background - logits[1] = *(mutable_ptr + step); // face - logits[2] = *(mutable_ptr + 2 * step); // hair - float label = 0.f; - float max_logit = logits[0]; - for (unsigned int i = 1; i < 3; ++i) - { - if (logits[i] > max_logit) - { - max_logit = logits[i]; - label = (float) i; - } - } - // normalize -> 0.~1. - return label / 2.f; // 0. bgr 0.5 face 1. hair -} - -void MNNFaceHairSeg::generate_mask(const std::map &output_tensors, - const cv::Mat &mat, types::FaceHairSegContent &content, - bool remove_noise) -{ - auto device_output_ptr = output_tensors.at("output"); // e.g (1,3,224,224) - MNN::Tensor host_output_tensor(device_output_ptr, device_output_ptr->getDimensionType()); - device_output_ptr->copyToHostTensor(&host_output_tensor); - const unsigned int h = mat.rows; - const unsigned int w = mat.cols; - - auto output_dims = host_output_tensor.shape(); - const unsigned int out_h = output_dims.at(2); - const unsigned int out_w = output_dims.at(3); - const unsigned int channel_step = out_h * out_w; - - float *output_ptr = host_output_tensor.host(); - - std::vector elements(channel_step, 0.f); // allocate - for (unsigned int i = 0; i < channel_step; ++i) - elements[i] = (float) argmax(output_ptr + i, channel_step); // with normalize - - cv::Mat mask(out_h, out_w, CV_32FC1, elements.data()); - // post process - if (remove_noise) lite::utils::remove_small_connected_area(mask, 0.05f); - // already allocated a new continuous memory after resize. - if (out_h != h || out_w != w) cv::resize(mask, mask, cv::Size(w, h)); - // need clone to allocate a new continuous memory if not performed resize. - // The memory elements point to will release after return. - else mask = mask.clone(); - - content.mask = mask; // auto handle the memory inside ocv with smart ref. - content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_face_hair_seg.h b/lite/mnn/cv/mnn_face_hair_seg.h deleted file mode 100644 index cecf835f..00000000 --- a/lite/mnn/cv/mnn_face_hair_seg.h +++ /dev/null @@ -1,37 +0,0 @@ -// -// Created by DefTruth on 2022/6/19. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_FACE_HAIR_SEG_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_FACE_HAIR_SEG_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNFaceHairSeg : public BasicMNNHandler - { - public: - explicit MNNFaceHairSeg(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNFaceHairSeg() override = default; - - private: - const float mean_vals[3] = {0.f, 0.f, 0.f}; - const float norm_vals[3] = {1.f / 255.f, 1.f / 255.f, 1.f / 255.f}; - - private: - void initialize_pretreat(); - - void transform(const cv::Mat &mat) override; // resize & normalize. - - void generate_mask(const std::map &output_tensors, - const cv::Mat &mat, types::FaceHairSegContent &content, - bool remove_noise = false); - - public: - void detect(const cv::Mat &mat, types::FaceHairSegContent &content, - bool remove_noise = false); - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_FACE_HAIR_SEG_H diff --git a/lite/mnn/cv/mnn_face_landmarks_1000.cpp b/lite/mnn/cv/mnn_face_landmarks_1000.cpp deleted file mode 100644 index 805c94df..00000000 --- a/lite/mnn/cv/mnn_face_landmarks_1000.cpp +++ /dev/null @@ -1,68 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#include "mnn_face_landmarks_1000.h" - -using mnncv::MNNFaceLandmark1000; - -MNNFaceLandmark1000::MNNFaceLandmark1000(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNFaceLandmark1000::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::GRAY, - mean_vals, 1, - norm_vals, 1 - ) - ); -} - -void MNNFaceLandmark1000::transform(const cv::Mat &mat) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNFaceLandmark1000::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - // this->transform(mat); - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch landmarks. - auto device_landmarks_ptr = output_tensors.at("output0"); // (1,1953) - MNN::Tensor host_landmarks_tensor(device_landmarks_ptr, device_landmarks_ptr->getDimensionType()); - device_landmarks_ptr->copyToHostTensor(&host_landmarks_tensor); - auto landmark_dims = host_landmarks_tensor.shape(); - - unsigned int num_landmarks = landmark_dims.at(1); - if (num_landmarks > 1946) num_landmarks = 1946; - - const float *landmarks_ptr = host_landmarks_tensor.host(); - - for (unsigned int i = 0; i < num_landmarks; i += 2) - { - float x = landmarks_ptr[i]; - float y = landmarks_ptr[i + 1]; - - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - landmarks.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_face_landmarks_1000.h b/lite/mnn/cv/mnn_face_landmarks_1000.h deleted file mode 100644 index 351d1816..00000000 --- a/lite/mnn/cv/mnn_face_landmarks_1000.h +++ /dev/null @@ -1,32 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_FACE_LANDMARKS_1000_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_FACE_LANDMARKS_1000_H - - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNFaceLandmark1000 : public BasicMNNHandler - { - public: - explicit MNNFaceLandmark1000(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNFaceLandmark1000() override = default; - - private: - const float mean_vals[1] = {0.0f}; - const float norm_vals[1] = {1.0f}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - }; -} -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_FACE_LANDMARKS_1000_H diff --git a/lite/mnn/cv/mnn_face_parsing_bisenet.cpp b/lite/mnn/cv/mnn_face_parsing_bisenet.cpp deleted file mode 100644 index 2a367ca0..00000000 --- a/lite/mnn/cv/mnn_face_parsing_bisenet.cpp +++ /dev/null @@ -1,182 +0,0 @@ -// -// Created by DefTruth on 2022/6/30. -// - -#include "mnn_face_parsing_bisenet.h" - -using mnncv::MNNFaceParsingBiSeNet; - -MNNFaceParsingBiSeNet::MNNFaceParsingBiSeNet(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -void MNNFaceParsingBiSeNet::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNFaceParsingBiSeNet::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,3,512,512) deepcopy inside - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNFaceParsingBiSeNet::detect(const cv::Mat &mat, types::FaceParsingContent &content, - bool minimum_post_process) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. generate mask - this->generate_mask(output_tensors, mat, content, minimum_post_process); -} - -static inline uchar argmax(float *mutable_ptr, const unsigned int &step) -{ - std::vector logits(19, 0.f); - for (unsigned int i = 0; i < 19; ++i) - logits[i] = *(mutable_ptr + i * step); - uchar label = 0; - float max_logit = logits[0]; - for (unsigned int i = 1; i < 19; ++i) - { - if (logits[i] > max_logit) - { - max_logit = logits[i]; - label = (uchar) i; - } - } - return label; -} - -static const uchar part_colors[20][3] = { - {255, 0, 0}, - {255, 85, 0}, - {255, 170, 0}, - {255, 0, 85}, - {255, 0, 170}, - {0, 255, 0}, - {85, 255, 0}, - {170, 255, 0}, - {0, 255, 85}, - {0, 255, 170}, - {0, 0, 255}, - {85, 0, 255}, - {170, 0, 255}, - {0, 85, 255}, - {0, 170, 255}, - {255, 255, 0}, - {255, 255, 85}, - {255, 255, 170}, - {255, 0, 255}, - {255, 85, 255} -}; - -void MNNFaceParsingBiSeNet::generate_mask(const std::map &output_tensors, - const cv::Mat &mat, types::FaceParsingContent &content, - bool minimum_post_process) -{ - auto device_output_ptr = output_tensors.at("out"); // e.g (1,19,h,w) - MNN::Tensor host_output_tensor(device_output_ptr, device_output_ptr->getDimensionType()); - device_output_ptr->copyToHostTensor(&host_output_tensor); - const unsigned int h = mat.rows; - const unsigned int w = mat.cols; - - auto output_dims = host_output_tensor.shape(); - const unsigned int out_h = output_dims.at(2); - const unsigned int out_w = output_dims.at(3); - const unsigned int channel_step = out_h * out_w; - - float *output_ptr = host_output_tensor.host(); - std::vector elements(channel_step, 0); // allocate - for (unsigned int i = 0; i < channel_step; ++i) - elements[i] = argmax(output_ptr + i, channel_step); - - cv::Mat label(out_h, out_w, CV_8UC1, elements.data()); - - if (!minimum_post_process) - { - // FaceParsingBiSeNet only predict integer label mask, - // no fgr. So, the fake fgr and merge mat may not need, - // let the fgr mat and merge mat empty to - // Speed up the post processes. - const uchar *label_ptr = label.data; - cv::Mat color_mat(out_h, out_w, CV_8UC3, cv::Scalar(255, 255, 255)); - for (unsigned int i = 0; i < color_mat.rows; ++i) - { - cv::Vec3b *p = color_mat.ptr(i); - for (unsigned int j = 0; j < color_mat.cols; ++j) - { - if (label_ptr[i * out_w + j] == 0) continue; - p[j][0] = part_colors[label_ptr[i * out_w + j]][0]; - p[j][1] = part_colors[label_ptr[i * out_w + j]][1]; - p[j][2] = part_colors[label_ptr[i * out_w + j]][2]; - } - } - if (out_h != h || out_w != w) - cv::resize(color_mat, color_mat, cv::Size(w, h)); - cv::addWeighted(mat, 0.4, color_mat, 0.6, 0., content.merge); - } - // already allocated a new continuous memory after resize. - if (out_h != h || out_w != w) cv::resize(label, label, cv::Size(w, h)); - // need clone to allocate a new continuous memory if not performed resize. - // The memory elements point to will release after return. - else label = label.clone(); - - content.label = label; // auto handle the memory inside ocv with smart ref. - content.flag = true; -} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lite/mnn/cv/mnn_face_parsing_bisenet.h b/lite/mnn/cv/mnn_face_parsing_bisenet.h deleted file mode 100644 index 51d69235..00000000 --- a/lite/mnn/cv/mnn_face_parsing_bisenet.h +++ /dev/null @@ -1,37 +0,0 @@ -// -// Created by DefTruth on 2022/6/30. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_FACE_PARSING_BISENET_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_FACE_PARSING_BISENET_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNFaceParsingBiSeNet : public BasicMNNHandler - { - public: - explicit MNNFaceParsingBiSeNet(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNFaceParsingBiSeNet() override = default; - - private: - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; // RGB - const float norm_vals[3] = {1.f / (0.229f * 255.f), 1.f / (0.224f * 255.f), 1.f / (0.225f * 255.f)}; - - private: - void initialize_pretreat(); - - void transform(const cv::Mat &mat) override; // resize & normalize. - - void generate_mask(const std::map &output_tensors, - const cv::Mat &mat, types::FaceParsingContent &content, - bool minimum_post_process = false); - - public: - void detect(const cv::Mat &mat, types::FaceParsingContent &content, - bool minimum_post_process = false); - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_FACE_PARSING_BISENET_H diff --git a/lite/mnn/cv/mnn_faceboxes.cpp b/lite/mnn/cv/mnn_faceboxes.cpp deleted file mode 100644 index 5e67f8d2..00000000 --- a/lite/mnn/cv/mnn_faceboxes.cpp +++ /dev/null @@ -1,244 +0,0 @@ -// -// Created by DefTruth on 2021/11/20. -// - -#include "mnn_faceboxes.h" -#include "lite/utils.h" - -using mnncv::MNNFaceBoxes; - -MNNFaceBoxes::MNNFaceBoxes(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNFaceBoxes::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::BGR, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNFaceBoxes::transform(const cv::Mat &mat) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - // normalize & HWC -> CHW & BGR -> BGR - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNFaceBoxes::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - // this->transform(mat); - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - this->transform(mat); - // 2. inference scores & boxes. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(bbox_collection, output_tensors, score_threshold, img_height, img_width); - // 4. hard|blend nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void MNNFaceBoxes::generate_anchors(const int target_height, const int target_width, - std::vector &anchors) -{ - std::vector> feature_maps; - for (auto step: steps) - { - feature_maps.push_back( - { - (int) std::ceil((float) target_height / (float) step), - (int) std::ceil((float) target_width / (float) step) - } // ceil - ); - } - - anchors.clear(); - const int num_feature_map = feature_maps.size(); - - for (int k = 0; k < num_feature_map; ++k) - { - auto f_map = feature_maps.at(k); // e.g [640//32,640/32] - auto tmp_min_sizes = min_sizes.at(k); // e.g [32,64,128] - int f_h = f_map.at(0); - int f_w = f_map.at(1); - std::vector offset_32 = {0.f, 0.25f, 0.5f, 0.75f}; - std::vector offset_64 = {0.f, 0.5f}; - - for (int i = 0; i < f_h; ++i) - { - for (int j = 0; j < f_w; ++j) - { - for (auto min_size: tmp_min_sizes) - { - float s_kx = (float) min_size / (float) target_width; // e.g 32/w - float s_ky = (float) min_size / (float) target_height; // e.g 32/h - - // 32 anchor size - if (min_size == 32) - { - // range y offsets first and then x - for (auto offset_y: offset_32) - { - for (auto offset_x: offset_32) - { - // (x or y + offset) * step / w or h normalized loc mapping to input size. - float cx = ((float) j + offset_x) * (float) steps.at(k) / (float) target_width; - float cy = ((float) i + offset_y) * (float) steps.at(k) / (float) target_height; - anchors.push_back(FaceBoxesAnchor{cx, cy, s_kx, s_ky}); // without clip - } - } - - } // 64 anchor size - else if (min_size == 64) - { - // range y offsets first and then x - for (auto offset_y: offset_64) - { - for (auto offset_x: offset_64) - { - float cx = ((float) j + offset_x) * (float) steps.at(k) / (float) target_width; - float cy = ((float) i + offset_y) * (float) steps.at(k) / (float) target_height; - anchors.push_back(FaceBoxesAnchor{cx, cy, s_kx, s_ky}); // without clip - } - } - - } // other anchor size - else - { - float cx = ((float) j + 0.5f) * (float) steps.at(k) / (float) target_width; - float cy = ((float) i + 0.5f) * (float) steps.at(k) / (float) target_height; - anchors.push_back(FaceBoxesAnchor{cx, cy, s_kx, s_ky}); // without clip - } - } - } - } - } -} - -void MNNFaceBoxes::generate_bboxes(std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, float img_height, - float img_width) -{ - auto device_bboxes_ptr = output_tensors.at("bbox"); // e.g (1,16800,4) - auto device_probs_ptr = output_tensors.at("conf"); // e.g (1,16800,2) after softmax - MNN::Tensor host_bboxes_tensor(device_bboxes_ptr, device_bboxes_ptr->getDimensionType()); - MNN::Tensor host_probs_tensor(device_probs_ptr, device_probs_ptr->getDimensionType()); - device_bboxes_ptr->copyToHostTensor(&host_bboxes_tensor); - device_probs_ptr->copyToHostTensor(&host_probs_tensor); - - auto bbox_dims = host_bboxes_tensor.shape(); - const unsigned int bbox_num = bbox_dims.at(1); // n = ? - - std::vector anchors; - this->generate_anchors(input_height, input_width, anchors); - - const unsigned int num_anchors = anchors.size(); - if (num_anchors != bbox_num) - throw std::runtime_error("mismatch num_anchors != bbox_num"); - - const float *bboxes_ptr = host_bboxes_tensor.host(); - const float *probs_ptr = host_probs_tensor.host(); - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - float conf = probs_ptr[2 * i + 1]; - if (conf < score_threshold) continue; // filter first. - - float prior_cx = anchors.at(i).cx; - float prior_cy = anchors.at(i).cy; - float prior_s_kx = anchors.at(i).s_kx; - float prior_s_ky = anchors.at(i).s_ky; - - float dx = bboxes_ptr[4 * i + 0]; - float dy = bboxes_ptr[4 * i + 1]; - float dw = bboxes_ptr[4 * i + 2]; - float dh = bboxes_ptr[4 * i + 3]; - // ref: https://github.com/zisianw/FaceBoxes.PyTorch/blob/master/utils/box_utils.py - float cx = prior_cx + dx * variance[0] * prior_s_kx; - float cy = prior_cy + dy * variance[0] * prior_s_ky; - float w = prior_s_kx * std::exp(dw * variance[1]); - float h = prior_s_ky * std::exp(dh * variance[1]); // norm coor (0.,1.) - - types::Boxf box; - box.x1 = (cx - w / 2.f) * img_width; - box.y1 = (cy - h / 2.f) * img_height; - box.x2 = (cx + w / 2.f) * img_width; - box.y2 = (cy + h / 2.f) * img_height; - box.score = conf; - box.label = 1; - box.label_text = "face"; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITEMNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void MNNFaceBoxes::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lite/mnn/cv/mnn_faceboxes.h b/lite/mnn/cv/mnn_faceboxes.h deleted file mode 100644 index ba6eebc9..00000000 --- a/lite/mnn/cv/mnn_faceboxes.h +++ /dev/null @@ -1,70 +0,0 @@ -// -// Created by DefTruth on 2021/11/20. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_FACEBOXES_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_FACEBOXES_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNFaceBoxes : public BasicMNNHandler - { - public: - explicit MNNFaceBoxes(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNFaceBoxes() override = default; - - private: - // nested classes - struct FaceBoxesAnchor - { - float cx; - float cy; - float s_kx; - float s_ky; - }; - - private: - const float mean_vals[3] = {104.f, 117.f, 123.f}; // bgr order - const float norm_vals[3] = {1.f, 1.f, 1.f}; - const float variance[2] = {0.1f, 0.2f}; - std::vector steps = {32, 64, 128}; - std::vector> min_sizes = { - {32, 64, 128}, - {256}, - {512} - }; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // - - void generate_anchors(const int target_height, - const int target_width, - std::vector &anchors); - - void generate_bboxes(std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.7f, float iou_threshold = 0.3f, - unsigned int topk = 300, unsigned int nms_type = 0); - - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_FACEBOXES_H diff --git a/lite/mnn/cv/mnn_faceboxesv2.cpp b/lite/mnn/cv/mnn_faceboxesv2.cpp deleted file mode 100644 index 819ff16f..00000000 --- a/lite/mnn/cv/mnn_faceboxesv2.cpp +++ /dev/null @@ -1,208 +0,0 @@ -// -// Created by DefTruth on 2022/3/19. -// - -#include "mnn_faceboxesv2.h" -#include "lite/utils.h" - -using mnncv::MNNFaceBoxesV2; - -MNNFaceBoxesV2::MNNFaceBoxesV2(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNFaceBoxesV2::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::BGR, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNFaceBoxesV2::transform(const cv::Mat &mat) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - // normalize & HWC -> CHW & BGR -> BGR - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNFaceBoxesV2::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - // this->transform(mat); - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - this->transform(mat); - // 2. inference scores & boxes. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(bbox_collection, output_tensors, score_threshold, img_height, img_width); - // 4. hard|blend nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void MNNFaceBoxesV2::generate_anchors(const int target_height, const int target_width, - std::vector &anchors) -{ - std::vector> feature_maps; - for (auto step: steps) - { - feature_maps.push_back( - { - (int) std::ceil((float) target_height / (float) step), - (int) std::ceil((float) target_width / (float) step) - } // ceil - ); - } - - anchors.clear(); - const int num_feature_map = feature_maps.size(); - - for (int k = 0; k < num_feature_map; ++k) - { - auto f_map = feature_maps.at(k); // e.g [640//32,640/32] - auto tmp_min_sizes = min_sizes.at(k); // e.g [32,64,128] - int f_h = f_map.at(0); - int f_w = f_map.at(1); - std::vector offset_32 = {0.f, 0.25f, 0.5f, 0.75f}; - std::vector offset_64 = {0.f, 0.5f}; - - for (int i = 0; i < f_h; ++i) - { - for (int j = 0; j < f_w; ++j) - { - for (auto min_size: tmp_min_sizes) - { - float s_kx = (float) min_size / (float) target_width; // e.g 32/w - float s_ky = (float) min_size / (float) target_height; // e.g 32/h - - // 32 anchor size - if (min_size == 32) - { - // range y offsets first and then x - for (auto offset_y: offset_32) - { - for (auto offset_x: offset_32) - { - // (x or y + offset) * step / w or h normalized loc mapping to input size. - float cx = ((float) j + offset_x) * (float) steps.at(k) / (float) target_width; - float cy = ((float) i + offset_y) * (float) steps.at(k) / (float) target_height; - anchors.push_back(FaceBoxesAnchorV2{cx, cy, s_kx, s_ky}); // without clip - } - } - - } // 64 anchor size - else if (min_size == 64) - { - // range y offsets first and then x - for (auto offset_y: offset_64) - { - for (auto offset_x: offset_64) - { - float cx = ((float) j + offset_x) * (float) steps.at(k) / (float) target_width; - float cy = ((float) i + offset_y) * (float) steps.at(k) / (float) target_height; - anchors.push_back(FaceBoxesAnchorV2{cx, cy, s_kx, s_ky}); // without clip - } - } - - } // other anchor size - else - { - float cx = ((float) j + 0.5f) * (float) steps.at(k) / (float) target_width; - float cy = ((float) i + 0.5f) * (float) steps.at(k) / (float) target_height; - anchors.push_back(FaceBoxesAnchorV2{cx, cy, s_kx, s_ky}); // without clip - } - } - } - } - } -} - -void MNNFaceBoxesV2::generate_bboxes(std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, float img_height, - float img_width) -{ - auto device_bboxes_ptr = output_tensors.at("loc"); // e.g (1,16800,4) - auto device_probs_ptr = output_tensors.at("conf"); // e.g (1,16800,2) after softmax - MNN::Tensor host_bboxes_tensor(device_bboxes_ptr, device_bboxes_ptr->getDimensionType()); - MNN::Tensor host_probs_tensor(device_probs_ptr, device_probs_ptr->getDimensionType()); - device_bboxes_ptr->copyToHostTensor(&host_bboxes_tensor); - device_probs_ptr->copyToHostTensor(&host_probs_tensor); - - auto bbox_dims = host_bboxes_tensor.shape(); - const unsigned int bbox_num = bbox_dims.at(1); // n = ? - - std::vector anchors; - this->generate_anchors(input_height, input_width, anchors); - - const unsigned int num_anchors = anchors.size(); - if (num_anchors != bbox_num) - throw std::runtime_error("mismatch num_anchors != bbox_num"); - - const float *bboxes_ptr = host_bboxes_tensor.host(); - const float *probs_ptr = host_probs_tensor.host(); - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - float conf = probs_ptr[2 * i + 1]; - if (conf < score_threshold) continue; // filter first. - - float prior_cx = anchors.at(i).cx; - float prior_cy = anchors.at(i).cy; - float prior_s_kx = anchors.at(i).s_kx; - float prior_s_ky = anchors.at(i).s_ky; - - float dx = bboxes_ptr[4 * i + 0]; - float dy = bboxes_ptr[4 * i + 1]; - float dw = bboxes_ptr[4 * i + 2]; - float dh = bboxes_ptr[4 * i + 3]; - float cx = prior_cx + dx * variance[0] * prior_s_kx; - float cy = prior_cy + dy * variance[0] * prior_s_ky; - float w = prior_s_kx * std::exp(dw * variance[1]); - float h = prior_s_ky * std::exp(dh * variance[1]); // norm coor (0.,1.) - - types::Boxf box; - box.x1 = (cx - w / 2.f) * img_width; - box.y1 = (cy - h / 2.f) * img_height; - box.x2 = (cx + w / 2.f) * img_width; - box.y2 = (cy + h / 2.f) * img_height; - box.score = conf; - box.label = 1; - box.label_text = "face"; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITEMNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void MNNFaceBoxesV2::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} diff --git a/lite/mnn/cv/mnn_faceboxesv2.h b/lite/mnn/cv/mnn_faceboxesv2.h deleted file mode 100644 index 0c50b8c0..00000000 --- a/lite/mnn/cv/mnn_faceboxesv2.h +++ /dev/null @@ -1,71 +0,0 @@ -// -// Created by DefTruth on 2022/3/19. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_FACEBOXESV2_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_FACEBOXESV2_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNFaceBoxesV2 : public BasicMNNHandler - { - public: - explicit MNNFaceBoxesV2(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNFaceBoxesV2() override = default; - - private: - // nested classes - struct FaceBoxesAnchorV2 - { - float cx; - float cy; - float s_kx; - float s_ky; - }; - - private: - const float mean_vals[3] = {104.f, 117.f, 123.f}; // bgr order - const float norm_vals[3] = {1.f, 1.f, 1.f}; - const float variance[2] = {0.1f, 0.2f}; - std::vector steps = {32, 64, 128}; - std::vector> min_sizes = { - {32, 64, 128}, - {256}, - {512} - }; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // - - void generate_anchors(const int target_height, - const int target_width, - std::vector &anchors); - - void generate_bboxes(std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.35f, float iou_threshold = 0.3f, - unsigned int topk = 300, unsigned int nms_type = 0); - - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_FACEBOXESV2_H diff --git a/lite/mnn/cv/mnn_facenet.cpp b/lite/mnn/cv/mnn_facenet.cpp deleted file mode 100644 index b96048b2..00000000 --- a/lite/mnn/cv/mnn_facenet.cpp +++ /dev/null @@ -1,58 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "mnn_facenet.h" - -using mnncv::MNNFaceNet; - -MNNFaceNet::MNNFaceNet(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - - -inline void MNNFaceNet::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNFaceNet::transform(const cv::Mat &mat) -{ - // normalize & HWC -> CHW & BGR -> RGB - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNFaceNet::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - - auto device_embedding_ptr = output_tensors.at("embedding"); - MNN::Tensor host_embedding_tensor(device_embedding_ptr, device_embedding_ptr->getDimensionType()); // NCHW - device_embedding_ptr->copyToHostTensor(&host_embedding_tensor); - - auto embedding_dims = host_embedding_tensor.shape(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = host_embedding_tensor.host(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_facenet.h b/lite/mnn/cv/mnn_facenet.h deleted file mode 100644 index 4f6a4a42..00000000 --- a/lite/mnn/cv/mnn_facenet.h +++ /dev/null @@ -1,32 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_FACENET_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_FACENET_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNFaceNet : public BasicMNNHandler - { - public: - explicit MNNFaceNet(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNFaceNet() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 128.0f, 1.f / 128.0f, 1.f / 128.0f}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_FACENET_H diff --git a/lite/mnn/cv/mnn_fast_portrait_seg.cpp b/lite/mnn/cv/mnn_fast_portrait_seg.cpp deleted file mode 100644 index 1daf8778..00000000 --- a/lite/mnn/cv/mnn_fast_portrait_seg.cpp +++ /dev/null @@ -1,141 +0,0 @@ -// -// Created by DefTruth on 2022/6/18. -// - -#include "mnn_fast_portrait_seg.h" -#include "lite/utils.h" - -using mnncv::MNNFastPortraitSeg; - -MNNFastPortraitSeg::MNNFastPortraitSeg(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ initialize_pretreat(); } - -void MNNFastPortraitSeg::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::BGR, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNFastPortraitSeg::transform(const cv::Mat &mat_rs) -{ - pretreat->convert(mat_rs.data, input_width, input_height, - mat_rs.step[0], input_tensor); -} - -void MNNFastPortraitSeg::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - FastPortraitSegScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(0, 0, 0)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat = mat.clone(); - cv::resize(new_unpad_mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void MNNFastPortraitSeg::detect(const cv::Mat &mat, types::PortraitSegContent &content, - float score_threshold, bool remove_noise) -{ - if (mat.empty()) return; - - // resize & unscale - cv::Mat mat_rs; - FastPortraitSegScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - this->transform(mat_rs); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. generate mask - this->generate_mask(scale_params, output_tensors, mat, content, score_threshold, remove_noise); -} - -static inline void softmax_inplace(float *mutable_ptr_bgr, float *mutable_ptr_fgr) -{ - const float bgr_exp = std::exp(*mutable_ptr_bgr); - const float fgr_exp = std::exp(*mutable_ptr_fgr); - *mutable_ptr_bgr = bgr_exp / (bgr_exp + fgr_exp + 1e-10f); - *mutable_ptr_fgr = 1.f - *mutable_ptr_bgr; -} - -static inline void zero_if_small_inplace(float *mutable_ptr, float &score) -{ if (*(mutable_ptr) < score) *(mutable_ptr) = 0.f; } - -void MNNFastPortraitSeg::generate_mask(const FastPortraitSegScaleParams &scale_params, - const std::map &output_tensors, - const cv::Mat &mat, types::PortraitSegContent &content, - float score_threshold, bool remove_noise) -{ - auto device_output_ptr = output_tensors.at("948"); // e.g (1,2,256,320) - MNN::Tensor host_output_tensor(device_output_ptr, device_output_ptr->getDimensionType()); - device_output_ptr->copyToHostTensor(&host_output_tensor); - const unsigned int h = mat.rows; - const unsigned int w = mat.cols; - auto output_dims = host_output_tensor.shape(); - const unsigned int out_h = output_dims.at(2); // e.g 256 - const unsigned int out_w = output_dims.at(3); // e.g 320 - const unsigned int channel_step = out_h * out_w; - - float *output_ptr = host_output_tensor.host(); - - // softmax - for (unsigned int i = 0; i < channel_step; ++i) - softmax_inplace(output_ptr + i, output_ptr + i + channel_step); // bgr & fgr - - // remove small values - for (unsigned int i = 0; i < channel_step; ++i) - zero_if_small_inplace(output_ptr + channel_step + i, score_threshold); - - // fetch foreground score - const int dw = scale_params.dw; - const int dh = scale_params.dh; - const int nw = scale_params.new_unpad_w; - const int nh = scale_params.new_unpad_h; - - cv::Mat alpha_pred(out_h, out_w, CV_32FC1, output_ptr + channel_step); // only need prob of fgr - cv::Mat mask = alpha_pred(cv::Rect(dw, dh, nw, nh)); // 0. ~ 1. - if (remove_noise) lite::utils::remove_small_connected_area(mask, 0.05f); - // already allocated a new continuous memory after resize. - if (nh != h || nw != w) cv::resize(mask, mask, cv::Size(w, h)); - // need clone to allocate a new continuous memory if not performed resize. - // The memory elements point to will release after return. - else mask = mask.clone(); - - content.mask = mask; // auto handle the memory inside ocv with smart ref. - content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_fast_portrait_seg.h b/lite/mnn/cv/mnn_fast_portrait_seg.h deleted file mode 100644 index 91b2f4ac..00000000 --- a/lite/mnn/cv/mnn_fast_portrait_seg.h +++ /dev/null @@ -1,55 +0,0 @@ -// -// Created by DefTruth on 2022/6/18. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_FAST_PORTRAIT_SEG_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_FAST_PORTRAIT_SEG_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNFastPortraitSeg : public BasicMNNHandler - { - public: - explicit MNNFastPortraitSeg(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNFastPortraitSeg() override = default; - - private: - // nested classes - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } FastPortraitSegScaleParams; - - private: - const float mean_vals[3] = {107.304565f, 115.69884f, 132.35703f}; // BGR - const float norm_vals[3] = {1.f / (63.97182f * 255.f), 1.f / (65.1337f * 255.f), - 1.f / (68.29726f * 255.f)}; - private: - void initialize_pretreat(); - - void transform(const cv::Mat &mat_rs) override; - - void resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - FastPortraitSegScaleParams &scale_params); - - void generate_mask(const FastPortraitSegScaleParams &scale_params, - const std::map &output_tensors, - const cv::Mat &mat, types::PortraitSegContent &content, - float score_threshold = 0.02f, bool remove_noise = false); - - public: - void detect(const cv::Mat &mat, types::PortraitSegContent &content, - float score_threshold = 0.02f, bool remove_noise = false); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_FAST_PORTRAIT_SEG_H diff --git a/lite/mnn/cv/mnn_fast_style_transfer.cpp b/lite/mnn/cv/mnn_fast_style_transfer.cpp deleted file mode 100644 index ebd51798..00000000 --- a/lite/mnn/cv/mnn_fast_style_transfer.cpp +++ /dev/null @@ -1,69 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "mnn_fast_style_transfer.h" - -using mnncv::MNNFastStyleTransfer; - -MNNFastStyleTransfer::MNNFastStyleTransfer(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNFastStyleTransfer::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNFastStyleTransfer::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,3,224,224) - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNFastStyleTransfer::detect(const cv::Mat &mat, types::StyleContent &style_content) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch. - auto device_pred_ptr = output_tensors.at("output1"); - MNN::Tensor host_pred_tensor(device_pred_ptr, device_pred_ptr->getDimensionType()); - device_pred_ptr->copyToHostTensor(&host_pred_tensor); - - auto pred_dims = host_pred_tensor.shape(); // (1,3,224,224) - const unsigned int rows = pred_dims.at(2); // H - const unsigned int cols = pred_dims.at(3); // W - const unsigned int channel_step = rows * cols; - - float *pred_ptr = host_pred_tensor.host(); - - // fast copy & channel transpose(CHW->HWC). - cv::Mat rmat(rows, cols, CV_32FC1, pred_ptr); // ref only, zero-copy. - cv::Mat gmat(rows, cols, CV_32FC1, pred_ptr + channel_step); - cv::Mat bmat(rows, cols, CV_32FC1, pred_ptr + 2 * channel_step); - std::vector channel_mats; - channel_mats.push_back(bmat); - channel_mats.push_back(gmat); - channel_mats.push_back(rmat); - - cv::merge(channel_mats, style_content.mat); // BGR - - style_content.mat.convertTo(style_content.mat, CV_8UC3); - - style_content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_fast_style_transfer.h b/lite/mnn/cv/mnn_fast_style_transfer.h deleted file mode 100644 index 16ba1878..00000000 --- a/lite/mnn/cv/mnn_fast_style_transfer.h +++ /dev/null @@ -1,33 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_FAST_STYLE_TRANSFER_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_FAST_STYLE_TRANSFER_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNFastStyleTransfer : public BasicMNNHandler - { - public: - explicit MNNFastStyleTransfer(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNFastStyleTransfer() override = default; - - private: - const float mean_vals[3] = {0.f, 0.f, 0.f}; - const float norm_vals[3] = {1.f, 1.f, 1.f}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // resize & normalize. - - public: - void detect(const cv::Mat &mat, types::StyleContent &style_content); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_FAST_STYLE_TRANSFER_H diff --git a/lite/mnn/cv/mnn_fcn_resnet101.cpp b/lite/mnn/cv/mnn_fcn_resnet101.cpp deleted file mode 100644 index 7c39994d..00000000 --- a/lite/mnn/cv/mnn_fcn_resnet101.cpp +++ /dev/null @@ -1,167 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "mnn_fcn_resnet101.h" -#include "lite/utils.h" - -using mnncv::MNNFCNResNet101; - -MNNFCNResNet101::MNNFCNResNet101( - const std::string &_mnn_path, unsigned int _num_threads -) : log_id(_mnn_path.data()), - mnn_path(_mnn_path.data()), - num_threads(_num_threads) -{ - initialize_interpreter(); - initialize_pretreat(); -} - -MNNFCNResNet101::~MNNFCNResNet101() -{ - mnn_interpreter->releaseModel(); - if (mnn_session) - mnn_interpreter->releaseSession(mnn_session); -} - -void MNNFCNResNet101::initialize_interpreter() -{ - mnn_interpreter = std::shared_ptr(MNN::Interpreter::createFromFile(mnn_path)); - // 2. init schedule_config - schedule_config.numThread = (int) num_threads; - MNN::BackendConfig backend_config; - backend_config.precision = MNN::BackendConfig::Precision_High; // default Precision_High - schedule_config.backendConfig = &backend_config; - // 3. create session - mnn_session = mnn_interpreter->createSession(schedule_config); - // 4. init input tensor - input_tensor = mnn_interpreter->getSessionInput(mnn_session, nullptr); - // 5. init input dims - dynamic_input_height = input_tensor->height(); - dynamic_input_width = input_tensor->width(); - dimension_type = input_tensor->getDimensionType(); // CAFFE(NCHW) - mnn_interpreter->resizeTensor(input_tensor, {1, 3, dynamic_input_height, dynamic_input_width}); - mnn_interpreter->resizeSession(mnn_session); -#ifdef LITEMNN_DEBUG - this->print_debug_string(); -#endif -} - -void MNNFCNResNet101::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNFCNResNet101::transform(const cv::Mat &mat) -{ - const int img_width = mat.cols; - const int img_height = mat.rows; - // update dynamic input dims - dynamic_input_height = img_height; - dynamic_input_width = img_width; - - // update input tensor and resize Session - mnn_interpreter->resizeTensor(input_tensor, {1, 3, dynamic_input_height, dynamic_input_width}); - mnn_interpreter->resizeSession(mnn_session); - - // push data into input tensor - pretreat->convert(mat.data, dynamic_input_width, dynamic_input_height, mat.step[0], input_tensor); -} - -void MNNFCNResNet101::detect(const cv::Mat &mat, types::SegmentContent &content) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference & run session - mnn_interpreter->runSession(mnn_session); - - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch - auto device_scores_ptr = output_tensors.at("out"); // (1,21,h,w) - MNN::Tensor host_scores_tensor(device_scores_ptr, device_scores_ptr->getDimensionType()); - device_scores_ptr->copyToHostTensor(&host_scores_tensor); -#ifdef LITEMNN_DEBUG - host_scores_tensor.printShape(); -#endif - - auto scores_dims = host_scores_tensor.shape(); - const unsigned int output_classes = scores_dims.at(1); - const unsigned int output_height = scores_dims.at(2); - const unsigned int output_width = scores_dims.at(3); - - const float *scores_ptr = host_scores_tensor.host(); - // time cost! - content.names_map.clear(); - content.class_mat = cv::Mat(output_height, output_width, CV_8UC1, cv::Scalar(0)); - content.color_mat = mat.clone(); - - const unsigned int scores_step = output_height * output_width; // h x w - - for (unsigned int i = 0; i < output_height; ++i) - { - - uchar *p_class = content.class_mat.ptr(i); - cv::Vec3b *p_color = content.color_mat.ptr(i); - - for (unsigned int j = 0; j < output_width; ++j) - { - // argmax - unsigned int max_label = 0; - float max_conf = scores_ptr[0 * scores_step + i * output_width + j]; - - for (unsigned int l = 0; l < output_classes; ++l) - { - float conf = scores_ptr[l * scores_step + i * output_width + j]; - if (conf > max_conf) - { - max_conf = conf; - max_label = l; - } - } - - if (max_label == 0) continue; - - // assign label for pixel(i,j) - p_class[j] = cv::saturate_cast(max_label); - // assign color for detected class at pixel(i,j). - p_color[j][0] = cv::saturate_cast((max_label % 10) * 20); - p_color[j][1] = cv::saturate_cast((max_label % 5) * 40); - p_color[j][2] = cv::saturate_cast((max_label % 10) * 20); - // assign names map - content.names_map[max_label] = class_names[max_label - 1]; // max_label >= 1 - } - - } - - content.flag = true; -} - -void MNNFCNResNet101::print_debug_string() -{ - std::cout << "LITEMNN_DEBUG LogId: " << log_id << "\n"; - std::cout << "=============== Input-Dims ==============\n"; - if (input_tensor) input_tensor->printShape(); - if (dimension_type == MNN::Tensor::CAFFE) - std::cout << "Dimension Type: (CAFFE/PyTorch/ONNX)NCHW" << "\n"; - else if (dimension_type == MNN::Tensor::TENSORFLOW) - std::cout << "Dimension Type: (TENSORFLOW)NHWC" << "\n"; - else if (dimension_type == MNN::Tensor::CAFFE_C4) - std::cout << "Dimension Type: (CAFFE_C4)NC4HW4" << "\n"; - std::cout << "=============== Output-Dims ==============\n"; - auto tmp_output_map = mnn_interpreter->getSessionOutputAll(mnn_session); - std::cout << "getSessionOutputAll done!\n"; - for (auto it = tmp_output_map.cbegin(); it != tmp_output_map.cend(); ++it) - { - std::cout << "Output: " << it->first << ": "; - it->second->printShape(); - } - std::cout << "========================================\n"; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_fcn_resnet101.h b/lite/mnn/cv/mnn_fcn_resnet101.h deleted file mode 100644 index de3b89ee..00000000 --- a/lite/mnn/cv/mnn_fcn_resnet101.h +++ /dev/null @@ -1,69 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_FCN_RESNET101_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_FCN_RESNET101_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNFCNResNet101 - { - public: - explicit MNNFCNResNet101(const std::string &_mnn_path, - unsigned int _num_threads = 8); // - ~MNNFCNResNet101(); - - private: - std::shared_ptr mnn_interpreter; - MNN::Session *mnn_session = nullptr; - MNN::ScheduleConfig schedule_config; - std::shared_ptr pretreat; // init at runtime - const char *log_id = nullptr; - const char *mnn_path = nullptr; - MNN::Tensor *input_tensor = nullptr; - - private: - const float norm_vals[3] = {(1.f / 0.229f) * (1.f / 255.f), - (1.f / 0.224f) * (1.f / 255.f), - (1.f / 0.225f) * (1.f / 255.f)}; - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; // RGB - - private: - const unsigned int num_threads; // initialize at runtime. - int dynamic_input_height = 512; // init only, will change according to input mat. - int dynamic_input_width = 512; // init only, will change according to input mat. - int dimension_type; // hint only - - const char *class_names[20] = { - "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", - "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", - "train", "tvmonitor" - }; // 20 classes - - // un-copyable - protected: - MNNFCNResNet101(const MNNFCNResNet101 &) = delete; // - MNNFCNResNet101(MNNFCNResNet101 &&) = delete; // - MNNFCNResNet101 &operator=(const MNNFCNResNet101 &) = delete; // - MNNFCNResNet101 &operator=(MNNFCNResNet101 &&) = delete; // - - private: - void print_debug_string(); - - private: - void transform(const cv::Mat &mat); - - void initialize_pretreat(); // - - void initialize_interpreter(); - - public: - void detect(const cv::Mat &mat, types::SegmentContent &content); - - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_FCN_RESNET101_H diff --git a/lite/mnn/cv/mnn_female_photo2cartoon.cpp b/lite/mnn/cv/mnn_female_photo2cartoon.cpp deleted file mode 100644 index 5bd54be1..00000000 --- a/lite/mnn/cv/mnn_female_photo2cartoon.cpp +++ /dev/null @@ -1,133 +0,0 @@ -// -// Created by DefTruth on 2022/6/12. -// - -#include "mnn_female_photo2cartoon.h" - -using mnncv::MNNFemalePhoto2Cartoon; - -MNNFemalePhoto2Cartoon::MNNFemalePhoto2Cartoon(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNFemalePhoto2Cartoon::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNFemalePhoto2Cartoon::transform(const cv::Mat &mat_merged_rs) -{ - // (1,3,256,256) deepcopy inside - pretreat->convert(mat_merged_rs.data, input_width, input_height, mat_merged_rs.step[0], input_tensor); -} - -void MNNFemalePhoto2Cartoon::detect( - const cv::Mat &mat, const cv::Mat &mask, - types::FemalePhoto2CartoonContent &content) -{ - if (mat.empty() || mask.empty()) return; - const unsigned int channels = mat.channels(); - if (channels != 3) return; - const unsigned int mask_channels = mask.channels(); - if (mask_channels != 1 && mask_channels != 3) return; - // model input size - const unsigned int input_h = input_height; // 256 - const unsigned int input_w = input_width; // 256 - // resize before merging mat and mask - cv::Mat mat_rs, mask_rs; - cv::resize(mat, mat_rs, cv::Size(input_w, input_h)); - cv::resize(mask, mask_rs, cv::Size(input_w, input_h)); // CV_32FC1 - if (mask_channels != 3) cv::cvtColor(mask_rs, mask_rs, cv::COLOR_GRAY2BGR); // CV_32FC3 - mat_rs.convertTo(mat_rs, CV_32FC3, 1.f, 0.f); // CV_32FC3 - // merge mat_rs and mask_rs - cv::Mat mat_merged_rs = mat_rs.mul(mask_rs) + (1.f - mask_rs) * 255.f; - mat_merged_rs.convertTo(mat_merged_rs, CV_8UC3); // keep CV_8UC3 BGR - - // 1. make input tensor - this->transform(mat_merged_rs); - // 2. inference cartoon (1,3,256,256) - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. generate cartoon - this->generate_cartoon(output_tensors, mask_rs, content); -} - -void MNNFemalePhoto2Cartoon::generate_cartoon( - const std::map &output_tensors, - const cv::Mat &mask_rs, types::FemalePhoto2CartoonContent &content) -{ - auto device_cartoon_pred = output_tensors.at("output"); - MNN::Tensor host_cartoon_tensor(device_cartoon_pred, device_cartoon_pred->getDimensionType()); - device_cartoon_pred->copyToHostTensor(&host_cartoon_tensor); - - auto cartoon_dims = host_cartoon_tensor.shape(); - const unsigned int out_h = cartoon_dims.at(2); - const unsigned int out_w = cartoon_dims.at(3); - const unsigned int channel_step = out_h * out_w; - const unsigned int mask_h = mask_rs.rows; - const unsigned int mask_w = mask_rs.cols; - // fast assign & channel transpose(CHW->HWC). - float *cartoon_ptr = host_cartoon_tensor.host(); - std::vector cartoon_channel_mats; - cv::Mat rmat(out_h, out_w, CV_32FC1, cartoon_ptr); // R - cv::Mat gmat(out_h, out_w, CV_32FC1, cartoon_ptr + channel_step); // G - cv::Mat bmat(out_h, out_w, CV_32FC1, cartoon_ptr + 2 * channel_step); // B - rmat = (rmat + 1.f) * 127.5f; - gmat = (gmat + 1.f) * 127.5f; - bmat = (bmat + 1.f) * 127.5f; - cartoon_channel_mats.push_back(rmat); - cartoon_channel_mats.push_back(gmat); - cartoon_channel_mats.push_back(bmat); - cv::Mat cartoon; - cv::merge(cartoon_channel_mats, cartoon); // CV_32FC3 allocated - if (out_h != mask_h || out_w != mask_w) - cv::resize(cartoon, cartoon, cv::Size(mask_w, mask_h)); - // combine & RGB -> BGR -> uint8 - cartoon = cartoon.mul(mask_rs) + (1.f - mask_rs) * 255.f; - cv::cvtColor(cartoon, cartoon, cv::COLOR_RGB2BGR); - cartoon.convertTo(cartoon, CV_8UC3); - - content.cartoon = cartoon; - content.flag = true; -} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lite/mnn/cv/mnn_female_photo2cartoon.h b/lite/mnn/cv/mnn_female_photo2cartoon.h deleted file mode 100644 index 46591483..00000000 --- a/lite/mnn/cv/mnn_female_photo2cartoon.h +++ /dev/null @@ -1,35 +0,0 @@ -// -// Created by DefTruth on 2022/6/12. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_FEMALE_PHOTO2CARTOON_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_FEMALE_PHOTO2CARTOON_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNFemalePhoto2Cartoon : public BasicMNNHandler - { - public: - explicit MNNFemalePhoto2Cartoon(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNFemalePhoto2Cartoon() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; - const float norm_vals[3] = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - - private: - void initialize_pretreat(); - - void transform(const cv::Mat &mat_merged_rs /*merged & resized mat*/) override; - - void generate_cartoon(const std::map &output_tensors, - const cv::Mat &mask_rs, types::FemalePhoto2CartoonContent &content); - - public: - void detect(const cv::Mat &mat, const cv::Mat &mask, types::FemalePhoto2CartoonContent &content); - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_FEMALE_PHOTO2CARTOON_H diff --git a/lite/mnn/cv/mnn_focal_arcface.cpp b/lite/mnn/cv/mnn_focal_arcface.cpp deleted file mode 100644 index 9db4ded7..00000000 --- a/lite/mnn/cv/mnn_focal_arcface.cpp +++ /dev/null @@ -1,58 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "mnn_focal_arcface.h" - -using mnncv::MNNFocalArcFace; - -MNNFocalArcFace::MNNFocalArcFace(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - - -inline void MNNFocalArcFace::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNFocalArcFace::transform(const cv::Mat &mat) -{ - // normalize & HWC -> CHW & BGR -> RGB - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNFocalArcFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - - auto device_embedding_ptr = output_tensors.at("embedding"); - MNN::Tensor host_embedding_tensor(device_embedding_ptr, device_embedding_ptr->getDimensionType()); // NCHW - device_embedding_ptr->copyToHostTensor(&host_embedding_tensor); - - auto embedding_dims = host_embedding_tensor.shape(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = host_embedding_tensor.host(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_focal_arcface.h b/lite/mnn/cv/mnn_focal_arcface.h deleted file mode 100644 index 9fcb9ae7..00000000 --- a/lite/mnn/cv/mnn_focal_arcface.h +++ /dev/null @@ -1,33 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_FOCAL_ARCFACE_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_FOCAL_ARCFACE_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNFocalArcFace : public BasicMNNHandler - { - public: - explicit MNNFocalArcFace(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNFocalArcFace() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_FOCAL_ARCFACE_H diff --git a/lite/mnn/cv/mnn_focal_asia_arcface.cpp b/lite/mnn/cv/mnn_focal_asia_arcface.cpp deleted file mode 100644 index 254dcf32..00000000 --- a/lite/mnn/cv/mnn_focal_asia_arcface.cpp +++ /dev/null @@ -1,58 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "mnn_focal_asia_arcface.h" - -using mnncv::MNNFocalAsiaArcFace; - -MNNFocalAsiaArcFace::MNNFocalAsiaArcFace(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - - -inline void MNNFocalAsiaArcFace::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNFocalAsiaArcFace::transform(const cv::Mat &mat) -{ - // normalize & HWC -> CHW & BGR -> RGB - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNFocalAsiaArcFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - - auto device_embedding_ptr = output_tensors.at("embedding"); - MNN::Tensor host_embedding_tensor(device_embedding_ptr, device_embedding_ptr->getDimensionType()); // NCHW - device_embedding_ptr->copyToHostTensor(&host_embedding_tensor); - - auto embedding_dims = host_embedding_tensor.shape(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = host_embedding_tensor.host(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_focal_asia_arcface.h b/lite/mnn/cv/mnn_focal_asia_arcface.h deleted file mode 100644 index b8929b98..00000000 --- a/lite/mnn/cv/mnn_focal_asia_arcface.h +++ /dev/null @@ -1,33 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_FOCAL_ASIA_ARCFACE_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_FOCAL_ASIA_ARCFACE_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNFocalAsiaArcFace : public BasicMNNHandler - { - public: - explicit MNNFocalAsiaArcFace(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNFocalAsiaArcFace() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_FOCAL_ASIA_ARCFACE_H diff --git a/lite/mnn/cv/mnn_fsanet.cpp b/lite/mnn/cv/mnn_fsanet.cpp deleted file mode 100644 index f2c0ec01..00000000 --- a/lite/mnn/cv/mnn_fsanet.cpp +++ /dev/null @@ -1,66 +0,0 @@ -// -// Created by DefTruth on 2021/11/25. -// - -#include "mnn_fsanet.h" - -using mnncv::MNNFSANet; - -MNNFSANet::MNNFSANet(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNFSANet::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::BGR, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNFSANet::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - // 0. padding - const int h = mat.rows; - const int w = mat.cols; - const int nh = static_cast((static_cast(h) + pad * static_cast(h))); - const int nw = static_cast((static_cast(w) + pad * static_cast(w))); - - const int nx1 = std::max(0, static_cast((nw - w) / 2)); - const int ny1 = std::max(0, static_cast((nh - h) / 2)); - - canvas = cv::Mat(nh, nw, CV_8UC3, cv::Scalar(0, 0, 0)); - mat.copyTo(canvas(cv::Rect(nx1, ny1, w, h))); - cv::resize(canvas, canvas, cv::Size(input_width, input_height)); - - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNFSANet::detect(const cv::Mat &mat, types::EulerAngles &euler_angles) -{ - if (mat.empty()) return; - - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch angles. - auto device_angles_ptr = output_tensors.at("output"); // (1,3) - MNN::Tensor host_angles_tensor(device_angles_ptr, device_angles_ptr->getDimensionType()); - device_angles_ptr->copyToHostTensor(&host_angles_tensor); - - const float *angles_ptr = host_angles_tensor.host(); - - euler_angles.yaw = angles_ptr[0]; - euler_angles.pitch = angles_ptr[1]; - euler_angles.roll = angles_ptr[2]; - euler_angles.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_fsanet.h b/lite/mnn/cv/mnn_fsanet.h deleted file mode 100644 index dc3c4f7a..00000000 --- a/lite/mnn/cv/mnn_fsanet.h +++ /dev/null @@ -1,34 +0,0 @@ -// -// Created by DefTruth on 2021/11/25. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_FSANET_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_FSANET_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNFSANet : public BasicMNNHandler - { - public: - explicit MNNFSANet(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNFSANet() override = default; - - private: - static constexpr const float pad = 0.3f; - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; - const float norm_vals[3] = {1.0f / 127.5f, 1.0f / 127.5f, 1.0f / 127.5f}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // padding & resize & normalize. - - public: - void detect(const cv::Mat &mat, types::EulerAngles &euler_angles); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_FSANET_H diff --git a/lite/mnn/cv/mnn_gender_googlenet.cpp b/lite/mnn/cv/mnn_gender_googlenet.cpp deleted file mode 100644 index 0529caec..00000000 --- a/lite/mnn/cv/mnn_gender_googlenet.cpp +++ /dev/null @@ -1,60 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#include "mnn_gender_googlenet.h" -#include "lite/utils.h" - -using mnncv::MNNGenderGoogleNet; - -MNNGenderGoogleNet::MNNGenderGoogleNet(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNGenderGoogleNet::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNGenderGoogleNet::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - // (1,3,224,224) - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNGenderGoogleNet::detect(const cv::Mat &mat, types::Gender &gender) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch. - auto device_gender_logits_ptr = output_tensors.at("loss3/loss3_Y"); // (1,2) - MNN::Tensor host_gender_logits_tensor(device_gender_logits_ptr, device_gender_logits_ptr->getDimensionType()); - device_gender_logits_ptr->copyToHostTensor(&host_gender_logits_tensor); - - auto gender_dims = host_gender_logits_tensor.shape(); - const unsigned int num_genders = gender_dims.at(1); // 2 - const float *pred_logits_ptr = host_gender_logits_tensor.host(); - - unsigned int pred_gender = 0; - auto softmax_probs = lite::utils::math::softmax(pred_logits_ptr, num_genders, pred_gender); - unsigned int gender_label = pred_gender == 1 ? 0 : 1; - gender.label = gender_label; - gender.text = gender_texts[gender_label]; - gender.score = softmax_probs[pred_gender]; - gender.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_gender_googlenet.h b/lite/mnn/cv/mnn_gender_googlenet.h deleted file mode 100644 index 24cfa08f..00000000 --- a/lite/mnn/cv/mnn_gender_googlenet.h +++ /dev/null @@ -1,33 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_GENDER_GOOGLENET_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_GENDER_GOOGLENET_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNGenderGoogleNet : public BasicMNNHandler - { - public: - explicit MNNGenderGoogleNet(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNGenderGoogleNet() override = default; - - private: - const float mean_vals[3] = {104.0f, 117.0f, 123.0f}; - const float norm_vals[3] = {1.0f, 1.0f, 1.0f}; - const char *gender_texts[2] = {"female", "male"}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; - - public: - void detect(const cv::Mat &mat, types::Gender &gender); - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_GENDER_GOOGLENET_H diff --git a/lite/mnn/cv/mnn_ghostnet.cpp b/lite/mnn/cv/mnn_ghostnet.cpp deleted file mode 100644 index a7adb3a1..00000000 --- a/lite/mnn/cv/mnn_ghostnet.cpp +++ /dev/null @@ -1,68 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "mnn_ghostnet.h" -#include "lite/utils.h" - -using mnncv::MNNGhostNet; - -MNNGhostNet::MNNGhostNet(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNGhostNet::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNGhostNet::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,3,224,224) - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNGhostNet::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch. - auto device_logits_ptr = output_tensors.at("logits"); - MNN::Tensor host_logits_tensor(device_logits_ptr, device_logits_ptr->getDimensionType()); - device_logits_ptr->copyToHostTensor(&host_logits_tensor); - - auto logits_dims = host_logits_tensor.shape(); - const unsigned int num_classes = logits_dims.at(1); // 1000 - const float *logits = host_logits_tensor.host(); - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_ghostnet.h b/lite/mnn/cv/mnn_ghostnet.h deleted file mode 100644 index 8db3e60d..00000000 --- a/lite/mnn/cv/mnn_ghostnet.h +++ /dev/null @@ -1,409 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_GHOSTNET_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_GHOSTNET_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNGhostNet : public BasicMNNHandler - { - public: - explicit MNNGhostNet(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNGhostNet() override = default; - - private: - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // resize & normalize. - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_GHOSTNET_H diff --git a/lite/mnn/cv/mnn_glint_arcface.cpp b/lite/mnn/cv/mnn_glint_arcface.cpp deleted file mode 100644 index dca8fb18..00000000 --- a/lite/mnn/cv/mnn_glint_arcface.cpp +++ /dev/null @@ -1,58 +0,0 @@ -// -// Created by DefTruth on 2021/11/13. -// - -#include "mnn_glint_arcface.h" - -using mnncv::MNNGlintArcFace; - -MNNGlintArcFace::MNNGlintArcFace(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - - -inline void MNNGlintArcFace::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNGlintArcFace::transform(const cv::Mat &mat) -{ - // normalize & HWC -> CHW & BGR -> RGB - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNGlintArcFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - - auto device_embedding_ptr = output_tensors.at("embedding"); - MNN::Tensor host_embedding_tensor(device_embedding_ptr, device_embedding_ptr->getDimensionType()); // NCHW - device_embedding_ptr->copyToHostTensor(&host_embedding_tensor); - - auto embedding_dims = host_embedding_tensor.shape(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = host_embedding_tensor.host(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_glint_arcface.h b/lite/mnn/cv/mnn_glint_arcface.h deleted file mode 100644 index cbec2b9e..00000000 --- a/lite/mnn/cv/mnn_glint_arcface.h +++ /dev/null @@ -1,32 +0,0 @@ -// -// Created by DefTruth on 2021/11/13. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_GLINT_ARCFACE_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_GLINT_ARCFACE_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNGlintArcFace : public BasicMNNHandler - { - public: - explicit MNNGlintArcFace(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNGlintArcFace() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_GLINT_ARCFACE_H diff --git a/lite/mnn/cv/mnn_glint_cosface.cpp b/lite/mnn/cv/mnn_glint_cosface.cpp deleted file mode 100644 index 9b518a85..00000000 --- a/lite/mnn/cv/mnn_glint_cosface.cpp +++ /dev/null @@ -1,58 +0,0 @@ -// -// Created by DefTruth on 2021/11/13. -// - -#include "mnn_glint_cosface.h" - -using mnncv::MNNGlintCosFace; - -MNNGlintCosFace::MNNGlintCosFace(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - - -inline void MNNGlintCosFace::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNGlintCosFace::transform(const cv::Mat &mat) -{ - // normalize & HWC -> CHW & BGR -> RGB - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNGlintCosFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - - auto device_embedding_ptr = output_tensors.at("embedding"); - MNN::Tensor host_embedding_tensor(device_embedding_ptr, device_embedding_ptr->getDimensionType()); // NCHW - device_embedding_ptr->copyToHostTensor(&host_embedding_tensor); - - auto embedding_dims = host_embedding_tensor.shape(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = host_embedding_tensor.host(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_glint_cosface.h b/lite/mnn/cv/mnn_glint_cosface.h deleted file mode 100644 index 8bf1d041..00000000 --- a/lite/mnn/cv/mnn_glint_cosface.h +++ /dev/null @@ -1,33 +0,0 @@ -// -// Created by DefTruth on 2021/11/13. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_GLINT_COSFACE_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_GLINT_COSFACE_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNGlintCosFace : public BasicMNNHandler - { - public: - explicit MNNGlintCosFace(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNGlintCosFace() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_GLINT_COSFACE_H diff --git a/lite/mnn/cv/mnn_glint_partial_fc.cpp b/lite/mnn/cv/mnn_glint_partial_fc.cpp deleted file mode 100644 index fc604f1a..00000000 --- a/lite/mnn/cv/mnn_glint_partial_fc.cpp +++ /dev/null @@ -1,58 +0,0 @@ -// -// Created by DefTruth on 2021/11/13. -// - -#include "mnn_glint_partial_fc.h" - -using mnncv::MNNGlintPartialFC; - -MNNGlintPartialFC::MNNGlintPartialFC(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - - -inline void MNNGlintPartialFC::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNGlintPartialFC::transform(const cv::Mat &mat) -{ - // normalize & HWC -> CHW & BGR -> RGB - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNGlintPartialFC::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - - auto device_embedding_ptr = output_tensors.at("embedding"); - MNN::Tensor host_embedding_tensor(device_embedding_ptr, device_embedding_ptr->getDimensionType()); // NCHW - device_embedding_ptr->copyToHostTensor(&host_embedding_tensor); - - auto embedding_dims = host_embedding_tensor.shape(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = host_embedding_tensor.host(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_glint_partial_fc.h b/lite/mnn/cv/mnn_glint_partial_fc.h deleted file mode 100644 index f802560c..00000000 --- a/lite/mnn/cv/mnn_glint_partial_fc.h +++ /dev/null @@ -1,33 +0,0 @@ -// -// Created by DefTruth on 2021/11/13. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_GLINT_PARTIAL_FC_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_GLINT_PARTIAL_FC_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNGlintPartialFC : public BasicMNNHandler - { - public: - explicit MNNGlintPartialFC(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNGlintPartialFC() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_GLINT_PARTIAL_FC_H diff --git a/lite/mnn/cv/mnn_hair_seg.cpp b/lite/mnn/cv/mnn_hair_seg.cpp deleted file mode 100644 index 983f3dca..00000000 --- a/lite/mnn/cv/mnn_hair_seg.cpp +++ /dev/null @@ -1,85 +0,0 @@ -// -// Created by DefTruth on 2022/6/19. -// - -#include "mnn_hair_seg.h" -#include "lite/utils.h" - -using mnncv::MNNHairSeg; - -MNNHairSeg::MNNHairSeg(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -void MNNHairSeg::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNHairSeg::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,3,224,224) deepcopy inside - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNHairSeg::detect(const cv::Mat &mat, types::HairSegContent &content, - float score_threshold, bool remove_noise) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. generate mask - this->generate_mask(output_tensors, mat, content, score_threshold, remove_noise); -} - -static inline void zero_if_small_inplace(float *mutable_ptr, float &score) -{ if ((*mutable_ptr) < score) *mutable_ptr = 0.f; } - -void MNNHairSeg::generate_mask(const std::map &output_tensors, - const cv::Mat &mat, types::HairSegContent &content, - float score_threshold, bool remove_noise) -{ - auto device_output_ptr = output_tensors.at("output"); // e.g (1,1,224,224) - MNN::Tensor host_output_tensor(device_output_ptr, device_output_ptr->getDimensionType()); - device_output_ptr->copyToHostTensor(&host_output_tensor); - const unsigned int h = mat.rows; - const unsigned int w = mat.cols; - - auto output_dims = host_output_tensor.shape(); - const unsigned int out_h = output_dims.at(2); - const unsigned int out_w = output_dims.at(3); - const unsigned int element_size = out_h * out_w; - - float *output_ptr = host_output_tensor.host(); - - // remove small values - if (score_threshold > 0.001f) - for (unsigned int i = 0; i < element_size; ++i) - zero_if_small_inplace(output_ptr + i, score_threshold); - - cv::Mat mask(out_h, out_w, CV_32FC1, output_ptr); - // post process - if (remove_noise) lite::utils::remove_small_connected_area(mask, 0.05f); - // already allocated a new continuous memory after resize. - if (out_h != h || out_w != w) cv::resize(mask, mask, cv::Size(w, h)); - // need clone to allocate a new continuous memory if not performed resize. - // The memory elements point to will release after return. - else mask = mask.clone(); - - content.mask = mask; // auto handle the memory inside ocv with smart ref. - content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_hair_seg.h b/lite/mnn/cv/mnn_hair_seg.h deleted file mode 100644 index c7a7c792..00000000 --- a/lite/mnn/cv/mnn_hair_seg.h +++ /dev/null @@ -1,38 +0,0 @@ -// -// Created by DefTruth on 2022/6/19. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_HAIR_SEG_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_HAIR_SEG_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNHairSeg : public BasicMNNHandler - { - public: - explicit MNNHairSeg(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNHairSeg() override = default; - - private: - const float mean_vals[3] = {0.f, 0.f, 0.f}; - const float norm_vals[3] = {1.f / 255.f, 1.f / 255.f, 1.f / 255.f}; - - private: - void initialize_pretreat(); - - void transform(const cv::Mat &mat) override; // resize & normalize. - - void generate_mask(const std::map &output_tensors, - const cv::Mat &mat, types::HairSegContent &content, - float score_threshold = 0.0f, bool remove_noise = false); - - public: - void detect(const cv::Mat &mat, types::HairSegContent &content, - float score_threshold = 0.0f, bool remove_noise = false); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_HAIR_SEG_H diff --git a/lite/mnn/cv/mnn_hdrdnet.cpp b/lite/mnn/cv/mnn_hdrdnet.cpp deleted file mode 100644 index 1852307b..00000000 --- a/lite/mnn/cv/mnn_hdrdnet.cpp +++ /dev/null @@ -1,68 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "mnn_hdrdnet.h" -#include "lite/utils.h" - -using mnncv::MNNHdrDNet; - -MNNHdrDNet::MNNHdrDNet(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNHdrDNet::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNHdrDNet::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,3,224,224) - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNHdrDNet::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch. - auto device_logits_ptr = output_tensors.at("logits"); - MNN::Tensor host_logits_tensor(device_logits_ptr, device_logits_ptr->getDimensionType()); - device_logits_ptr->copyToHostTensor(&host_logits_tensor); - - auto logits_dims = host_logits_tensor.shape(); - const unsigned int num_classes = logits_dims.at(1); // 1000 - const float *logits = host_logits_tensor.host(); - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_hdrdnet.h b/lite/mnn/cv/mnn_hdrdnet.h deleted file mode 100644 index d77ab5a1..00000000 --- a/lite/mnn/cv/mnn_hdrdnet.h +++ /dev/null @@ -1,409 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_HDRDNET_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_HDRDNET_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNHdrDNet : public BasicMNNHandler - { - public: - explicit MNNHdrDNet(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNHdrDNet() override = default; - - private: - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // resize & normalize. - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_HDRDNET_H diff --git a/lite/mnn/cv/mnn_head_seg.cpp b/lite/mnn/cv/mnn_head_seg.cpp deleted file mode 100644 index 10b47c85..00000000 --- a/lite/mnn/cv/mnn_head_seg.cpp +++ /dev/null @@ -1,102 +0,0 @@ -// -// Created by DefTruth on 2022/6/11. -// - -#include "mnn_head_seg.h" - -using mnncv::MNNHeadSeg; - -MNNHeadSeg::MNNHeadSeg(const std::string &_mnn_path, unsigned int _num_threads) : - mnn_path(_mnn_path.data()), log_id(_mnn_path.data()), num_threads(_num_threads) -{ - mnn_interpreter = std::shared_ptr(MNN::Interpreter::createFromFile(mnn_path)); - schedule_config.numThread = (int) num_threads; - MNN::BackendConfig backend_config; - backend_config.precision = MNN::BackendConfig::Precision_High; // default Precision_High - schedule_config.backendConfig = &backend_config; - mnn_session = mnn_interpreter->createSession(schedule_config); - // resize tensor & session (NHWC) (1,384,384,3) - input_tensor = mnn_interpreter->getSessionInput(mnn_session, nullptr); - dimension_type = input_tensor->getDimensionType(); - mnn_interpreter->resizeTensor( - input_tensor, {input_batch, input_height, input_width, input_channel}); - mnn_interpreter->resizeSession(mnn_session); // may not need -#ifdef LITEMNN_DEBUG - this->print_debug_string(); -#endif -} - -MNNHeadSeg::~MNNHeadSeg() -{ - mnn_interpreter->releaseModel(); - if (mnn_session) - mnn_interpreter->releaseSession(mnn_session); -} - -void MNNHeadSeg::print_debug_string() -{ - std::cout << "LITEMNN_DEBUG LogId: " << log_id << "\n"; - std::cout << "=============== Input-Dims ==============\n"; - input_tensor->printShape(); - if (dimension_type == MNN::Tensor::CAFFE) - std::cout << "Dimension Type: (CAFFE/PyTorch/ONNX)NCHW" << "\n"; - else if (dimension_type == MNN::Tensor::TENSORFLOW) - std::cout << "Dimension Type: (TENSORFLOW)NHWC" << "\n"; - else if (dimension_type == MNN::Tensor::CAFFE_C4) - std::cout << "Dimension Type: (CAFFE_C4)NC4HW4" << "\n"; - std::cout << "=============== Output-Dims ==============\n"; - auto tmp_output_map = mnn_interpreter->getSessionOutputAll(mnn_session); - std::cout << "getSessionOutputAll done!\n"; - for (auto it = tmp_output_map.cbegin(); it != tmp_output_map.cend(); ++it) - { - std::cout << "Output: " << it->first << ": "; - it->second->printShape(); - } - std::cout << "========================================\n"; -} - -void MNNHeadSeg::transform(const cv::Mat &mat_rs) -{ - cv::Mat canvas; - cv::cvtColor(mat_rs, canvas, cv::COLOR_BGR2RGB); - canvas.convertTo(canvas, CV_32FC3, 1.f / 255.f, 0.f); - MNN::Tensor tmp_host_tensor(input_tensor, input_tensor->getDimensionType()); - std::memcpy(tmp_host_tensor.host(), (void *) canvas.data, - 3 * input_height * input_width * sizeof(float)); - input_tensor->copyFromHostTensor(&tmp_host_tensor); // deep copy -} - -void MNNHeadSeg::detect(const cv::Mat &mat, types::HeadSegContent &content) -{ - if (mat.empty()) return; - const unsigned int img_h = mat.rows; - const unsigned int img_w = mat.cols; - const unsigned int channels = mat.channels(); - if (channels != 3) return; - const unsigned int input_h = input_height; // 384 - const unsigned int input_w = input_width; // 384 - - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_w, input_h)); - // 1. make input tensor - this->transform(mat_rs); - // 2. inference mask (1,384,384,1) - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. post process. - auto device_mask_pred = output_tensors.at("sigmoid/Sigmoid:0"); - MNN::Tensor host_mask_tensor(device_mask_pred, device_mask_pred->getDimensionType()); - device_mask_pred->copyToHostTensor(&host_mask_tensor); - - auto mask_dims = host_mask_tensor.shape(); - const unsigned int out_h = mask_dims.at(1); // 384 - const unsigned int out_w = mask_dims.at(2); // 384 - float *mask_ptr = host_mask_tensor.host(); - - cv::Mat mask_adj; - cv::Mat mask_out(out_h, out_w, CV_32FC1, mask_ptr); - cv::resize(mask_out, mask_adj, cv::Size(img_w, img_h)); // (img_h,img_w,1) - - content.mask = mask_adj; - content.flag = true; -} diff --git a/lite/mnn/cv/mnn_head_seg.h b/lite/mnn/cv/mnn_head_seg.h deleted file mode 100644 index 5b96bb2a..00000000 --- a/lite/mnn/cv/mnn_head_seg.h +++ /dev/null @@ -1,46 +0,0 @@ -// -// Created by DefTruth on 2022/6/11. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_HEAD_SEG_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_HEAD_SEG_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNHeadSeg - { - public: - explicit MNNHeadSeg(const std::string &_mnn_path, unsigned int _num_threads = 1); - - ~MNNHeadSeg(); - - private: - std::shared_ptr mnn_interpreter; - MNN::Session *mnn_session = nullptr; - MNN::Tensor *input_tensor = nullptr; // assume single input. - MNN::ScheduleConfig schedule_config; - const char *mnn_path = nullptr; - const char *log_id = nullptr; - const unsigned int num_threads; // initialize at runtime. - int dimension_type; // hint only - - private: - // hardcode input size - static constexpr const int input_batch = 1; - static constexpr const int input_channel = 3; - static constexpr const int input_height = 384; - static constexpr const int input_width = 384; - - private: - void transform(const cv::Mat &mat_rs); - - void print_debug_string(); - - public: - void detect(const cv::Mat &mat, types::HeadSegContent &content); - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_HEAD_SEG_H diff --git a/lite/mnn/cv/mnn_ibnnet.cpp b/lite/mnn/cv/mnn_ibnnet.cpp deleted file mode 100644 index d1facc19..00000000 --- a/lite/mnn/cv/mnn_ibnnet.cpp +++ /dev/null @@ -1,68 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "mnn_ibnnet.h" -#include "lite/utils.h" - -using mnncv::MNNIBNNet; - -MNNIBNNet::MNNIBNNet(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNIBNNet::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNIBNNet::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,3,224,224) - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNIBNNet::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch. - auto device_logits_ptr = output_tensors.at("logits"); - MNN::Tensor host_logits_tensor(device_logits_ptr, device_logits_ptr->getDimensionType()); - device_logits_ptr->copyToHostTensor(&host_logits_tensor); - - auto logits_dims = host_logits_tensor.shape(); - const unsigned int num_classes = logits_dims.at(1); // 1000 - const float *logits = host_logits_tensor.host(); - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_ibnnet.h b/lite/mnn/cv/mnn_ibnnet.h deleted file mode 100644 index 31ee2f57..00000000 --- a/lite/mnn/cv/mnn_ibnnet.h +++ /dev/null @@ -1,409 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_IBNNET_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_IBNNET_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNIBNNet : public BasicMNNHandler - { - public: - explicit MNNIBNNet(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNIBNNet() override = default; - - private: - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // resize & normalize. - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_IBNNET_H diff --git a/lite/mnn/cv/mnn_insectdet.cpp b/lite/mnn/cv/mnn_insectdet.cpp deleted file mode 100644 index ad18fe1d..00000000 --- a/lite/mnn/cv/mnn_insectdet.cpp +++ /dev/null @@ -1,159 +0,0 @@ -// -// Created by DefTruth on 2022/3/27. -// - -#include "mnn_insectdet.h" -#include "lite/utils.h" - -using mnncv::MNNInsectDet; - -MNNInsectDet::MNNInsectDet(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNInsectDet::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -inline void MNNInsectDet::transform(const cv::Mat &mat_rs) -{ - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNInsectDet::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - InsectDetScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.ratio = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.flag = true; -} - -void MNNInsectDet::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - InsectDetScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - this->transform(mat_rs); - // 2. inference scores & boxes. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, output_tensors, score_threshold, img_height, img_width); - // 4. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk); -} - -void MNNInsectDet::generate_bboxes(const InsectDetScaleParams &scale_params, - std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, int img_height, - int img_width) -{ - auto device_output_pred = output_tensors.at("output"); - MNN::Tensor host_output_pred(device_output_pred, device_output_pred->getDimensionType()); - device_output_pred->copyToHostTensor(&host_output_pred); - - auto output_dims = host_output_pred.shape(); // (1,n,6) - const unsigned int num_anchors = output_dims.at(1); // n = ? - const float *output_ptr = host_output_pred.host(); - - float r_ = scale_params.ratio; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *row_ptr = output_ptr + i * 6; - float obj_conf = row_ptr[4]; - if (obj_conf < score_threshold) continue; // filter first. - float cls_conf = row_ptr[5]; - if (cls_conf < score_threshold) continue; // insect score. - - // bounding box - const float *offsets = row_ptr; - float cx = offsets[0]; - float cy = offsets[1]; - float w = offsets[2]; - float h = offsets[3]; - - types::Boxf box; - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min((float) img_width - 1.f, x2); - box.y2 = std::min((float) img_height - 1.f, y2); - box.score = cls_conf; - box.label = 1; - box.label_text = "insect"; - box.flag = true; - - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - -#if LITEMNN_DEBUG - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void MNNInsectDet::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk) -{ - lite::utils::hard_nms(input, output, iou_threshold, topk); -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_insectdet.h b/lite/mnn/cv/mnn_insectdet.h deleted file mode 100644 index 00585f46..00000000 --- a/lite/mnn/cv/mnn_insectdet.h +++ /dev/null @@ -1,62 +0,0 @@ -// -// Created by DefTruth on 2022/3/27. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_INSECTDET_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_INSECTDET_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNInsectDet : public BasicMNNHandler - { - public: - explicit MNNInsectDet(const std::string &_mnn_path, unsigned int _num_threads = 1); - - ~MNNInsectDet() override = default; - - private: - // nested classes - typedef struct - { - float ratio; - int dw; - int dh; - bool flag; - } InsectDetScaleParams; - - private: - const float mean_vals[3] = {0.f, 0.f, 0.f}; // RGB - const float norm_vals[3] = {1.f / 255.f, 1.f / 255.f, 1.f / 255.f}; - static constexpr const unsigned int max_nms = 30000; - - private: - void transform(const cv::Mat &mat_rs) override; // without resize - - void initialize_pretreat(); - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - InsectDetScaleParams &scale_params); - - void generate_bboxes(const InsectDetScaleParams &scale_params, - std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, int img_height, - int img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.5f, float iou_threshold = 0.45f, - unsigned int topk = 100); - - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_INSECTDET_H diff --git a/lite/mnn/cv/mnn_insectid.cpp b/lite/mnn/cv/mnn_insectid.cpp deleted file mode 100644 index 47956deb..00000000 --- a/lite/mnn/cv/mnn_insectid.cpp +++ /dev/null @@ -1,68 +0,0 @@ -// -// Created by DefTruth on 2022/3/27. -// - -#include "mnn_insectid.h" -#include "lite/utils.h" - -using mnncv::MNNInsectID; - -MNNInsectID::MNNInsectID(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNInsectID::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -inline void MNNInsectID::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,3,224,224) - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNInsectID::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch. - auto device_logits_ptr = output_tensors.at("477"); - MNN::Tensor host_logits_tensor(device_logits_ptr, device_logits_ptr->getDimensionType()); - device_logits_ptr->copyToHostTensor(&host_logits_tensor); - - auto logits_dims = host_logits_tensor.shape(); - const unsigned int num_classes = logits_dims.at(1); // 1000 - const float *logits = host_logits_tensor.host(); - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_insectid.h b/lite/mnn/cv/mnn_insectid.h deleted file mode 100644 index d862793a..00000000 --- a/lite/mnn/cv/mnn_insectid.h +++ /dev/null @@ -1,372 +0,0 @@ -// -// Created by DefTruth on 2022/3/27. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_INSECTID_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_INSECTID_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNInsectID : public BasicMNNHandler - { - public: - explicit MNNInsectID(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNInsectID() override = default; - - private: - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // resize & normalize. - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[2037] = { - "Pseudoscorpiones", "Diplopoda", "Megymenum", "Cicadellidae", "Bothrogonia addita", "Bothrogonia ferruginea", "Cicadella viridis", - "Maiestas dorsalis", "Nephotettix cincticeps", "Mileewa", "Ledra", "Olidiana brevis", "Acanthosoma denticaudum", - "Sastragala esakii", "Neolethaeus dallasi", "Metochus uniguttatus", "Metochus abbreviatus", "Horridipamera inconspicua", - "Geocoris pallidipennis", "Geocoris varius", "Clovia", "Omalophora pectoralis", "Ricaniidae", "Ricaniidae", "Ricanula pulverosa", - "Ricania speculum", "Euricania facialis", "Ricania guttata", "Ricanula sublimata", "Euricania ocella", "Ricania taeniata", - "Euricania clara", "Ricania simulans", "Urochela quadrinotata", "Cercopidae", "Cosmoscarta", "Cosmoscarta abdominalis", - "Cosmoscarta exultans", "Cosmoscarta dimidiata", "Cosmoscarta dorsimacula", "Callitettix versicolor", "Reduviidae", - "Haematoloecha nigrorufa", "Platymeris", "Agriosphodrus dohrni", "Euagoras plagiatus", "Yolinus albopustulatus", - "Sycanus croceovittatus", "Sphedanolestes impressicollis", "Epidaus", "Epidaus sexspinus", "Vesbius sanguinosus", "Acanthaspis", - "Isyndus obscurus", "Sirthenea flavipes", "Ectrychotes andreae", "Sclomina erinacea", "Issidae", "Phymatidae", "Miridae", - "Eurystylus coelestialium", "Apolygus lucorum", "Helopeltis cinchonae", "Eucorysses grandis", "Hyperoncus lateritius", - "Poecilocoris nepalensis", "Poecilocoris sanszeusignatus", "Poecilocoris druraei", "Poecilocoris latus", "", "Poecilocoris lewisi", - "", "Tetrarthria variegata", "Sphaerocoris annulus", "Scutellera amethystina(Scutellera fasciata)", "Chrysocoris stollii", - "Lamprocoris lateralis", "Calliphara nobilis", "Cantao ocellatus", "Pyrrhocoridae", "Pyrrhocoris sibiricus", "Macrocheraia grandis", - "Physopelta quadriguttata", "Physopelta gutta", "", "Dysdercus decussatus", "Dysdercus cingulatus", "Dysdercus poecilus", - "Dindymus rubiginosus", "Dindymus brevis", "Antilochus coquebertii", "Coreidae", "Mictis tenebrosa", "Mictis gallina", - "Mictis serina", "Mictis fuscipes", "Paradasynus spinosus", "Homoeocerus unipunctatus", "Homoeocerus dilatatus", - "Homoeocerus striicornis", "Molipteryx", "Molipteryx lunata", "Cletus", "Acanthocoris scaber", "Riptortus", "Riptortus pedestris", - "Plinachtus bicoloripes", "Notobitus meleagris", "Tingidae", "Corythucha ciliata", "Corythucha marmorata", "Anthocoris confusus", - "Eurostus", "", "Tessaratoma papillosa", "", "Borysthenes maculatus", "Flatidae", "Cerynia maria", "Lawana imitata", - "Geisha distinctissima", "Salurnis marginella", "Pyrops", "Pyrops spinolae", "Pyrops watanabei", "Pyrops watanabei", - "Pyrops candelaria", "Penthicodes atomaria", "Lycorma delicatula", "Lycorma delicatula", "Penthicodes pulchella", "Saiva bullata", - "Cicadidae", "Cicadidae", "Talainga chinensis", "Meimuna", "Gaeana maculata", "Hyalessa maculaticollis", "Scieroptera", - "Sulphogaeana sulphurea", "Polymeura chenni", "Chremistica ochracea", "Platypleura kaempferi", "Tacua speciosa", - "Formotosena seebohmi", "Huechys sanguinea", "Cryptotympana atrata", "Nepidae", "Eysarcoris", "Eysarcoris guttigerus", - "Eysarcoris aeneus", "Eysarcoris ventralis", "Metonymia glandulosa", "Palomena viridissima", "Priassus spiniger", "Dalpada", - "Lelia decempunctata", "Dolycoris baccarum", "Eurydema gebleri", "Plautia", "Cazira", "Nezara", "Carpocoris purpureipennis", - "Menida violacea", "Palomena prasina", "Catacanthus incarnatus", "Alcimocoris", "Halyomorpha halys", "Eurydema dominulus", - "Zicrona caerulea", "", "Graphosoma rubrolineatum", "Erthesina fullo", "", "Derbidae", "Diostrombus politus", "Membracidae", - "Dictyopharidae", "Kirkaldyia deyrollei", "Berytidae", "Lygaeus equestris", "Spilostethus hospes", "Tropidothorax elegans", - "Lygaeus hanseni", "Graptostethus servus", "Gerridae", "Plataspidae", "Tipulidae", "", "Tephritidae", "Tachinidae", "Chironomidae", - "Stratiomyidae", "Ptecticus aurifer", "Hermetia illucens", "Liriomyza sativae", "Anthomyia illocata", "Culicidae", "Psychodidae", - "Bombyliidae", "Muscidae", "Asilidae", "Microstylum oberthurii", "Syrphidae", "Eupeodes nitens", "Eupeodes corollae", - "Eristalinus arvorum", "Eristalis cerealis", "Ischiodon scutellaris", "Eristalis arbustorum", "Phytomia zonata", "Phytomia errans", - "Syrphus torvus", "Paragus crenulatus", "Syrphus ribesii", "Eristalinus quinquestriatus", "Episyrphus balteatus", - "Helophilus pendulus", "Corydalidae", "", "Neochauliodes", "", "", "Trichoptera", "Opiliones", "Ornebius kanetataki", - "Eucriotettix oculatus", "Tetrix japonica", "Erianthus dohrni", "Acrida cinerea", "Oedaleus infernalis", "Chondracris rosea", - "Trilophidia annulata", "Xenocatantops brachycerus", "Oxya chinensis", "Shirakiacris", "Stauroderus scalaris", - "Aiolopus thalassinus tamulus", "Pseudoxya diminuta", "Ceracris nigricornis", "Locusta migratoria", "Aularches miliaris", "Patanga", - "", "Tettigoniidae", "Pseudophyllus titan", "", "Ducetia japonica", "Hexacentrus unicolor", "", "", "Conocephalus melaenus", "", - "Gampsocleis sedakovii", "Phaneroptera falcata", "Sanaa intermedia", "Gryllacrididae", "Xenogryllus marmoratus", - "Teleogryllus mitratus", "Gryllus bimaculatus", "Teleogryllus emma", "Atractomorpha sinensis", "", "", "", "Ixodida", "Phasmatodea", - "Porcellio", "Lepismatidae", "Nemopteridae", "Chrysopidae", "Myrmeleontidae", "Psychopsidae", "Ascalaphidae", - "Ascalaphus sibiricus", "Mantispidae", "Hemerobiidae", "Tenthredinidae", "Scolia superciliaris", "Ichneumonidae", "Megarhyssa", - "Xanthopimpla", "Brachymeria minuta", "Liris aurulentus", "", "Ampulex compressa", "Sphex argentatus", "Sceliphron madraspatanum", - "Sphex subtruncatus", "Sceliphron javanum", "Vespidae", "Parapolybia nodosa", "Parapolybia varia", "Polistes snelleni", - "Polistes japonicus", "Polistes gigas", "Polistes jokahamae", "Vespa velutina", "Vespa mandarinia", "Vespa affinis", "Polistinae", - "Vespula flaviceps", "Formicidae", "Pseudoneoponera rufipes", "Oecophylla smaragdina", "Mutillidae", "Pompilidae", "Apidae", - "Xylocopinae", "Bombus", "Bombus pyrosoma", "Bombus picipes", "Amegilla calceifera", "Delta esuriens", "Phimenes flavopictus", - "Oreumenes decoratus", "Delta pyriforme", "Chrysididae", "Scutigeridae", "Scolopendridae", "Ephemeroptera", "Araneae", "Araneidae", - "Araneus diadematus", "Araneus ventricosus", "Macracantha arcuata", "Neoscona mellotteei", "Gasteracantha hasselti", - "Gasteracantha kuhli", "Gasteracantha diadesmia", "Nephila pilipes", "", "Neoscona vigilans", "Argiope", "Argiope amoena", - "Araneus ejusmodi", "Araneus mitificus", "Heteropoda venatoria", "Pholcidae", "Macrothele raveni", "Agelenidae", "Lycosidae", - "Steatoda nobilis", "Latrodectus tredecimguttatus", "Tetragnathidae", "Leucauge tessellata", "", "Ebrechtella tricuspidata", - "Salticidae", "Thiania bhamoensis", "Telamonia caprina", "Plexippoides", "Siler semiglaucus", "Pancorius crassipes", "Epeus", - "Hasarius adansoni", "Phintella bifurcilinea", "Cheliceroides longipalpis", "Plexippus paykulli", "", "Eresidae", "Blattodea", - "Periplaneta australasiae", "Periplaneta americana", "Periplaneta fuliginosa", "Blattella germanica", "Corydidae", - "Indolestes peregrinus", "Indolestes cyaneus", "Chlorogomphus papilio", "", "Platycnemididae", "Copera annulata", - "Coeliccia cyanomelas", "Pseudolestes mirabilis", "Gomphidae", "Sinictinogomphus clavatus", "Ictinogomphus rapax", - "Gomphidia confluens", "", "Philoganga vetusta", "Euphaea decorata", "Calopterygidae", "Calopteryx splendens", - "Neurobasis chinensis", "Matrona basilaris", "Calopteryx virgo", "Mnais", "Mnais mneme", "Archineura incarnata", - "Atrocalopteryx atrata", "Anax guttatus", "Anax parthenope", "Anax immaculifrons", "Anax nigrofasciatus", "Gynacantha japonica", - "Gynacantha subinterrupta", "Aeshna mixta", "Rhyothemis", "Rhyothemis variegata", "Rhyothemis fuliginosa", "Tholymis tillarga", - "Palpopleura sexmaculata", "Tramea virginia", "Deielia phaon", "Tetrathemis platyptera", "Sympetrum vulgatum", - "Indothemis carnatica", "Potamarcha congener", "Orthetrum", "Orthetrum chrysis", "Orthetrum luzonicum", "Orthetrum melania", - "Orthetrum poecilops", "Orthetrum sabina", "Orthetrum albistylum", "Orthetrum cancellatum", "Orthetrum lineostigma", - "Orthetrum pruinosum", "Orthetrum glaucum", "Orthetrum triangulare", "Pseudothemis zonata", "Crocothemis servilia", - "Zyxomma petiolatum", "Neurothemis taiwanensis", "Neurothemis tullia", "Neurothemis fulvia", "Neurothemis intermedia", - "Diplacodes trivialis", "Brachydiplax chalybea", "Trithemis festiva", "Trithemis aurora", "Sympetrum croceolum", - "Sympetrum parvulum", "Sympetrum risi", "Sympetrum eroticum", "Sympetrum pedemontanum", "Sympetrum danae", "Acisoma panorpoides", - "Lyriothemis pachygastra", "Epophthalmia elegans", "Brachythemis contaminata", "Pantala flavescens", "Selysiothemis nigra", - "Pseudagrion rubriceps", "Ceriagrion fallax", "Ischnura asiatica", "Ischnura senegalensis", "Ischnura rufostigma", - "Ischnura aurora", "Agriocnemis femina", "Enallagma cyathigerum", "Paracercion calamorum", "Ceriagrion nipponicum", - "Agriocnemis pygmaea", "Chlorocyphidae", "Heliocypha perforata", "Scorpiones", "Heterometrus petersii", "Mantodea", - "Pseudocreobotra wahlbergi", "Phyllocrania paradoxa", "Acromantis japonica", "Creobroter", "Sibylla pretiosa", - "Hymenopus coronatus", "Tenodera sinensis", "Tenodera aridifolia", "Phyllothelys", "Hierodula patellifera", "Mantis religiosa", - "Statilia maculata", "Plecoptera", "Mecoptera", "", "Trictenotomidae", "Rutelidae", "Anomala", "Popillia", - "Eumorphus quadriguttatus", "Attelabidae", "Byctiscus betulae", "Paratrachelophorus nodicornis", "Tomapoderus ruficollis", - "Apoderus coryli", "Aspidobyctiscus lacunipennis", "Trachelophorus giraffa", "Elateridae", "Campsosternus", "Campsosternus gemma", - "Chrysomelidae", "Gallerucida bifasciata", "Monolepta quadriguttata", "Chrysomela populi", "Chrysomela vigintipunctata", - "Plagiodera versicolora", "Oides decempunctata", "Oides bowringii", "Colasposoma dauricum", "Leptinotarsa decemlineata", - "Sagra femorata", "Agasicles hygrophila", "Criocerinae", "", "Chrysolina polita", "Chaetocnema hortensis", "Aulacophora indica", - "Monolepta signata", "Phyllotreta striolata", "Diabrotica undecimpunctata", "Podontia lutea", "Aulacophora lewisii", - "Gastrolina thoracica", "Aulacophora nigripennis", "Buprestidae", "Chrysochroa fulgidissima", "Agrilus planipennis", "Chalcophora", - "Cerambycidae", "Thysia", "Monochamus saltuarius", "Leptura duodecimguttata", "Lamiomimus gottschei", "Moechotypa diphysis", - "Xystrocera globosa", "Mesosa myops", "Dorysthenes", "Monochamus alternatus", "Polyzonus fasciatus", "Agapanthia amurensis", - "Stenocorus meridianus", "Acanthocinus griseus", "Leptura thoracica", "Apomecyna saltator", "Anoplophora", "Anoplophora horsfieldi", - "Leptura annularis", "Rhytiphora bankii", "Semanotus bifasciatus", "Strangalia attenuata", "Neocerambyx raddei", - "Pterolophia annulata", "Glenea relicta", "Imantocera penicillata", "Eupromus ruber", "Aristobia horridula", - "Dicelosternus corallinus", "Batocera", "", "Batocera rubus", "Glenea cantor", "Oberea", "Olenecamptus", "Apriona rugicollis", - "Apriona swainson", "Purpuricenus temminckii", "Callidium violaceum", "Chlorophorus", "Chlorophorus douei", - "Chlorophorus annularis", "Chlorophorus signaticollis", "Eucomatocera vittata", "Xylotrechus", "Xylotrechus yanoi", - "Xylotrechus rusticus", "Asemum striatum", "Paraglenea fortunei", "Phytoecia rufiventris", "Xylorhiza", "", "Aegosoma", - "Arhopalus rusticus", "Stromatium longicorne", "Macrochenus guerini", "Euryphagus", "Saperda populnea", "Aromia bungii", - "Tetraopes tetrophthalmus", "Thyestilla gebleri", "Psacothea", "Paraleprodera diophthalma", "", "", "Tenebrionidae", "Lagriinae", - "Blaps rynchopetera", "", "", "Carabidae", "Therates fruhstorferi", "Pheropsophus", "Carabus lafossei", "Carabus elysii", - "Carabus smaragdinus", "Scarites", "Dolichus halensis", "Chlaenius", "Carabus brandti", "Dynastidae", "Allomyrina dichotoma", - "Oryctes rhinoceros", "Xylotrupes gideon", "", "Eupatorus gracilicornis", "Trichogomphus mongol", "Oryctes nasicornis", - "Dynastes hercules", "Coccinellidae", "Coccinellidae", "Coccinella septempunctata", "Aiolocaria hexaspilota", - "Cheilomenes sexmaculata", "Oenopia formosana", "Vibidia duodecimguttata", "Coccinula quatuordecimpustulata", - "Coelophora biplagiata", "Calvia muiri", "Propylaea quatuordecimpunctata", "Illeis koebelei", "Henosepilachna vigintioctopunctata", - "Oenopia conglobata", "Halmus chalybeus", "Henosepilachna vigintioctomaculata", "Propylea japonica", "Lasioderma serricorne", - "Geotrupidae", "Eumolpidae", "Platycorynus parryi", "Smaragdina nigrifrons", "Euchiridae", "Cheirotonus gestroi", - "Cheirotonus jansoni", "Meloidae", "Lytta caraganae", "Epicauta", "", "Themus", "Cetoniidae", "Euselates", "Goliathus", - "Gametis jucunda", "Pseudotorynorrhina japonica", "Protaetia", "Clinterocera mandarina", "Dicronorhina derbyana", - "Glycyphana horsfieldi", "Agestrata orichalca", "Rhomborhina", "Campsiura mirabilis", "Dicronocephalus adamsi", - "Dicronocephalus wallichii", "Dicronocephalus bowringi", "Pyrocoelia", "Pyrocoelia analis", "Silphidae", "Collyris", "Tricondyla", - "Cicindela", "Cicindela chinenesis", "Cicindela separata", "Cicindela gemmata", "Cicindela aurulenta", "Aphodius fimetarius", - "Bruchidae", "Curculionidae", "Cryptorhynchus lapathi", "Sipalinus gigas", "Eucryptorrhynchus", "Cylas formicarius", "", - "Sitophilus oryzae", "Rhynchophorus ferrugineus", "Hypomeces pulviger", "Pyrochroidae", "Cleridae", "Trichodes sinae", - "Scarabaeoidea", "Hispidae", "Cassida rubiginosa", "Chiridopsis bowringii", "Aspidimorpha miliaris", "Aspidimorpha furcata", - "Aspidimorpha sanctaecrucis", "Taiwania circumdata", "Laccoptera nepalensis(Laccoptera quadrimaculata)", "Cassida nebulosa", - "Lucanidae", "Dorcus titanus", "Dorcus hopei", "Neolucanus", "Neolucanus swinhoei", "", "Lucanus", "Prosopocoilus confucius", - "Prosopocoilus astacoides", "Prosopocoilus girafa", "Prosopocoilus biplagiatus", "Odontolabis cuvera", "Odontolabis siva", - "Eucorynus crassicornis", "Bolboceratidae", "Staphylinidae", "Melolonthidae", "Polyphylla", "Polyphylla decemlineata", - "Melolontha hippocastani", "Amphimallon solstitiale", "Dytiscidae", "Uropygi", "Heliodinidae", "Epicopeia mencia", - "Epicopeia hainesii", "Papilionidae", "Sericinus montelus", "Papilio krishna", "Papilio glaucus", "", "Papilio multicaudata", - "Papilio hermosanus", "Papilio ulysses", "Papilio nephelus", "Papilio paris", "Papilio dehaanii", "Papilio prexaspes", - "Papilio xuthus", "", "Papilio polytes", "Papilio helenus", "Papilio castor", "Papilio bianor", "Papilio dialis", - "Papilio arcturus", "Papilio alcmenor", "Papilio maackii", "Papilio memnon", "Papilio macilentus", "Papilio cresphontes", - "Papilio protenor", "Papilio demoleus", "Papilio hoppo", "Papilio machaon", "", "Papilio troilus", "Pazala", "Pazala eurous", - "Pazala mullah", "Teinopalpus imperialis", "Teinopalpus aureus", "Agehana elwesi", "Bhutanitis thaidina", "Bhutanitis ludlowi", - "Bhutanitis lidderdalii", "Chilasa clytia", "Chilasa clytia", "Iphiclides podalirius", "Atrophaneura horishana", - "Atrophaneura varuna", "Lamproptera curius", "Lamproptera meges", "Pachliopta aristolochiae", "Trogonoptera brookiana", - "Pathysa agetes", "Pathysa_antiphates", "Luehdorfia chinensis", "Troides magellanus", "Troides helena", "Troides aeacus", - "Meandrusa sciron", "Meandrusa payeni", "Losaria coon", "Graphium", "Graphium cloanthus", "Graphium doson", "Graphium chironides", - "Graphium nomius", "Graphium megarus", "Graphium agamemnon", "Graphium sarpedon", "Graphium leechi", "Eurytides marcellus", "Byasa", - "Byasa confusa", "Byasa hedistus", "Byasa polyeuctes", "Byasa mencius", "Byasa dasarada", "Byasa impediens", "Byasa alcinous", - "Limacodidae", "", "Chalcoscelides castaneipars", "Ceratonema", "Thosea", "Matsumurides", "Iragoides conjuncta", "", - "Narosoideus flavidorsalis", "Iraga rugosa", "Rhamnosa uniformis", "Scopelodes venosa", "Scopelodes contracta", "", "Narosa", - "Phocoderma velutina", "Parasa", "Parasa bicolor", "Parasa bicolor", "Parasa lepida", "", "Parasa darma", "Parasa consocia", "", - "Parasa pastoralis", "", "Belippa horrida", "Demonarosa rufotessellata", "Setora postornata", "", "Setora baibarana", - "Miresa bracteata", "Miresa fulgida", "Hyphorma minax", "Monema flavescens", "Monema flavescens", "Thosea sinensis", - "Thosea sinensis", "Tortricidae", "Gypsonoma minutana", "Loboschiza koenigiana", "Eupoecilia ambiguella", "Epiblema foenella", - "Eucosma campoliliana", "Cerace xanthocosma", "Grapholita delineana", "Libythea lepita", "Libythea myrrha", "Noctuidae", - "Chalciope geometrica", "Chalciope mygdon", "Chalciope hyppasia", "Anomis mesogona", "Hadjina chinensis", - "Thysanoplusia intermixta", "Sphragifera sigillata", "Chytonix segregata", "Anisoneura aluco", "Sarbanissa subflava", - "Daddala lucilla", "Cucullia fraterna", "Pericyma cruegeri", "Acronicta tridens", "Acronicta tridens", "Acronicta cuspis", - "Acronicta euphorbiae", "Acronicta euphorbiae", "Acronicta alni", "Acronicta alni", "Acronicta rumicis", "Acronicta rumicis", - "Acronicta hercules", "Acronicta denticulata", "Acronicta psi", "Acronicta psi", "Acronicta pruinosa", "Acronicta pruinosa", - "Acronicta megacephala", "Acronicta megacephala", "Supersypnoides simplex", "Conservula indica", "Hypopyra vespertilio", - "Mimeusemia vilemani", "Mimeusemia vilemani", "Asota heliconia", "Asota heliconia", "Hylophilodes tsukusensis", "Paracolax fentoni", - "Paracolax sugii", "Corgatha nitens", "Corgatha dictaria", "Ophiusa coronata", "Ophiusa tirhaca", "Protoschinia scutosa", - "Agrotis ipsilon", "Oruza albigutta", "Parallelia arctotaenia", "Parallelia stuposa", "Parallelia maturata", "Phyllodes imperialis", - "Staurophora celsia", "Episteme vetula", "Episteme lectrix", "Episteme adulatrix", "Lopharthrum comprimens", "Asota tortuosa", - "Mimeusemia persimilis", "Tiracola plagiata", "Callopistria nobilior", "Callopistria repleta", "Eligma narcissus", "", - "Spirama retorta", "Sphragifera biplagiata", "Lophoptera squamigera", "Ercheia cyllaria", "Axylia putris", "Ramadasa pavo", - "Adris tyrannus", "Hydrillodes lentalis", "Diarsia canescens", "Diarsia subtincta", "Brithys crini", "", "Mocis frugalis", - "Mocis undata", "Spodoptera depravata", "Macdunnoughia purissima", "Spodoptera picta", "Spodoptera litura", "Spodoptera pecten", - "Narangodes argyrostrigatus", "Athetis lepigone", "Xanthodes transversa", "", "Mamestra brassicae", "Spodoptera exigua", "Bocula", - "Cosmia restituta", "Aedia leucomelas", "Phlogophora albovittata", "Trachea auriplena", "Ctenoplusia albostriata", - "Pangrapta lunulata", "Edessena gentiusalis", "Erebus macrops", "Erebus pilosa", "Erebus albicincta", "Erebus caprimulgus", - "Erebus crepuscularis", "Erebus ephesperis", "Ommatophora luminosa", "Cruriopsis funebris", "Checupa stegeri", - "Ischyja ferrifracta", "Narangodes confluens", "Adris okurai", "Sarcopteron punctimargo", "Catocala fraxini", "Thyas honesta", - "Eudocima salaminia", "", "Eudocima phalonia", "Yepcalphis dilectissima", "Arcte coerula", "", "Spodoptera frugiperda", - "Xylostola indistincta", "Achaea janata", "Ischyja manlia", "Catocala electa", "Heliophobus dissectus", "Baorisa hieroglyphica", - "Scrobigera", "Sinna extrema", "Sinna floralis", "Apsarasa radians", "Thysanoplusia daubei", "Tiracola aureata", - "Anacronicta nitida", "Anacronicta horishana", "Edessena hamada", "Serrodes campana", "Gabala argentata", "Othreis homaena", "", - "Asota plana", "Asota plana", "Daseochaeta pulchra", "Diphtherocome", "Hypena", "Hypena trigonalis", "Hypena vestita", - "Hypena lignealis", "Hypena amica", "Hypena indicatalis", "Hypena albopunctalis", "Hypena strigatus", "Hypena perspicua", - "Hypena obesalis", "Hypena lividalis", "Hypena laceratalis", "Sympis rufibasis", "Saturniidae", "Attacus atlas", - "Graellsia isabellae", "Antheraea yamamai", "Actias sinensis", "Caligula simla", "Antheraea polyphemus", "Actias maenas", - "Cricula andrei", "", "Argema mittrei", "Actias luna", "Antheraea pernyi", "Samia", "", "Automeris io", "", "", "Saturnia thibeta", - "Loepa", "Loepa oberthuri", "Loepa megacore", "Antheraea assamensis", "Dictyoploca japonica(Caligula japonica)", "", "Sphingidae", - "Marumba saishiuana", "Marumba sperchius", "Marumba dyras", "Marumba cristata", "Meganoton analis", "Hayesiana triopus", - "Eupanacra mydon", "Theretra oldenlandiae", "", "Theretra alecto subsp. cretica", "Theretra latreillei", "Theretra silhetensis", "", - "Theretra tibetiana", "Theretra pallicosta", "Theretra japonica", "Theretra nessus", "Hippotion rafflesii", "Hippotion rosetta", - "Hippotion celerio", "Pergesa acteus", "", "Dolbina inexacta", "Dolbina tancrei", "Sphecodina caudata", "Parum colligata", "", - "Cypoides", "Callambulyx tatarinovii", "Agrius convolvuli", "", "Rhagastis", "Daphnis nerii", "", "Daphnis hypothous", - "Smerinthus caecus", "Smerinthus planus", "Phyllosphingia", "Deilephila elpenor", "Angonyx testacea", "Acosmeryx formosana", - "Acosmeryx castanea", "Acosmeryx naga", "Acosmeryx miskini", "Cechenena minor", "Cechenena lineosa", "Cechenena subangustata", - "Amplypterus panopus", "Ampelophaga rubiginosa", "Clanis", "Cephonodes hylas", "Nephele hespera", "Langia zenzeroides", - "Macroglossum", "Macroglossum fritzei", "Macroglossum stellatarum", "Macroglossum passalus", "", "Macroglossum bombylans", - "Macroglossum pyrrhosticta", "", "Psilogramma increta", "Psilogramma menephron", "Acherontia styx", "Acherontia atropos", "", - "Acherontia lachesis", "", "Ambulyx", "Haemorrhagiae", "Ethmia lineatonotella", "Labdia semicoccinea", "Geometridae", - "Mixochlora vittata", "Sarcinodes aequilinearia", "Abraxas suspecta", "Xanthabraxas hemionata", "Plutodes", "Plutodes flavescens", - "Plutodes exquisita", "Plutodes costatus", "Gandaritis fixseni", "Semiothisa emersaria", "Paramaxates", "Biston comitata", - "Megaspilates mundataria", "Neohipparchus vallata", "Cleora cinctaria", "Chlorodontopera discospilata", "Semiothisa intermediaria", - "Dalima patularia", "Terpna subtrita", "Ectropis excellens", "Percnia cordiforma", "Naxa seriaria", "Herochroma cristata", - "Herochroma supraviridaria", "Psyra conferta", "Jankowskia fuscaria", "Idaea muricata", "Hypomecis punctinalis", - "Ourapteryx sambucaria", "Ourapteryx nigrociliaris", "Ourapteryx clara", "Ourapteryx nivea", "Scopula yamanei", "Dindica taiwana", - "Dindica polyphaenaria", "Ophthalmitis cordularia", "Agnibesa pictaria", "Eucyclodes semialba", - "Eucyclodes gavissima(Chloromachia gavissima)", "Antipercnia albinigrata", "Plagodis dolabraria", "Telenomeuta punctimarginaria", - "Hemithea tritonaria", "Oxymacaria temeraria", "Dooabia lunifera", "Biston panterinaria", "Deileptenia ribeata", - "Percnia giraffata", "", "Erebomorpha fulguraria", "Ophthalmitis albosignaria", "Chiasmia hebesata", "Phthonandria atrilineata", - "Apochima excavata", "", "Abraxas sylvata", "Thalassodes antiquadraria", "Inurois membranaria", "Chiasmia defixaria", - "Catoria olivescens", "Myrteta angelica", "Hydrelia bicauliata", "Hydrelia bicolorata", "Hydrelia ulula", "Hydrelia enisaria", - "Hydrelia flammeolaria", "Evecliptopera decurrens", "Biston suppressaria", "Biston marginata", "Uliocnemis castalaria", - "Nycterosea obstipata", "Ninodes splendens", "Tyloptera bella", "Chartographa", "Ectropis bhurmitra", "Biston perclara", - "Myrteta tinagmaria", "Thalassodes immissaria", "Percnia suffusa", "Bizia aexaria", "Electrophaes zaphenges", - "Electrophaes corylata", "Xandrames latiferaria", "Xandrames dholaria", "Cyclothea disjuncta", "Stegania cararia", - "Lophomachia lalashana", "Abraxaphantes perampla", "Operophtera relegata", "Krananda latimarginaria", "Krananda semihyalina", - "Krananda lucidaria", "Colotois pennaria", "Amblychia angeronaria", "Dischidesia cinerea", "Problepsis", "Problepsis vulgaris", - "Problepsis superans", "Problepsis albidior", "Ennomos autumnaria", "Corymica", "Pingasa ruginaria", "Pingasa alba", "Idaea impexa", - "Fascellina chromataria", "", "Palpoctenidia phoenicosoma", "Berta rugosivalva", "Timandra dichela", "Timandra stueningi", - "Timandra convectaria", "Timandra synthaca", "Timandra comptaria", "Timandra recompta", "Comibaena", "Comibaena pictipennis", - "Comostola subtiliaria", "Comibaena nigromacularia", "Comibaena procumbaria", "Hemistola monotona", "Fascellina plagiata", - "Tanaoctenia haliaria", "Episothalma robustaria", "Aporandria specularia", "Hypochrosis hyadaria", "Capasa festivaria", - "Gnamptoloma aventiaria", "", "Timandromorpha discolor", "Laciniodes plurilinearia", "Ascotis selenaria", "Xenoplia trivialis", - "Agathia", "Agathia lycaenaria", "Agathia hilarata", "Agathia arcuata", "Agathia laetata", "Agathia diversiformis", - "Agathia carissima", "Milionia basalis", "Cystidia", "Pseudomiza aurata", "Chorodna creataria", "Hydatocapnia gemina", - "Tephrina inchoata", "Metallolophia arenaria", "Dysphania militaris", "Obeidia tigrata", "Obeidia gigantearia", "Obeidia lucifera", - "Odontopera insulata", "Odontopera bilinearia", "Culpinia diffusa", "Iotaphora", "Spilopera divaricata", "Plesiomorpha flaviceps", - "", "Acolutha pulchella subsp. semifulva", "Hyposidra aquilaria", "Heterolocha aristonaria", "Ophthalmitis herbidaria", - "Auaxa cesadaria", "Tanaorhinus viridiluteata", "Tanaorhinus kina", "Tanaorhinus rafflesii", "Tanaorhinus reciprocata", - "Sibatania arizana", "Eumelea ludovicata", "Alcis angulifera", "Alcis repandata", "Heterolocha coccinea", - "Trichopteryx polycommata", "Opisthograptis moelleri", "Garaeus specularis", "Zanclopera falcata", "Arichanna melanaria", - "Nothomiza flavicosta", "", "Thinopteryx crocoptera", "Eilicrinia flava", "Borbacha pardaria", "Hyposidra infixaria", - "Cleora fraterna", "Medasina corticaria", "Yponomeutidae", "Yponomeuta evonymella", "Yponomeuta padella", "Hesperiidae", - "Burara gomata", "Baoris farri", "Udaspes folus", "Polytremis lubricans", "Badamia exclamationis", "Isoteinon lamprospilus", - "Celaenorrhinus maculosus", "Mooreana trichoneura", "Matapa aria", "Erynnis montanus", "Erynnis tages", "Seseria dohertyi", - "Abraximorpha davidii", "Parnara naso", "Parnara ganga", "Parnara guttata", "Borbo cinnara", "Suastus gremius", "", - "Astictopterus jama", "Erionota torus", "Notocrypta curvifascia", "Tagiades litigiosa", "Tagiades menaka", "Pseudocoladenia dan", - "Odontoptilum angulatum", "Pelopidas", "Pelopidas agna", "Pelopidas conjuncta", "Pelopidas mathias", "Hasora badra", - "Hasora chromus", "Hasora anura", "Hasora vitta", "Halpe porus", "Ancistroides nigrita", "Telicota besta", "Telicota colon", - "Telicota ohara", "Iambrix salsala", "Potanthus confucius", "Potanthus trachala", "Ampittia virgata", "Daimio tethys", "Zygaenidae", - "", "Erasmia pulchella", "", "Pryeria sinica", "Pidorus", "Campylotes", "Phauda flammans", "", "Elcysma westwoodi", - "Thyrassia penangae", "", "Artona hainana", "Trypanophora semihyalina", "", "Eterusia aedea", "", "Clelea sapphirina", - "Cyclosia midama", "Cyclosia papilionaris", "Cyclosia papilionaris", "Cyclosia panthona", "Amesia sanguiflua", "Histia rhodope", - "Gynautocera papilionaria", "Soritia strandi", "Soritia strandi", "Rhodopsona rubiginosa", "Idea leuconoe", "Danaus genutia", - "Danaus chrysippus", "", "Danaus plexippus", "Ideopsis similis", "Ideopsis vulgaris", "Euploea", "Euploea sylvester", - "Euploea tulliolus", "Euploea core", "Euploea mulciber", "Euploea midamus", "Parantica", "Parantica sita", "Parantica swinhoei", - "Parantica aglea", "Parantica melaneus", "Tirumala septentrionis", "Tirumala limniace", "Cossidae", "Zeuzera coffeae", - "Zeuzera multistrigata", "Zeuzera pyrina", "Lasiocampidae", "Gastropacha quercifolia", "Gastropacha populifolia", "Trabala vishnou", - "", "Gastropacha pardale", "Lebeda nobilis", "", "Euthrix laeta", "Metanastria gemella", "", "Odonestis pruni", "Euthrix isocyma", - "Cosmotriche discitincta", "Lymantriidae", "Calliteara pudibunda", "Calliteara horsfieldii", "Calliteara horsfieldii", - "Calliteara grotei", "Calliteara grotei", "Arna bipunctapex", "Orgyia antiqua", "Orgyia antiqua", "Orgyia postica", - "Orgyia postica", "Olene mendosa", "Olene mendosa", "Leucoma salicis", "Lymantria mathura", "Lymantria mathura#幼虫", - "Lymantria concolor", "Lymantria dispar", "Lymantria dispar", "Lymantria marginata", "Dasychira suisharyonis", - "Dasychira suisharyonis", "Arctornis l-nigrum", "Laelia coenosa", "Olene dudgeoni", "Olene dudgeoni", "Cifuna locuples", - "Euproctis similis", "Euproctis similis", "Habrosyne pyritoides", "Parapsestis tomponis", "Thyatira batis", "Tethea consimilis", - "Arctiidae", "Phragmatobia luctifera", "Areas galactina", "Peridrome subfascia", "Phragmatobia fuliginosa", - "Phragmatobia fuliginosa", "Ammatho tairadiata", "Peridrome orbicularis", "Eilema costipuncta", "Nudaria ranruna", - "Aglaomorpha histrio", "Utetheisa lotrix", "Pericallia matronula", "Asota plaginota", "Spilosoma lubricipeda", "Asota ficus", - "Asota egens", "Pelosia muscerda", "Arctia flavia", "Arctia caja", "Eilema griseola", "Creatonotus transiens", "Creatonotos gangis", - "Stictane rectilinea", "Rhyparioides metelkana", "Agrisius fuliginosus", "Stigmatophora palmata", "Stigmatophora flava", - "Vamuna remelana", "Aloa lactinea", "Spilosoma subcarnea", "", "Tyria jacobaeae", "", "Macrobrochis gigas", "", "Hyphantria cunea", - "Hyphantria cunea", "Miltochrista", "Miltochrista sauteri(Barsine sauteri)", "Miltochrista ziczac", "Miltochrista convexa", - "Miltochrista fuscozonata", "Miltochrista miniata", "Mangina argus", "Teulisna tumida", "Eugoa grisea", "", "Nyctemera lacticinia", - "Nyctemera lacticinia", "Nyctemera baulus", "Nyctemera tripunctaria", "Nyctemera adversata", "Euplocia membliaria", - "Amerila astreus", "Chrysaeglia magnifica", "Neochera dominia", "Paraona staudingeri", "Cyana", "Cyana hamata", "Cyana propinqua", - "Spilosoma taiwanensis", "Lycaenidae", "Ticherra acte", "Amblopala_avidiena", "Miletus_chinensis", "Lampides boeticus", - "Creon cleobis", "Tajuria cippus", "Zizeeria karsandra", "Catochrysops strabo", "Catochrysops panormus", "Poritia erycinoides", - "Udara dilectus", "Udara albocaerulea", "Arhopala paramuta", "Arhopala bazala", "Arhopala rama", "Nacaduba kurava", - "Nacaduba berenice", "Plebejus orbitulus", "Ancema blanka", "Iraota timoleon", "Heliophorus", "Heliophorus brahma", - "Heliophorus epicles", "Heliophorus ila", "heliophorus saphir", "Caleta roxus", "Horaga onyx", "Horaga albimacula", - "Yasoda tripunctata", "Zizeeria otis", "Prosotas nora", "Lycaena dispar", "Lycaena phlaeas", "Neopithecops zalmora", "Rapala", - "Rapala suffusa", "Rapala nissa", "Tongeia potanini", "Tongeia filicaudis", "Tongeia fischeri", "Mahathala ameria", - "Deudorix epijarbas", "Pratapa deva", "Zeltus amasa", "Scolitantides orion", "Celastrina argiolus", "Sinthusa chandrana", - "Chilades pandava", "Tarucus plinius", "Artipe eryx", "Megisba malaya", "Remelana jangala", "Everes argiades", "Taraka hamada", - "Plebejus argyrognomon", "Ussuriana michaelis", "Pseudozizeeria maha", "Acytolepis puspa", "Teratozephyrus arisanus", - "Curetis acuta", "Spindasis", "Spindasis syama", "Allotinus_drumila", "Aeromachus pygmaeus", "Aeromachus inachus", "Zizula hylax", - "Jamides alecto", "Jamides celeno", "Jamides bochus", "Spialia galba", "Loxura atymnus", "Niphanda fusca", "Dysaethria erasaria", - "Urapteroides astheniata", "Orudiza protheclaria", "Lyssa zampa", "Acropteris leptaliata", "Acropteris iphiata", - "Warreniplema fumicosta", "Urania leilus", "Chrysiridia rhipheus", "Amathusiidae", "Faunis eumeus", "Faunis aerope", - "Faunis canens", "Thauria lathyi", "Thaumantis diores", "Discophora sondaica", "Stichophthalma howqua", "Aemona amathusia", - "Acraea violae", "Acraea terpsicore", "Acraea issoria", "", "Siglophora sanguinolenta", "Westermannia elliptica", - "Risoba prominens", "Blenina quinaria", "Blenina senex", "Iragaodes nobilis", "Carea varipes", "Satyridae", "Neorina patria", - "Mandarinia regalis", "Penthema formosanum", "Penthema darlisa", "Penthema adelma", "Melanitis leda", "Melanitis phedima", - "Coenonympha amaryllis", "Melanargia", "Melanargia galathea", "Mycalesis intermedia", "Mycalesis sangaica", "Mycalesis anaxias", - "Mycalesis mineus", "Mycalesis zonata", "Mycalesis francisca", "Mycalesis gotama", "Mycalesis perseus", "Ypthima", - "Ypthima motschulskyi", "Ypthima praenubila", "Ypthima baldus", "Callerebia", "Neope", "Neope bremeri", "Neope muirheadii", - "Neope pulaha", "Elymnias hypermnestra", "Aphantopus hyperantus", "Lethe", "Lethe mekara", "Lethe butleri", "Lethe gemina", - "Lethe sinorix", "Lethe vindhya", "Lethe chandica", "Lethe christophi", "Lethe rohria", "Lethe insana", "Lethe verma", - "Lethe confusa", "Lethe lanaris", "Lethe syrcis", "Lethe europa", "Lethe dura", "Brahmaeidae", "Brahmaea wallichii", - "Brahmaea porphyrio", "Brahmaea hearseyi", "Brahmaea certhia", "Pieridae", "Pontia daplidice", "Pontia chloridice", - "Leptidea sinapis", "Leptidea amurensis", "Leptidea morsei", "Appias libythea", "Appias lyncida", "Appias albina", "Appias nero", - "Delias hyparete", "Delias pasithoe", "Delias descombesi", "Delias acalis", "Delias belladonna", "Dercas verhuelli", "Ixias pyrene", - "Gandaca harina", "Pieris canidia", "Pieris napi", "Pieris rapae", "Pieris melete", "Leptosia nina", "Aporia", "Aporia agathon", - "Aporia crataegi", "Anthocharis bambusarum", "Anthocharis scolymus", "Colias erate", "Colias fieldii", "Colias hyale", - "Colias palaeno", "Catopsilia pyranthe", "Catopsilia pomona", "Catopsilia scylla", "Gonepteryx amintha", "Gonepteryx rhamni", - "Prioneris thestylis", "Pareronia valeria", "Hebomoia glaucippe", "Eurema mandarina", "Eurema andersoni", "Eurema hecabe", - "Eurema laeta", "Eurema brigitta", "Eurema blanda", "Cepora nerissa", "Promalactis suzukiella", "Scythris sinensis", - "Eretmocera impactella", "Parnassius", "Parnassius citrinarius", "Parnassius nomion", "Parnassius phoebus", "Parnassius bremeri", - "Parnassius apollonius", "Parnassius apollo", "Thyrididae", "Striglina scitaria", "Thyris fenestrella", "Pyrinioides sinuosa", - "Pterophoridae", "Saptha divitiosa", "Notodontidae", "Gazalina chrysolopha", "Cerura menciana", "Cerura vinula", "", - "Syntypistis subgeneris", "Shachihoka formosana", "Clostera anastomosis", "Formofentonia orbifer", "Quadricalcarifera viridipicta", - "Mimopydna", "Phalera", "Phalera grotei", "Phalera bucephala", "Phalera assimilis", "Phalera flavescens", "Pheosia rimosa", - "Clostera anachoreta", "Fentonia ocypete", "Netria viridescens", "Syntypistis comatus", "Clostera albosigma", "Rachia striata", - "Ptilodon saturata", "Uropyia meticulodina", "Spatalia doerriesi", "Stauropus fagi", "Syntypistis pallidifascia", - "Gonoclostera timoniorum", "Gangarides", "Euhampsonia splendida", "Ginshachia elongata", "Euhampsonia cristata", - "Dudusa sphingiformis", "Patania chlorophanta", "Paracymoriza cataclystalis", "Pycnarmon lactiferalis", "Heterocnephes lymphatalis", - "Pagyda quinquelineata", "Cotachena histricalis", "Anania funebris", "Talanga sexpunctalis", "Agathodes ostentalis", - "Syllepte taiwanalis", "Nagiella quadrimaculalis", "Glyphodes quadrimaculalis", "Cirrhochrista brizoalis", "Polythlipta liquidalis", - "Botyodes principalis", "Eoophyla gibbosalis", "Eoophyla conjunctalis", "Parapediasia teterrellus", "Syllepte iophanes", - "Glyphodes duplicalis", "Pleuroptya balteata", "Glyphodes pyloalis", "Syllepte derogata", "Ramila acciusalis", "Tyspanodes striata", - "Cotachena pubescens", "Herpetogramma licarsisalis", "Pachynoa sabelialis", "Pycnarmon cribrata", "Paracymoriza prodigalis", - "Diaphania indica", "Omphisa anastomosalis", "Botyodes asialis", "Cangetta rectilinea", "Agrioglypta itysalis", - "Cnaphalocrocis medinalis", "Crypsiptya coclesalis", "Parapoynx stagnalis", "Parapoynx fluctuosalis", "Parapoynx vittalis", - "Parapoynx crisonalis", "Parapoynx villidalis", "Parapoynx diminutalis", "Pleuroptya iopasalis", "Palpita", - "Palpita nigropunctalis", "Nevrina procopia", "Nosophora semitritalis", "Loxostege sticticalis", "Poliobotys ablactalis", - "Diplopseustis perieresalis", "Pagyda nebulosa", "Cyrtogramme turbata", "Agrotera scissalis", "Pleuroptya ruralis", - "Maruca vitrata", "Pycnarmon pantherata", "Pseudargyria interruptella", "Eumorphobotys eumorphalis", "Botyodes diniasalis", - "Goniorhynchus butyrosa", "Triuncina brunnea", "Bombyx mandarina", "Bombyx mandarina", "Rondotia menciana", "", "Riodinidae", - "Dodona", "Dodona egeon", "Dodona maculosa", "Dodona durga", "Dodona eugenes", "Zemeros flegyas", "Stiboges nymphidia", - "Abisara saturata", "Abisara fylloides", "Abisara burnii", "Abisara echerius", "Abisara bifasciata", "Abisara neophron", - "Abisara fylla", "Nymphalidae", "银纹红袖蝶 Agraulis vanillae", "Cyrestis cocles", "Cyrestis thyodamas", "Cyrestis nivea", - "Parthenos syvia", "Parasarpa dudu", "Chersonesia risa", "Chalinga", "Abrota ganga", "Siproeta stelenes", "Boloria titania", - "Brenthis daphne", "Polyura narcaea", "Polyura eudamippus", "Polyura nepenthes", "Polyura athamas", "Sephisa chandra", - "Sephisa princeps", "Pararge aegeria", "Terinos atlita", "Athyma", "Athyma cama", "Athyma zeroca", "Athyma selenophora", - "Athyma perius", "Athyma asura", "Athyma nefte", "Athyma ranga", "Athyma opalina", "Vagrans egista", "Lexias pardalis", - "Vindula erota", "Argyreus hyperbius", "Asterocampa celtis", "Hypolimnas bolina", "Hypolimnas missipus", "Kallima inachus", - "Euphaedra themis", "Ariadne ariadne", "Ariadne merione", "Diaethria", "Herona marathus", "Timelaea", "Timelaea albescens", - "Neptis", "Neptis hylas", "Neptis soma", "Neptis namba", "Neptis nata", "Neptis sappho", "Neptis miah", "Neptis sankara", - "Neptis clinia", "Neptis pryeri", "Tanaecia julii", "Tanaecia jahnu", "Clossiana freija", "Clossiana euphrosyne", "Clossiana dia", - "Phalanta phalantha", "Issoria eugenia", "Issoria lathonia", "Kaniska canace", "Prothoe franck", "Dichorragia nesimachus", - "Helcyra subalba", "Symbrenthia lilaea", "Symbrenthia brabira", "Junonia atlites", "Junonia almana", "Junonia orithya", - "Junonia lemonias", "Junonia iphita", "Junonia coenia", "Junonia coenia", "Junonia hierta", "Fabriciana adippe", - "Pseudergolis wedah", "Moduza procris", "Dilipa fenestra", "Sasakia charonda", "Sasakia funebris", "Vanessa atalanta", - "Vanessa indica", "Vanessa cardui", "Vanessa virginiensis", "Limenitis", "Limenitis doerriesi", "Limenitis sulpitia", - "Limenitis populi", "Calinaga buddha", "Dophla evelina", "Melitaea", "Rohana parisatis", "Euthalia", "Euthalia", "Euthalia phemius", - "Euthalia pratti", "Euthalia aconthea", "Euthalia lubentina", "Euthalia niepelti", "Argyronome laodice", "Bhagadatta austenia", - "Hestina persimilis", "Hestina nama", "Hestina assimilis", "Phaedyma columella", "Hamadryas", "Nymphalis xanthomelas", - "Nymphalis vau-album", "Nymphalis antiopa", "", "Araschnia doris", "Araschnia prorsoides", "Araschnia levana", "Charaxes bernardus", - "Charaxes bernardus", "Pantoporia hordonia", "Doleschallia bisaltide", "Heliconius erato", "Heliconius charithonia", - "Cupha erymanthis", "Cupha erymanthis", "Argynnis paphia", "Argynnis aglaja", "Mimathyma schrenckii", "Polygonia c-album", - "Polygonia c-aureum", "Proclossiana eunomia", "Chitoria ulupi", "Cethosia cyane", "Cethosia biblis", "Apatura ilia", "Apatura iris", - "Damora sagana", "Stibochiona nicea", "Aglais io", "Aglais urticae", "Lebadea martha", "Pyralidae", "Mabra charonialis", - "Plodia interpunctella", "Eurrhyparodes bracteolalis", "Aethaloessa calidalis", "Endotricha olivacealis", "Ostrinia palustralis", - "Spoladea recurvalis", "Bocchoris inspersalis", "Arippara indicator", "Ancylolomia japonica", "Circobotys aurealis", - "Oncocera semirubella", "Heortia vitessoides", "Locastra muscosalis", "Nosophora insignis", "Orybina regalis", - "Rhectothyris gratiosalis", "Leucinodes orbonalis", "Herpetogramma luctuosalis", "Conogethes punctiferalis", "Pyralis pictalis", - "Pyralis farinalis", "Pyralis regalis", "Diasemia accalis", "Apomyelois ceratoniae", "Omiodes indicata", "Orybina flaviplaga", - "Lista haraldusalis", "Eurrhyparodes tricoloralis", "Rehimena phrynealis", "Cydalima perspectalis", "", "Tyspanodes hypsalis", - "Lamprosema commixta", "Bocchoris onychinalis", "Ericeia inangulata", "Gesonia obeditalis", "Eublemma anachoresis", - "Nagadeba indecoralis", "Lagoptera juno", "Artena dotata", "Scoliopteryx libatrix", "Eublemma cochylioides", "Oruza glaucotorna", - "Autoba tristalis", "Paracolax pryeri", "Ercheia umbrosa", "Cruxoruza decorata", "Opogona nipponica", "Sesiidae", - "Paranthrene tabaniformis", "Drepanidae", "Drepana pallida", "Pseudalbara parvula", "Canucha miranda", "Callidrepana patrana", - "Oreta insignis", "Cyclidia substigmaria", "Cyclidia orciferaria", "Macrauzata maxima", "Oreta loochooana", "Nordstromia japonica", - "Ditrigona triangularia", "Macrocilix mysticata", "Deroca hidda", "Drepana curvatula", "Agnidra scabiosa", "Macrocilix maia", - "Drapetodes mitaria", "", "Petavia attenuata", "Tetragonus catamitus", "Adelidae", "Lepidotarphius perornatellus", "Ctenuchidae", - "Syntomoides imaon", "Amata sperbius", "Amata germana", "Amata fortunei", "Amata grotei", "Anacampsis populella", - "Dichomeris sandycitis" - }; - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_INSECTID_H diff --git a/lite/mnn/cv/mnn_mg_matting.cpp b/lite/mnn/cv/mnn_mg_matting.cpp deleted file mode 100644 index 007afe82..00000000 --- a/lite/mnn/cv/mnn_mg_matting.cpp +++ /dev/null @@ -1,381 +0,0 @@ -// -// Created by DefTruth on 2021/12/5. -// - -#include "mnn_mg_matting.h" -#include "lite/utils.h" - -using mnncv::MNNMGMatting; - -MNNMGMatting::MNNMGMatting( - const std::string &_mnn_path, unsigned int _num_threads -) : log_id(_mnn_path.data()), - mnn_path(_mnn_path.data()), - num_threads(_num_threads) -{ - initialize_interpreter(); - initialize_pretreat(); -} - -MNNMGMatting::~MNNMGMatting() -{ - mnn_interpreter->releaseModel(); - if (mnn_session) - mnn_interpreter->releaseSession(mnn_session); -} - -void MNNMGMatting::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNMGMatting::initialize_interpreter() -{ - mnn_interpreter = std::shared_ptr(MNN::Interpreter::createFromFile(mnn_path)); - // 2. init schedule_config - schedule_config.numThread = (int) num_threads; - MNN::BackendConfig backend_config; - backend_config.precision = MNN::BackendConfig::Precision_High; // default Precision_High - schedule_config.backendConfig = &backend_config; - // 3. create session - mnn_session = mnn_interpreter->createSession(schedule_config); - // 4. init input tensor - image_tensor = mnn_interpreter->getSessionInput(mnn_session, "image"); - mask_tensor = mnn_interpreter->getSessionInput(mnn_session, "mask"); - dimension_type = image_tensor->getDimensionType(); // CAFFE(NCHW) -#ifdef LITEMNN_DEBUG - this->print_debug_string(); -#endif -} - -void MNNMGMatting::transform(const cv::Mat &mat, const cv::Mat &mask) -{ - auto padded_mat = this->padding(mat); // 0-255 int8 - auto padded_mask = this->padding(mask); // 0-1.0 float32 - - // update input tensor and resize Session - mnn_interpreter->resizeTensor(image_tensor, {1, 3, dynamic_input_height, dynamic_input_width}); - mnn_interpreter->resizeTensor(mask_tensor, {1, 1, dynamic_input_height, dynamic_input_width}); - mnn_interpreter->resizeSession(mnn_session); - - // push data into image tensor - pretreat->convert(padded_mat.data, dynamic_input_width, dynamic_input_height, - padded_mat.step[0], image_tensor); - - // push data into mask tensor - auto tmp_host_nchw_tensor = new MNN::Tensor(mask_tensor, MNN::Tensor::CAFFE); // tmp - std::memcpy(tmp_host_nchw_tensor->host(), padded_mask.data, - dynamic_input_mask_size * sizeof(float)); - mask_tensor->copyFromHostTensor(tmp_host_nchw_tensor); - - delete tmp_host_nchw_tensor; -} - -cv::Mat MNNMGMatting::padding(const cv::Mat &unpad_mat) -{ - const unsigned int h = unpad_mat.rows; - const unsigned int w = unpad_mat.cols; - - // aligned - if (h % align_val == 0 && w % align_val == 0) - { - unsigned int target_h = h + 2 * align_val; - unsigned int target_w = w + 2 * align_val; - cv::Mat pad_mat(target_h, target_w, unpad_mat.type()); - - cv::copyMakeBorder(unpad_mat, pad_mat, align_val, align_val, - align_val, align_val, cv::BORDER_REFLECT); - return pad_mat; - } // un-aligned - else - { - // align & padding - unsigned int align_h = align_val * ((h - 1) / align_val + 1); - unsigned int align_w = align_val * ((w - 1) / align_val + 1); - unsigned int pad_h = align_h - h; // >= 0 - unsigned int pad_w = align_w - w; // >= 0 - unsigned int target_h = h + align_val + (pad_h + align_val); - unsigned int target_w = w + align_val + (pad_w + align_val); - - cv::Mat pad_mat(target_h, target_w, unpad_mat.type()); - - cv::copyMakeBorder(unpad_mat, pad_mat, align_val, pad_h + align_val, - align_val, pad_w + align_val, cv::BORDER_REFLECT); - return pad_mat; - } -} - -void MNNMGMatting::update_guidance_mask(cv::Mat &mask, unsigned int guidance_threshold) -{ - if (mask.type() != CV_32FC1) mask.convertTo(mask, CV_32FC1); - const unsigned int h = mask.rows; - const unsigned int w = mask.cols; - if (mask.isContinuous()) - { - const unsigned int data_size = h * w * 1; - float *mutable_data_ptr = (float *) mask.data; - float guidance_threshold_ = (float) guidance_threshold; - for (unsigned int i = 0; i < data_size; ++i) - { - if (mutable_data_ptr[i] >= guidance_threshold_) - mutable_data_ptr[i] = 1.0f; - else - mutable_data_ptr[i] = 0.0f; - } - } // - else - { - float guidance_threshold_ = (float) guidance_threshold; - for (unsigned int i = 0; i < h; ++i) - { - float *p = mask.ptr(i); - for (unsigned int j = 0; j < w; ++j) - { - if (p[j] >= guidance_threshold_) - p[j] = 1.0; - else - p[j] = 0.; - } - } - } -} - -void MNNMGMatting::detect(const cv::Mat &mat, cv::Mat &mask, types::MattingContent &content, - bool remove_noise, unsigned int guidance_threshold, - bool minimum_post_process) -{ - if (mat.empty() || mask.empty()) return; - const unsigned int img_height = mat.rows; - const unsigned int img_width = mat.cols; - this->update_dynamic_shape(img_height, img_width); - this->update_guidance_mask(mask, guidance_threshold); // -> float32 hw1 0~1.0 - - // 1. make input tensors, image, mask - this->transform(mat, mask); - // 2. inference & run session - mnn_interpreter->runSession(mnn_session); - - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. generate matting - this->generate_matting(output_tensors, mat, content, remove_noise, minimum_post_process); -} - -void MNNMGMatting::generate_matting( - const std::map &output_tensors, - const cv::Mat &mat, types::MattingContent &content, - bool remove_noise, bool minimum_post_process) -{ - // https://github.com/yucornetto/MGMatting/blob/main/code-base/infer.py - auto device_alpha_os1_ptr = output_tensors.at("alpha_os1"); // e.g (1,1,h+2*pad_val,w+2*pad_val) - auto device_alpha_os4_ptr = output_tensors.at("alpha_os4"); // e.g (1,1,h+2*pad_val,w+2*pad_val) - auto device_alpha_os8_ptr = output_tensors.at("alpha_os8"); // e.g (1,1,h+2*pad_val,w+2*pad_val) - MNN::Tensor host_alpha_os1_tensor(device_alpha_os1_ptr, device_alpha_os1_ptr->getDimensionType()); - MNN::Tensor host_alpha_os4_tensor(device_alpha_os4_ptr, device_alpha_os4_ptr->getDimensionType()); - MNN::Tensor host_alpha_os8_tensor(device_alpha_os8_ptr, device_alpha_os8_ptr->getDimensionType()); - device_alpha_os1_ptr->copyToHostTensor(&host_alpha_os1_tensor); - device_alpha_os4_ptr->copyToHostTensor(&host_alpha_os4_tensor); - device_alpha_os8_ptr->copyToHostTensor(&host_alpha_os8_tensor); - - const unsigned int h = mat.rows; - const unsigned int w = mat.cols; - - auto output_dims = host_alpha_os1_tensor.shape(); - const unsigned int out_h = output_dims.at(2); - const unsigned int out_w = output_dims.at(3); - float *alpha_os1_ptr = host_alpha_os1_tensor.host(); - float *alpha_os4_ptr = host_alpha_os4_tensor.host(); - float *alpha_os8_ptr = host_alpha_os8_tensor.host(); - - cv::Mat alpha_os1_pred(out_h, out_w, CV_32FC1, alpha_os1_ptr); - cv::Mat alpha_os4_pred(out_h, out_w, CV_32FC1, alpha_os4_ptr); - cv::Mat alpha_os8_pred(out_h, out_w, CV_32FC1, alpha_os8_ptr); - - cv::Mat alpha_pred(out_h, out_w, CV_32FC1, alpha_os8_ptr); - cv::Mat weight_os4 = this->get_unknown_tensor_from_pred(alpha_pred, 30); - this->update_alpha_pred(alpha_pred, weight_os4, alpha_os4_pred); - cv::Mat weight_os1 = this->get_unknown_tensor_from_pred(alpha_pred, 15); - this->update_alpha_pred(alpha_pred, weight_os1, alpha_os1_pred); - if (remove_noise) lite::utils::remove_small_connected_area(alpha_pred, 0.05f); - // need clone to allocate a new continuous memory. - cv::Mat pmat = alpha_pred(cv::Rect(align_val, align_val, w, h)).clone(); // allocated - content.pha_mat = pmat; - - if (!minimum_post_process) - { - // MGMatting only predict Alpha, no fgr. So, - // the fake fgr and merge mat may not need, - // let the fgr mat and merge mat empty to - // Speed up the post processes. - cv::Mat mat_copy; - mat.convertTo(mat_copy, CV_32FC3); - // merge mat and fgr mat may not need - std::vector mat_channels; - cv::split(mat_copy, mat_channels); - cv::Mat bmat = mat_channels.at(0); - cv::Mat gmat = mat_channels.at(1); - cv::Mat rmat = mat_channels.at(2); // ref only, zero-copy. - bmat = bmat.mul(pmat); - gmat = gmat.mul(pmat); - rmat = rmat.mul(pmat); - cv::Mat rest = 1.f - pmat; - cv::Mat mbmat = bmat.mul(pmat) + rest * 153.f; - cv::Mat mgmat = gmat.mul(pmat) + rest * 255.f; - cv::Mat mrmat = rmat.mul(pmat) + rest * 120.f; - std::vector fgr_channel_mats, merge_channel_mats; - fgr_channel_mats.push_back(bmat); - fgr_channel_mats.push_back(gmat); - fgr_channel_mats.push_back(rmat); - merge_channel_mats.push_back(mbmat); - merge_channel_mats.push_back(mgmat); - merge_channel_mats.push_back(mrmat); - - cv::merge(fgr_channel_mats, content.fgr_mat); // allocated - cv::merge(merge_channel_mats, content.merge_mat); // allocated - - content.fgr_mat.convertTo(content.fgr_mat, CV_8UC3); - content.merge_mat.convertTo(content.merge_mat, CV_8UC3); - } - - content.flag = true; -} - -// https://github.com/yucornetto/MGMatting/issues/11 -// https://github.com/yucornetto/MGMatting/blob/main/code-base/utils/util.py#L225 -cv::Mat MNNMGMatting::get_unknown_tensor_from_pred(const cv::Mat &alpha_pred, unsigned int rand_width) -{ - const unsigned int h = alpha_pred.rows; - const unsigned int w = alpha_pred.cols; - const unsigned int data_size = h * w; - cv::Mat uncertain_area(h, w, CV_32FC1, cv::Scalar(1.0f)); // continuous - const float *pred_ptr = (float *) alpha_pred.data; - float *uncertain_ptr = (float *) uncertain_area.data; - // threshold - if (alpha_pred.isContinuous() && uncertain_area.isContinuous()) - { - for (unsigned int i = 0; i < data_size; ++i) - if ((pred_ptr[i] < 1.0f / 255.0f) || (pred_ptr[i] > 1.0f - 1.0f / 255.0f)) - uncertain_ptr[i] = 0.f; - } // - else - { - for (unsigned int i = 0; i < h; ++i) - { - const float *pred_row_ptr = alpha_pred.ptr(i); - float *uncertain_row_ptr = uncertain_area.ptr(i); - for (unsigned int j = 0; j < w; ++j) - { - if ((pred_row_ptr[j] < 1.0f / 255.0f) || (pred_row_ptr[j] > 1.0f - 1.0f / 255.0f)) - uncertain_row_ptr[j] = 0.f; - } - } - } - // dilate - unsigned int size = rand_width / 2; - auto kernel = cv::getStructuringElement(cv::MORPH_ELLIPSE, cv::Size(size, size)); - cv::dilate(uncertain_area, uncertain_area, kernel); - - // weight - cv::Mat weight(h, w, CV_32FC1, uncertain_area.data); // ref only, zero copy. - float *weight_ptr = (float *) weight.data; - if (weight.isContinuous()) - { - for (unsigned int i = 0; i < data_size; ++i) - if (weight_ptr[i] != 1.0f) weight_ptr[i] = 0; - } // - else - { - for (unsigned int i = 0; i < h; ++i) - { - float *weight_row_ptr = weight.ptr(i); - for (unsigned int j = 0; j < w; ++j) - if (weight_row_ptr[j] != 1.0f) weight_row_ptr[j] = 0.f; - - } - } - - return weight; -} - -void MNNMGMatting::update_alpha_pred(cv::Mat &alpha_pred, const cv::Mat &weight, const cv::Mat &other_alpha_pred) -{ - const unsigned int h = alpha_pred.rows; - const unsigned int w = alpha_pred.cols; - const unsigned int data_size = h * w; - const float *weight_ptr = (float *) weight.data; - float *mutable_alpha_ptr = (float *) alpha_pred.data; - const float *other_alpha_ptr = (float *) other_alpha_pred.data; - - if (alpha_pred.isContinuous() && weight.isContinuous() && other_alpha_pred.isContinuous()) - { - for (unsigned int i = 0; i < data_size; ++i) - if (weight_ptr[i] > 0.f) mutable_alpha_ptr[i] = other_alpha_ptr[i]; - } // - else - { - for (unsigned int i = 0; i < h; ++i) - { - const float *weight_row_ptr = weight.ptr(i); - float *mutable_alpha_row_ptr = alpha_pred.ptr(i); - const float *other_alpha_row_ptr = other_alpha_pred.ptr(i); - for (unsigned int j = 0; j < w; ++j) - if (weight_row_ptr[j] > 0.f) mutable_alpha_row_ptr[j] = other_alpha_row_ptr[j]; - } - } -} - -void MNNMGMatting::update_dynamic_shape(unsigned int img_height, unsigned int img_width) -{ - // update dynamic input dims - unsigned int h = img_height; - unsigned int w = img_width; - // update dynamic input dims - if (h % align_val == 0 && w % align_val == 0) - { - // aligned - dynamic_input_height = h + 2 * align_val; - dynamic_input_width = w + 2 * align_val; - } // un-aligned - else - { - // align first - unsigned int align_h = align_val * ((h - 1) / align_val + 1); - unsigned int align_w = align_val * ((w - 1) / align_val + 1); - unsigned int pad_h = align_h - h; // >= 0 - unsigned int pad_w = align_w - w; // >= 0 - dynamic_input_height = h + align_val + (pad_h + align_val); - dynamic_input_width = w + align_val + (pad_w + align_val); - } - - dynamic_input_image_size = 1 * 3 * dynamic_input_height * dynamic_input_width; - dynamic_input_mask_size = 1 * 1 * dynamic_input_height * dynamic_input_width; -} - -void MNNMGMatting::print_debug_string() -{ - std::cout << "LITEMNN_DEBUG LogId: " << log_id << "\n"; - std::cout << "=============== Input-Dims ==============\n"; - if (image_tensor) image_tensor->printShape(); - if (mask_tensor) mask_tensor->printShape(); - if (dimension_type == MNN::Tensor::CAFFE) - std::cout << "Dimension Type: (CAFFE/PyTorch/ONNX)NCHW" << "\n"; - else if (dimension_type == MNN::Tensor::TENSORFLOW) - std::cout << "Dimension Type: (TENSORFLOW)NHWC" << "\n"; - else if (dimension_type == MNN::Tensor::CAFFE_C4) - std::cout << "Dimension Type: (CAFFE_C4)NC4HW4" << "\n"; - std::cout << "=============== Output-Dims ==============\n"; - auto tmp_output_map = mnn_interpreter->getSessionOutputAll(mnn_session); - std::cout << "getSessionOutputAll done!\n"; - for (auto it = tmp_output_map.cbegin(); it != tmp_output_map.cend(); ++it) - { - std::cout << "Output: " << it->first << ": "; - it->second->printShape(); - } - std::cout << "========================================\n"; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_mg_matting.h b/lite/mnn/cv/mnn_mg_matting.h deleted file mode 100644 index 52afe36f..00000000 --- a/lite/mnn/cv/mnn_mg_matting.h +++ /dev/null @@ -1,90 +0,0 @@ -// -// Created by DefTruth on 2021/12/5. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_MG_MATTING_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_MG_MATTING_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNMGMatting - { - public: - explicit MNNMGMatting(const std::string &_mnn_path, unsigned int _num_threads = 8); // - ~MNNMGMatting(); - - private: - std::shared_ptr mnn_interpreter; - MNN::Session *mnn_session = nullptr; - MNN::ScheduleConfig schedule_config; - std::shared_ptr pretreat; // init at runtime - const char *log_id = nullptr; - const char *mnn_path = nullptr; - MNN::Tensor *image_tensor = nullptr; - MNN::Tensor *mask_tensor = nullptr; - - private: - const float norm_vals[3] = {(1.f / 0.229f) * (1.f / 255.f), - (1.f / 0.224f) * (1.f / 255.f), - (1.f / 0.225f) * (1.f / 255.f)}; - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; // RGB - - private: - const unsigned int num_threads; // initialize at runtime. - int dynamic_input_height = 512; // init only, will change according to input mat. - int dynamic_input_width = 512; // init only, will change according to input mat. - unsigned int dynamic_input_image_size = 1 * 3 * 512 * 512; // init only, will change according to input mat. - unsigned int dynamic_input_mask_size = 1 * 1 * 512 * 512; // init only, will change according to input mat. - int dimension_type; // hint only - static constexpr const unsigned int align_val = 32; - - // un-copyable - protected: - MNNMGMatting(const MNNMGMatting &) = delete; // - MNNMGMatting(MNNMGMatting &&) = delete; // - MNNMGMatting &operator=(const MNNMGMatting &) = delete; // - MNNMGMatting &operator=(MNNMGMatting &&) = delete; // - - private: - void print_debug_string(); - - private: - void transform(const cv::Mat &mat, const cv::Mat &mask); - - void initialize_pretreat(); // - - void initialize_interpreter(); - - cv::Mat padding(const cv::Mat &unpad_mat); - - void update_guidance_mask(cv::Mat &mask, unsigned int guidance_threshold = 128); - - void update_dynamic_shape(unsigned int img_height, unsigned int img_width); - - void update_alpha_pred(cv::Mat &alpha_pred, const cv::Mat &weight, const cv::Mat &other_alpha_pred); - - cv::Mat get_unknown_tensor_from_pred(const cv::Mat &alpha_pred, unsigned int rand_width = 30); - - void generate_matting(const std::map &output_tensors, - const cv::Mat &mat, types::MattingContent &content, - bool remove_noise = false, bool minimum_post_process = false); - - public: - /** - * Image Matting Using MGMatting(https://github.com/yucornetto/MGMatting) - * @param mat: cv::Mat BGR HWC, source image - * @param mask: cv::Mat Gray, guidance mask. - * @param guidance_threshold: int, guidance threshold.. - * @param content: types::MattingContent to catch the detected results. - */ - void detect(const cv::Mat &mat, cv::Mat &mask, types::MattingContent &content, - bool remove_noise = false, unsigned int guidance_threshold = 128, - bool minimum_post_process = false); - - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_MG_MATTING_H diff --git a/lite/mnn/cv/mnn_mobile_emotion7.cpp b/lite/mnn/cv/mnn_mobile_emotion7.cpp deleted file mode 100644 index 3a827261..00000000 --- a/lite/mnn/cv/mnn_mobile_emotion7.cpp +++ /dev/null @@ -1,81 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#include "mnn_mobile_emotion7.h" - -using mnncv::MNNMobileEmotion7; - -MNNMobileEmotion7::MNNMobileEmotion7(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - // re-init with fixed input shape, due to the error of input - // shape auto-detection while using MNN with NHWC input. - // TODO: pre-process bug fix - dimension_type = MNN::Tensor::TENSORFLOW; - input_batch = 1; - input_channel = 3; - input_width = 224; - input_height = 224; - mnn_interpreter->resizeTensor( - input_tensor, {input_batch, input_height, input_width, input_channel}); - mnn_interpreter->resizeSession(mnn_session); - - initialize_pretreat(); -} - -inline void MNNMobileEmotion7::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNMobileEmotion7::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,224,224,3) - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNMobileEmotion7::detect(const cv::Mat &mat, types::Emotions &emotions) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch. - auto device_emotion_probs_ptr = output_tensors.at("emotion_preds"); // (1,7) - MNN::Tensor host_emotion_probs_tensor(device_emotion_probs_ptr, device_emotion_probs_ptr->getDimensionType()); - device_emotion_probs_ptr->copyToHostTensor(&host_emotion_probs_tensor); - - auto emotion_dims = host_emotion_probs_tensor.shape(); - const unsigned int num_emotions = emotion_dims.at(1); // 7 - - unsigned int pred_label = 0; - const float *pred_probs_ptr = host_emotion_probs_tensor.host(); - - float pred_score = pred_probs_ptr[0]; - - for (unsigned int i = 0; i < num_emotions; ++i) - { - if (pred_probs_ptr[i] > pred_score) - { - pred_score = pred_probs_ptr[i]; - pred_label = i; - } - } - - emotions.label = pred_label; - emotions.score = pred_score; - emotions.text = emotion_texts[pred_label]; - emotions.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_mobile_emotion7.h b/lite/mnn/cv/mnn_mobile_emotion7.h deleted file mode 100644 index 1b9d4e4d..00000000 --- a/lite/mnn/cv/mnn_mobile_emotion7.h +++ /dev/null @@ -1,36 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_MOBILE_EMOTION7_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_MOBILE_EMOTION7_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNMobileEmotion7 : public BasicMNNHandler - { - public: - explicit MNNMobileEmotion7(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNMobileEmotion7() override = default; - - private: - const float mean_vals[3] = {103.939f, 116.779f, 123.68f}; - const float norm_vals[3] = {1.f, 1.f, 1.f}; - const char *emotion_texts[7] = { - "angry", "disgust", "fear", "happiness", "neutral", "sadness", "surprise" - }; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // padding & resize & normalize. - - public: - void detect(const cv::Mat &mat, types::Emotions &emotions); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_MOBILE_EMOTION7_H diff --git a/lite/mnn/cv/mnn_mobile_facenet.cpp b/lite/mnn/cv/mnn_mobile_facenet.cpp deleted file mode 100644 index 355ef130..00000000 --- a/lite/mnn/cv/mnn_mobile_facenet.cpp +++ /dev/null @@ -1,58 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "mnn_mobile_facenet.h" - -using mnncv::MNNMobileFaceNet; - -MNNMobileFaceNet::MNNMobileFaceNet(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - - -inline void MNNMobileFaceNet::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNMobileFaceNet::transform(const cv::Mat &mat) -{ - // normalize & HWC -> CHW & BGR -> RGB - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNMobileFaceNet::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - - auto device_embedding_ptr = output_tensors.at("embedding"); - MNN::Tensor host_embedding_tensor(device_embedding_ptr, device_embedding_ptr->getDimensionType()); // NCHW - device_embedding_ptr->copyToHostTensor(&host_embedding_tensor); - - auto embedding_dims = host_embedding_tensor.shape(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = host_embedding_tensor.host(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_mobile_facenet.h b/lite/mnn/cv/mnn_mobile_facenet.h deleted file mode 100644 index db0613a3..00000000 --- a/lite/mnn/cv/mnn_mobile_facenet.h +++ /dev/null @@ -1,33 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_MOBILE_FACENET_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_MOBILE_FACENET_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNMobileFaceNet : public BasicMNNHandler - { - public: - explicit MNNMobileFaceNet(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNMobileFaceNet() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 128.0f, 1.f / 128.0f, 1.f / 128.0f}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_MOBILE_FACENET_H diff --git a/lite/mnn/cv/mnn_mobile_hair_seg.cpp b/lite/mnn/cv/mnn_mobile_hair_seg.cpp deleted file mode 100644 index 9133988c..00000000 --- a/lite/mnn/cv/mnn_mobile_hair_seg.cpp +++ /dev/null @@ -1,85 +0,0 @@ -// -// Created by DefTruth on 2022/6/22. -// - -#include "mnn_mobile_hair_seg.h" -#include "lite/utils.h" - -using mnncv::MNNMobileHairSeg; - -MNNMobileHairSeg::MNNMobileHairSeg(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -void MNNMobileHairSeg::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNMobileHairSeg::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,3,224,224) deepcopy inside - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNMobileHairSeg::detect(const cv::Mat &mat, types::HairSegContent &content, - float score_threshold, bool remove_noise) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. generate mask - this->generate_mask(output_tensors, mat, content, score_threshold, remove_noise); -} - -static inline void zero_if_small_inplace(float *mutable_ptr, float &score) -{ if ((*mutable_ptr) < score) *mutable_ptr = 0.f; } - -void MNNMobileHairSeg::generate_mask(const std::map &output_tensors, - const cv::Mat &mat, types::HairSegContent &content, - float score_threshold, bool remove_noise) -{ - auto device_output_ptr = output_tensors.at("output"); // e.g (1,2,224,224) - MNN::Tensor host_output_tensor(device_output_ptr, device_output_ptr->getDimensionType()); - device_output_ptr->copyToHostTensor(&host_output_tensor); - const unsigned int h = mat.rows; - const unsigned int w = mat.cols; - - auto output_dims = host_output_tensor.shape(); - const unsigned int out_h = output_dims.at(2); - const unsigned int out_w = output_dims.at(3); - const unsigned int element_size = out_h * out_w; - - float *output_ptr = host_output_tensor.host() + element_size; // only need positive prob - - // remove small values - if (score_threshold > 0.001f) - for (unsigned int i = 0; i < element_size; ++i) - zero_if_small_inplace(output_ptr + i, score_threshold); - - cv::Mat mask(out_h, out_w, CV_32FC1, output_ptr); - // post process - if (remove_noise) lite::utils::remove_small_connected_area(mask, 0.05f); - // already allocated a new continuous memory after resize. - if (out_h != h || out_w != w) cv::resize(mask, mask, cv::Size(w, h)); - // need clone to allocate a new continuous memory if not performed resize. - // The memory elements point to will release after return. - else mask = mask.clone(); - - content.mask = mask; // auto handle the memory inside ocv with smart ref. - content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_mobile_hair_seg.h b/lite/mnn/cv/mnn_mobile_hair_seg.h deleted file mode 100644 index cfdac94a..00000000 --- a/lite/mnn/cv/mnn_mobile_hair_seg.h +++ /dev/null @@ -1,37 +0,0 @@ -// -// Created by DefTruth on 2022/6/22. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_MOBILE_HAIR_SEG_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_MOBILE_HAIR_SEG_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNMobileHairSeg : public BasicMNNHandler - { - public: - explicit MNNMobileHairSeg(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNMobileHairSeg() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; - const float norm_vals[3] = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - - private: - void initialize_pretreat(); - - void transform(const cv::Mat &mat) override; // resize & normalize. - - void generate_mask(const std::map &output_tensors, - const cv::Mat &mat, types::HairSegContent &content, - float score_threshold = 0.0f, bool remove_noise = false); - - public: - void detect(const cv::Mat &mat, types::HairSegContent &content, - float score_threshold = 0.0f, bool remove_noise = false); - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_MOBILE_HAIR_SEG_H diff --git a/lite/mnn/cv/mnn_mobile_human_matting.cpp b/lite/mnn/cv/mnn_mobile_human_matting.cpp deleted file mode 100644 index 9271dc92..00000000 --- a/lite/mnn/cv/mnn_mobile_human_matting.cpp +++ /dev/null @@ -1,115 +0,0 @@ -// -// Created by DefTruth on 2022/6/20. -// - -#include "mnn_mobile_human_matting.h" -#include "lite/utils.h" - -using mnncv::MNNMobileHumanMatting; - -MNNMobileHumanMatting::MNNMobileHumanMatting(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNMobileHumanMatting::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::BGR, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNMobileHumanMatting::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,3,256,256) deepcopy inside - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNMobileHumanMatting::detect(const cv::Mat &mat, types::MattingContent &content, - bool remove_noise, bool minimum_post_process) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. generate matting - this->generate_matting(output_tensors, mat, content, remove_noise, minimum_post_process); -} - -void MNNMobileHumanMatting::generate_matting(const std::map &output_tensors, - const cv::Mat &mat, types::MattingContent &content, - bool remove_noise, bool minimum_post_process) -{ - auto device_output_ptr = output_tensors.at("alpha"); // e.g (1,1,256,256) - MNN::Tensor host_output_tensor(device_output_ptr, device_output_ptr->getDimensionType()); - device_output_ptr->copyToHostTensor(&host_output_tensor); - const unsigned int h = mat.rows; - const unsigned int w = mat.cols; - - auto output_dims = host_output_tensor.shape(); - const unsigned int out_h = output_dims.at(2); - const unsigned int out_w = output_dims.at(3); - - float *output_ptr = host_output_tensor.host(); - - cv::Mat alpha_pred(out_h, out_w, CV_32FC1, output_ptr); - if (remove_noise) lite::utils::remove_small_connected_area(alpha_pred, 0.05f); - // resize alpha - if (out_h != h || out_w != w) - // already allocated a new continuous memory after resize. - cv::resize(alpha_pred, alpha_pred, cv::Size(w, h)); - // need clone to allocate a new continuous memory if not performed resize. - // The memory elements point to will release after return. - else alpha_pred = alpha_pred.clone(); - - cv::Mat pmat = alpha_pred; // ref - content.pha_mat = pmat; // auto handle the memory inside ocv with smart ref. - - if (!minimum_post_process) - { - // MobileHumanMatting only predict Alpha, no fgr. So, - // the fake fgr and merge mat may not need, - // let the fgr mat and merge mat empty to - // Speed up the post processes. - cv::Mat mat_copy; - mat.convertTo(mat_copy, CV_32FC3); - // merge mat and fgr mat may not need - std::vector mat_channels; - cv::split(mat_copy, mat_channels); - cv::Mat bmat = mat_channels.at(0); - cv::Mat gmat = mat_channels.at(1); - cv::Mat rmat = mat_channels.at(2); // ref only, zero-copy. - bmat = bmat.mul(pmat); - gmat = gmat.mul(pmat); - rmat = rmat.mul(pmat); - cv::Mat rest = 1.f - pmat; - cv::Mat mbmat = bmat.mul(pmat) + rest * 153.f; - cv::Mat mgmat = gmat.mul(pmat) + rest * 255.f; - cv::Mat mrmat = rmat.mul(pmat) + rest * 120.f; - std::vector fgr_channel_mats, merge_channel_mats; - fgr_channel_mats.push_back(bmat); - fgr_channel_mats.push_back(gmat); - fgr_channel_mats.push_back(rmat); - merge_channel_mats.push_back(mbmat); - merge_channel_mats.push_back(mgmat); - merge_channel_mats.push_back(mrmat); - - cv::merge(fgr_channel_mats, content.fgr_mat); - cv::merge(merge_channel_mats, content.merge_mat); - - content.fgr_mat.convertTo(content.fgr_mat, CV_8UC3); - content.merge_mat.convertTo(content.merge_mat, CV_8UC3); - } - - content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_mobile_human_matting.h b/lite/mnn/cv/mnn_mobile_human_matting.h deleted file mode 100644 index 885adb72..00000000 --- a/lite/mnn/cv/mnn_mobile_human_matting.h +++ /dev/null @@ -1,39 +0,0 @@ -// -// Created by DefTruth on 2022/6/20. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_MOBILE_HUMAN_MATTING_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_MOBILE_HUMAN_MATTING_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNMobileHumanMatting : public BasicMNNHandler - { - public: - explicit MNNMobileHumanMatting(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNMobileHumanMatting() override = default; - - private: - const float mean_vals[3] = {104.f, 112.f, 121.f}; //BGR - const float norm_vals[3] = {1.f / 255.f, 1.f / 255.f, 1.f / 255.f}; - - private: - void initialize_pretreat(); - - void transform(const cv::Mat &mat) override; // resize & normalize. - - void generate_matting(const std::map &output_tensors, - const cv::Mat &mat, types::MattingContent &content, - bool remove_noise = false, bool minimum_post_process = false); - - public: - void detect(const cv::Mat &mat, types::MattingContent &content, bool remove_noise = false, - bool minimum_post_process = false); - - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_MOBILE_HUMAN_MATTING_H diff --git a/lite/mnn/cv/mnn_mobilenetv2.cpp b/lite/mnn/cv/mnn_mobilenetv2.cpp deleted file mode 100644 index 52eb2e82..00000000 --- a/lite/mnn/cv/mnn_mobilenetv2.cpp +++ /dev/null @@ -1,68 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "mnn_mobilenetv2.h" -#include "lite/utils.h" - -using mnncv::MNNMobileNetV2; - -MNNMobileNetV2::MNNMobileNetV2(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNMobileNetV2::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNMobileNetV2::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,3,224,224) - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNMobileNetV2::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch. - auto device_logits_ptr = output_tensors.at("logits"); - MNN::Tensor host_logits_tensor(device_logits_ptr, device_logits_ptr->getDimensionType()); - device_logits_ptr->copyToHostTensor(&host_logits_tensor); - - auto logits_dims = host_logits_tensor.shape(); - const unsigned int num_classes = logits_dims.at(1); // 1000 - const float *logits = host_logits_tensor.host(); - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_mobilenetv2.h b/lite/mnn/cv/mnn_mobilenetv2.h deleted file mode 100644 index f7cadabc..00000000 --- a/lite/mnn/cv/mnn_mobilenetv2.h +++ /dev/null @@ -1,410 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_MOBILENETV2_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_MOBILENETV2_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNMobileNetV2 : public BasicMNNHandler - { - public: - explicit MNNMobileNetV2(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNMobileNetV2() override = default; - - private: - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // resize & normalize. - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_MOBILENETV2_H diff --git a/lite/mnn/cv/mnn_mobilenetv2_68.cpp b/lite/mnn/cv/mnn_mobilenetv2_68.cpp deleted file mode 100644 index 61f36df5..00000000 --- a/lite/mnn/cv/mnn_mobilenetv2_68.cpp +++ /dev/null @@ -1,68 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// -#include "mnn_mobilenetv2_68.h" - -using mnncv::MNNMobileNetV268; - -MNNMobileNetV268::MNNMobileNetV268(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNMobileNetV268::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNMobileNetV268::transform(const cv::Mat &mat) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNMobileNetV268::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - // this->transform(mat); - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch landmarks. - auto device_landmarks_ptr = output_tensors.at("output"); // (1,68*2) - MNN::Tensor host_landmarks_tensor(device_landmarks_ptr, device_landmarks_ptr->getDimensionType()); - device_landmarks_ptr->copyToHostTensor(&host_landmarks_tensor); - auto landmark_dims = host_landmarks_tensor.shape(); - - const unsigned int num_landmarks = landmark_dims.at(1); // 68*2=136 - const float *landmarks_ptr = host_landmarks_tensor.host(); - - for (unsigned int i = 0; i < num_landmarks; i += 2) - { - float x = landmarks_ptr[i]; - float y = landmarks_ptr[i + 1]; - - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - landmarks.flag = true; -} - - - diff --git a/lite/mnn/cv/mnn_mobilenetv2_68.h b/lite/mnn/cv/mnn_mobilenetv2_68.h deleted file mode 100644 index 5018dca3..00000000 --- a/lite/mnn/cv/mnn_mobilenetv2_68.h +++ /dev/null @@ -1,32 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_MOBILENETV2_68_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_MOBILENETV2_68_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNMobileNetV268 : public BasicMNNHandler - { - public: - explicit MNNMobileNetV268(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNMobileNetV268() override = default; - - private: - const float mean_vals[3] = {255.f * 0.485f, 255.f * 0.456, 255.f * 0.406f}; - const float norm_vals[3] = {1.0f / (255.f * 0.229f), 1.0f / (255.f * 0.224f), 1.0f / (255.f * 0.225f)}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_MOBILENETV2_68_H diff --git a/lite/mnn/cv/mnn_mobilenetv2_se_68.cpp b/lite/mnn/cv/mnn_mobilenetv2_se_68.cpp deleted file mode 100644 index 818cb4cf..00000000 --- a/lite/mnn/cv/mnn_mobilenetv2_se_68.cpp +++ /dev/null @@ -1,66 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#include "mnn_mobilenetv2_se_68.h" - -using mnncv::MNNMobileNetV2SE68; - -MNNMobileNetV2SE68::MNNMobileNetV2SE68(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNMobileNetV2SE68::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNMobileNetV2SE68::transform(const cv::Mat &mat) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNMobileNetV2SE68::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - // this->transform(mat); - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch landmarks. - auto device_landmarks_ptr = output_tensors.at("output"); // (1,68*2) - MNN::Tensor host_landmarks_tensor(device_landmarks_ptr, device_landmarks_ptr->getDimensionType()); - device_landmarks_ptr->copyToHostTensor(&host_landmarks_tensor); - auto landmark_dims = host_landmarks_tensor.shape(); - - const unsigned int num_landmarks = landmark_dims.at(1); // 68*2=136 - const float *landmarks_ptr = host_landmarks_tensor.host(); - - for (unsigned int i = 0; i < num_landmarks; i += 2) - { - float x = landmarks_ptr[i]; - float y = landmarks_ptr[i + 1]; - - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - landmarks.flag = true; -} diff --git a/lite/mnn/cv/mnn_mobilenetv2_se_68.h b/lite/mnn/cv/mnn_mobilenetv2_se_68.h deleted file mode 100644 index ab23686e..00000000 --- a/lite/mnn/cv/mnn_mobilenetv2_se_68.h +++ /dev/null @@ -1,33 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_MOBILENETV2_SE_68_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_MOBILENETV2_SE_68_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNMobileNetV2SE68 : public BasicMNNHandler - { - public: - explicit MNNMobileNetV2SE68(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNMobileNetV2SE68() override = default; - - private: - const float mean_vals[3] = {255.f * 0.485f, 255.f * 0.456, 255.f * 0.406f}; - const float norm_vals[3] = {1.0f / (255.f * 0.229f), 1.0f / (255.f * 0.224f), 1.0f / (255.f * 0.225f)}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_MOBILENETV2_SE_68_H diff --git a/lite/mnn/cv/mnn_mobilese_focal_face.cpp b/lite/mnn/cv/mnn_mobilese_focal_face.cpp deleted file mode 100644 index b8ee7fcc..00000000 --- a/lite/mnn/cv/mnn_mobilese_focal_face.cpp +++ /dev/null @@ -1,57 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "mnn_mobilese_focal_face.h" - -using mnncv::MNNMobileSEFocalFace; - -MNNMobileSEFocalFace::MNNMobileSEFocalFace(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNMobileSEFocalFace::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNMobileSEFocalFace::transform(const cv::Mat &mat) -{ - // normalize & HWC -> CHW & BGR -> RGB - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNMobileSEFocalFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - - auto device_embedding_ptr = output_tensors.at("embedding"); - MNN::Tensor host_embedding_tensor(device_embedding_ptr, device_embedding_ptr->getDimensionType()); // NCHW - device_embedding_ptr->copyToHostTensor(&host_embedding_tensor); - - auto embedding_dims = host_embedding_tensor.shape(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = host_embedding_tensor.host(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_mobilese_focal_face.h b/lite/mnn/cv/mnn_mobilese_focal_face.h deleted file mode 100644 index 7b7c1f70..00000000 --- a/lite/mnn/cv/mnn_mobilese_focal_face.h +++ /dev/null @@ -1,33 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_MOBILESE_FOCAL_FACE_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_MOBILESE_FOCAL_FACE_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNMobileSEFocalFace : public BasicMNNHandler - { - public: - explicit MNNMobileSEFocalFace(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNMobileSEFocalFace() override = default; - - private: - const float mean_vals[3] = {0.f, 0.f, 0.f}; // RGB - const float norm_vals[3] = {1.f / 255.0f, 1.f / 255.0f, 1.f / 255.0f}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_MOBILESE_FOCAL_FACE_H diff --git a/lite/mnn/cv/mnn_modnet.cpp b/lite/mnn/cv/mnn_modnet.cpp deleted file mode 100644 index 22c36f40..00000000 --- a/lite/mnn/cv/mnn_modnet.cpp +++ /dev/null @@ -1,116 +0,0 @@ -// -// Created by DefTruth on 2022/3/27. -// - -#include "mnn_modnet.h" -#include "lite/utils.h" - -using mnncv::MNNMODNet; - -MNNMODNet::MNNMODNet(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNMODNet::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNMODNet::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,3,256,256) deepcopy inside - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNMODNet::detect(const cv::Mat &mat, types::MattingContent &content, bool remove_noise, - bool minimum_post_process) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. generate matting - this->generate_matting(output_tensors, mat, content, remove_noise, minimum_post_process); -} - - -void MNNMODNet::generate_matting(const std::map &output_tensors, - const cv::Mat &mat, types::MattingContent &content, - bool remove_noise, bool minimum_post_process) -{ - auto device_output_ptr = output_tensors.at("output"); // e.g (1,1,256,256) - MNN::Tensor host_output_tensor(device_output_ptr, device_output_ptr->getDimensionType()); - device_output_ptr->copyToHostTensor(&host_output_tensor); - const unsigned int h = mat.rows; - const unsigned int w = mat.cols; - - auto output_dims = host_output_tensor.shape(); - const unsigned int out_h = output_dims.at(2); - const unsigned int out_w = output_dims.at(3); - - float *output_ptr = host_output_tensor.host(); - - cv::Mat alpha_pred(out_h, out_w, CV_32FC1, output_ptr); - if (remove_noise) lite::utils::remove_small_connected_area(alpha_pred, 0.05f); - // resize alpha - if (out_h != h || out_w != w) - // already allocated a new continuous memory after resize. - cv::resize(alpha_pred, alpha_pred, cv::Size(w, h)); - // need clone to allocate a new continuous memory if not performed resize. - // The memory elements point to will release after return. - else alpha_pred = alpha_pred.clone(); - - cv::Mat pmat = alpha_pred; // ref - content.pha_mat = pmat; // auto handle the memory inside ocv with smart ref. - - if (!minimum_post_process) - { - // MODNet only predict Alpha, no fgr. So, - // the fake fgr and merge mat may not need, - // let the fgr mat and merge mat empty to - // speed up the post processes. - cv::Mat mat_copy; - mat.convertTo(mat_copy, CV_32FC3); - // merge mat and fgr mat may not need - std::vector mat_channels; - cv::split(mat_copy, mat_channels); - cv::Mat bmat = mat_channels.at(0); - cv::Mat gmat = mat_channels.at(1); - cv::Mat rmat = mat_channels.at(2); // ref only, zero-copy. - bmat = bmat.mul(pmat); - gmat = gmat.mul(pmat); - rmat = rmat.mul(pmat); - cv::Mat rest = 1.f - pmat; - cv::Mat mbmat = bmat.mul(pmat) + rest * 153.f; - cv::Mat mgmat = gmat.mul(pmat) + rest * 255.f; - cv::Mat mrmat = rmat.mul(pmat) + rest * 120.f; - std::vector fgr_channel_mats, merge_channel_mats; - fgr_channel_mats.push_back(bmat); - fgr_channel_mats.push_back(gmat); - fgr_channel_mats.push_back(rmat); - merge_channel_mats.push_back(mbmat); - merge_channel_mats.push_back(mgmat); - merge_channel_mats.push_back(mrmat); - - cv::merge(fgr_channel_mats, content.fgr_mat); - cv::merge(merge_channel_mats, content.merge_mat); - - content.fgr_mat.convertTo(content.fgr_mat, CV_8UC3); - content.merge_mat.convertTo(content.merge_mat, CV_8UC3); - } - - content.flag = true; -} diff --git a/lite/mnn/cv/mnn_modnet.h b/lite/mnn/cv/mnn_modnet.h deleted file mode 100644 index c072542a..00000000 --- a/lite/mnn/cv/mnn_modnet.h +++ /dev/null @@ -1,38 +0,0 @@ -// -// Created by DefTruth on 2022/3/27. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_MODNET_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_MODNET_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNMODNet : public BasicMNNHandler - { - public: - explicit MNNMODNet(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNMODNet() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; - const float norm_vals[3] = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - - private: - void initialize_pretreat(); - - void transform(const cv::Mat &mat) override; // resize & normalize. - - void generate_matting(const std::map &output_tensors, - const cv::Mat &mat, types::MattingContent &content, - bool remove_noise = false, bool minimum_post_process = false); - - public: - void detect(const cv::Mat &mat, types::MattingContent &content, bool remove_noise = false, - bool minimum_post_process = false); - - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_MODNET_H diff --git a/lite/mnn/cv/mnn_nanodet.cpp b/lite/mnn/cv/mnn_nanodet.cpp deleted file mode 100644 index df6af2c7..00000000 --- a/lite/mnn/cv/mnn_nanodet.cpp +++ /dev/null @@ -1,249 +0,0 @@ -// -// Created by DefTruth on 2021/10/6. -// - -#include "mnn_nanodet.h" -#include "lite/utils.h" - -using mnncv::MNNNanoDet; - -MNNNanoDet::MNNNanoDet(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNNanoDet::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::BGR, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -inline void MNNNanoDet::transform(const cv::Mat &mat_rs) -{ - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - - -void MNNNanoDet::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - NanoScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(0, 0, 0)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.ratio = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.flag = true; -} - -void MNNNanoDet::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - auto img_height = static_cast(mat.rows); - auto img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - NanoScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - this->transform(mat_rs); - - // 2. inference scores & boxes. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, output_tensors, score_threshold, img_height, img_width); - // 4. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void MNNNanoDet::generate_points(unsigned int target_height, unsigned int target_width) -{ - if (center_points_is_update) return; - - for (auto stride : strides) - { - unsigned int num_grid_w = target_width / stride; - unsigned int num_grid_h = target_height / stride; - std::vector points; - - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - float grid0 = (float) g0 + 0.5f; - float grid1 = (float) g1 + 0.5f; -#ifdef LITE_WIN32 - NanoCenterPoint point; - point.grid0 = grid0; - point.grid1 = grid1; - point.stride = (float) stride; - points.push_back(point); -#else - points.push_back((NanoCenterPoint) {grid0, grid1, (float) stride}); -#endif - } - } - center_points[stride] = points; - } - - center_points_is_update = true; -} - -void MNNNanoDet::generate_bboxes(const NanoScaleParams &scale_params, - std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, float img_height, - float img_width) -{ - // device tensor - auto cls_pred_stride_8 = output_tensors.at("cls_pred_stride_8"); // e.g (1,1600,80) - auto cls_pred_stride_16 = output_tensors.at("cls_pred_stride_16"); // e.g (1,400,80) - auto cls_pred_stride_32 = output_tensors.at("cls_pred_stride_32"); // e.g (1,100,80) - auto dis_pred_stride_8 = output_tensors.at("dis_pred_stride_8"); // e.g (1,1600,4) xyxy (l,t,r,b) - auto dis_pred_stride_16 = output_tensors.at("dis_pred_stride_16"); // e.g (1,400,4) xyxy (l,t,r,b) - auto dis_pred_stride_32 = output_tensors.at("dis_pred_stride_32"); // e.g (1,100,4) xyxy (l,t,r,b) - this->generate_points(input_height, input_width); // e.g 320 320 - - bbox_collection.clear(); - // level 8 & 16 & 32 - this->generate_bboxes_single_stride(scale_params, cls_pred_stride_8, dis_pred_stride_8, 8, - score_threshold, img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, cls_pred_stride_16, dis_pred_stride_16, 16, - score_threshold, img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, cls_pred_stride_32, dis_pred_stride_32, 32, - score_threshold, img_height, img_width, bbox_collection); -#if LITEMNN_DEBUG - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - - -void MNNNanoDet::generate_bboxes_single_stride(const NanoScaleParams &scale_params, - const MNN::Tensor *device_cls_pred, - const MNN::Tensor *device_dis_pred, - unsigned int stride, - float score_threshold, - float img_height, - float img_width, - std::vector &bbox_collection) -{ - unsigned int nms_pre_ = (stride / 8) * nms_pre; // 1 * 1000,2*1000,... - nms_pre_ = nms_pre_ >= nms_pre ? nms_pre_ : nms_pre; - MNN::Tensor host_cls_pred(device_cls_pred, device_cls_pred->getDimensionType()); // e.g (1,1600,80) - MNN::Tensor host_dis_pred(device_dis_pred, device_dis_pred->getDimensionType()); // e.g (1,1600,4) - device_cls_pred->copyToHostTensor(&host_cls_pred); - device_dis_pred->copyToHostTensor(&host_dis_pred); - - auto cls_pred_dims = host_cls_pred.shape(); // e.g (1,1600,80) - const unsigned int num_points = cls_pred_dims.at(1); // e.g 1600 - const unsigned int num_classes = cls_pred_dims.at(2); // e.g 80 - - float ratio = scale_params.ratio; - int dw = scale_params.dw; - int dh = scale_params.dh; - - unsigned int count = 0; - auto &stride_points = center_points[stride]; - for (unsigned int i = 0; i < num_points; ++i) - { - const float *scores = host_cls_pred.host() + (i * num_classes); // row ptr - float cls_conf = scores[0]; - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = scores[j]; - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - if (cls_conf < score_threshold) continue; // filter - - auto &point = stride_points.at(i); - const float cx = point.grid0; // cx - const float cy = point.grid1; // cy - const float s = point.stride; // stride - - const float *offsets = host_dis_pred.host() + (i * 4); - - float l = offsets[0]; // left - float t = offsets[1]; // top - float r = offsets[2]; // right - float b = offsets[3]; // bottom - - types::Boxf box; - float x1 = ((cx - l) * s - (float) dw) / ratio; // cx - l x1 - float y1 = ((cy - t) * s - (float) dh) / ratio; // cy - t y1 - float x2 = ((cx + r) * s - (float) dw) / ratio; // cx + r x2 - float y2 = ((cy + b) * s - (float) dh) / ratio; // cy + b y2 - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(img_width - 1.f, x2); - box.y2 = std::min(img_height - 1.f, y2); - box.score = cls_conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - - if (bbox_collection.size() > nms_pre_) - { - std::sort(bbox_collection.begin(), bbox_collection.end(), - [](const types::Boxf &a, const types::Boxf &b) - { return a.score > b.score; }); // sort inplace - // trunc - bbox_collection.resize(nms_pre_); - } -} - -void MNNNanoDet::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} diff --git a/lite/mnn/cv/mnn_nanodet.h b/lite/mnn/cv/mnn_nanodet.h deleted file mode 100644 index 50e6c2c0..00000000 --- a/lite/mnn/cv/mnn_nanodet.h +++ /dev/null @@ -1,109 +0,0 @@ -// -// Created by DefTruth on 2021/10/6. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_NANODET_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_NANODET_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNNanoDet : public BasicMNNHandler - { - public: - explicit MNNNanoDet(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNNanoDet() override = default; - - private: - // nested classes - typedef struct - { - float grid0; - float grid1; - float stride; - } NanoCenterPoint; - - typedef struct - { - float ratio; - int dw; - int dh; - bool flag; - } NanoScaleParams; - - private: - const float mean_vals[3] = {103.53f, 116.28f, 123.675f}; // BGR - const float norm_vals[3] = {0.017429f, 0.017507f, 0.017125f}; - - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int nms_pre = 1000; - static constexpr const unsigned int max_nms = 30000; - - // multi-levels center points - std::vector strides = {8, 16, 32}; - std::unordered_map> center_points; - bool center_points_is_update = false; - - private: - void transform(const cv::Mat &mat_rs) override; // without resize - - void initialize_pretreat(); // - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - NanoScaleParams &scale_params); - - // only generate once - void generate_points(unsigned int target_height, unsigned int target_width); - - void generate_bboxes_single_stride(const NanoScaleParams &scale_params, - const MNN::Tensor *device_cls_pred, - const MNN::Tensor *device_dis_pred, - unsigned int stride, - float score_threshold, - float img_height, - float img_width, - std::vector &bbox_collection); - - void generate_bboxes(const NanoScaleParams &scale_params, - std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - /** - * @param mat cv::Mat BGR format - * @param detected_boxes vector of Boxf to catch detected boxes. - * @param score_threshold default 0.45f, only keep the result which >= score_threshold. - * @param iou_threshold default 0.3f, iou threshold for NMS. - * @param topk default 100, maximum output boxes after NMS. - * @param nms_type the method. - */ - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.45f, float iou_threshold = 0.3f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_NANODET_H diff --git a/lite/mnn/cv/mnn_nanodet_efficientnet_lite.cpp b/lite/mnn/cv/mnn_nanodet_efficientnet_lite.cpp deleted file mode 100644 index 00b4e0d6..00000000 --- a/lite/mnn/cv/mnn_nanodet_efficientnet_lite.cpp +++ /dev/null @@ -1,250 +0,0 @@ -// -// Created by DefTruth on 2021/10/6. -// - -#include "mnn_nanodet_efficientnet_lite.h" -#include "lite/utils.h" - -using mnncv::MNNNanoDetEfficientNetLite; - -MNNNanoDetEfficientNetLite::MNNNanoDetEfficientNetLite( - const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNNanoDetEfficientNetLite::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::BGR, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -inline void MNNNanoDetEfficientNetLite::transform(const cv::Mat &mat_rs) -{ - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - - -void MNNNanoDetEfficientNetLite::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - NanoLiteScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(0, 0, 0)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.ratio = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.flag = true; -} - -void MNNNanoDetEfficientNetLite::detect( - const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - auto img_height = static_cast(mat.rows); - auto img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - NanoLiteScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - this->transform(mat_rs); - // 2. inference scores & boxes. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, output_tensors, score_threshold, img_height, img_width); - // 4. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void MNNNanoDetEfficientNetLite::generate_points(unsigned int target_height, unsigned int target_width) -{ - if (center_points_is_update) return; - - for (auto stride : strides) - { - unsigned int num_grid_w = target_width / stride; - unsigned int num_grid_h = target_height / stride; - std::vector points; - - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - float grid0 = (float) g0 + 0.5f; - float grid1 = (float) g1 + 0.5f; -#ifdef LITE_WIN32 - NanoLiteCenterPoint point; - point.grid0 = grid0; - point.grid1 = grid1; - point.stride = (float) stride; - points.push_back(point); -#else - points.push_back((NanoLiteCenterPoint) {grid0, grid1, (float) stride}); -#endif - } - } - center_points[stride] = points; - } - - center_points_is_update = true; -} - -void MNNNanoDetEfficientNetLite::generate_bboxes(const NanoLiteScaleParams &scale_params, - std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, float img_height, - float img_width) -{ - // device tensor - auto cls_pred_stride_8 = output_tensors.at("cls_pred_stride_8"); // e.g (1,1600,80) - auto cls_pred_stride_16 = output_tensors.at("cls_pred_stride_16"); // e.g (1,400,80) - auto cls_pred_stride_32 = output_tensors.at("cls_pred_stride_32"); // e.g (1,100,80) - auto dis_pred_stride_8 = output_tensors.at("dis_pred_stride_8"); // e.g (1,1600,4) xyxy (l,t,r,b) - auto dis_pred_stride_16 = output_tensors.at("dis_pred_stride_16"); // e.g (1,400,4) xyxy (l,t,r,b) - auto dis_pred_stride_32 = output_tensors.at("dis_pred_stride_32"); // e.g (1,100,4) xyxy (l,t,r,b) - this->generate_points(input_height, input_width); // e.g 320 320 - - bbox_collection.clear(); - // level 8 & 16 & 32 - this->generate_bboxes_single_stride(scale_params, cls_pred_stride_8, dis_pred_stride_8, 8, - score_threshold, img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, cls_pred_stride_16, dis_pred_stride_16, 16, - score_threshold, img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, cls_pred_stride_32, dis_pred_stride_32, 32, - score_threshold, img_height, img_width, bbox_collection); -#if LITEMNN_DEBUG - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - - -void MNNNanoDetEfficientNetLite::generate_bboxes_single_stride(const NanoLiteScaleParams &scale_params, - const MNN::Tensor *device_cls_pred, - const MNN::Tensor *device_dis_pred, - unsigned int stride, - float score_threshold, - float img_height, - float img_width, - std::vector &bbox_collection) -{ - unsigned int nms_pre_ = (stride / 8) * nms_pre; // 1 * 1000,2*1000,... - nms_pre_ = nms_pre_ >= nms_pre ? nms_pre_ : nms_pre; - MNN::Tensor host_cls_pred(device_cls_pred, device_cls_pred->getDimensionType()); // e.g (1,1600,80) - MNN::Tensor host_dis_pred(device_dis_pred, device_dis_pred->getDimensionType()); // e.g (1,1600,4) - device_cls_pred->copyToHostTensor(&host_cls_pred); - device_dis_pred->copyToHostTensor(&host_dis_pred); - - auto cls_pred_dims = host_cls_pred.shape(); // e.g (1,1600,80) - const unsigned int num_points = cls_pred_dims.at(1); // e.g 1600 - const unsigned int num_classes = cls_pred_dims.at(2); // e.g 80 - - float ratio = scale_params.ratio; - int dw = scale_params.dw; - int dh = scale_params.dh; - - unsigned int count = 0; - auto &stride_points = center_points[stride]; - for (unsigned int i = 0; i < num_points; ++i) - { - const float *scores = host_cls_pred.host() + (i * num_classes); // row ptr - float cls_conf = scores[0]; - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = scores[j]; - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - if (cls_conf < score_threshold) continue; // filter - - auto &point = stride_points.at(i); - const float cx = point.grid0; // cx - const float cy = point.grid1; // cy - const float s = point.stride; // stride - - const float *offsets = host_dis_pred.host() + (i * 4); - - float l = offsets[0]; // left - float t = offsets[1]; // top - float r = offsets[2]; // right - float b = offsets[3]; // bottom - - types::Boxf box; - float x1 = ((cx - l) * s - (float) dw) / ratio; // cx - l x1 - float y1 = ((cy - t) * s - (float) dh) / ratio; // cy - t y1 - float x2 = ((cx + r) * s - (float) dw) / ratio; // cx + r x2 - float y2 = ((cy + b) * s - (float) dh) / ratio; // cy + b y2 - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(img_width - 1.f, x2); - box.y2 = std::min(img_height - 1.f, y2); - box.score = cls_conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - - if (bbox_collection.size() > nms_pre_) - { - std::sort(bbox_collection.begin(), bbox_collection.end(), - [](const types::Boxf &a, const types::Boxf &b) - { return a.score > b.score; }); // sort inplace - // trunc - bbox_collection.resize(nms_pre_); - } -} - -void MNNNanoDetEfficientNetLite::nms( - std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} diff --git a/lite/mnn/cv/mnn_nanodet_efficientnet_lite.h b/lite/mnn/cv/mnn_nanodet_efficientnet_lite.h deleted file mode 100644 index b4b4befb..00000000 --- a/lite/mnn/cv/mnn_nanodet_efficientnet_lite.h +++ /dev/null @@ -1,108 +0,0 @@ -// -// Created by DefTruth on 2021/10/6. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_NANODET_EFFICIENTNET_LITE_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_NANODET_EFFICIENTNET_LITE_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNNanoDetEfficientNetLite : public BasicMNNHandler - { - public: - explicit MNNNanoDetEfficientNetLite(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNNanoDetEfficientNetLite() override = default; - - private: - // nested classes - typedef struct - { - float grid0; - float grid1; - float stride; - } NanoLiteCenterPoint; - - typedef struct - { - float ratio; - int dw; - int dh; - bool flag; - } NanoLiteScaleParams; - - private: - const float mean_vals[3] = {127.f, 127.f, 127.f}; // BGR - const float norm_vals[3] = {1.f / 128.0f, 1.f / 128.0f, 1.f / 128.0f}; - - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int nms_pre = 1000; - static constexpr const unsigned int max_nms = 30000; - - // multi-levels center points - std::vector strides = {8, 16, 32}; - std::unordered_map> center_points; - bool center_points_is_update = false; - - private: - void transform(const cv::Mat &mat_rs) override; // without resize - - void initialize_pretreat(); // - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - NanoLiteScaleParams &scale_params); - - // only generate once - void generate_points(unsigned int target_height, unsigned int target_width); - - void generate_bboxes_single_stride(const NanoLiteScaleParams &scale_params, - const MNN::Tensor *device_cls_pred, - const MNN::Tensor *device_dis_pred, - unsigned int stride, - float score_threshold, - float img_height, - float img_width, - std::vector &bbox_collection); - - void generate_bboxes(const NanoLiteScaleParams &scale_params, - std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - /** - * @param mat cv::Mat BGR format - * @param detected_boxes vector of Boxf to catch detected boxes. - * @param score_threshold default 0.45f, only keep the result which >= score_threshold. - * @param iou_threshold default 0.3f, iou threshold for NMS. - * @param topk default 100, maximum output boxes after NMS. - * @param nms_type the method. - */ - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.45f, float iou_threshold = 0.3f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - }; -} -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_NANODET_EFFICIENTNET_LITE_H diff --git a/lite/mnn/cv/mnn_nanodet_plus.cpp b/lite/mnn/cv/mnn_nanodet_plus.cpp deleted file mode 100644 index df326c4c..00000000 --- a/lite/mnn/cv/mnn_nanodet_plus.cpp +++ /dev/null @@ -1,224 +0,0 @@ -// -// Created by DefTruth on 2021/12/27. -// - -#include "mnn_nanodet_plus.h" -#include "lite/utils.h" - -using mnncv::MNNNanoDetPlus; - -MNNNanoDetPlus::MNNNanoDetPlus(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNNanoDetPlus::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::BGR, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -inline void MNNNanoDetPlus::transform(const cv::Mat &mat_rs) -{ - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - - -void MNNNanoDetPlus::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - NanoPlusScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(0, 0, 0)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.ratio = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.flag = true; -} - -void MNNNanoDetPlus::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - auto img_height = static_cast(mat.rows); - auto img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - NanoPlusScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - this->transform(mat_rs); - - // 2. inference scores & boxes. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, output_tensors, score_threshold, img_height, img_width); - // 4. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void MNNNanoDetPlus::generate_points(unsigned int target_height, unsigned int target_width) -{ - if (center_points_is_update) return; - // 8, 16, 32, 64 - for (auto stride : strides) - { - unsigned int num_grid_w = target_width / stride; - unsigned int num_grid_h = target_height / stride; - - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - float grid0 = (float) g0; - float grid1 = (float) g1; -#ifdef LITE_WIN32 - NanoPlusCenterPoint point; - point.grid0 = grid0; - point.grid1 = grid1; - point.stride = (float) stride; - center_points.push_back(point); -#else - center_points.push_back((NanoPlusCenterPoint) {grid0, grid1, (float) stride}); -#endif - } - } - } - - center_points_is_update = true; -} - -void MNNNanoDetPlus::generate_bboxes(const NanoPlusScaleParams &scale_params, - std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, float img_height, - float img_width) -{ - // device tensor - auto device_output_pred = output_tensors.at("output"); // [1,2125,112] - MNN::Tensor host_output_pred(device_output_pred, device_output_pred->getDimensionType()); - device_output_pred->copyToHostTensor(&host_output_pred); - this->generate_points(input_height, input_width); // e.g 320 320 - - auto output_pred_dims = host_output_pred.shape(); // e.g [1,2125,112] - const unsigned int num_classes = 80; - const unsigned int num_cls_reg = output_pred_dims.at(2); // 112 - const unsigned int reg_max = (num_cls_reg - num_classes) / 4; // e.g 8=7+1 - const unsigned int num_points = center_points.size(); - const float *output_pred_ptr = host_output_pred.host(); - - float ratio = scale_params.ratio; - int dw = scale_params.dw; - int dh = scale_params.dh; - - unsigned int count = 0; - - bbox_collection.clear(); - for (unsigned int i = 0; i < num_points; ++i) - { - const float *scores = output_pred_ptr + i * num_cls_reg; // row ptr - float cls_conf = scores[0]; - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = scores[j]; - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - if (cls_conf < score_threshold) continue; // filter - - auto &point = center_points.at(i); - const float cx = point.grid0; // cx - const float cy = point.grid1; // cy - const float s = point.stride; // stride - - const float *logits = output_pred_ptr + i * num_cls_reg + num_classes; // 32|44... - std::vector offsets(4); - for (unsigned int k = 0; k < 4; ++k) - { - float offset = 0.f; - unsigned int max_id; - auto probs = lite::utils::math::softmax( - logits + (k * reg_max), reg_max, max_id); - for (unsigned int l = 0; l < reg_max; ++l) - offset += (float) l * probs[l]; - offsets[k] = offset; - } - - float l = offsets[0]; // left - float t = offsets[1]; // top - float r = offsets[2]; // right - float b = offsets[3]; // bottom - - types::Boxf box; - float x1 = ((cx - l) * s - (float) dw) / ratio; // cx - l x1 - float y1 = ((cy - t) * s - (float) dh) / ratio; // cy - t y1 - float x2 = ((cx + r) * s - (float) dw) / ratio; // cx + r x2 - float y2 = ((cy + b) * s - (float) dh) / ratio; // cy + b y2 - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(img_width - 1.f, x2); - box.y2 = std::min(img_height - 1.f, y2); - box.score = cls_conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - -#if LITEMNN_DEBUG - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void MNNNanoDetPlus::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} diff --git a/lite/mnn/cv/mnn_nanodet_plus.h b/lite/mnn/cv/mnn_nanodet_plus.h deleted file mode 100644 index 93e9002e..00000000 --- a/lite/mnn/cv/mnn_nanodet_plus.h +++ /dev/null @@ -1,99 +0,0 @@ -// -// Created by DefTruth on 2021/12/27. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_NANODET_PLUS_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_NANODET_PLUS_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNNanoDetPlus : public BasicMNNHandler - { - public: - explicit MNNNanoDetPlus(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNNanoDetPlus() override = default; - - private: - // nested classes - typedef struct - { - float grid0; - float grid1; - float stride; - } NanoPlusCenterPoint; - - typedef struct - { - float ratio; - int dw; - int dh; - bool flag; - } NanoPlusScaleParams; - - private: - const float mean_vals[3] = {103.53f, 116.28f, 123.675f}; // BGR - const float norm_vals[3] = {0.017429f, 0.017507f, 0.017125f}; - - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - // multi-levels center points - std::vector strides = {8, 16, 32, 64}; - std::vector center_points; - bool center_points_is_update = false; - - private: - void transform(const cv::Mat &mat_rs) override; // without resize - - void initialize_pretreat(); // - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - NanoPlusScaleParams &scale_params); - - // only generate once - void generate_points(unsigned int target_height, unsigned int target_width); - - void generate_bboxes(const NanoPlusScaleParams &scale_params, - std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - /** - * @param mat cv::Mat BGR format - * @param detected_boxes vector of Boxf to catch detected boxes. - * @param score_threshold default 0.45f, only keep the result which >= score_threshold. - * @param iou_threshold default 0.3f, iou threshold for NMS. - * @param topk default 100, maximum output boxes after NMS. - * @param nms_type the method. - */ - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.45f, float iou_threshold = 0.3f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_NANODET_PLUS_H diff --git a/lite/mnn/cv/mnn_pfld.cpp b/lite/mnn/cv/mnn_pfld.cpp deleted file mode 100644 index 3a37cd59..00000000 --- a/lite/mnn/cv/mnn_pfld.cpp +++ /dev/null @@ -1,83 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#include "mnn_pfld.h" - -using mnncv::MNNPFLD; - -MNNPFLD::MNNPFLD(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNPFLD::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::BGR, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNPFLD::transform(const cv::Mat &mat) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNPFLD::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - // this->transform(mat); - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch landmarks. - auto device_landmarks_ptr = output_tensors.at("output"); // (1,212) - MNN::Tensor host_landmarks_tensor(device_landmarks_ptr, device_landmarks_ptr->getDimensionType()); - device_landmarks_ptr->copyToHostTensor(&host_landmarks_tensor); - auto landmark_dims = host_landmarks_tensor.shape(); - - const unsigned int num_landmarks = landmark_dims.at(1); // 106*2=212 - const float *landmarks_ptr = host_landmarks_tensor.host(); - - for (unsigned int i = 0; i < num_landmarks; i += 2) - { - float x = landmarks_ptr[i]; - float y = landmarks_ptr[i + 1]; - - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - landmarks.flag = true; -} - - - - - - - - - - - - - - - - - diff --git a/lite/mnn/cv/mnn_pfld.h b/lite/mnn/cv/mnn_pfld.h deleted file mode 100644 index 6ad27f36..00000000 --- a/lite/mnn/cv/mnn_pfld.h +++ /dev/null @@ -1,32 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_PFLD_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_PFLD_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNPFLD : public BasicMNNHandler - { - public: - explicit MNNPFLD(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNPFLD() override = default; - - private: - const float mean_vals[3] = {0.0f, 0.0f, 0.0f}; - const float norm_vals[3] = {1.0f / 255.0f, 1.0f / 255.0f, 1.0f / 255.0f}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_PFLD_H diff --git a/lite/mnn/cv/mnn_pfld68.cpp b/lite/mnn/cv/mnn_pfld68.cpp deleted file mode 100644 index cb6b352e..00000000 --- a/lite/mnn/cv/mnn_pfld68.cpp +++ /dev/null @@ -1,67 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#include "mnn_pfld68.h" - -using mnncv::MNNPFLD68; - -MNNPFLD68::MNNPFLD68(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNPFLD68::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNPFLD68::transform(const cv::Mat &mat) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNPFLD68::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - // this->transform(mat); - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch landmarks. - auto device_landmarks_ptr = output_tensors.at("output"); // (1,68*2) - MNN::Tensor host_landmarks_tensor(device_landmarks_ptr, device_landmarks_ptr->getDimensionType()); - device_landmarks_ptr->copyToHostTensor(&host_landmarks_tensor); - auto landmark_dims = host_landmarks_tensor.shape(); - - const unsigned int num_landmarks = landmark_dims.at(1); // 68*2=136 - const float *landmarks_ptr = host_landmarks_tensor.host(); - - for (unsigned int i = 0; i < num_landmarks; i += 2) - { - float x = landmarks_ptr[i]; - float y = landmarks_ptr[i + 1]; - - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - landmarks.flag = true; -} - diff --git a/lite/mnn/cv/mnn_pfld68.h b/lite/mnn/cv/mnn_pfld68.h deleted file mode 100644 index a8435a82..00000000 --- a/lite/mnn/cv/mnn_pfld68.h +++ /dev/null @@ -1,33 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_PFLD68_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_PFLD68_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNPFLD68 : public BasicMNNHandler - { - public: - explicit MNNPFLD68(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNPFLD68() override = default; - - private: - const float mean_vals[3] = {0.0f, 0.0f, 0.0f}; - const float norm_vals[3] = {1.0f / 255.0f, 1.0f / 255.0f, 1.0f / 255.0f}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_PFLD68_H diff --git a/lite/mnn/cv/mnn_pfld98.cpp b/lite/mnn/cv/mnn_pfld98.cpp deleted file mode 100644 index 194610ec..00000000 --- a/lite/mnn/cv/mnn_pfld98.cpp +++ /dev/null @@ -1,66 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#include "mnn_pfld98.h" - -using mnncv::MNNPFLD98; - -MNNPFLD98::MNNPFLD98(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNPFLD98::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::BGR, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNPFLD98::transform(const cv::Mat &mat) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNPFLD98::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - // this->transform(mat); - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch landmarks. - auto device_landmarks_ptr = output_tensors.at("landmarks"); // (1,98*2) - MNN::Tensor host_landmarks_tensor(device_landmarks_ptr, device_landmarks_ptr->getDimensionType()); - device_landmarks_ptr->copyToHostTensor(&host_landmarks_tensor); - auto landmark_dims = host_landmarks_tensor.shape(); - - const unsigned int num_landmarks = landmark_dims.at(1); // (1,98*2) - const float *landmarks_ptr = host_landmarks_tensor.host(); - - for (unsigned int i = 0; i < num_landmarks; i += 2) - { - float x = landmarks_ptr[i]; - float y = landmarks_ptr[i + 1]; - - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - landmarks.flag = true; -} diff --git a/lite/mnn/cv/mnn_pfld98.h b/lite/mnn/cv/mnn_pfld98.h deleted file mode 100644 index 27f8ae62..00000000 --- a/lite/mnn/cv/mnn_pfld98.h +++ /dev/null @@ -1,32 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_PFLD98_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_PFLD98_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNPFLD98 : public BasicMNNHandler - { - public: - explicit MNNPFLD98(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNPFLD98() override = default; - - private: - const float mean_vals[3] = {0.0f, 0.0f, 0.0f}; - const float norm_vals[3] = {1.0f / 255.0f, 1.0f / 255.0f, 1.0f / 255.0f}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_PFLD98_H diff --git a/lite/mnn/cv/mnn_pipnet19.cpp b/lite/mnn/cv/mnn_pipnet19.cpp deleted file mode 100644 index 5b4f8f07..00000000 --- a/lite/mnn/cv/mnn_pipnet19.cpp +++ /dev/null @@ -1,206 +0,0 @@ -// -// Created by DefTruth on 2022/3/20. -// - -#include "mnn_pipnet19.h" - -using mnncv::MNNPIPNet19; - -MNNPIPNet19::MNNPIPNet19(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNPIPNet19::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNPIPNet19::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,3,256,256) will do deepcopy inside MNN convert process - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNPIPNet19::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - // this->transform(mat); - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. generate landmarks - this->generate_landmarks(landmarks, output_tensors, img_height, img_width); -} - -void MNNPIPNet19::generate_landmarks(types::Landmarks &landmarks, - const std::map &output_tensors, - float img_height, float img_width) -{ - auto device_outputs_cls_ptr = output_tensors.at("outputs_cls"); // (1,19,8,8) - auto device_outputs_x_ptr = output_tensors.at("outputs_x"); // (1,19,8,8) - auto device_outputs_y_ptr = output_tensors.at("outputs_y"); // (1,19,8,8) - auto device_outputs_nb_x_ptr = output_tensors.at("outputs_nb_x"); // (1,19*10,8,8) - auto device_outputs_nb_y_ptr = output_tensors.at("outputs_nb_y"); // (1,19*10,8,8) - MNN::Tensor host_outputs_cls_tensor(device_outputs_cls_ptr, device_outputs_cls_ptr->getDimensionType()); - MNN::Tensor host_outputs_x_tensor(device_outputs_x_ptr, device_outputs_x_ptr->getDimensionType()); - MNN::Tensor host_outputs_y_tensor(device_outputs_y_ptr, device_outputs_y_ptr->getDimensionType()); - MNN::Tensor host_outputs_nb_x_tensor(device_outputs_nb_x_ptr, device_outputs_nb_x_ptr->getDimensionType()); - MNN::Tensor host_outputs_nb_y_tensor(device_outputs_nb_y_ptr, device_outputs_nb_y_ptr->getDimensionType()); - device_outputs_cls_ptr->copyToHostTensor(&host_outputs_cls_tensor); - device_outputs_x_ptr->copyToHostTensor(&host_outputs_x_tensor); - device_outputs_y_ptr->copyToHostTensor(&host_outputs_y_tensor); - device_outputs_nb_x_ptr->copyToHostTensor(&host_outputs_nb_x_tensor); - device_outputs_nb_y_ptr->copyToHostTensor(&host_outputs_nb_y_tensor); - - auto cls_shape = host_outputs_cls_tensor.shape(); - const unsigned int grid_h = cls_shape.at(2); // 8 - const unsigned int grid_w = cls_shape.at(3); // 8 - const unsigned int grid_length = grid_h * grid_w; // 8 * 8 = 64 - const unsigned int input_h = input_height; - const unsigned int input_w = input_width; - - const float *outputs_cls_ptr = host_outputs_cls_tensor.host(); - const float *outputs_x_ptr = host_outputs_x_tensor.host(); - const float *outputs_y_ptr = host_outputs_y_tensor.host(); - const float *outputs_nb_x_ptr = host_outputs_nb_x_tensor.host(); - const float *outputs_nb_y_ptr = host_outputs_nb_y_tensor.host(); - - // find max_ids - std::vector max_ids(num_lms); - for (unsigned int i = 0; i < num_lms; ++i) - { - const float *score_ptr = outputs_cls_ptr + i * grid_length; - unsigned int max_id = 0; - float max_score = score_ptr[0]; - for (unsigned int j = 0; j < grid_length; ++j) - { - if (score_ptr[j] > max_score) - { - max_score = score_ptr[j]; - max_id = j; - } - } - max_ids[i] = max_id; // range 0~64 - } - - // find x & y offsets - std::vector output_x_select(num_lms); - std::vector output_y_select(num_lms); - for (unsigned int i = 0; i < num_lms; ++i) - { - const float *offset_x_ptr = outputs_x_ptr + i * grid_length; - const float *offset_y_ptr = outputs_y_ptr + i * grid_length; - const unsigned int max_id = max_ids.at(i); - output_x_select[i] = offset_x_ptr[max_id]; - output_y_select[i] = offset_y_ptr[max_id]; - } - - // find nb_x & nb_y offsets - std::unordered_map> output_nb_x_select; - std::unordered_map> output_nb_y_select; - // initialize offsets map - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector nb_x_offset(num_nb); - std::vector nb_y_offset(num_nb); - output_nb_x_select[i] = nb_x_offset; - output_nb_y_select[i] = nb_y_offset; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - for (unsigned int j = 0; j < num_nb; ++j) - { - const float *offset_nb_x_ptr = outputs_nb_x_ptr + (i * num_nb + j) * grid_length; - const float *offset_nb_y_ptr = outputs_nb_y_ptr + (i * num_nb + j) * grid_length; - const unsigned int max_id = max_ids.at(i); - output_nb_x_select[i][j] = offset_nb_x_ptr[max_id]; - output_nb_y_select[i][j] = offset_nb_y_ptr[max_id]; - } - } - - // calculate coords - std::vector lms_pred_x(num_lms); // 19 - std::vector lms_pred_y(num_lms); // 19 - std::unordered_map> lms_pred_nb_x; // 19,10 - std::unordered_map> lms_pred_nb_y; // 19,10 - // initialize pred maps - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector nb_x_offset(num_nb); - std::vector nb_y_offset(num_nb); - lms_pred_nb_x[i] = nb_x_offset; - lms_pred_nb_y[i] = nb_y_offset; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - float cx = static_cast(max_ids.at(i) % grid_w); - float cy = static_cast(max_ids.at(i) / grid_w); - // calculate coords & normalize - lms_pred_x[i] = ((cx + output_x_select[i]) * (float) net_stride) / (float) input_w; - lms_pred_y[i] = ((cy + output_y_select[i]) * (float) net_stride) / (float) input_h; - for (unsigned int j = 0; j < num_nb; ++j) - { - lms_pred_nb_x[i][j] = ((cx + output_nb_x_select[i][j]) * (float) net_stride) / (float) input_w; - lms_pred_nb_y[i][j] = ((cy + output_nb_y_select[i][j]) * (float) net_stride) / (float) input_h; - } - } - - // reverse indexes - std::unordered_map> tmp_nb_x; // 19,max_len - std::unordered_map> tmp_nb_y; // 19,max_len - // initialize reverse maps - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector tmp_x(max_len); - std::vector tmp_y(max_len); - tmp_nb_x[i] = tmp_x; - tmp_nb_y[i] = tmp_y; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - for (unsigned int j = 0; j < max_len; ++j) - { - unsigned int ri = reverse_index1[i * max_len + j]; - unsigned int rj = reverse_index2[i * max_len + j]; - tmp_nb_x[i][j] = lms_pred_nb_x[ri][rj]; - tmp_nb_y[i][j] = lms_pred_nb_y[ri][rj]; - } - } - - // merge predictions - landmarks.points.clear(); - for (unsigned int i = 0; i < num_lms; ++i) - { - float total_x = lms_pred_x[i]; - float total_y = lms_pred_y[i]; - for (unsigned int j = 0; j < max_len; ++j) - { - total_x += tmp_nb_x[i][j]; - total_y += tmp_nb_y[i][j]; - } - float x = total_x / ((float) max_len + 1.f); - float y = total_y / ((float) max_len + 1.f); - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - - landmarks.flag = true; -} diff --git a/lite/mnn/cv/mnn_pipnet19.h b/lite/mnn/cv/mnn_pipnet19.h deleted file mode 100644 index 92f03be5..00000000 --- a/lite/mnn/cv/mnn_pipnet19.h +++ /dev/null @@ -1,66 +0,0 @@ -// -// Created by DefTruth on 2022/3/20. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_PIPNET19_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_PIPNET19_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNPIPNet19 : public BasicMNNHandler - { - public: - explicit MNNPIPNet19(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNPIPNet19() override = default; - - private: - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - static constexpr const unsigned int num_nb = 10; - static constexpr const unsigned int num_lms = 19; - static constexpr const unsigned int max_len = 18; - static constexpr const unsigned int net_stride = 32; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // resize & normalize. - - void generate_landmarks(types::Landmarks &landmarks, - const std::map &output_tensors, - float img_height, float img_width); - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - - private: - const unsigned int reverse_index1[19 * 18] = { - 1, 2, 6, 7, 8, 1, 2, 6, 7, 8, 1, 2, 6, 7, 8, 1, 2, 6, 0, 2, 3, 4, 6, 7, 8, 0, 2, 3, 4, 6, 7, 8, 0, 2, 3, 4, 0, 1, 3, 4, 5, 6, 7, 8, - 9, 10, 11, 12, 0, 1, 3, 4, 5, 6, 0, 1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 14, 0, 1, 2, 4, 5, 6, 1, 2, 3, 5, 9, 10, 11, 1, 2, 3, 5, 9, 10, - 11, 1, 2, 3, 5, 3, 4, 9, 10, 11, 3, 4, 9, 10, 11, 3, 4, 9, 10, 11, 3, 4, 9, 0, 1, 2, 3, 7, 8, 12, 13, 15, 0, 1, 2, 3, 7, 8, 12, 13, - 15, 0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 15, 16, 18, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 0, - 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15, 16, 17, 18, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 13, 14, 16, 17, 18, 0, 1, 3, 4, 5, 9, - 10, 14, 17, 3, 4, 5, 9, 10, 14, 17, 3, 4, 5, 9, 0, 1, 6, 7, 8, 13, 14, 15, 16, 17, 18, 0, 1, 6, 7, 8, 13, 14, 0, 2, 5, 6, 7, 8, 9, - 10, 11, 12, 14, 15, 16, 17, 18, 0, 2, 5, 4, 5, 9, 10, 11, 12, 13, 15, 16, 17, 18, 4, 5, 9, 10, 11, 12, 13, 12, 13, 14, 16, 17, 18, - 12, 13, 14, 16, 17, 18, 12, 13, 14, 16, 17, 18, 12, 13, 14, 15, 17, 18, 12, 13, 14, 15, 17, 18, 12, 13, 14, 15, 17, 18, 12, 13, 14, - 15, 16, 18, 12, 13, 14, 15, 16, 18, 12, 13, 14, 15, 16, 18, 15, 16, 17, 15, 16, 17, 15, 16, 17, 15, 16, 17, 15, 16, 17, 15, 16, 17 - }; - const unsigned int reverse_index2[19 * 18] = { - 0, 6, 1, 4, 6, 0, 6, 1, 4, 6, 0, 6, 1, 4, 6, 0, 6, 1, 0, 1, 8, 7, 2, 2, 3, 0, 1, 8, 7, 2, 2, 3, 0, 1, 8, 7, 3, 1, 3, 5, 5, 4, 3, 1, - 5, 6, 6, 9, 3, 1, 3, 5, 5, 4, 5, 5, 3, 1, 3, 7, 5, 5, 1, 3, 4, 9, 5, 5, 3, 1, 3, 7, 7, 8, 1, 0, 3, 2, 2, 7, 8, 1, 0, 3, 2, 2, 7, 8, - 1, 0, 6, 0, 6, 4, 1, 6, 0, 6, 4, 1, 6, 0, 6, 4, 1, 6, 0, 6, 1, 3, 4, 9, 1, 2, 6, 9, 8, 1, 3, 4, 9, 1, 2, 6, 9, 8, 2, 2, 2, 7, 8, 9, - 0, 0, 9, 9, 9, 5, 7, 7, 8, 8, 2, 2, 4, 4, 0, 5, 6, 6, 3, 0, 4, 5, 7, 4, 3, 8, 6, 6, 9, 6, 7, 6, 5, 0, 4, 4, 8, 6, 4, 0, 3, 8, 4, 4, - 9, 7, 6, 7, 9, 8, 7, 2, 2, 2, 9, 9, 9, 0, 0, 8, 5, 9, 7, 9, 9, 8, 4, 3, 1, 2, 1, 6, 8, 4, 3, 1, 2, 1, 6, 8, 4, 3, 1, 2, 6, 9, 5, 7, - 8, 0, 2, 1, 3, 4, 4, 6, 9, 5, 7, 8, 0, 2, 8, 9, 8, 6, 8, 7, 7, 8, 8, 0, 0, 2, 2, 2, 5, 8, 9, 8, 9, 7, 8, 7, 5, 2, 1, 4, 4, 1, 3, 9, - 7, 8, 7, 5, 2, 1, 1, 5, 7, 0, 3, 1, 1, 5, 7, 0, 3, 1, 1, 5, 7, 0, 3, 1, 3, 2, 3, 0, 0, 0, 3, 2, 3, 0, 0, 0, 3, 2, 3, 0, 0, 0, 7, 6, - 1, 3, 1, 2, 7, 6, 1, 3, 1, 2, 7, 6, 1, 3, 1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 - }; - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_PIPNET19_H diff --git a/lite/mnn/cv/mnn_pipnet29.cpp b/lite/mnn/cv/mnn_pipnet29.cpp deleted file mode 100644 index 095fa434..00000000 --- a/lite/mnn/cv/mnn_pipnet29.cpp +++ /dev/null @@ -1,206 +0,0 @@ -// -// Created by DefTruth on 2022/3/20. -// - -#include "mnn_pipnet29.h" - -using mnncv::MNNPIPNet29; - -MNNPIPNet29::MNNPIPNet29(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNPIPNet29::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNPIPNet29::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,3,256,256) will do deepcopy inside MNN convert process - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNPIPNet29::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - // this->transform(mat); - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. generate landmarks - this->generate_landmarks(landmarks, output_tensors, img_height, img_width); -} - -void MNNPIPNet29::generate_landmarks(types::Landmarks &landmarks, - const std::map &output_tensors, - float img_height, float img_width) -{ - auto device_outputs_cls_ptr = output_tensors.at("outputs_cls"); // (1,29,8,8) - auto device_outputs_x_ptr = output_tensors.at("outputs_x"); // (1,29,8,8) - auto device_outputs_y_ptr = output_tensors.at("outputs_y"); // (1,29,8,8) - auto device_outputs_nb_x_ptr = output_tensors.at("outputs_nb_x"); // (1,29*10,8,8) - auto device_outputs_nb_y_ptr = output_tensors.at("outputs_nb_y"); // (1,29*10,8,8) - MNN::Tensor host_outputs_cls_tensor(device_outputs_cls_ptr, device_outputs_cls_ptr->getDimensionType()); - MNN::Tensor host_outputs_x_tensor(device_outputs_x_ptr, device_outputs_x_ptr->getDimensionType()); - MNN::Tensor host_outputs_y_tensor(device_outputs_y_ptr, device_outputs_y_ptr->getDimensionType()); - MNN::Tensor host_outputs_nb_x_tensor(device_outputs_nb_x_ptr, device_outputs_nb_x_ptr->getDimensionType()); - MNN::Tensor host_outputs_nb_y_tensor(device_outputs_nb_y_ptr, device_outputs_nb_y_ptr->getDimensionType()); - device_outputs_cls_ptr->copyToHostTensor(&host_outputs_cls_tensor); - device_outputs_x_ptr->copyToHostTensor(&host_outputs_x_tensor); - device_outputs_y_ptr->copyToHostTensor(&host_outputs_y_tensor); - device_outputs_nb_x_ptr->copyToHostTensor(&host_outputs_nb_x_tensor); - device_outputs_nb_y_ptr->copyToHostTensor(&host_outputs_nb_y_tensor); - - auto cls_shape = host_outputs_cls_tensor.shape(); - const unsigned int grid_h = cls_shape.at(2); // 8 - const unsigned int grid_w = cls_shape.at(3); // 8 - const unsigned int grid_length = grid_h * grid_w; // 8 * 8 = 64 - const unsigned int input_h = input_height; - const unsigned int input_w = input_width; - - const float *outputs_cls_ptr = host_outputs_cls_tensor.host(); - const float *outputs_x_ptr = host_outputs_x_tensor.host(); - const float *outputs_y_ptr = host_outputs_y_tensor.host(); - const float *outputs_nb_x_ptr = host_outputs_nb_x_tensor.host(); - const float *outputs_nb_y_ptr = host_outputs_nb_y_tensor.host(); - - // find max_ids - std::vector max_ids(num_lms); - for (unsigned int i = 0; i < num_lms; ++i) - { - const float *score_ptr = outputs_cls_ptr + i * grid_length; - unsigned int max_id = 0; - float max_score = score_ptr[0]; - for (unsigned int j = 0; j < grid_length; ++j) - { - if (score_ptr[j] > max_score) - { - max_score = score_ptr[j]; - max_id = j; - } - } - max_ids[i] = max_id; // range 0~64 - } - - // find x & y offsets - std::vector output_x_select(num_lms); - std::vector output_y_select(num_lms); - for (unsigned int i = 0; i < num_lms; ++i) - { - const float *offset_x_ptr = outputs_x_ptr + i * grid_length; - const float *offset_y_ptr = outputs_y_ptr + i * grid_length; - const unsigned int max_id = max_ids.at(i); - output_x_select[i] = offset_x_ptr[max_id]; - output_y_select[i] = offset_y_ptr[max_id]; - } - - // find nb_x & nb_y offsets - std::unordered_map> output_nb_x_select; - std::unordered_map> output_nb_y_select; - // initialize offsets map - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector nb_x_offset(num_nb); - std::vector nb_y_offset(num_nb); - output_nb_x_select[i] = nb_x_offset; - output_nb_y_select[i] = nb_y_offset; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - for (unsigned int j = 0; j < num_nb; ++j) - { - const float *offset_nb_x_ptr = outputs_nb_x_ptr + (i * num_nb + j) * grid_length; - const float *offset_nb_y_ptr = outputs_nb_y_ptr + (i * num_nb + j) * grid_length; - const unsigned int max_id = max_ids.at(i); - output_nb_x_select[i][j] = offset_nb_x_ptr[max_id]; - output_nb_y_select[i][j] = offset_nb_y_ptr[max_id]; - } - } - - // calculate coords - std::vector lms_pred_x(num_lms); // 29 - std::vector lms_pred_y(num_lms); // 29 - std::unordered_map> lms_pred_nb_x; // 29,10 - std::unordered_map> lms_pred_nb_y; // 29,10 - // initialize pred maps - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector nb_x_offset(num_nb); - std::vector nb_y_offset(num_nb); - lms_pred_nb_x[i] = nb_x_offset; - lms_pred_nb_y[i] = nb_y_offset; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - float cx = static_cast(max_ids.at(i) % grid_w); - float cy = static_cast(max_ids.at(i) / grid_w); - // calculate coords & normalize - lms_pred_x[i] = ((cx + output_x_select[i]) * (float) net_stride) / (float) input_w; - lms_pred_y[i] = ((cy + output_y_select[i]) * (float) net_stride) / (float) input_h; - for (unsigned int j = 0; j < num_nb; ++j) - { - lms_pred_nb_x[i][j] = ((cx + output_nb_x_select[i][j]) * (float) net_stride) / (float) input_w; - lms_pred_nb_y[i][j] = ((cy + output_nb_y_select[i][j]) * (float) net_stride) / (float) input_h; - } - } - - // reverse indexes - std::unordered_map> tmp_nb_x; // 29,max_len - std::unordered_map> tmp_nb_y; // 29,max_len - // initialize reverse maps - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector tmp_x(max_len); - std::vector tmp_y(max_len); - tmp_nb_x[i] = tmp_x; - tmp_nb_y[i] = tmp_y; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - for (unsigned int j = 0; j < max_len; ++j) - { - unsigned int ri = reverse_index1[i * max_len + j]; - unsigned int rj = reverse_index2[i * max_len + j]; - tmp_nb_x[i][j] = lms_pred_nb_x[ri][rj]; - tmp_nb_y[i][j] = lms_pred_nb_y[ri][rj]; - } - } - - // merge predictions - landmarks.points.clear(); - for (unsigned int i = 0; i < num_lms; ++i) - { - float total_x = lms_pred_x[i]; - float total_y = lms_pred_y[i]; - for (unsigned int j = 0; j < max_len; ++j) - { - total_x += tmp_nb_x[i][j]; - total_y += tmp_nb_y[i][j]; - } - float x = total_x / ((float) max_len + 1.f); - float y = total_y / ((float) max_len + 1.f); - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - - landmarks.flag = true; -} diff --git a/lite/mnn/cv/mnn_pipnet29.h b/lite/mnn/cv/mnn_pipnet29.h deleted file mode 100644 index 71cc6ea0..00000000 --- a/lite/mnn/cv/mnn_pipnet29.h +++ /dev/null @@ -1,78 +0,0 @@ -// -// Created by DefTruth on 2022/3/20. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_PIPNET29_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_PIPNET29_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNPIPNet29 : public BasicMNNHandler - { - public: - explicit MNNPIPNet29(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNPIPNet29() override = default; - - private: - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - static constexpr const unsigned int num_nb = 10; - static constexpr const unsigned int num_lms = 29; - static constexpr const unsigned int max_len = 19; - static constexpr const unsigned int net_stride = 32; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // resize & normalize. - - void generate_landmarks(types::Landmarks &landmarks, - const std::map &output_tensors, - float img_height, float img_width); - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - - private: - const unsigned int reverse_index1[29 * 19] = { - 2, 4, 5, 8, 12, 13, 16, 2, 4, 5, 8, 12, 13, 16, 2, 4, 5, 8, 12, 3, 6, 7, 9, 14, 15, 17, 3, 6, 7, 9, 14, 15, 17, 3, 6, 7, 9, 14, 0, - 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 16, 0, 3, 4, 5, 6, 7, 0, 1, 2, 4, 5, 6, 7, 9, 10, 11, 12, 14, 15, 17, 0, 1, 2, 4, 5, 0, 2, 5, - 8, 10, 12, 13, 16, 0, 2, 5, 8, 10, 12, 13, 16, 0, 2, 5, 0, 2, 4, 8, 10, 12, 13, 16, 0, 2, 4, 8, 10, 12, 13, 16, 0, 2, 4, 1, 3, 7, 9, - 11, 14, 15, 17, 1, 3, 7, 9, 11, 14, 15, 17, 1, 3, 7, 1, 3, 6, 9, 11, 14, 15, 17, 1, 3, 6, 9, 11, 14, 15, 17, 1, 3, 6, 0, 2, 4, 5, - 10, 12, 13, 16, 0, 2, 4, 5, 10, 12, 13, 16, 0, 2, 4, 1, 3, 6, 7, 11, 14, 15, 17, 1, 3, 6, 7, 11, 14, 15, 17, 1, 3, 6, 0, 2, 3, 4, 5, - 8, 12, 13, 16, 18, 20, 0, 2, 3, 4, 5, 8, 12, 13, 1, 2, 3, 6, 7, 9, 14, 15, 17, 19, 20, 21, 1, 2, 3, 6, 7, 9, 14, 0, 2, 4, 5, 8, 10, - 13, 16, 0, 2, 4, 5, 8, 10, 13, 16, 0, 2, 4, 0, 2, 4, 5, 8, 10, 12, 16, 18, 22, 0, 2, 4, 5, 8, 10, 12, 16, 18, 1, 3, 6, 7, 9, 11, 15, - 17, 1, 3, 6, 7, 9, 11, 15, 17, 1, 3, 6, 1, 3, 6, 7, 9, 11, 14, 17, 19, 23, 1, 3, 6, 7, 9, 11, 14, 17, 19, 0, 2, 4, 5, 8, 10, 12, 13, - 18, 0, 2, 4, 5, 8, 10, 12, 13, 18, 0, 1, 3, 6, 7, 9, 11, 14, 15, 19, 1, 3, 6, 7, 9, 11, 14, 15, 19, 1, 0, 4, 5, 8, 10, 12, 13, 16, - 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 0, 1, 6, 7, 9, 11, 14, 15, 17, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 1, 1, 8, 9, 10, 11, - 13, 15, 16, 17, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 18, 19, 20, 22, 23, 24, 25, 26, 27, 28, 18, 19, 20, 22, 23, 24, 25, 26, 27, - 18, 20, 21, 24, 25, 26, 27, 28, 18, 20, 21, 24, 25, 26, 27, 28, 18, 20, 21, 19, 21, 24, 25, 26, 27, 28, 19, 21, 24, 25, 26, 27, 28, - 19, 21, 24, 25, 26, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 18, 19, 20, 21, 22, 23, 25, 26, 27, 18, 19, 20, 21, 22, 23, 24, 26, 27, - 28, 18, 19, 20, 21, 22, 23, 24, 26, 27, 18, 19, 20, 21, 22, 23, 24, 25, 27, 28, 18, 19, 20, 21, 22, 23, 24, 25, 27, 20, 21, 22, 23, - 24, 25, 26, 28, 20, 21, 22, 23, 24, 25, 26, 28, 20, 21, 22, 22, 23, 24, 25, 26, 27, 22, 23, 24, 25, 26, 27, 22, 23, 24, 25, 26, 27, - 22 - }; - const unsigned int reverse_index2[29 * 19] = { - 9, 3, 5, 3, 7, 7, 7, 9, 3, 5, 3, 7, 7, 7, 9, 3, 5, 3, 7, 9, 3, 5, 3, 7, 7, 7, 9, 3, 5, 3, 7, 7, 7, 9, 3, 5, 3, 7, 7, 6, 6, 6, 8, 9, - 7, 0, 9, 6, 5, 9, 6, 7, 6, 6, 6, 8, 9, 9, 7, 6, 8, 9, 6, 6, 7, 8, 0, 9, 6, 6, 6, 9, 7, 6, 8, 9, 2, 5, 0, 5, 5, 3, 6, 5, 2, 5, 0, 5, - 5, 3, 6, 5, 2, 5, 0, 1, 3, 0, 4, 4, 2, 4, 2, 1, 3, 0, 4, 4, 2, 4, 2, 1, 3, 0, 2, 4, 0, 5, 5, 3, 5, 5, 2, 4, 0, 5, 5, 3, 5, 5, 2, 4, - 0, 1, 3, 0, 4, 4, 2, 4, 2, 1, 3, 0, 4, 4, 2, 4, 2, 1, 3, 0, 0, 7, 4, 3, 6, 5, 3, 4, 0, 7, 4, 3, 6, 5, 3, 4, 0, 7, 4, 0, 7, 4, 3, 6, - 5, 2, 4, 0, 7, 4, 3, 6, 5, 2, 4, 0, 7, 4, 6, 0, 8, 7, 7, 6, 4, 2, 3, 5, 6, 6, 0, 8, 7, 7, 6, 4, 2, 6, 8, 0, 7, 7, 6, 4, 3, 3, 5, 7, - 9, 6, 8, 0, 7, 7, 6, 4, 3, 1, 1, 1, 2, 3, 1, 0, 3, 1, 1, 1, 2, 3, 1, 0, 3, 1, 1, 5, 4, 5, 4, 0, 2, 1, 1, 6, 9, 5, 4, 5, 4, 0, 2, 1, - 1, 6, 3, 1, 1, 1, 2, 3, 1, 0, 3, 1, 1, 1, 2, 3, 1, 0, 3, 1, 1, 5, 5, 5, 4, 0, 2, 1, 1, 7, 9, 5, 5, 5, 4, 0, 2, 1, 1, 7, 4, 2, 2, 2, - 1, 1, 0, 0, 9, 4, 2, 2, 2, 1, 1, 0, 0, 9, 4, 4, 2, 2, 2, 1, 1, 0, 0, 9, 4, 2, 2, 2, 1, 1, 0, 0, 9, 4, 8, 9, 8, 8, 7, 8, 8, 8, 8, 1, - 3, 0, 8, 5, 8, 9, 9, 9, 8, 8, 9, 8, 8, 7, 8, 8, 8, 8, 2, 4, 8, 0, 6, 7, 8, 8, 7, 8, 9, 9, 9, 9, 8, 9, 9, 9, 9, 0, 0, 0, 6, 6, 4, 4, - 6, 7, 8, 1, 1, 0, 5, 5, 2, 3, 3, 4, 6, 1, 1, 0, 5, 5, 2, 3, 3, 4, 2, 8, 7, 7, 5, 4, 6, 5, 2, 8, 7, 7, 5, 4, 6, 5, 2, 8, 7, 2, 8, 8, - 6, 5, 5, 4, 2, 8, 8, 6, 5, 5, 4, 2, 8, 8, 6, 5, 3, 3, 3, 1, 2, 3, 0, 2, 2, 3, 3, 3, 3, 1, 2, 3, 0, 2, 2, 4, 4, 4, 2, 1, 1, 0, 0, 1, - 2, 4, 4, 4, 2, 1, 1, 0, 0, 1, 7, 6, 5, 5, 3, 2, 1, 1, 0, 1, 7, 6, 5, 5, 3, 2, 1, 1, 0, 9, 6, 4, 4, 3, 2, 1, 0, 9, 6, 4, 4, 3, 2, 1, - 0, 9, 6, 4, 7, 7, 9, 9, 7, 3, 7, 7, 9, 9, 7, 3, 7, 7, 9, 9, 7, 3, 7 - }; - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_PIPNET29_H diff --git a/lite/mnn/cv/mnn_pipnet68.cpp b/lite/mnn/cv/mnn_pipnet68.cpp deleted file mode 100644 index 33f32bc9..00000000 --- a/lite/mnn/cv/mnn_pipnet68.cpp +++ /dev/null @@ -1,206 +0,0 @@ -// -// Created by DefTruth on 2022/3/20. -// - -#include "mnn_pipnet68.h" - -using mnncv::MNNPIPNet68; - -MNNPIPNet68::MNNPIPNet68(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNPIPNet68::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNPIPNet68::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,3,256,256) will do deepcopy inside MNN convert process - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNPIPNet68::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - // this->transform(mat); - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. generate landmarks - this->generate_landmarks(landmarks, output_tensors, img_height, img_width); -} - -void MNNPIPNet68::generate_landmarks(types::Landmarks &landmarks, - const std::map &output_tensors, - float img_height, float img_width) -{ - auto device_outputs_cls_ptr = output_tensors.at("outputs_cls"); // (1,68,8,8) - auto device_outputs_x_ptr = output_tensors.at("outputs_x"); // (1,68,8,8) - auto device_outputs_y_ptr = output_tensors.at("outputs_y"); // (1,68,8,8) - auto device_outputs_nb_x_ptr = output_tensors.at("outputs_nb_x"); // (1,68*10,8,8) - auto device_outputs_nb_y_ptr = output_tensors.at("outputs_nb_y"); // (1,68*10,8,8) - MNN::Tensor host_outputs_cls_tensor(device_outputs_cls_ptr, device_outputs_cls_ptr->getDimensionType()); - MNN::Tensor host_outputs_x_tensor(device_outputs_x_ptr, device_outputs_x_ptr->getDimensionType()); - MNN::Tensor host_outputs_y_tensor(device_outputs_y_ptr, device_outputs_y_ptr->getDimensionType()); - MNN::Tensor host_outputs_nb_x_tensor(device_outputs_nb_x_ptr, device_outputs_nb_x_ptr->getDimensionType()); - MNN::Tensor host_outputs_nb_y_tensor(device_outputs_nb_y_ptr, device_outputs_nb_y_ptr->getDimensionType()); - device_outputs_cls_ptr->copyToHostTensor(&host_outputs_cls_tensor); - device_outputs_x_ptr->copyToHostTensor(&host_outputs_x_tensor); - device_outputs_y_ptr->copyToHostTensor(&host_outputs_y_tensor); - device_outputs_nb_x_ptr->copyToHostTensor(&host_outputs_nb_x_tensor); - device_outputs_nb_y_ptr->copyToHostTensor(&host_outputs_nb_y_tensor); - - auto cls_shape = host_outputs_cls_tensor.shape(); - const unsigned int grid_h = cls_shape.at(2); // 8 - const unsigned int grid_w = cls_shape.at(3); // 8 - const unsigned int grid_length = grid_h * grid_w; // 8 * 8 = 64 - const unsigned int input_h = input_height; - const unsigned int input_w = input_width; - - const float *outputs_cls_ptr = host_outputs_cls_tensor.host(); - const float *outputs_x_ptr = host_outputs_x_tensor.host(); - const float *outputs_y_ptr = host_outputs_y_tensor.host(); - const float *outputs_nb_x_ptr = host_outputs_nb_x_tensor.host(); - const float *outputs_nb_y_ptr = host_outputs_nb_y_tensor.host(); - - // find max_ids - std::vector max_ids(num_lms); - for (unsigned int i = 0; i < num_lms; ++i) - { - const float *score_ptr = outputs_cls_ptr + i * grid_length; - unsigned int max_id = 0; - float max_score = score_ptr[0]; - for (unsigned int j = 0; j < grid_length; ++j) - { - if (score_ptr[j] > max_score) - { - max_score = score_ptr[j]; - max_id = j; - } - } - max_ids[i] = max_id; // range 0~64 - } - - // find x & y offsets - std::vector output_x_select(num_lms); - std::vector output_y_select(num_lms); - for (unsigned int i = 0; i < num_lms; ++i) - { - const float *offset_x_ptr = outputs_x_ptr + i * grid_length; - const float *offset_y_ptr = outputs_y_ptr + i * grid_length; - const unsigned int max_id = max_ids.at(i); - output_x_select[i] = offset_x_ptr[max_id]; - output_y_select[i] = offset_y_ptr[max_id]; - } - - // find nb_x & nb_y offsets - std::unordered_map> output_nb_x_select; - std::unordered_map> output_nb_y_select; - // initialize offsets map - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector nb_x_offset(num_nb); - std::vector nb_y_offset(num_nb); - output_nb_x_select[i] = nb_x_offset; - output_nb_y_select[i] = nb_y_offset; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - for (unsigned int j = 0; j < num_nb; ++j) - { - const float *offset_nb_x_ptr = outputs_nb_x_ptr + (i * num_nb + j) * grid_length; - const float *offset_nb_y_ptr = outputs_nb_y_ptr + (i * num_nb + j) * grid_length; - const unsigned int max_id = max_ids.at(i); - output_nb_x_select[i][j] = offset_nb_x_ptr[max_id]; - output_nb_y_select[i][j] = offset_nb_y_ptr[max_id]; - } - } - - // calculate coords - std::vector lms_pred_x(num_lms); // 68 - std::vector lms_pred_y(num_lms); // 68 - std::unordered_map> lms_pred_nb_x; // 68,10 - std::unordered_map> lms_pred_nb_y; // 68,10 - // initialize pred maps - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector nb_x_offset(num_nb); - std::vector nb_y_offset(num_nb); - lms_pred_nb_x[i] = nb_x_offset; - lms_pred_nb_y[i] = nb_y_offset; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - float cx = static_cast(max_ids.at(i) % grid_w); - float cy = static_cast(max_ids.at(i) / grid_w); - // calculate coords & normalize - lms_pred_x[i] = ((cx + output_x_select[i]) * (float) net_stride) / (float) input_w; - lms_pred_y[i] = ((cy + output_y_select[i]) * (float) net_stride) / (float) input_h; - for (unsigned int j = 0; j < num_nb; ++j) - { - lms_pred_nb_x[i][j] = ((cx + output_nb_x_select[i][j]) * (float) net_stride) / (float) input_w; - lms_pred_nb_y[i][j] = ((cy + output_nb_y_select[i][j]) * (float) net_stride) / (float) input_h; - } - } - - // reverse indexes - std::unordered_map> tmp_nb_x; // 68,max_len - std::unordered_map> tmp_nb_y; // 68,max_len - // initialize reverse maps - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector tmp_x(max_len); - std::vector tmp_y(max_len); - tmp_nb_x[i] = tmp_x; - tmp_nb_y[i] = tmp_y; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - for (unsigned int j = 0; j < max_len; ++j) - { - unsigned int ri = reverse_index1[i * max_len + j]; - unsigned int rj = reverse_index2[i * max_len + j]; - tmp_nb_x[i][j] = lms_pred_nb_x[ri][rj]; - tmp_nb_y[i][j] = lms_pred_nb_y[ri][rj]; - } - } - - // merge predictions - landmarks.points.clear(); - for (unsigned int i = 0; i < num_lms; ++i) - { - float total_x = lms_pred_x[i]; - float total_y = lms_pred_y[i]; - for (unsigned int j = 0; j < max_len; ++j) - { - total_x += tmp_nb_x[i][j]; - total_y += tmp_nb_y[i][j]; - } - float x = total_x / ((float) max_len + 1.f); - float y = total_y / ((float) max_len + 1.f); - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - - landmarks.flag = true; -} diff --git a/lite/mnn/cv/mnn_pipnet68.h b/lite/mnn/cv/mnn_pipnet68.h deleted file mode 100644 index 2b99a4e5..00000000 --- a/lite/mnn/cv/mnn_pipnet68.h +++ /dev/null @@ -1,127 +0,0 @@ -// -// Created by DefTruth on 2022/3/20. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_PIPNET68_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_PIPNET68_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNPIPNet68 : public BasicMNNHandler - { - public: - explicit MNNPIPNet68(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNPIPNet68() override = default; - - private: - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - static constexpr const unsigned int num_nb = 10; - static constexpr const unsigned int num_lms = 68; - static constexpr const unsigned int max_len = 22; - static constexpr const unsigned int net_stride = 32; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // resize & normalize. - - void generate_landmarks(types::Landmarks &landmarks, - const std::map &output_tensors, - float img_height, float img_width); - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - - private: - const unsigned int reverse_index1[68 * 22] = { - 1, 2, 17, 18, 36, 1, 2, 17, 18, 36, 1, 2, 17, 18, 36, 1, 2, 17, 18, 36, 1, 2, 0, 2, 3, 17, 0, 2, 3, 17, 0, 2, 3, 17, 0, 2, 3, 17, 0, - 2, 3, 17, 0, 2, 0, 1, 3, 4, 0, 1, 3, 4, 0, 1, 3, 4, 0, 1, 3, 4, 0, 1, 3, 4, 0, 1, 1, 2, 4, 5, 1, 2, 4, 5, 1, 2, 4, 5, 1, 2, 4, 5, 1, - 2, 4, 5, 1, 2, 2, 3, 5, 6, 2, 3, 5, 6, 2, 3, 5, 6, 2, 3, 5, 6, 2, 3, 5, 6, 2, 3, 3, 4, 6, 7, 3, 4, 6, 7, 3, 4, 6, 7, 3, 4, 6, 7, 3, - 4, 6, 7, 3, 4, 3, 4, 5, 7, 8, 3, 4, 5, 7, 8, 3, 4, 5, 7, 8, 3, 4, 5, 7, 8, 3, 4, 5, 6, 8, 9, 5, 6, 8, 9, 5, 6, 8, 9, 5, 6, 8, 9, 5, - 6, 8, 9, 5, 6, 6, 7, 9, 10, 6, 7, 9, 10, 6, 7, 9, 10, 6, 7, 9, 10, 6, 7, 9, 10, 6, 7, 7, 8, 10, 11, 7, 8, 10, 11, 7, 8, 10, 11, 7, - 8, 10, 11, 7, 8, 10, 11, 7, 8, 8, 9, 11, 12, 13, 8, 9, 11, 12, 13, 8, 9, 11, 12, 13, 8, 9, 11, 12, 13, 8, 9, 9, 10, 12, 13, 9, 10, - 12, 13, 9, 10, 12, 13, 9, 10, 12, 13, 9, 10, 12, 13, 9, 10, 10, 11, 13, 14, 10, 11, 13, 14, 10, 11, 13, 14, 10, 11, 13, 14, 10, 11, - 13, 14, 10, 11, 11, 12, 14, 15, 11, 12, 14, 15, 11, 12, 14, 15, 11, 12, 14, 15, 11, 12, 14, 15, 11, 12, 12, 13, 15, 16, 12, 13, 15, - 16, 12, 13, 15, 16, 12, 13, 15, 16, 12, 13, 15, 16, 12, 13, 13, 14, 16, 26, 13, 14, 16, 26, 13, 14, 16, 26, 13, 14, 16, 26, 13, 14, - 16, 26, 13, 14, 14, 15, 25, 26, 45, 14, 15, 25, 26, 45, 14, 15, 25, 26, 45, 14, 15, 25, 26, 45, 14, 15, 0, 1, 2, 18, 19, 36, 37, 41, - 0, 1, 2, 18, 19, 36, 37, 41, 0, 1, 2, 18, 19, 36, 0, 1, 17, 19, 20, 36, 37, 38, 41, 0, 1, 17, 19, 20, 36, 37, 38, 41, 0, 1, 17, 19, - 0, 17, 18, 20, 21, 36, 37, 38, 40, 41, 0, 17, 18, 20, 21, 36, 37, 38, 40, 41, 0, 17, 17, 18, 19, 21, 36, 37, 38, 39, 40, 41, 17, 18, - 19, 21, 36, 37, 38, 39, 40, 41, 17, 18, 18, 19, 20, 22, 27, 28, 37, 38, 39, 40, 41, 18, 19, 20, 22, 27, 28, 37, 38, 39, 40, 41, 21, - 23, 24, 25, 27, 28, 42, 43, 44, 46, 47, 21, 23, 24, 25, 27, 28, 42, 43, 44, 46, 47, 22, 24, 25, 26, 42, 43, 44, 45, 46, 47, 22, 24, - 25, 26, 42, 43, 44, 45, 46, 47, 22, 24, 16, 22, 23, 25, 26, 43, 44, 45, 46, 47, 16, 22, 23, 25, 26, 43, 44, 45, 46, 47, 16, 22, 15, - 16, 23, 24, 26, 43, 44, 45, 46, 15, 16, 23, 24, 26, 43, 44, 45, 46, 15, 16, 23, 24, 14, 15, 16, 24, 25, 44, 45, 46, 14, 15, 16, 24, - 25, 44, 45, 46, 14, 15, 16, 24, 25, 44, 20, 21, 22, 23, 28, 29, 38, 39, 40, 42, 43, 47, 20, 21, 22, 23, 28, 29, 38, 39, 40, 42, 21, - 22, 27, 29, 30, 39, 40, 42, 47, 21, 22, 27, 29, 30, 39, 40, 42, 47, 21, 22, 27, 29, 27, 28, 30, 31, 35, 39, 42, 27, 28, 30, 31, 35, - 39, 42, 27, 28, 30, 31, 35, 39, 42, 27, 28, 29, 31, 32, 33, 34, 35, 28, 29, 31, 32, 33, 34, 35, 28, 29, 31, 32, 33, 34, 35, 28, 2, - 3, 29, 30, 32, 33, 48, 49, 2, 3, 29, 30, 32, 33, 48, 49, 2, 3, 29, 30, 32, 33, 29, 30, 31, 33, 34, 35, 49, 50, 29, 30, 31, 33, 34, - 35, 49, 50, 29, 30, 31, 33, 34, 35, 29, 30, 31, 32, 34, 35, 50, 51, 52, 29, 30, 31, 32, 34, 35, 50, 51, 52, 29, 30, 31, 32, 29, 30, - 31, 32, 33, 35, 52, 53, 29, 30, 31, 32, 33, 35, 52, 53, 29, 30, 31, 32, 33, 35, 13, 14, 29, 30, 32, 33, 34, 53, 54, 13, 14, 29, 30, - 32, 33, 34, 53, 54, 13, 14, 29, 30, 0, 1, 2, 17, 18, 19, 20, 37, 38, 39, 40, 41, 0, 1, 2, 17, 18, 19, 20, 37, 38, 39, 0, 1, 17, 18, - 19, 20, 21, 36, 38, 39, 40, 41, 0, 1, 17, 18, 19, 20, 21, 36, 38, 39, 0, 1, 17, 18, 19, 20, 21, 27, 28, 36, 37, 39, 40, 41, 0, 1, - 17, 18, 19, 20, 21, 27, 19, 20, 21, 27, 28, 29, 36, 37, 38, 40, 41, 19, 20, 21, 27, 28, 29, 36, 37, 38, 40, 41, 0, 1, 17, 18, 19, - 20, 21, 27, 28, 36, 37, 38, 39, 41, 0, 1, 17, 18, 19, 20, 21, 27, 0, 1, 2, 17, 18, 19, 20, 21, 36, 37, 38, 39, 40, 0, 1, 2, 17, 18, - 19, 20, 21, 36, 22, 23, 24, 27, 28, 29, 43, 44, 45, 46, 47, 22, 23, 24, 27, 28, 29, 43, 44, 45, 46, 47, 15, 16, 22, 23, 24, 25, 26, - 27, 42, 44, 45, 46, 47, 15, 16, 22, 23, 24, 25, 26, 27, 42, 15, 16, 22, 23, 24, 25, 26, 42, 43, 45, 46, 47, 15, 16, 22, 23, 24, 25, - 26, 42, 43, 45, 14, 15, 16, 23, 24, 25, 26, 42, 43, 44, 46, 47, 14, 15, 16, 23, 24, 25, 26, 42, 43, 44, 14, 15, 16, 22, 23, 24, 25, - 26, 42, 43, 44, 45, 47, 14, 15, 16, 22, 23, 24, 25, 26, 42, 15, 16, 22, 23, 24, 25, 26, 27, 28, 42, 43, 44, 45, 46, 15, 16, 22, 23, - 24, 25, 26, 27, 2, 3, 4, 5, 6, 49, 59, 60, 2, 3, 4, 5, 6, 49, 59, 60, 2, 3, 4, 5, 6, 49, 3, 4, 5, 31, 32, 48, 50, 51, 59, 60, 61, - 67, 3, 4, 5, 31, 32, 48, 50, 51, 59, 60, 30, 31, 32, 33, 34, 48, 49, 51, 52, 58, 59, 60, 61, 62, 66, 67, 30, 31, 32, 33, 34, 48, 30, - 31, 32, 33, 34, 35, 48, 49, 50, 52, 53, 54, 56, 58, 60, 61, 62, 63, 64, 65, 66, 67, 30, 32, 33, 34, 35, 50, 51, 53, 54, 55, 56, 62, - 63, 64, 65, 30, 32, 33, 34, 35, 50, 51, 11, 12, 13, 34, 35, 52, 54, 55, 63, 64, 65, 11, 12, 13, 34, 35, 52, 54, 55, 63, 64, 65, 10, - 11, 12, 13, 14, 53, 55, 64, 10, 11, 12, 13, 14, 53, 55, 64, 10, 11, 12, 13, 14, 53, 8, 9, 10, 11, 12, 13, 53, 54, 56, 57, 63, 64, - 65, 8, 9, 10, 11, 12, 13, 53, 54, 56, 7, 8, 9, 10, 11, 12, 54, 55, 57, 58, 63, 64, 65, 66, 7, 8, 9, 10, 11, 12, 54, 55, 6, 7, 8, 9, - 10, 55, 56, 58, 59, 62, 65, 66, 67, 6, 7, 8, 9, 10, 55, 56, 58, 59, 4, 5, 6, 7, 8, 9, 48, 56, 57, 59, 60, 61, 62, 66, 67, 4, 5, 6, - 7, 8, 9, 48, 3, 4, 5, 6, 7, 8, 48, 49, 57, 58, 60, 61, 67, 3, 4, 5, 6, 7, 8, 48, 49, 57, 2, 3, 4, 5, 6, 31, 48, 49, 59, 2, 3, 4, 5, - 6, 31, 48, 49, 59, 2, 3, 4, 5, 31, 32, 33, 48, 49, 50, 51, 52, 57, 58, 59, 60, 62, 63, 66, 67, 31, 32, 33, 48, 49, 50, 33, 34, 48, - 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 63, 64, 65, 66, 67, 33, 34, 35, 50, 51, 52, 53, 54, 55, 56, 57, 61, 62, 64, 65, - 66, 34, 35, 50, 51, 52, 53, 54, 10, 11, 12, 13, 14, 35, 53, 54, 55, 10, 11, 12, 13, 14, 35, 53, 54, 55, 10, 11, 12, 13, 9, 10, 11, - 12, 51, 52, 53, 54, 55, 56, 57, 58, 61, 62, 63, 64, 66, 67, 9, 10, 11, 12, 7, 8, 9, 50, 51, 52, 55, 56, 57, 58, 59, 60, 61, 62, 63, - 64, 65, 67, 7, 8, 9, 50, 4, 5, 6, 7, 48, 49, 50, 51, 56, 57, 58, 59, 60, 61, 62, 63, 65, 66, 4, 5, 6, 7 - }; - const unsigned int reverse_index2[68 * 22] = { - 0, 3, 1, 7, 8, 0, 3, 1, 7, 8, 0, 3, 1, 7, 8, 0, 3, 1, 7, 8, 0, 3, 1, 1, 4, 9, 1, 1, 4, 9, 1, 1, 4, 9, 1, 1, 4, 9, 1, 1, 4, 9, 1, 1, - 6, 1, 1, 5, 6, 1, 1, 5, 6, 1, 1, 5, 6, 1, 1, 5, 6, 1, 1, 5, 6, 1, 5, 0, 0, 6, 5, 0, 0, 6, 5, 0, 0, 6, 5, 0, 0, 6, 5, 0, 0, 6, 5, 0, - 2, 0, 1, 7, 2, 0, 1, 7, 2, 0, 1, 7, 2, 0, 1, 7, 2, 0, 1, 7, 2, 0, 2, 1, 1, 6, 2, 1, 1, 6, 2, 1, 1, 6, 2, 1, 1, 6, 2, 1, 1, 6, 2, 1, - 9, 4, 0, 1, 4, 9, 4, 0, 1, 4, 9, 4, 0, 1, 4, 9, 4, 0, 1, 4, 9, 4, 5, 0, 1, 3, 5, 0, 1, 3, 5, 0, 1, 3, 5, 0, 1, 3, 5, 0, 1, 3, 5, 0, - 4, 0, 0, 4, 4, 0, 0, 4, 4, 0, 0, 4, 4, 0, 0, 4, 4, 0, 0, 4, 4, 0, 3, 0, 0, 5, 3, 0, 0, 5, 3, 0, 0, 5, 3, 0, 0, 5, 3, 0, 0, 5, 3, 0, - 3, 1, 0, 4, 9, 3, 1, 0, 4, 9, 3, 1, 0, 4, 9, 3, 1, 0, 4, 9, 3, 1, 6, 1, 0, 2, 6, 1, 0, 2, 6, 1, 0, 2, 6, 1, 0, 2, 6, 1, 0, 2, 6, 1, - 7, 1, 0, 2, 7, 1, 0, 2, 7, 1, 0, 2, 7, 1, 0, 2, 7, 1, 0, 2, 7, 1, 6, 1, 1, 4, 6, 1, 1, 4, 6, 1, 1, 4, 6, 1, 1, 4, 6, 1, 1, 4, 6, 1, - 5, 1, 0, 6, 5, 1, 0, 6, 5, 1, 0, 6, 5, 1, 0, 6, 5, 1, 0, 6, 5, 1, 3, 0, 0, 9, 3, 0, 0, 9, 3, 0, 0, 9, 3, 0, 0, 9, 3, 0, 0, 9, 3, 0, - 3, 1, 7, 2, 8, 3, 1, 7, 2, 8, 3, 1, 7, 2, 8, 3, 1, 7, 2, 8, 3, 1, 0, 3, 9, 0, 4, 4, 8, 6, 0, 3, 9, 0, 4, 4, 8, 6, 0, 3, 9, 0, 4, 4, - 3, 8, 0, 0, 6, 5, 7, 9, 7, 3, 8, 0, 0, 6, 5, 7, 9, 7, 3, 8, 0, 0, 7, 4, 1, 1, 6, 6, 5, 7, 9, 5, 7, 4, 1, 1, 6, 6, 5, 7, 9, 5, 7, 4, - 8, 4, 1, 0, 9, 6, 4, 7, 6, 8, 8, 4, 1, 0, 9, 6, 4, 7, 6, 8, 8, 4, 9, 6, 0, 4, 2, 7, 9, 6, 5, 5, 9, 9, 6, 0, 4, 2, 7, 9, 6, 5, 5, 9, - 4, 1, 6, 9, 3, 8, 5, 6, 9, 9, 6, 4, 1, 6, 9, 3, 8, 5, 6, 9, 9, 6, 0, 1, 4, 8, 7, 5, 7, 9, 8, 5, 0, 1, 4, 8, 7, 5, 7, 9, 8, 5, 0, 1, - 7, 6, 0, 1, 4, 7, 5, 6, 6, 9, 7, 6, 0, 1, 4, 7, 5, 6, 6, 9, 7, 6, 8, 3, 5, 0, 0, 9, 6, 5, 7, 8, 3, 5, 0, 0, 9, 6, 5, 7, 8, 3, 5, 0, - 8, 3, 1, 4, 0, 8, 4, 5, 8, 3, 1, 4, 0, 8, 4, 5, 8, 3, 1, 4, 0, 8, 9, 1, 1, 9, 1, 2, 8, 4, 7, 2, 8, 7, 9, 1, 1, 9, 1, 2, 8, 4, 7, 2, - 8, 8, 0, 0, 6, 6, 8, 6, 8, 8, 8, 0, 0, 6, 6, 8, 6, 8, 8, 8, 0, 0, 5, 0, 0, 9, 9, 9, 9, 5, 0, 0, 9, 9, 9, 9, 5, 0, 0, 9, 9, 9, 9, 5, - 4, 1, 2, 2, 2, 2, 2, 4, 1, 2, 2, 2, 2, 2, 4, 1, 2, 2, 2, 2, 2, 4, 8, 8, 6, 5, 0, 7, 7, 9, 8, 8, 6, 5, 0, 7, 7, 9, 8, 8, 6, 5, 0, 7, - 4, 3, 0, 0, 4, 5, 8, 7, 4, 3, 0, 0, 4, 5, 8, 7, 4, 3, 0, 0, 4, 5, 7, 2, 1, 1, 1, 1, 5, 8, 5, 7, 2, 1, 1, 1, 1, 5, 8, 5, 7, 2, 1, 1, - 3, 1, 5, 4, 1, 0, 6, 9, 3, 1, 5, 4, 1, 0, 6, 9, 3, 1, 5, 4, 1, 0, 8, 9, 5, 4, 9, 6, 0, 8, 7, 8, 9, 5, 4, 9, 6, 0, 8, 7, 8, 9, 5, 4, - 2, 2, 4, 2, 3, 5, 8, 1, 5, 8, 4, 1, 2, 2, 4, 2, 3, 5, 8, 1, 5, 8, 5, 6, 3, 2, 2, 3, 7, 1, 1, 3, 3, 0, 5, 6, 3, 2, 2, 3, 7, 1, 1, 3, - 9, 9, 6, 6, 3, 2, 2, 7, 9, 3, 2, 1, 0, 3, 9, 9, 6, 6, 3, 2, 2, 7, 9, 4, 3, 4, 3, 9, 7, 4, 2, 1, 4, 9, 4, 3, 4, 3, 9, 7, 4, 2, 1, 4, - 8, 7, 7, 8, 8, 5, 5, 8, 5, 2, 3, 0, 0, 2, 8, 7, 7, 8, 8, 5, 5, 8, 4, 4, 5, 5, 5, 7, 7, 9, 0, 0, 3, 2, 2, 4, 4, 5, 5, 5, 7, 7, 9, 0, - 3, 4, 9, 1, 2, 8, 2, 4, 7, 4, 2, 3, 4, 9, 1, 2, 8, 2, 4, 7, 4, 2, 9, 9, 2, 2, 3, 6, 6, 6, 1, 2, 3, 3, 0, 9, 9, 2, 2, 3, 6, 6, 6, 1, - 6, 5, 7, 3, 2, 2, 3, 4, 1, 1, 1, 3, 6, 5, 7, 3, 2, 2, 3, 4, 1, 1, 4, 2, 2, 8, 5, 3, 1, 8, 4, 1, 0, 4, 4, 2, 2, 8, 5, 3, 1, 8, 4, 1, - 5, 5, 4, 9, 7, 7, 5, 5, 3, 3, 0, 0, 1, 5, 5, 4, 9, 7, 7, 5, 5, 3, 7, 8, 5, 6, 8, 8, 7, 9, 6, 0, 0, 3, 2, 2, 7, 8, 5, 6, 8, 8, 7, 9, - 6, 3, 2, 2, 5, 3, 3, 0, 6, 3, 2, 2, 5, 3, 3, 0, 6, 3, 2, 2, 5, 3, 6, 7, 8, 4, 6, 1, 3, 9, 4, 1, 5, 8, 6, 7, 8, 4, 6, 1, 3, 9, 4, 1, - 7, 3, 3, 4, 8, 5, 1, 1, 7, 9, 8, 5, 1, 6, 9, 5, 7, 3, 3, 4, 8, 5, 9, 6, 5, 3, 5, 6, 9, 6, 1, 1, 6, 9, 8, 8, 8, 3, 0, 3, 8, 6, 6, 6, - 8, 8, 5, 3, 3, 8, 2, 1, 5, 8, 9, 7, 1, 5, 4, 8, 8, 5, 3, 3, 8, 2, 8, 7, 6, 6, 4, 3, 1, 3, 5, 1, 8, 8, 7, 6, 6, 4, 3, 1, 3, 5, 1, 8, - 5, 2, 2, 4, 6, 2, 4, 0, 5, 2, 2, 4, 6, 2, 4, 0, 5, 2, 2, 4, 6, 2, 7, 5, 2, 3, 6, 7, 5, 2, 2, 9, 8, 2, 5, 7, 5, 2, 3, 6, 7, 5, 2, 2, - 7, 5, 2, 3, 7, 8, 6, 0, 1, 5, 7, 6, 3, 8, 7, 5, 2, 3, 7, 8, 6, 0, 8, 4, 2, 4, 8, 7, 0, 0, 7, 8, 7, 4, 7, 8, 4, 2, 4, 8, 7, 0, 0, 7, - 9, 7, 3, 2, 6, 7, 6, 5, 0, 0, 6, 7, 9, 7, 3, 9, 7, 3, 2, 6, 7, 6, 7, 6, 3, 2, 5, 8, 2, 5, 8, 2, 2, 8, 4, 7, 6, 3, 2, 5, 8, 2, 5, 8, - 7, 5, 3, 4, 6, 8, 0, 0, 1, 7, 5, 3, 4, 6, 8, 0, 0, 1, 7, 5, 3, 4, 7, 7, 9, 3, 2, 0, 3, 9, 6, 4, 5, 3, 2, 6, 3, 0, 7, 7, 9, 3, 2, 0, - 8, 9, 8, 7, 2, 0, 2, 7, 8, 9, 6, 5, 6, 9, 7, 2, 2, 7, 2, 0, 2, 8, 7, 7, 9, 4, 0, 3, 3, 5, 4, 7, 6, 3, 3, 0, 5, 7, 7, 9, 4, 0, 3, 3, - 6, 4, 3, 5, 7, 8, 0, 0, 1, 6, 4, 3, 5, 7, 8, 0, 0, 1, 6, 4, 3, 5, 8, 9, 9, 9, 7, 4, 4, 4, 2, 1, 4, 7, 9, 5, 0, 4, 2, 9, 8, 9, 9, 9, - 9, 9, 9, 6, 5, 8, 6, 3, 2, 3, 6, 9, 4, 1, 4, 9, 1, 1, 9, 9, 9, 6, 8, 9, 9, 8, 4, 4, 4, 6, 7, 3, 1, 2, 4, 0, 4, 9, 9, 1, 8, 9, 9, 8 - }; - - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_PIPNET68_H diff --git a/lite/mnn/cv/mnn_pipnet98.cpp b/lite/mnn/cv/mnn_pipnet98.cpp deleted file mode 100644 index 15b6c320..00000000 --- a/lite/mnn/cv/mnn_pipnet98.cpp +++ /dev/null @@ -1,206 +0,0 @@ -// -// Created by DefTruth on 2022/3/20. -// - -#include "mnn_pipnet98.h" - -using mnncv::MNNPIPNet98; - -MNNPIPNet98::MNNPIPNet98(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNPIPNet98::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNPIPNet98::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,3,256,256) will do deepcopy inside MNN convert process - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNPIPNet98::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - // this->transform(mat); - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. generate landmarks - this->generate_landmarks(landmarks, output_tensors, img_height, img_width); -} - -void MNNPIPNet98::generate_landmarks(types::Landmarks &landmarks, - const std::map &output_tensors, - float img_height, float img_width) -{ - auto device_outputs_cls_ptr = output_tensors.at("outputs_cls"); // (1,98,8,8) - auto device_outputs_x_ptr = output_tensors.at("outputs_x"); // (1,98,8,8) - auto device_outputs_y_ptr = output_tensors.at("outputs_y"); // (1,98,8,8) - auto device_outputs_nb_x_ptr = output_tensors.at("outputs_nb_x"); // (1,98*10,8,8) - auto device_outputs_nb_y_ptr = output_tensors.at("outputs_nb_y"); // (1,98*10,8,8) - MNN::Tensor host_outputs_cls_tensor(device_outputs_cls_ptr, device_outputs_cls_ptr->getDimensionType()); - MNN::Tensor host_outputs_x_tensor(device_outputs_x_ptr, device_outputs_x_ptr->getDimensionType()); - MNN::Tensor host_outputs_y_tensor(device_outputs_y_ptr, device_outputs_y_ptr->getDimensionType()); - MNN::Tensor host_outputs_nb_x_tensor(device_outputs_nb_x_ptr, device_outputs_nb_x_ptr->getDimensionType()); - MNN::Tensor host_outputs_nb_y_tensor(device_outputs_nb_y_ptr, device_outputs_nb_y_ptr->getDimensionType()); - device_outputs_cls_ptr->copyToHostTensor(&host_outputs_cls_tensor); - device_outputs_x_ptr->copyToHostTensor(&host_outputs_x_tensor); - device_outputs_y_ptr->copyToHostTensor(&host_outputs_y_tensor); - device_outputs_nb_x_ptr->copyToHostTensor(&host_outputs_nb_x_tensor); - device_outputs_nb_y_ptr->copyToHostTensor(&host_outputs_nb_y_tensor); - - auto cls_shape = host_outputs_cls_tensor.shape(); - const unsigned int grid_h = cls_shape.at(2); // 8 - const unsigned int grid_w = cls_shape.at(3); // 8 - const unsigned int grid_length = grid_h * grid_w; // 8 * 8 = 64 - const unsigned int input_h = input_height; - const unsigned int input_w = input_width; - - const float *outputs_cls_ptr = host_outputs_cls_tensor.host(); - const float *outputs_x_ptr = host_outputs_x_tensor.host(); - const float *outputs_y_ptr = host_outputs_y_tensor.host(); - const float *outputs_nb_x_ptr = host_outputs_nb_x_tensor.host(); - const float *outputs_nb_y_ptr = host_outputs_nb_y_tensor.host(); - - // find max_ids - std::vector max_ids(num_lms); - for (unsigned int i = 0; i < num_lms; ++i) - { - const float *score_ptr = outputs_cls_ptr + i * grid_length; - unsigned int max_id = 0; - float max_score = score_ptr[0]; - for (unsigned int j = 0; j < grid_length; ++j) - { - if (score_ptr[j] > max_score) - { - max_score = score_ptr[j]; - max_id = j; - } - } - max_ids[i] = max_id; // range 0~64 - } - - // find x & y offsets - std::vector output_x_select(num_lms); - std::vector output_y_select(num_lms); - for (unsigned int i = 0; i < num_lms; ++i) - { - const float *offset_x_ptr = outputs_x_ptr + i * grid_length; - const float *offset_y_ptr = outputs_y_ptr + i * grid_length; - const unsigned int max_id = max_ids.at(i); - output_x_select[i] = offset_x_ptr[max_id]; - output_y_select[i] = offset_y_ptr[max_id]; - } - - // find nb_x & nb_y offsets - std::unordered_map> output_nb_x_select; - std::unordered_map> output_nb_y_select; - // initialize offsets map - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector nb_x_offset(num_nb); - std::vector nb_y_offset(num_nb); - output_nb_x_select[i] = nb_x_offset; - output_nb_y_select[i] = nb_y_offset; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - for (unsigned int j = 0; j < num_nb; ++j) - { - const float *offset_nb_x_ptr = outputs_nb_x_ptr + (i * num_nb + j) * grid_length; - const float *offset_nb_y_ptr = outputs_nb_y_ptr + (i * num_nb + j) * grid_length; - const unsigned int max_id = max_ids.at(i); - output_nb_x_select[i][j] = offset_nb_x_ptr[max_id]; - output_nb_y_select[i][j] = offset_nb_y_ptr[max_id]; - } - } - - // calculate coords - std::vector lms_pred_x(num_lms); // 98 - std::vector lms_pred_y(num_lms); // 98 - std::unordered_map> lms_pred_nb_x; // 98,10 - std::unordered_map> lms_pred_nb_y; // 98,10 - // initialize pred maps - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector nb_x_offset(num_nb); - std::vector nb_y_offset(num_nb); - lms_pred_nb_x[i] = nb_x_offset; - lms_pred_nb_y[i] = nb_y_offset; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - float cx = static_cast(max_ids.at(i) % grid_w); - float cy = static_cast(max_ids.at(i) / grid_w); - // calculate coords & normalize - lms_pred_x[i] = ((cx + output_x_select[i]) * (float) net_stride) / (float) input_w; - lms_pred_y[i] = ((cy + output_y_select[i]) * (float) net_stride) / (float) input_h; - for (unsigned int j = 0; j < num_nb; ++j) - { - lms_pred_nb_x[i][j] = ((cx + output_nb_x_select[i][j]) * (float) net_stride) / (float) input_w; - lms_pred_nb_y[i][j] = ((cy + output_nb_y_select[i][j]) * (float) net_stride) / (float) input_h; - } - } - - // reverse indexes - std::unordered_map> tmp_nb_x; // 98,max_len - std::unordered_map> tmp_nb_y; // 98,max_len - // initialize reverse maps - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector tmp_x(max_len); - std::vector tmp_y(max_len); - tmp_nb_x[i] = tmp_x; - tmp_nb_y[i] = tmp_y; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - for (unsigned int j = 0; j < max_len; ++j) - { - unsigned int ri = reverse_index1[i * max_len + j]; - unsigned int rj = reverse_index2[i * max_len + j]; - tmp_nb_x[i][j] = lms_pred_nb_x[ri][rj]; - tmp_nb_y[i][j] = lms_pred_nb_y[ri][rj]; - } - } - - // merge predictions - landmarks.points.clear(); - for (unsigned int i = 0; i < num_lms; ++i) - { - float total_x = lms_pred_x[i]; - float total_y = lms_pred_y[i]; - for (unsigned int j = 0; j < max_len; ++j) - { - total_x += tmp_nb_x[i][j]; - total_y += tmp_nb_y[i][j]; - } - float x = total_x / ((float) max_len + 1.f); - float y = total_y / ((float) max_len + 1.f); - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - - landmarks.flag = true; -} diff --git a/lite/mnn/cv/mnn_pipnet98.h b/lite/mnn/cv/mnn_pipnet98.h deleted file mode 100644 index 7c9b1875..00000000 --- a/lite/mnn/cv/mnn_pipnet98.h +++ /dev/null @@ -1,135 +0,0 @@ -// -// Created by DefTruth on 2022/3/20. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_PIPNET98_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_PIPNET98_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNPIPNet98 : public BasicMNNHandler - { - public: - explicit MNNPIPNet98(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNPIPNet98() override = default; - - private: - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - static constexpr const unsigned int num_nb = 10; - static constexpr const unsigned int num_lms = 98; - static constexpr const unsigned int max_len = 17; - static constexpr const unsigned int net_stride = 32; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // resize & normalize. - - void generate_landmarks(types::Landmarks &landmarks, - const std::map &output_tensors, - float img_height, float img_width); - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - - private: - const unsigned int reverse_index1[98 * 17] = { - 1, 2, 3, 4, 5, 33, 1, 2, 3, 4, 5, 33, 1, 2, 3, 4, 5, 0, 2, 3, 4, 5, 6, 33, 0, 2, 3, 4, 5, 6, 33, 0, 2, 3, 0, 1, 3, 4, 5, 6, 0, 1, 3, - 4, 5, 6, 0, 1, 3, 4, 5, 0, 1, 2, 4, 5, 6, 7, 0, 1, 2, 4, 5, 6, 7, 0, 1, 2, 0, 1, 2, 3, 5, 6, 7, 8, 0, 1, 2, 3, 5, 6, 7, 8, 0, 1, 2, - 3, 4, 6, 7, 8, 9, 1, 2, 3, 4, 6, 7, 8, 9, 1, 2, 3, 4, 5, 7, 8, 9, 10, 2, 3, 4, 5, 7, 8, 9, 10, 2, 3, 4, 5, 6, 8, 9, 10, 3, 4, 5, 6, - 8, 9, 10, 3, 4, 5, 4, 5, 6, 7, 9, 10, 11, 4, 5, 6, 7, 9, 10, 11, 4, 5, 6, 4, 5, 6, 7, 8, 10, 11, 12, 4, 5, 6, 7, 8, 10, 11, 12, 4, - 5, 6, 7, 8, 9, 11, 12, 13, 76, 5, 6, 7, 8, 9, 11, 12, 13, 7, 8, 9, 10, 12, 13, 14, 76, 88, 7, 8, 9, 10, 12, 13, 14, 76, 8, 9, 10, - 11, 13, 14, 15, 8, 9, 10, 11, 13, 14, 15, 8, 9, 10, 10, 11, 12, 14, 15, 16, 10, 11, 12, 14, 15, 16, 10, 11, 12, 14, 15, 11, 12, 13, - 15, 16, 17, 11, 12, 13, 15, 16, 17, 11, 12, 13, 15, 16, 12, 13, 14, 16, 17, 18, 12, 13, 14, 16, 17, 18, 12, 13, 14, 16, 17, 13, 14, - 15, 17, 18, 19, 13, 14, 15, 17, 18, 19, 13, 14, 15, 17, 18, 14, 15, 16, 18, 19, 20, 14, 15, 16, 18, 19, 20, 14, 15, 16, 18, 19, 15, - 16, 17, 19, 20, 21, 15, 16, 17, 19, 20, 21, 15, 16, 17, 19, 20, 16, 17, 18, 20, 21, 22, 16, 17, 18, 20, 21, 22, 16, 17, 18, 20, 21, - 17, 18, 19, 21, 22, 23, 24, 17, 18, 19, 21, 22, 23, 24, 17, 18, 19, 18, 19, 20, 22, 23, 24, 25, 82, 18, 19, 20, 22, 23, 24, 25, 82, - 18, 19, 20, 21, 23, 24, 25, 26, 27, 19, 20, 21, 23, 24, 25, 26, 27, 19, 20, 21, 22, 24, 25, 26, 27, 28, 20, 21, 22, 24, 25, 26, 27, - 28, 20, 21, 22, 23, 25, 26, 27, 28, 21, 22, 23, 25, 26, 27, 28, 21, 22, 23, 21, 22, 23, 24, 26, 27, 28, 29, 21, 22, 23, 24, 26, 27, - 28, 29, 21, 22, 23, 24, 25, 27, 28, 29, 30, 22, 23, 24, 25, 27, 28, 29, 30, 22, 23, 24, 25, 26, 28, 29, 30, 31, 23, 24, 25, 26, 28, - 29, 30, 31, 23, 24, 25, 26, 27, 29, 30, 31, 32, 24, 25, 26, 27, 29, 30, 31, 32, 24, 25, 26, 27, 28, 30, 31, 32, 25, 26, 27, 28, 30, - 31, 32, 25, 26, 27, 26, 27, 28, 29, 31, 32, 26, 27, 28, 29, 31, 32, 26, 27, 28, 29, 31, 26, 27, 28, 29, 30, 32, 46, 26, 27, 28, 29, - 30, 32, 46, 26, 27, 28, 27, 28, 29, 30, 31, 46, 27, 28, 29, 30, 31, 46, 27, 28, 29, 30, 31, 0, 1, 2, 3, 34, 41, 60, 0, 1, 2, 3, 34, - 41, 60, 0, 1, 2, 0, 33, 35, 40, 41, 60, 0, 33, 35, 40, 41, 60, 0, 33, 35, 40, 41, 33, 34, 36, 37, 39, 40, 41, 60, 61, 62, 33, 34, - 36, 37, 39, 40, 41, 34, 35, 37, 38, 39, 40, 63, 64, 34, 35, 37, 38, 39, 40, 63, 64, 34, 36, 38, 39, 51, 64, 36, 38, 39, 51, 64, 36, - 38, 39, 51, 64, 36, 38, 36, 37, 39, 51, 52, 63, 64, 65, 36, 37, 39, 51, 52, 63, 64, 65, 36, 35, 36, 37, 38, 40, 62, 63, 64, 65, 66, - 67, 96, 35, 36, 37, 38, 40, 33, 34, 35, 36, 37, 38, 39, 41, 60, 61, 62, 63, 65, 66, 67, 96, 33, 0, 1, 2, 33, 34, 35, 40, 60, 61, 67, - 0, 1, 2, 33, 34, 35, 40, 43, 49, 50, 51, 68, 43, 49, 50, 51, 68, 43, 49, 50, 51, 68, 43, 49, 42, 44, 45, 48, 49, 50, 68, 69, 42, 44, - 45, 48, 49, 50, 68, 69, 42, 42, 43, 45, 46, 47, 48, 49, 70, 42, 43, 45, 46, 47, 48, 49, 70, 42, 32, 44, 46, 47, 48, 71, 72, 73, 32, - 44, 46, 47, 48, 71, 72, 73, 32, 29, 30, 31, 32, 45, 47, 72, 29, 30, 31, 32, 45, 47, 72, 29, 30, 31, 30, 31, 32, 44, 45, 46, 48, 71, - 72, 73, 30, 31, 32, 44, 45, 46, 48, 42, 43, 44, 45, 46, 47, 49, 50, 69, 70, 71, 72, 73, 74, 75, 97, 42, 42, 43, 44, 48, 50, 68, 69, - 70, 74, 75, 97, 42, 43, 44, 48, 50, 68, 42, 43, 49, 51, 52, 68, 69, 75, 42, 43, 49, 51, 52, 68, 69, 75, 42, 37, 38, 42, 50, 52, 53, - 64, 68, 37, 38, 42, 50, 52, 53, 64, 68, 37, 51, 53, 54, 51, 53, 54, 51, 53, 54, 51, 53, 54, 51, 53, 54, 51, 53, 51, 52, 54, 55, 56, - 57, 59, 51, 52, 54, 55, 56, 57, 59, 51, 52, 54, 52, 53, 55, 56, 57, 58, 59, 52, 53, 55, 56, 57, 58, 59, 52, 53, 55, 53, 54, 56, 57, - 76, 77, 78, 88, 53, 54, 56, 57, 76, 77, 78, 88, 53, 53, 54, 55, 57, 58, 77, 78, 79, 88, 53, 54, 55, 57, 58, 77, 78, 79, 53, 54, 55, - 56, 58, 59, 78, 79, 80, 90, 53, 54, 55, 56, 58, 59, 78, 53, 54, 56, 57, 59, 79, 80, 81, 82, 92, 53, 54, 56, 57, 59, 79, 80, 53, 54, - 57, 58, 80, 81, 82, 92, 53, 54, 57, 58, 80, 81, 82, 92, 53, 0, 1, 2, 3, 4, 33, 34, 41, 61, 62, 66, 67, 96, 0, 1, 2, 3, 0, 1, 33, 34, - 35, 40, 41, 60, 62, 63, 65, 66, 67, 96, 0, 1, 33, 33, 34, 35, 36, 37, 38, 39, 40, 41, 60, 61, 63, 64, 65, 66, 67, 96, 35, 36, 37, - 38, 39, 40, 51, 52, 61, 62, 64, 65, 66, 67, 96, 35, 36, 36, 37, 38, 39, 51, 52, 53, 63, 65, 66, 96, 36, 37, 38, 39, 51, 52, 36, 37, - 38, 39, 52, 61, 62, 63, 64, 66, 67, 96, 36, 37, 38, 39, 52, 41, 60, 61, 62, 63, 64, 65, 67, 96, 41, 60, 61, 62, 63, 64, 65, 67, 0, - 1, 2, 3, 33, 34, 35, 40, 41, 60, 61, 62, 65, 66, 96, 0, 1, 42, 43, 49, 50, 51, 52, 53, 69, 74, 75, 97, 42, 43, 49, 50, 51, 52, 42, - 43, 44, 48, 49, 50, 51, 68, 70, 71, 73, 74, 75, 97, 42, 43, 44, 42, 43, 44, 45, 46, 47, 48, 49, 50, 68, 69, 71, 72, 73, 74, 75, 97, - 31, 32, 44, 45, 46, 47, 48, 69, 70, 72, 73, 74, 75, 97, 31, 32, 44, 28, 29, 30, 31, 32, 45, 46, 47, 70, 71, 73, 74, 97, 28, 29, 30, - 31, 29, 30, 31, 32, 44, 45, 46, 47, 48, 70, 71, 72, 74, 75, 97, 29, 30, 47, 68, 69, 70, 71, 72, 73, 75, 97, 47, 68, 69, 70, 71, 72, - 73, 75, 42, 43, 49, 50, 52, 68, 69, 70, 71, 72, 73, 74, 97, 42, 43, 49, 50, 6, 7, 8, 9, 10, 11, 12, 55, 77, 87, 88, 89, 95, 6, 7, 8, - 9, 55, 56, 76, 78, 86, 87, 88, 89, 95, 55, 56, 76, 78, 86, 87, 88, 89, 54, 55, 56, 57, 58, 76, 77, 79, 80, 85, 86, 87, 88, 89, 90, - 94, 95, 54, 55, 56, 57, 58, 59, 77, 78, 80, 81, 84, 85, 86, 89, 90, 91, 94, 54, 57, 58, 59, 78, 79, 81, 82, 83, 84, 85, 90, 91, 92, - 93, 94, 54, 58, 59, 80, 82, 83, 84, 91, 92, 93, 58, 59, 80, 82, 83, 84, 91, 92, 20, 21, 22, 23, 24, 25, 26, 59, 81, 83, 91, 92, 93, - 20, 21, 22, 23, 17, 18, 19, 20, 21, 22, 23, 81, 82, 84, 91, 92, 93, 17, 18, 19, 20, 16, 17, 18, 19, 20, 81, 82, 83, 85, 91, 92, 93, - 94, 16, 17, 18, 19, 14, 15, 16, 17, 18, 83, 84, 86, 87, 90, 93, 94, 95, 14, 15, 16, 17, 11, 12, 13, 14, 15, 16, 76, 77, 85, 87, 88, - 89, 94, 95, 11, 12, 13, 9, 10, 11, 12, 13, 14, 76, 77, 86, 88, 89, 95, 9, 10, 11, 12, 13, 7, 8, 9, 10, 11, 12, 13, 55, 76, 77, 86, - 87, 89, 95, 7, 8, 9, 55, 56, 76, 77, 78, 79, 86, 87, 88, 90, 95, 55, 56, 76, 77, 78, 79, 56, 57, 58, 78, 79, 80, 83, 84, 85, 86, 87, - 89, 91, 92, 93, 94, 95, 58, 59, 79, 80, 81, 82, 83, 84, 85, 90, 92, 93, 94, 58, 59, 79, 80, 19, 20, 21, 22, 23, 24, 25, 59, 81, 82, - 83, 84, 91, 93, 19, 20, 21, 18, 19, 79, 80, 81, 82, 83, 84, 85, 90, 91, 92, 94, 18, 19, 79, 80, 15, 16, 17, 78, 79, 80, 83, 84, 85, - 86, 87, 89, 90, 91, 93, 95, 15, 13, 14, 15, 76, 77, 78, 85, 86, 87, 88, 89, 90, 94, 13, 14, 15, 76, 34, 35, 36, 38, 39, 40, 41, 60, - 61, 62, 63, 64, 65, 66, 67, 34, 35, 43, 44, 45, 47, 48, 49, 50, 68, 69, 70, 71, 72, 73, 74, 75, 43, 44 - }; - const unsigned int reverse_index2[98 * 17] = { - 0, 2, 4, 6, 8, 4, 0, 2, 4, 6, 8, 4, 0, 2, 4, 6, 8, 0, 0, 2, 4, 6, 8, 8, 0, 0, 2, 4, 6, 8, 8, 0, 0, 2, 1, 1, 0, 2, 4, 6, 1, 1, 0, 2, - 4, 6, 1, 1, 0, 2, 4, 3, 2, 1, 0, 2, 4, 6, 3, 2, 1, 0, 2, 4, 6, 3, 2, 1, 6, 3, 3, 1, 0, 2, 4, 7, 6, 3, 3, 1, 0, 2, 4, 7, 6, 6, 4, 3, - 1, 0, 2, 4, 8, 6, 4, 3, 1, 0, 2, 4, 8, 6, 7, 5, 3, 1, 0, 2, 4, 9, 7, 5, 3, 1, 0, 2, 4, 9, 7, 6, 5, 3, 1, 0, 2, 4, 6, 5, 3, 1, 0, 2, - 4, 6, 5, 3, 7, 5, 3, 1, 0, 2, 4, 7, 5, 3, 1, 0, 2, 4, 7, 5, 3, 9, 7, 5, 3, 1, 0, 2, 5, 9, 7, 5, 3, 1, 0, 2, 5, 9, 9, 7, 5, 3, 1, 0, - 2, 5, 8, 9, 7, 5, 3, 1, 0, 2, 5, 7, 5, 3, 1, 0, 2, 5, 9, 9, 7, 5, 3, 1, 0, 2, 5, 9, 9, 5, 3, 1, 0, 2, 4, 9, 5, 3, 1, 0, 2, 4, 9, 5, - 3, 6, 3, 1, 0, 2, 6, 6, 3, 1, 0, 2, 6, 6, 3, 1, 0, 2, 7, 3, 1, 0, 3, 7, 7, 3, 1, 0, 3, 7, 7, 3, 1, 0, 3, 6, 3, 1, 1, 3, 6, 6, 3, 1, - 1, 3, 6, 6, 3, 1, 1, 3, 7, 3, 1, 1, 3, 7, 7, 3, 1, 1, 3, 7, 7, 3, 1, 1, 3, 6, 3, 0, 1, 3, 6, 6, 3, 0, 1, 3, 6, 6, 3, 0, 1, 3, 7, 2, - 0, 1, 3, 5, 7, 2, 0, 1, 3, 5, 7, 2, 0, 1, 3, 5, 2, 0, 1, 3, 5, 5, 2, 0, 1, 3, 5, 5, 2, 0, 1, 3, 4, 2, 0, 1, 3, 5, 8, 4, 2, 0, 1, 3, - 5, 8, 4, 2, 0, 5, 2, 0, 1, 3, 5, 7, 9, 5, 2, 0, 1, 3, 5, 7, 9, 5, 4, 2, 0, 1, 3, 5, 7, 9, 4, 2, 0, 1, 3, 5, 7, 9, 4, 4, 2, 0, 1, 3, - 5, 7, 9, 4, 2, 0, 1, 3, 5, 7, 9, 4, 4, 2, 0, 1, 3, 5, 7, 4, 2, 0, 1, 3, 5, 7, 4, 2, 0, 9, 4, 2, 0, 1, 3, 5, 6, 9, 4, 2, 0, 1, 3, 5, - 6, 9, 9, 4, 2, 0, 1, 3, 5, 6, 9, 4, 2, 0, 1, 3, 5, 6, 9, 8, 4, 2, 0, 1, 3, 4, 6, 8, 4, 2, 0, 1, 3, 4, 6, 8, 6, 4, 2, 0, 1, 3, 3, 5, - 6, 4, 2, 0, 1, 3, 3, 5, 6, 6, 4, 2, 0, 1, 2, 3, 6, 4, 2, 0, 1, 2, 3, 6, 4, 2, 6, 4, 2, 0, 1, 1, 6, 4, 2, 0, 1, 1, 6, 4, 2, 0, 1, 8, - 6, 4, 2, 0, 0, 9, 8, 6, 4, 2, 0, 0, 9, 8, 6, 4, 8, 6, 4, 2, 0, 6, 8, 6, 4, 2, 0, 6, 8, 6, 4, 2, 0, 2, 4, 5, 8, 3, 1, 6, 2, 4, 5, 8, - 3, 1, 6, 2, 4, 5, 7, 1, 1, 5, 0, 8, 7, 1, 1, 5, 0, 8, 7, 1, 1, 5, 0, 7, 1, 2, 8, 6, 0, 5, 9, 8, 8, 7, 1, 2, 8, 6, 0, 5, 8, 2, 1, 4, - 0, 6, 7, 9, 8, 2, 1, 4, 0, 6, 7, 9, 8, 1, 0, 5, 5, 7, 1, 0, 5, 5, 7, 1, 0, 5, 5, 7, 1, 0, 4, 0, 2, 2, 6, 6, 2, 8, 4, 0, 2, 2, 6, 6, - 2, 8, 4, 4, 0, 2, 1, 4, 7, 4, 4, 5, 9, 9, 7, 4, 0, 2, 1, 4, 5, 2, 0, 3, 9, 9, 4, 2, 7, 5, 4, 8, 9, 8, 6, 6, 5, 5, 7, 9, 0, 0, 3, 3, - 2, 6, 7, 5, 7, 9, 0, 0, 3, 3, 2, 5, 0, 6, 7, 2, 5, 0, 6, 7, 2, 5, 0, 6, 7, 2, 5, 1, 1, 8, 5, 0, 4, 9, 7, 1, 1, 8, 5, 0, 4, 9, 7, 1, - 8, 1, 1, 7, 4, 0, 6, 9, 8, 1, 1, 7, 4, 0, 6, 9, 8, 7, 2, 1, 0, 6, 9, 8, 9, 7, 2, 1, 0, 6, 9, 8, 9, 7, 8, 5, 4, 2, 2, 1, 6, 8, 5, 4, - 2, 2, 1, 6, 8, 5, 4, 9, 7, 6, 3, 0, 0, 3, 6, 2, 7, 9, 7, 6, 3, 0, 0, 3, 7, 3, 0, 3, 5, 2, 2, 9, 8, 4, 5, 7, 6, 7, 9, 6, 7, 2, 0, 4, - 2, 1, 3, 2, 7, 9, 5, 8, 2, 0, 4, 2, 1, 3, 0, 4, 3, 1, 5, 2, 6, 8, 0, 4, 3, 1, 5, 2, 6, 8, 0, 5, 6, 5, 5, 1, 5, 8, 8, 5, 6, 5, 5, 1, - 5, 8, 8, 5, 0, 1, 9, 0, 1, 9, 0, 1, 9, 0, 1, 9, 0, 1, 9, 0, 1, 7, 0, 1, 9, 9, 9, 9, 7, 0, 1, 9, 9, 9, 9, 7, 0, 1, 4, 0, 5, 2, 0, 2, - 4, 4, 0, 5, 2, 0, 2, 4, 4, 0, 5, 6, 5, 0, 8, 6, 6, 9, 6, 6, 5, 0, 8, 6, 6, 9, 6, 6, 3, 2, 0, 2, 7, 7, 5, 7, 8, 3, 2, 0, 2, 7, 7, 5, - 7, 2, 0, 2, 1, 1, 2, 4, 3, 5, 7, 2, 0, 2, 1, 1, 2, 4, 4, 3, 7, 1, 0, 5, 4, 8, 8, 8, 4, 3, 7, 1, 0, 5, 4, 7, 4, 7, 0, 9, 6, 6, 6, 7, - 4, 7, 0, 9, 6, 6, 6, 7, 4, 5, 6, 7, 8, 2, 5, 4, 1, 9, 6, 1, 9, 4, 5, 6, 7, 8, 9, 3, 4, 6, 2, 3, 1, 2, 9, 7, 4, 0, 5, 8, 9, 3, 9, 6, - 5, 6, 7, 7, 3, 1, 7, 4, 2, 3, 6, 4, 1, 4, 0, 8, 5, 3, 3, 1, 8, 8, 9, 7, 3, 1, 0, 5, 8, 3, 8, 5, 8, 4, 2, 8, 4, 3, 9, 1, 1, 7, 8, 8, - 4, 2, 8, 4, 3, 9, 6, 5, 9, 7, 9, 6, 0, 0, 3, 5, 2, 9, 6, 5, 9, 7, 9, 3, 4, 1, 5, 5, 3, 2, 1, 9, 3, 4, 1, 5, 5, 3, 2, 9, 8, 8, 9, 6, - 7, 9, 9, 6, 0, 0, 5, 6, 2, 4, 9, 8, 4, 8, 8, 2, 3, 2, 8, 1, 8, 1, 9, 4, 8, 8, 2, 3, 2, 3, 5, 8, 8, 1, 3, 9, 0, 3, 7, 8, 5, 0, 5, 3, - 5, 8, 9, 6, 5, 6, 8, 6, 1, 4, 7, 6, 4, 2, 5, 4, 2, 4, 0, 9, 8, 6, 4, 3, 3, 4, 9, 1, 1, 0, 4, 7, 2, 9, 8, 6, 8, 7, 7, 5, 4, 5, 2, 5, - 8, 1, 1, 6, 7, 8, 7, 7, 5, 9, 8, 8, 9, 9, 7, 4, 7, 9, 5, 0, 0, 1, 6, 3, 9, 8, 9, 5, 5, 2, 4, 3, 2, 3, 1, 9, 5, 5, 2, 4, 3, 2, 3, 6, - 9, 9, 6, 8, 1, 0, 6, 8, 9, 5, 3, 4, 6, 9, 9, 6, 9, 8, 6, 6, 5, 6, 7, 8, 4, 2, 0, 8, 7, 9, 8, 6, 6, 1, 5, 2, 7, 5, 3, 2, 0, 3, 1, 5, - 2, 7, 5, 3, 2, 0, 7, 4, 3, 4, 9, 7, 5, 1, 3, 7, 7, 6, 7, 2, 2, 3, 4, 6, 7, 4, 3, 4, 6, 9, 0, 0, 9, 9, 6, 9, 7, 0, 7, 2, 8, 5, 3, 3, - 3, 2, 5, 7, 6, 7, 8, 3, 2, 7, 4, 4, 8, 5, 1, 6, 2, 3, 5, 0, 2, 3, 5, 1, 6, 2, 3, 5, 0, 2, 7, 6, 6, 6, 7, 8, 9, 8, 4, 2, 8, 0, 8, 7, - 6, 6, 6, 8, 7, 6, 5, 7, 8, 9, 3, 1, 1, 3, 1, 2, 8, 7, 6, 5, 7, 5, 4, 5, 9, 7, 5, 5, 1, 4, 5, 1, 5, 7, 5, 4, 5, 8, 5, 4, 6, 8, 8, 2, - 2, 8, 4, 9, 0, 9, 8, 5, 4, 6, 9, 8, 4, 4, 6, 8, 5, 8, 2, 5, 5, 4, 6, 1, 9, 8, 4, 9, 8, 5, 4, 6, 7, 1, 3, 1, 1, 3, 2, 9, 8, 5, 4, 6, - 9, 8, 7, 7, 8, 9, 9, 6, 0, 2, 8, 1, 5, 5, 9, 8, 7, 3, 6, 3, 0, 2, 8, 3, 4, 3, 6, 0, 3, 6, 3, 0, 2, 8, 8, 6, 8, 1, 0, 1, 9, 6, 3, 6, - 9, 6, 6, 9, 7, 1, 8, 6, 5, 6, 2, 0, 3, 4, 3, 9, 5, 3, 0, 9, 6, 5, 6, 2, 9, 8, 8, 7, 7, 9, 9, 7, 2, 0, 1, 8, 5, 5, 9, 8, 8, 9, 8, 9, - 8, 1, 4, 0, 0, 4, 8, 1, 4, 7, 9, 8, 9, 8, 8, 9, 9, 6, 4, 7, 7, 4, 0, 4, 7, 9, 1, 9, 6, 6, 8, 8, 9, 9, 4, 1, 8, 5, 0, 0, 4, 1, 9, 8, - 8, 9, 9, 4, 9, 7, 7, 8, 7, 7, 8, 5, 3, 0, 2, 3, 2, 0, 3, 9, 7, 7, 7, 9, 8, 7, 7, 8, 4, 3, 0, 3, 4, 3, 0, 2, 7, 7 - }; - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_PIPNET98_H diff --git a/lite/mnn/cv/mnn_plantid.cpp b/lite/mnn/cv/mnn_plantid.cpp deleted file mode 100644 index 0d530074..00000000 --- a/lite/mnn/cv/mnn_plantid.cpp +++ /dev/null @@ -1,68 +0,0 @@ -// -// Created by DefTruth on 2022/3/27. -// - -#include "mnn_plantid.h" -#include "lite/utils.h" - -using mnncv::MNNPlantID; - -MNNPlantID::MNNPlantID(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNPlantID::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -inline void MNNPlantID::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,3,224,224) - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNPlantID::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch. - auto device_logits_ptr = output_tensors.at("477"); - MNN::Tensor host_logits_tensor(device_logits_ptr, device_logits_ptr->getDimensionType()); - device_logits_ptr->copyToHostTensor(&host_logits_tensor); - - auto logits_dims = host_logits_tensor.shape(); - const unsigned int num_classes = logits_dims.at(1); // 4066 - const float *logits = host_logits_tensor.host(); - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_plantid.h b/lite/mnn/cv/mnn_plantid.h deleted file mode 100644 index 616d2c6c..00000000 --- a/lite/mnn/cv/mnn_plantid.h +++ /dev/null @@ -1,816 +0,0 @@ -// -// Created by DefTruth on 2022/3/27. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_PLANTID_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_PLANTID_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNPlantID : public BasicMNNHandler - { - public: - explicit MNNPlantID(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNPlantID() override = default; - - private: - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // resize & normalize. - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[4066] = { - "Saururus chinensis", "Houttuynia cordata", "Aucuba chinensis", "Aucuba japonica var. variegata", "Aucuba obcordata", - "Blechnum novae-zelandiae", "Woodwardia fimbriata", "Woodwardia prolifera", "Pentaphylax euryoides", "Ternstroemia gymnanthera", - "Eurya chinensis", "Eurya distichophylla", "Eurya emarginata", "Eurya japonica", "Eurya macartneyi", "Eurya muricata", - "Eurya rubiginosa var. attenuata", "Eurya saxicola", "Cleyera japonica", "Anneslea fragrans", "Eleutherococcus nodiflorus", - "Eleutherococcus senticosus", "Eleutherococcus trifoliatus", "Panax ginseng", "Fatsia japonica", "Kalopanax septemlobus", - "Trevesia palmata", "Schefflera arboricola", "Schefflera elegantissima", "Schefflera heptaphylla", "Schefflera macrostachya", - "Hydrocotyle sibthorpioides", "Hydrocotyle sibthorpioides var. batrachium", "Hydrocotyle verticillata", "Hydrocotyle wilfordii", - "Hedera helix", "Hedera nepalensis var. sinensis", "Metapanax davidii", "Aralia cordata", "Aralia elata", "Aralia nudicaulis", - "Schisandra chinensis", "Schisandra sphenanthera", "Illicium angustisepalum", "Illicium dunnianum", "Illicium lanceolatum", - "Illicium verum", "Kadsura coccinea", "Kadsura heteroclita", "Kadsura longipedunculata", "Dillenia turbinata", - "Tetracera sarmentosa", "Adoxa moschatellina", "Sambucus adnata", "Sambucus javanica", "Sambucus nigra", "Sambucus nigra caerulea", - "Sambucus racemosa", "Sambucus williamsii", "Viburnum acerifolium", "Viburnum betulifolium", "Viburnum chinshanense", - "Viburnum dilatatum", "Viburnum foetidum var. rectangulatum", "Viburnum fordiae", "Viburnum lantanoides", "Viburnum macrocephalum", - "Viburnum macrocephalum f. keteleeri", "Viburnum melanocarpum", "Viburnum odoratissimum", "Viburnum odoratissimum var. awabuki", - "Viburnum opulus", "Viburnum opulus subsp. calvescens", "Viburnum plicatum", "Viburnum plicatum f. tomentosum", - "Viburnum prunifolium", "Viburnum setigerum", "Viburnum tinus", "Linum usitatissimum&perenne", "Reinwardtia indica", - "Lophophora williamsii", "Schlumbergera truncata", "Opuntia basilaris", "Opuntia ficus-indica", "Opuntia humifusa", - "Opuntia littoralis", "Opuntia microdasys", "Echinopsis chamaecereus", "Nopalxochia ackermannii", "Cylindropuntia imbricata", - "Cylindropuntia leptocaulis", "Ferocactus peninsulae", "Epiphyllum oxypetalum", "Astrophytum myriostigma", "Pereskia bleo", - "Cleistocactus colademononis", "Hylocereus undatus", "Echinocactus grusonii", "Aporocactus flagelliformis", "Curculigo capitulata", - "Hypoxis hirsuta", "Hypoxis juncea", "Pauridia capensis", "Eryngium leavenworthii", "Eryngium planum", "Eryngium yuccifolium", - "Sanicula lamelligera", "Sanicula orthacantha", "Angelica dahurica", "Angelica decursiva", "Angelica polymorpha", - "Changium smyrnioides", "Astrantia major", "Bupleurum smithii", "Pastinaca sativa", "Conium maculatum", "Oenanthe javanica", - "Heracleum maximum", "Glehnia littoralis", "Centella asiatica", "Torilis arvensis", "Torilis scabra", "Daucus carota", - "Daucus carota var. sativa", "Coriandrum sativum", "Apium graveolens", "Foeniculum vulgare", "Cnidium monnieri", "Zizia aurea", - "Quisqualis indica", "Terminalia arjuna", "Terminalia catappa", "Terminalia nigrovenulosa", "Combretum alfredii", - "Combretum constrictum", "", "", "Alstroemeria hybrida", "Isotria verticillata", "Sacoila lanceolata", "Limodorum abortivum", - "Anacamptis coriophora", "Anacamptis laxiflora", "Anacamptis morio", "Anacamptis palustris", "Anacamptis papilionacea", - "Anacamptis pyramidalis", "Eriochilus cucullatus", "Paphiopedilum", "Paphiopedilum emersonii", "Paphiopedilum hirsutissimum", - "Paphiopedilum purpuratum", "Neottianthe cucullata", "Cymbidium ensifolium", "Cymbidium faberi", "Cymbidium floribundum", - "Cymbidium goeringii", "Cymbidium kanran", "Cymbidium lancifolium", "Cymbidium serratum", "Cymbidium sinense", "Cattleya hybrida", - "Epigeneium fargesii", "Malaxis monophyllos", "Malaxis unifolia", "Cheirostylis yunnanensis", "Dipodium roseum", - "Chiloglottis valida", "Encyclia tampensis", "Polystachya concreta", "Cephalanthera damasonium", "Cephalanthera falcata", - "Cephalanthera longifolia", "Cephalanthera rubra", "Cryptochilus roseus", "Robiquetia succisa", "Oberonioides microtatantha", - "Ponerorchis brevicalcarata", "Dracula simia", "Oreorchis nana", "Galeola lindleyana", "Calypso bulbosa var. speciosa", - "Tainia dunnii", "Tainia hongkongensis", "Anoectochilus roxburghii", "Gymnadenia nigra", "Gymnadenia odoratissima", - "Gymnadenia rhellicani", "Bletia purpurea", "Aerides rosea", "Dactylorhiza fuchsii", "Dactylorhiza majalis", - "Dactylorhiza traunsteineri", "Dactylorhiza viridis", "Oncidium", "Goodyera foliosa", "Goodyera oblongifolia", "Goodyera procera", - "Goodyera pubescens", "Goodyera repens", "Goodyera schlechtendaliana", "Goodyera tesselata", "Goodyera viridiflora", - "Neotinea maculata", "Neotinea tridentata", "Amitostigma monanthum", "Amitostigma pinguicula", "Dienia ophrydis", - "Cypripedium acaule", "Cypripedium bardolphianum", "Cypripedium calceolus", "Cypripedium calcicola", "Cypripedium candidum", - "Cypripedium flavum", "Cypripedium franchetii", "Cypripedium guttatum", "Cypripedium henryi", "Cypripedium japonicum", - "Cypripedium lichiangense", "Cypripedium macranthos", "Cypripedium montanum", "Cypripedium parviflorum", - "Cypripedium plectrochilum", "Cypripedium reginae", "Cypripedium shanxiense", "Cypripedium tibeticum", "Cypripedium wardii", - "Cypripedium yunnanense", "Cypripedium × ventricosum", "Cremastra appendiculata", "Thelymitra antennifera", "Thelymitra longifolia", - "Epidendrum radicans", "Eria corneri", "Calopogon tuberosus", "Epipactis atrorubens", "Epipactis gigantea", "Epipactis helleborine", - "Epipactis mairei", "Epipactis microphylla", "Epipactis palustris", "Renanthera coccinea", "Appendicula cornuta", - "Pleione bulbocodioides", "Pleione formosana", "Habenaria ciliolaris", "Habenaria dentata", "Habenaria floribunda", - "Habenaria glaucifolia", "Habenaria leptoloba", "Habenaria limprichtii", "Habenaria monorrhiza", "Habenaria petelotii", - "Habenaria repens", "Habenaria rhodocheila", "Habenaria schindleri", "Corallorhiza maculata", "Corallorhiza mertensiana", - "Corallorhiza striata", "Corallorhiza trifida", "Corallorhiza wisteriana", "Bletilla ochracea", "Bletilla striata", - "Pseudorchis albida", "Pseudorchis straminea", "Thrixspermum centipeda", "Pecteilis susannae", "Gastrochilus calceolaris", - "Galearis rotundifolia", "Chamorchis alpina", "Pholidota articulata", "Pholidota cantonensis", "Pholidota chinensis", - "Dendrobium chrysotoxum", "Dendrobium crepidatum", "Dendrobium cucullatum", "Dendrobium densiflorum", "Dendrobium hancockii", - "Dendrobium henryi", "Dendrobium hercoglossum", "Dendrobium loddigesii", "Dendrobium moniliforme", "Dendrobium moschatum", - "Dendrobium officinale", "Dendrobium sinominutiflorum", "Dendrobium thyrsiflorum", "Bulbophyllum ambrosia", - "Bulbophyllum kwangtungense", "Bulbophyllum levinei", "Bulbophyllum odoratissimum", "Bulbophyllum orientale", - "Bulbophyllum pecten-veneris", "Bulbophyllum retusiusculum", "Prosthechea cochleata", "Arundina graminifolia", - "Orchis anthropophora", "Orchis italica", "Orchis mascula", "Orchis militaris", "Orchis pallens", "Orchis provincialis", - "Orchis simia", "Zeuxine parvifolia", "Zeuxine strateumatica", "Dendrolirium lasiopetalum", "Spiranthes cernua", - "Spiranthes lacera", "Spiranthes lucida", "Spiranthes magnicamporum", "Spiranthes praecox", "Spiranthes sinensis", - "Spiranthes spiralis", "Spiranthes tuberosa", "Spiranthes vernalis", "Liparis bootanensis", "Liparis nervosa", - "Liparis stricklandiana", "Liparis viridiflora", "Eulophia alta", "Eulophia cucullata", "Eulophia graminea", "Eulophia zollingeri", - "Arethusa bulbosa", "Pterostylis banksii", "Pterostylis nana", "Pterostylis nutans", "Acampe rigida", "Platanthera aquilonis", - "Platanthera blephariglottis", "Platanthera clavellata", "Platanthera dilatata", "Platanthera elegans", "Platanthera flava", - "Platanthera grandiflora", "Platanthera huronensis", "Platanthera hyperborea", "Platanthera lacera", "Platanthera minor", - "Platanthera obtusata", "Platanthera orbiculata", "Platanthera psycodes", "Platanthera sparsiflora", "Platanthera stricta", - "Platanthera ussuriensis", "Hemipilia flabellata", "Spathoglottis plicata", "Spathoglottis pubescens", "Disa bracteata", - "Microtis unifolia", "Traunsteinera globosa", "Ponthieva racemosa", "Epipogium aphyllum", "Epipogium roseum", "Calanthe brevicornu", - "Calanthe clavata", "Calanthe graciliflora", "Calanthe sylvatica", "Calanthe tricarinata", "Calanthe triplicata", - "Diploprora championii", "Conchidium pusillum", "Ophrys apifera", "Ophrys bertolonii", "Ophrys bombyliflora", "Ophrys fuciflora", - "Ophrys fusca", "Ophrys insectifera", "Ophrys lutea", "Ophrys scolopax", "Ophrys speculum", "Ophrys sphegodes", - "Ophrys tenthredinifera", "Arachnis labrosa", "Phalaenopsis aphrodite", "Ludisia discolor", "Caladenia caerulea", - "Caladenia carnea", "Caladenia flava", "Caladenia fuscata", "Caladenia major", "Caladenia tentaculata", "Herminium monorchis", - "Ansellia africana", "Coelogyne corymbosa", "Coelogyne fimbriata", "Acianthus exsertus", "Erythrodes blumei", "Corybas taliensis", - "Serapias cordigera", "Serapias lingua", "Serapias vomeracea", "Cleisostoma paniculatum", "Cleisostoma rostratum", - "Cleisostoma simondii var. guangdongense", "Neofinetia falcata", "Caleana major", "Neottia banksiana", "Neottia convallarioides", - "Neottia nidus-avis", "Neottia ovata", "Satyrium yunnanense", "Phaius", "Phaius flavus", "Phaius tancarvilleae", - "Cephalantheropsis obcordata", "Ilex aculeolata", "Ilex asprella", "Ilex centrochinensis", "Ilex cornuta", - "Ilex cornuta 'National'", "Ilex decidua", "Ilex latifolia", "Ilex macrocarpa", "Ilex opaca", "Ilex pubescens", "Ilex rotunda", - "Ilex verticillata", "Ilex vomitoria", "Impatiens arguta", "Impatiens balsamina", "Impatiens blepharosepala", "Impatiens capensis", - "Impatiens chekiangensis", "Impatiens chinensis", "Impatiens commelinoides", "Impatiens hawkeri", "Impatiens hongkongensis", - "Impatiens macrovexilla", "Impatiens niamniamensis", "Impatiens noli-tangere", "Impatiens pallida", "Impatiens platychlaena", - "Impatiens platysepala", "Impatiens tubulosa", "Impatiens walleriana", "Pellaea andromedifolia", "Adiantum aleuticum", - "Adiantum capillus-veneris", "Adiantum nelumboides", "Adiantum pedatum", "Aechmea fulgens", "Ananas comosus", "Cryptanthus acaulis", - "Billbergia pyramidalis", "Tillandsia cyanea", "Tillandsia recurvata", "Tillandsia usneoides", "Rehmannia chingii", - "Rehmannia glutinosa", "Cymbaria mongolica", "Euphrasia pectinata", "Euphrasia regelii", "Melampyrum laxum", "Melampyrum roseum", - "Brandisia hancei", "Phtheirospermum japonicum", "Phtheirospermum tenuisectum", "Castilleja exserta", "Castilleja indivisa", - "Striga asiatica", "Cistanche deserticola", "Conopholis americana", "Boschniakia himalaica", "Aeginetia indica", - "Siphonostegia chinensis", "Siphonostegia laeta", "Pedicularis cheilanthifolia", "Pedicularis chinensis", "Pedicularis cranolopha", - "Pedicularis davidii", "Pedicularis densiflora", "Pedicularis densispica", "Pedicularis kansuensis", "Pedicularis muscicola", - "Pedicularis rhinanthoides subsp. labellata", "Monochasma sheareri", "Portulacaria afra", "Portulacaria afra 'Variegata'", - "Solms-laubachia pulcherrima", "Pegaeophyton scapiflorum", "Iberis amara", "Barbarea orthoceras", "Barbarea vulgaris", - "Descurainia sophia", "Cakile maritima", "Lepidium apetalum", "Lepidium latifolium", "Lepidium virginicum", "Cardamine californica", - "Cardamine concatenata", "Cardamine diphylla", "Cardamine hirsuta", "Cardamine impatiens", "Cardamine leucantha", - "Cardamine lyrata", "Cardamine purpurascens", "Erysimum amurense", "Erysimum capitatum", "Erysimum × cheiri", "Matthiola incana", - "Eruca vesicaria subsp. sativa", "Dontostemon dentatus", "Dontostemon glandulosus", "Dontostemon tibeticus", "Brassica juncea", - "Brassica juncea var. gemmifera", "Brassica juncea var. multicep", "Brassica oleracea", "Brassica oleracea var. acephala", - "Brassica oleracea var. botrytis", "Brassica oleracea var. capitata", "Brassica oleracea var. gemmifera", - "Brassica oleracea var. gongylodes", "Brassica oleracea var. italica", "Brassica rapa var. chinensis", "Brassica rapa var. glabra", - "Brassica rapa var. oleifera", "Capsella bursa-pastoris", "Thlaspi arvense", "Raphanus raphanistrum", "Raphanus sativus", - "Alliaria petiolata", "Rorippa globosa", "Rorippa indica", "Orychophragmus violaceus", "Nasturtium officinale", - "Yinshania fumarioides", "Hesperis matronalis", "Lobularia maritima", "Megacarpaea delavayi", "Duabanga grandiflora", - "Lythrum salicaria", "Lawsonia inermis", "Sonneratia apetala", "Sonneratia caseolaris", "Punica granatum", - "Punica granatum 'Albescens'", "Lagerstroemia fordii", "Lagerstroemia indica", "Lagerstroemia indica f. alba", - "Lagerstroemia limii", "Lagerstroemia speciosa", "Lagerstroemia subcostata", "Rotala rotundifolia", "Trapa natans", - "Cuphea hookeriana", "Cuphea hyssopifolia", "Woodfordia fruticosa", "Heimia myrtifolia", "Celastrus monospermus", - "Celastrus orbiculatus", "Euonymus alatus", "Euonymus carnosus", "Euonymus centidens", "Euonymus cornutus", "Euonymus fortunei", - "Euonymus japonicus", "Euonymus japonicus 'Aurea-marginatus'", "Euonymus laxiflorus", "Euonymus maackii", "Euonymus myrianthus", - "Euonymus nitidus", "Euonymus phellomanus", "Euonymus schensianus", "Euonymus semenovii", "Parnassia wightiana", - "Brexia madagascariensis", "Tripterygium wilfordii", "Selaginella uncinata", "Bretschneidera sinensis", "", "", - "Erythroxylum sinense", "Antidesma bunius", "Antidesma japonicum", "Phyllanthus acidus", "Phyllanthus chekiangensis", - "Phyllanthus emblica", "Phyllanthus flexuosus", "Phyllanthus glaucus", "Phyllanthus hainanensis", "Phyllanthus pulcher", - "Phyllanthus sootepensis", "Phyllanthus urinaria", "Phyllanthus ussuriensis", "Actephila collinsiae", "Baccaurea ramiflora", - "Flueggea suffruticosa", "Bischofia polycarpa", "Glochidion eriocarpum", "Glochidion puberum", "Glochidion wrightii", - "Glochidion zeylanicum", "Aporosa dioica", "Cleistanthus sumatranus", "Breynia disticha", "Breynia fruticosa", "Rotheca myricoides", - "Petraeovitex wolfei", "Paraphlomis javanica", "Paraphlomis javanica var. angustifolia", "Paraphlomis javanica var. coronata", - "Physostegia virginiana", "Holmskioldia sanguinea", "Mesona chinensis", "Perovskia abrotanoides", "Pogostemon auricularius", - "Hanceola exserta", "Lycopus lucidus", "Lycopus lucidus var. hirtus", "Prunella hispida", "Prunella vulgaris", "Lagopsis supina", - "Clerodendrum bungei", "Clerodendrum canescens", "Clerodendrum chinense", "Clerodendrum chinense var. simplex", - "Clerodendrum cyrtophyllum", "Clerodendrum fortunatum", "Clerodendrum inerme", "Clerodendrum japonicum", "Clerodendrum lindleyi", - "Clerodendrum paniculatum", "Clerodendrum quadriloculare", "Clerodendrum serratum", "Clerodendrum speciosum", - "Clerodendrum splendens", "Clerodendrum thomsoniae", "Clerodendrum trichotomum", "Clerodendrum wallichii", "Galeobdolon chinense", - "Anisomeles indica", "Tectona grandis", "Phlomis fruticosa", "Phlomis mongolica", "Marrubium vulgare", "Stachys byzantina", - "Stachys geobombycis", "Stachys japonica", "Stachys oblongifolia", "Glechoma hederacea", "Glechoma longituba", - "Colquhounia seguinii", "Origanum vulgare", "Vitex agnus-castus", "Vitex negundo", "Vitex negundo var. cannabifolia", - "Vitex negundo var. heterophylla", "Vitex rotundifolia", "Vitex trifolia", "Lamiophlomis rotata", "Leonotis leonurus", - "Leonotis nepetifolia", "Leonurus japonicus", "Leonurus sibiricus", "Gmelina asiatica", "Gmelina hainanensis", - "Gmelina philippensis", "Mosla dianthera", "Mosla scabra", "Mosla soochowensis", "Karomia speciosa", "Ajuga ciliata", - "Ajuga decumbens", "Ajuga lupulina", "Ajuga reptans", "Callicarpa americana", "Callicarpa bodinieri&dichotoma", - "Callicarpa cathayana", "Callicarpa formosana", "Callicarpa giraldii", "Callicarpa rubella", "Perilla frutescens", - "Eriophyton wallichii", "Ocimum basilicum", "Monarda citriodora", "Monarda didyma", "Monarda fistulosa", "Monarda punctata", - "Clerodendranthus spicatus", "Nepeta cataria", "Nepeta × faassenii 'Six Hills Giant'", "Caryopteris incana", - "Caryopteris nepetifolia", "Caryopteris × clandonensis", "Mentha canadensis", "Lavandula dentata", "Lavandula stoechas", - "Agastache rugosa", "Premna microphylla", "Moluccella laevis", "Rosmarinus officinalis", "Lamium amplexicaule", "Lamium barbatum", - "Lamium purpureum", "Gomphostemma chinense", "Gomphostemma lucidum", "Dracocephalum heterophyllum", - "Coleus hybridu&scutellarioides", "Clinopodium chinense", "Clinopodium confine", "Clinopodium megalanthum", "Teucrium canadense", - "Teucrium fruticans", "Teucrium viscidum", "Keiskea elsholtzioides", "Isodon adenanthus", "Isodon amethystoides", - "Isodon lophanthoides", "Isodon sculponeatus", "Isodon serra", "Elsholtzia argyi", "Elsholtzia ciliata", "Elsholtzia fruticosa", - "Elsholtzia stauntonii", "Plectranthus ecklonii", "Plectranthus glabratus", "Plectranthus hadiensis var. tomentosus", - "Plectranthus prostratus", "Scutellaria baicalensis", "Scutellaria barbata", "Scutellaria indica", "Scutellaria viscidula", - "Scutellaria wongkei", "Salvia", "Salvia apiana", "Salvia bowleyana", "Salvia chinensis", "Salvia coccinea", "Salvia columbariae", - "Salvia farinacea", "Salvia greggii", "Salvia guaranitica 'Black and Blue'", "Salvia leucantha", "Salvia liguliloba", - "Salvia lyrata", "Salvia mellifera", "Salvia miltiorrhiza", "Salvia nemorosa", "Salvia plebeia", "Salvia pratensis", - "Salvia splendens", "Salvia uliginosa", "Meehania fargesii", "Meehania montis-koyae", "Phytolacca acinosa", "Phytolacca americana", - "Talinum paniculatum", "Marchantia polymorpha", "Rinorea bengalensis", "Viola acuminata", "Viola arcuata", "Viola betonicifolia", - "Viola cornuta", "Viola delavayi", "Viola diffusa", "Viola fargesii", "Viola grypoceras", "Viola inconspicua", "Viola japonica", - "Viola mongolica", "Viola philippica", "Viola sororia", "Viola stewardiana", "Viola tricolor", "Melicytus ramiflorus", - "Notholithocarpus densiflorus", "Lithocarpus corneus", "Lithocarpus glaber", "Lithocarpus hancei", "Quercus acutissima", - "Quercus agrifolia", "Quercus alba", "Quercus aliena", "Quercus kelloggii", "Quercus lobata", "Quercus macrocarpa", - "Quercus palustris", "Quercus phellos", "Quercus robur", "Quercus rubra", "Quercus stellata", "Quercus variabilis", - "Castanea dentata", "Castanea mollissima", "Castanea seguinii", "Fagus grandifolia", "Castanopsis fargesii", "Castanopsis fissa", - "Castanopsis lamontii", "Cyclobalanopsis fleuryi", "Trigonostemon chinensis", "Trigonostemon flavidus", "Triadica cochinchinensis", - "Triadica sebifera", "Codiaeum variegatum", "Codiaeum variegatum 'Excellent'", "Hura crepitans", "Euphorbia antiquorum", - "Euphorbia bicolor", "Euphorbia characias", "Euphorbia cotinifolia", "Euphorbia cyathophora", "Euphorbia dentata", - "Euphorbia helioscopia&esula", "Euphorbia humifusa", "Euphorbia hypericifolia", "Euphorbia kansuensis", "Euphorbia lathyris", - "Euphorbia leucocephala", "Euphorbia maculata", "Euphorbia marginata", "Euphorbia milii", "Euphorbia milii var. alba", - "Euphorbia neorubella", "Euphorbia obesa", "Euphorbia prostrata", "Euphorbia pulcherrima", "Euphorbia resinifera", - "Euphorbia tirucalli", "Euphorbia viguieri", "Sauropus androgynus", "Strophioblachia fimbricalyx", "Alchornea davidii", - "Alchornea trewioides", "Croton capitatus", "Croton setiger", "Croton tiglium", "Plukenetia volubilis", "Manihot esculenta", - "Garcia nutans", "Vernicia fordii&montana", "Excoecaria acerifolia", "Excoecaria agallocha", "Excoecaria cochinchinensis", - "Aleurites moluccana", "Pedilanthus tithymaloides", "Cnidoscolus texanus", "Ricinus communis", "Macaranga tanarius var. tomentosa", - "Mallotus apelta", "Mallotus barbatus", "Mallotus japonicus", "Mallotus paniculatus", "Mallotus philippensis", "Mallotus repandus", - "Mallotus repandus var. chrysocarpus", "Mallotus tenuifolius", "Acalypha australis", "Acalypha hispida", "Acalypha reptans", - "Jatropha curcas", "Jatropha integerrima", "Jatropha podagrica", "Cannabis sativa", "Trema cannabina var. dielsiana", - "Celtis biondii", "Celtis sinensis", "Humulus lupulus", "Humulus scandens", "Pteroceltis tatarinowii", "Caladium bicolor", - "Pinellia cordata", "Pinellia pedatisecta", "Pinellia ternata", "Syngonium podophyllum", "Philodendron erubescens", - "Philodendron selloum", "Pistia stratiotes", "Arisaema bockii", "Arisaema erubescens", "Arisaema heterophyllum", - "Arisaema hunanense", "Arisaema silvestrii", "Arisaema triphyllum", "Aglaonema modestum", "Lysichiton americanus", "Lemna minor", - "Alocasia 'Amazonica'", "Alocasia odora", "Typhonium blumei", "Spathiphyllum kochii", "Symplocarpus foetidus", - "Colocasia antiquorum", "Colocasia esculenta", "Anthurium andraeanum", "Zamioculcas zamiifolia", "Zantedeschia", - "Amorphophallus dunnii", "Amorphophallus kiusianus", "Amorphophallus konjac", "Amorphophallus paeoniifolius", "Epipremnum aureum", - "Dieffenbachia seguine", "Monstera deliciosa", "Yucca gloriosa", "Paradisea liliastrum", "Ruscus aculeatus", "Eucomis comosa", - "Chlorophytum comosum", "Albuca namaquensis", "Hesperocallis undulata", "Asparagus cochinchinensis", "Asparagus densiflorus", - "Asparagus officinalis", "Asparagus setaceus", "Liriope muscari", "Liriope spicata", "Campylandra delavayi", "Thysanotus chinensis", - "Triteleia laxa", "Ornithogalum caudatum", "Ornithogalum divergens", "Ornithogalum dubium", "Ornithogalum narbonense", - "Ornithogalum umbellatum", "Cordyline australis", "Cordyline fruticosa", "Ledebouria socialis", "Ophiopogon bodinieri", - "Ophiopogon chingii", "Ophiopogon japonicus", "Hosta albomarginata", "Hosta plantaginea", "Hosta ventricosa", "Speirantha gardenii", - "Chlorogalum pomeridianum", "Disporopsis aspersa", "Disporopsis fuscopicta", "Disporopsis longifolia", "Disporopsis pernyi", - "Dichopogon strictus", "Camassia leichtlinii", "Camassia quamash", "Camassia scilloides", "Lachenalia viridiflora", - "Barnardia japonica", "Maianthemum bifolium", "Maianthemum canadense", "Maianthemum henryi", "Maianthemum japonicum", - "Maianthemum racemosum", "Maianthemum stellatum", "Muscari botryoides", "Dichelostemma capitatum", "Scilla bifolia", - "Scilla luciliae", "Scilla siberica", "Scilla verna", "Hyacinthoides hispanica", "Hyacinthoides non-scripta", - "Sansevieria gracilis", "Sansevieria trifasciata", "Sansevieria trifasciata var. laurentii", "Puschkinia scilloides", - "Aspidistra fimbriata", "Aspidistra grandiflora", "Aspidistra minutiflora", "Hesperoyucca whipplei", "Beaucarnea recurvata", - "Convallaria majalis", "Hyacinthus orientalis", "Polygonatum cyrtonema", "Polygonatum filipes", "Polygonatum hookeri", - "Polygonatum odoratum", "Polygonatum sibiricum", "Polygonatum verticillatum", "Agave americana", "Dracaena cambodiana", - "Dracaena draco", "Dracaena fragrans", "Dracaena reflexa", "Dracaena sanderiana", "Dracaena surculosa var. maculata", - "Wrightia laevis", "Wrightia pubescens", "Wrightia religiosa", "Carissa macrocarpa", "Pseudolithos migiurtinus", - "Gymnema sylvestre", "Dregea sinensis", "Dregea volubilis", "Dregea yunnanensis", "Ceropegia trichantha", "Ceropegia woodii", - "Parsonsia alboflavescens", "Telosma cordata", "Graphistemma pictum", "Nerium oleander", "Nerium oleander 'Paihua'", - "Tylophora ovata", "Tylophora silvestris", "Melodinus suaveolens", "Tabernaemontana divaricata", "Periploca sepium", - "Cryptostegia grandiflora", "Pachypodium lamerei", "Urceola rosea", "Amsonia tabernaemontana", "Adenium obesum", "Cerbera manghas", - "Beaumontia brevituba", "Beaumontia grandiflora", "Calotropis gigantea", "Stapelia", "Hoya carnosa", "Hoya multiflora", - "Cryptolepis buchananii", "Dischidia chinensis", "Dischidia ruscifolia&nummularia", "Pentasachme caudatum", "Vallaris indecora", - "Trachelospermum axillare", "Trachelospermum jasminoides", "Trachelospermum jasminoides 'Flame'", "Apocynum androsaemifolium", - "Apocynum venetum", "Strophanthus divaricatus", "Strophanthus gratus", "Stephanotis floribunda", "Metaplexis japonica", - "Vinca major", "Vinca major 'Variegata'", "Vinca minor", "Kopsia arborea", "Kopsia fruticosa", "Heterostemma brownii", - "Gomphocarpus fruticosus", "Gomphocarpus physocarpus", "Catharanthus roseus", "Catharanthus roseus 'Albus'", "Mandevilla sanderi", - "Asclepias asperula", "Asclepias curassavica", "Asclepias curassavica 'Flaviflora'", "Asclepias fascicularis", - "Asclepias incarnata", "Asclepias oenotheroides", "Asclepias speciosa", "Asclepias syriaca", "Asclepias tuberosa", - "Asclepias verticillata", "Asclepias viridiflora", "Asclepias viridis", "Merrillanthus hainanensis", "Anodendron affine", - "Plumeria obtusa", "Plumeria pudica", "Plumeria rubra", "Plumeria rubra 'Acutifolia'", "Alstonia scholaris", - "Cynanchum acuminatifolium", "Cynanchum atratum", "Cynanchum auriculatum", "Cynanchum chekiangense", "Cynanchum chinense", - "Cynanchum corymbosum", "Cynanchum stauntonii", "Cynanchum thesioides", "Chonemorpha eriostylis", "Thevetia peruviana", - "Thevetia peruviana 'Aurantiaca'", "Allamanda blanchetii", "Allamanda schottii&cathartica", "Jasminanthes mucronata", - "Zingiber cochleariforme", "Zingiber mioga", "Zingiber officinale", "Zingiber striolatum", "Zingiber zerumbet", - "Hedychium coccineum", "Hedychium coronarium", "Hedychium flavescens", "Hedychium flavum", "Hedychium yunnanense", - "Curcuma alismatifolia", "Curcuma longa", "Curcuma phaeocaulis", "Curcuma wenyujin", "Alpinia hainanensis", "Alpinia japonica", - "Alpinia oblongifolia", "Alpinia officinarum", "Alpinia zerumbet", "Alpinia zerumbet 'Variegata'", "Kaempferia elegans", - "Kaempferia galanga", "Kaempferia rotunda", "Globba schomburgkii", "Etlingera elatior", "Amomum tsaoko", "Amomum villosum", - "Roscoea schneideriana", "Cheilocostus speciosus", "Styrax chinensis", "Styrax confusus", "Styrax faberi", "Styrax japonicus", - "Styrax odoratissimus", "Styrax suberifolius", "Huodendron biaristatum var. parviflorum", "Rehderodendron kwangtungense", - "Pterostyrax corymbosus", "Sinojackia xylocarpa", "Alniphyllum fortunei", "Halesia macgregorii", "Melliodendron xylocarpum", - "Myriophyllum aquaticum", "Myriophyllum verticillatum", "Podophyllum peltatum", "Mahonia bealei", "Mahonia fortunei", - "Mahonia oiwakensis", "Mahonia shenii", "Nandina domestica", "Berberis diaphana", "Berberis jamesiana", "Berberis julianae", - "Berberis lempergiana", "Berberis pruinosa", "Berberis thunbergii", "Berberis thunbergii 'Atropurpurea'", "Berberis trifoliolata", - "Berberis vulgaris", "Berberis wilsoniae", "Diphylleia grayi", "Sinopodophyllum hexandrum", "Epimedium brevicornu", - "Epimedium davidii", "Epimedium sagittatum", "Epimedium wushanense", "Gymnospermium kiangnanense", "Dysosma pleiantha", - "Dysosma versipellis", "Microdesmis caseariifolia", "Capparis acutifolia", "Capparis bodinieri", "Crateva formosensis", - "Crateva religiosa", "Crateva unilocularis", "Pouteria caimito", "Pouteria campechiana", "Synsepalum dulcificum", - "Madhuca pasquieri", "Chrysophyllum cainito", "Sinosideroxylon wightianum", "Manilkara zapota", "Mimusops elengi", - "Symplocos cochinchinensis", "Symplocos congesta", "Symplocos lancifolia", "Symplocos lucida", "Symplocos paniculata", - "Symplocos stellaris", "Symplocos sumuntia", "Alangium chinense", "Alangium kurzii", "Alangium platanifolium", - "Alangium salviifolium", "Cornus alba", "Cornus canadensis", "Cornus capitata", "Cornus controversa", "Cornus drummondii", - "Cornus florida", "Cornus hongkongensis", "Cornus hongkongensis subsp. elegans", "Cornus kousa subsp. chinensis", "Cornus mas", - "Cornus officinalis", "Cornus quinquenervis", "Cornus sanguinea", "Cornus sericea", "Polyspora axillaris", "Camellia amplexicaulis", - "Camellia azalea", "Camellia chekiangoleosa", "Camellia crapnelliana", "Camellia cuspidata", "Camellia grijsii", - "Camellia japonica", "Camellia oleifera", "Camellia petelotii", "Camellia pitardii", "Camellia salicifolia", "Camellia saluenensis", - "Camellia sasanqua", "Camellia sinensis", "Camellia sinensis var. assamica", "Camellia uraku", "Camellia yunnanensis", - "Schima superba", "Pyrenaria microcarpa", "Pyrenaria spectabilis", "Stewartia sinensis", "Helicia reticulata", "Protea cynaroides", - "Buckinghamia celsissima", "Macadamia integrifolia", "Leucospermum nutans", "Grevillea banksii", "Diapensia purpurea", - "Heptacodium miconioides", "Zabelia biflora", "Zabelia dielsii", "Acanthocalyx alba", "Linnaea borealis", "Dipsacus asper", - "Dipsacus fullonum", "Lonicera caerulea", "Lonicera chrysantha", "Lonicera elisae", "Lonicera ferdinandi", - "Lonicera fragrantissima", "Lonicera fragrantissima var. lancifolia", "Lonicera hispida", "Lonicera hispidula", - "Lonicera involucrata", "Lonicera japonica", "Lonicera japonica var. chinensis", "Lonicera korolkowi", "Lonicera maackii", - "Lonicera modesta", "Lonicera praeflorens", "Lonicera sempervirens", "Lonicera tangutica", "Lonicera tatarica", - "Lonicera tatarica 'Lutea'", "Lonicera trichosantha", "Symphoricarpos albus", "Symphoricarpos orbiculatus", "Kolkwitzia amabilis", - "Abelia chinensis", "Abelia macrotera", "Abelia uniflora", "Abelia × grandiflora", "Triosteum himalayanum", "Scabiosa atropurpurea", - "Scabiosa comosa", "Patrinia monandra", "Patrinia villosa", "Centranthus ruber", "Weigela coraeensis", "Weigela florida", - "Weigela florida 'Red Prince'", "Weigela florida 'Variegata'", "Weigela japonica var. sinica", "Leycesteria formosa", "Platanus", - "Platanus occidentalis", "Platanus racemosa", "Penthorum chinense", "Trientalis borealis", "Trientalis europaea", - "Trientalis latifolia", "Cyclamen persicum", "Stimpsonia chamaedryoides", "Primula acaulis", "Primula agleniana", - "Primula beesiana", "Primula bella", "Primula blinii", "Primula chionantha", "Primula cicutariifolia", "Primula denticulata", - "Primula denticulata subsp. sinodenticulata", "Primula dryadifolia subsp. jonardunii", "Primula hendersonii", - "Primula maximowiczii", "Primula nutans", "Primula obconica", "Primula palmata", "Primula pelargoniifolia", "Primula pinnatifida", - "Primula poissonii", "Primula polyneura", "Primula pseudodenticulata", "Primula pulverulenta", "Primula saxatilis", - "Primula secundiflora", "Primula sikkimensis", "Primula sinolisteri", "Primula sonchifolia", "Primula stenocalyx", - "Primula tangutica", "Primula valentiniana", "Primula vialii", "Maesa montana", "Maesa perlarius", "Androsace henryi", - "Androsace mariae", "Androsace rigida", "Androsace spinulifera", "Androsace umbellata", "Androsace wardii", - "Androsace yargongensis", "Omphalogramma vinciflorum", "Lysimachia alfredii", "Lysimachia barystachys", "Lysimachia candida", - "Lysimachia christiniae", "Lysimachia ciliata", "Lysimachia clethroides", "Lysimachia congestiflora", "Lysimachia fortunei", - "Lysimachia grammica", "Lysimachia hemsleyana", "Lysimachia heterogenea", "Lysimachia klattiana", "Lysimachia longipes", - "Lysimachia nanpingensis", "Lysimachia nummularia", "Lysimachia nummularia 'Aurea'", "Lysimachia paridiformis var. stenophylla", - "Lysimachia patungensis", "Lysimachia pseudohenryi", "Lysimachia pumila", "Lysimachia punctata", "Anagallis arvensis", - "Anagallis arvensis f. coerulea", "Ardisia crenata", "Ardisia crispa", "Ardisia elliptica", "Ardisia humilis", "Ardisia japonica", - "Ardisia mamillata", "Ardisia obtusa", "Ardisia villosa", "Aegiceras corniculatum", "Embelia parviflora", "Embelia ribes", - "Myrsine africana", "Myrsine seguinii", "Muntingia calabura", "Erycibe expansa", "Evolvulus alsinoides", "Evolvulus nuttallianus", - "Xenostegia tridentata", "Calystegia hederacea", "Calystegia sepium", "Calystegia soldanella", "Convolvulus arvensis", - "Convolvulus tragacanthoides", "Pharbitis limbata", "Operculina turpethum", "Quamoclit coccinea", "Quamoclit pennata", - "Quamoclit × sloteri", "Cuscuta japonica", "Ipomoea alba", "Ipomoea aquatica", "Ipomoea batatas", "Ipomoea biflora", - "Ipomoea cairica", "Ipomoea carnea subsp. fistulosa", "Ipomoea cordatotriloba", "Ipomoea indica", "Ipomoea lacunosa", - "Ipomoea nil&purpurea", "Ipomoea obscura", "Ipomoea pes-caprae", "Ipomoea triloba", "Dinetus racemosus", "Dichondra micrantha", - "Merremia dissecta", "Merremia hederacea", "Merremia sibirica", "Merremia tuberosa", "Merremia vitifolia", "Stachyurus chinensis", - "Stachyurus himalaicus", "Aesculus californica", "Aesculus chinensis", "Aesculus glabra", "Aesculus hippocastanum", - "Aesculus pavia", "Cardiospermum grandiflorum", "Cardiospermum halicacabum", "Blighia sapida", "Xanthoceras sorbifolium", - "Sapindus saponaria", "Koelreuteria bipinnata", "Koelreuteria paniculata", "Acer buergerianum", "Acer cordatum", "Acer davidii", - "Acer fabri", "Acer henryi", "Acer macrophyllum", "Acer negundo", "Acer palmatum", "Acer pensylvanicum", "Acer platanoides", - "Acer pubinerve", "Acer rubrum", "Acer saccharinum", "Acer saccharum", "Acer spicatum", "Acer tataricum subsp. ginnala", - "Acer tataricum subsp. theiferum", "Acer truncatum", "Arytera littoralis", "Delavaya toxocarpa", "Litchi chinensis", - "Dodonaea viscosa", "Nephelium lappaceum", "Dimocarpus longan", "Tropaeolum majus", "Kingdonia uniflora", "Kalanchoe beauverdii", - "Kalanchoe blossfeldiana", "Kalanchoe delagoensis", "Kalanchoe fedtschenkoi", "Kalanchoe marmorata", "Kalanchoe porphyrocalyx", - "Kalanchoe tomentosa", "Hylotelephium spectabile", "Hylotelephium tatarinowii", "× Pachyveria 'Powder Puff'", - "Adromischus cristatus var. clavifolius", "Greenovia", "Sedum acre 'Aurea'", "Sedum alfredii", "Sedum bulbiferum", - "Sedum drymarioides", "Sedum emarginatum", "Sedum lineare", "Sedum sarmentosum", "Sedum sediforme", "Sedum spathulifolium", - "Orostachys fimbriata", "Orostachys malacophylla", "Echeveria 'Neon Breakers'", "Echeveria 'Perle von Nurnberg'", - "Echeveria lilacina", "Echeveria pulidonis", "Echeveria pulvinata", "Echeveria runyonii 'Topsy Turvy'", "Rhodiola rosea", - "Rhodiola yunnanensis", "Aeonium 'Zwartkop'", "Bryophyllum pinnatum", "Phedimus aizoon", "Cotyledon tomentosa", - "Sempervivum arachnoideum subsp. tomentosum", "Crassula arborescens", "Crassula corymbulosa", "Crassula marnieriana", - "Crassula obliqua 'Gollum'", "Graptopetalum amethystinum", "Magnolia grandiflora", "Magnolia tripetala", "Houpoea officinalis", - "Michelia champaca", "Michelia chapensis", "Michelia crassipes", "Michelia figo", "Michelia foveolata", "Michelia guangdongensis", - "Michelia macclurei", "Michelia maudiae", "Michelia skinneriana", "Michelia × alba", "Oyama sieboldii", "Manglietia fordiana", - "Manglietia insignis", "Woonyoungia septentrionalis", "Yulania biondii", "Yulania denudata", "Yulania denudata 'Fei Huang'", - "Yulania liliiflora", "Yulania stellata", "Yulania zenii", "Yulania × soulangeana", "Talauma hodgsonii", "Lirianthe championii", - "Lirianthe coco", "Liriodendron chinense", "Liriodendron tulipifera", "Syringa", "Syringa meyeri", "Syringa oblata", - "Syringa pubescens", "Syringa reticulata subsp. amurensis", "Syringa reticulata subsp. pekinensis", - "Syringa reticulata subsp. pekinensis 'Jinyuan'", "Syringa vulgaris", "Syringa yunnanensis", "Syringa × persica", - "Ligustrum lucidum", "Ligustrum obtusifolium", "Ligustrum quihoui", "Ligustrum sinense", "Ligustrum × vicaryi", "Osmanthus armatus", - "Osmanthus fragrans", "Olea europaea", "Fraxinus chinensis", "Fraxinus pennsylvanica", "Fraxinus sieboldiana", - "Chionanthus retusus", "Jasminum elongatum", "Jasminum floridum", "Jasminum grandiflorum", "Jasminum humile", - "Jasminum lanceolaria", "Jasminum mesnyi", "Jasminum multiflorum", "Jasminum nervosum", "Jasminum nudiflorum", - "Jasminum odoratissimum", "Jasminum officinale", "Jasminum pentaneurum", "Jasminum polyanthum", "Jasminum sambac", - "Jasminum sinense", "Jasminum subhumile", "Forsythia suspensa", "Forsythia viridissima", "Fontanesia phillyreoides subsp. fortunei", - "", "Equisetum arvense", "Equisetum hyemale", "Equisetum ramosissimum", "Equisetum ramosissimum subsp. debile", "Akebia quinata", - "Akebia trifoliata", "Stauntonia chinensis", "Stauntonia obovatifoliola subsp. urophylla", "Eucommia ulmoides", - "Elaeocarpus apiculatus", "Elaeocarpus decipiens", "Elaeocarpus glabripetalus", "Elaeocarpus hainanensis", "Elaeocarpus serratus", - "Sloanea sinensis", "Monotropastrum humile", "Enkianthus campanulatus", "Enkianthus chinensis", "Enkianthus deflexus", - "Enkianthus quinqueflorus", "Enkianthus serrulatus", "Chimaphila maculata", "Kalmia latifolia", "Cassiope selaginoides", - "Diplarche multiflora", "Rhododendron argyrophyllum", "Rhododendron bachii", "Rhododendron campylogynum", "Rhododendron capitatum", - "Rhododendron championiae", "Rhododendron charitopes subsp. tsangpoense", "Rhododendron florulentum", "Rhododendron hongkongense", - "Rhododendron kwangtungense", "Rhododendron latoucheae", "Rhododendron lepidotum", "Rhododendron maculiferum subsp. anwheiense", - "Rhododendron micranthum", "Rhododendron molle", "Rhododendron mucronatum", "Rhododendron oreodoxa", "Rhododendron ovatum", - "Rhododendron rhuyuenense", "Rhododendron rivulare", "Rhododendron seniavinii", "Rhododendron simiarum", "Rhododendron simsii", - "Rhododendron stamineum", "Rhododendron yunnanense", "Rhododendron × pulchrum", "Pterospora andromedea", "Agapetes burmanica", - "Agapetes lacei", "Monotropa hypopitys", "Monotropa uniflora", "Moneses uniflora", "Lyonia ovalifolia var. hebecarpa", - "Gaultheria procumbens", "Gaultheria shallon", "Gaultheria trichophylla", "Arbutus menziesii", "Sarcodes sanguinea", - "Vaccinium bracteatum", "Vaccinium corymbosum", "Vaccinium macrocarpon", "Vaccinium mandarinorum", "Vaccinium ovatum", - "Vaccinium uliginosum", "Pieris formosa", "Pieris japonica", "Pyrola calliantha", "Homalium ceylanicum", "Homalium cochinchinense", - "Idesia polycarpa", "Populus alba", "Populus deltoides", "Populus simonii var. przewalskii", "Salix", "Salix integra", - "Salix integra 'Hakuro Nishiki'", "Salix wallichiana", "Itoa orientalis", "Casearia velutina", "Myrica rubra", "Picea abies", - "Picea likiangensis var. linzhiensis", "Abies balsamea", "Pinus densiflora", "Pinus massoniana", "Pinus palustris", - "Pinus parviflora", "Pinus ponderosa", "Pinus strobus", "Pinus sylvestris", "Pinus taeda", "Larix gmelinii", "Larix kaempferi", - "Pseudolarix amabilis", "Tsuga canadensis", "Pseudotsuga menziesii", "Platycladus orientalis", "Juniperus chinensis", - "Juniperus communis", "Juniperus virginiana", "Sequoia sempervirens", "Thuja occidentalis", "Taxodium distichum", - "Taxodium mucronatum", "Ludwigia adscendens", "Ludwigia octovalvis", "Ludwigia peploides subsp. stipulacea", "Ludwigia sedioides", - "Clarkia amoena", "Clarkia unguiculata", "Fuchsia hybrida", "Gaura lindheimeri", "Gaura parviflora", - "Oenothera biennis&glazioviana", "Oenothera drummondii", "Oenothera laciniata", "Oenothera macrocarpa", "Oenothera rosea", - "Oenothera speciosa", "Oenothera tetraptera", "Chamerion angustifolium", "Epilobium canum", "Epilobium hirsutum", - "Epilobium pyrricholophum", "Circaea cordata", "Tamarix chinensis", "Tamarix ramosissima", "Myricaria squamosa", "Diospyros armata", - "Diospyros cathayensis", "Diospyros japonica", "Diospyros kaki", "Diospyros lotus", "Diospyros nitida", "Diospyros rhombifolia", - "Diospyros vaccinioides", "Diospyros virginiana", "Corymbia ptychocarpa", "Plinia cauliflora", "Rhodomyrtus tomentosa", - "Eucalyptus cinerea", "Eugenia brasiliensis", "Eugenia uniflora", "Psidium guajava", "Melaleuca cajuputi subsp. cumingiana", - "Callistemon citrinus", "Callistemon rigidus", "Syzygium acuminatissimum", "Syzygium australe", "Syzygium cumini", - "Syzygium fluviatile", "Syzygium grijsii", "Syzygium jambos", "Syzygium malaccense", "Syzygium rehderianum", - "Syzygium samarangense", "Acca sellowiana", "Xanthostemon chrysanthus", "Chamelaucium uncinatum", "Myrtus communis", - "Leptospermum scoparium", "Dendrophthoe pentandra", "Scurrula parasitica", "Taxillus chinensis", "Broussonetia kaempferi", - "Broussonetia kaempferi var. australis", "Broussonetia kazinoki", "Broussonetia papyrifera", "Morus alba", "Morus australis", - "Ficus altissima", "Ficus auriculata", "Ficus carica", "Ficus deltoidea", "Ficus elastica", "Ficus erecta", - "Ficus gasparriniana var. laceratifolia", "Ficus hispida", "Ficus pandurata", "Ficus pumila", "Ficus racemosa", "Ficus religiosa", - "Ficus subpisocarpa", "Ficus vaccinioides", "Ficus virens", "Maclura cochinchinensis", "Maclura pomifera", "Maclura tricuspidata", - "Artocarpus communis", "Artocarpus heterophyllus", "Artocarpus hypargyreus", "Dorstenia elata", "Codonopsis lanceolata", - "Codonopsis subglobosa", "Lobelia cardinalis", "Lobelia chinensis", "Lobelia davidii", "Lobelia erinus", "Lobelia melliana", - "Lobelia nummularia", "Lobelia sessilifolia", "Lobelia siphilitica", "Lobelia zeylanica", "Triodanis perfoliata", - "Triodanis perfoliata subsp. biflora", "Platycodon grandiflorus", "Adenophora himalayana", "Adenophora petiolata subsp. hunanensis", - "Adenophora polyantha", "Adenophora potaninii", "Adenophora stricta", "Adenophora trachelioides", "Wahlenbergia marginata", - "Cyananthus formosus", "Cyananthus incanus", "Cyananthus macrocalyx", "Cyclocodon lancifolius", "Campanumoea javanica", - "Lithotoma axillaris", "Campanula", "Campanula glomerata subsp. speciosa", "Campanula punctata", "Campanula rotundifolia", - "Hippobroma longiflora", "Clethra barbinervis", "Clethra delavayi", "Alnus trabeculosa", "Corylus avellana", "Ostrya rehderiana", - "Washingtonia filifera", "Washingtonia robusta", "Chrysalidocarpus lutescens", "Trachycarpus fortunei", "Cocos nucifera", - "Areca catechu", "Phoenix sylvestris", "Wodyetia bifurcata", "Calamus thysanolepis", "Sabal minor", "Livistona chinensis", - "Salacca edulis", "Caryota maxima", "Aphanamixis polystachya", "Swietenia macrophylla", "Melia azedarach", "Aglaia odorata", - "Toona sinensis", "Heynea trijuga", "Chukrasia tabularis", "Ulmus americana", "Ulmus pumila", "Salvinia molesta", - "Azolla pinnata subsp. asiatica", "Umbellularia californica", "Lindera aggregata", "Lindera benzoin", "Lindera communis", - "Lindera megaphylla", "Laurus nobilis", "Litsea cubeba", "Litsea glutinosa", "Phoebe bournei", "Phoebe chekiangensis", - "Phoebe sheareri", "Cinnamomum burmannii", "Cinnamomum camphora", "Cinnamomum cassia", "Cinnamomum japonicum", - "Cinnamomum kotoense", "Sassafras albidum", "Sassafras tzumu", "Machilus grijsii", "Machilus leptophylla", "Machilus thunbergii", - "Machilus velutina", "Persea americana", "Canarium album", "Torenia concolor", "Torenia fournieri", "Torenia violacea", - "Lindernia anagallis", "Lindernia crustacea", "Lindernia ruellioides", "Aconitum barbatum var. puberulum", "Aconitum coreanum", - "Aconitum gymnandrum", "Aconitum hemsleyanum", "Aconitum kusnezoffii", "Aconitum tanguticum", "Dichocarpum dalzielii", "Adonis", - "Thalictrum acutifolium", "Thalictrum aquilegiifolium var. sibiricum", "Thalictrum delavayi", "Thalictrum fargesii", - "Thalictrum fortunei", "Thalictrum ichangense", "Thalictrum petaloideum", "Thalictrum thalictroides", "Semiaquilegia adoxoides", - "Paraquilegia microphylla", "Ficaria verna", "Ranunculus asiaticus", "Ranunculus cantoniensis", "Ranunculus muricatus", - "Ranunculus repens", "Ranunculus sceleratus", "Ranunculus ternatus", "Batrachium bungei", "Batrachium pekinense", - "Pulsatilla chinensis", "Actaea erythrocarpa", "Actaea pachypoda", "Actaea rubra", "Anemoclema glaucifolium", - "Delphinium anthriscifolium", "Delphinium anthriscifolium var. majus", "Delphinium anthriscifolium var. savatieri", - "Delphinium elatum", "Delphinium grandiflorum", "Aquilegia canadensis", "Aquilegia chrysantha", "Aquilegia ecalcarata", - "Aquilegia formosa", "Aquilegia oxysepala", "Aquilegia oxysepala var. oxysepala f. pallidiflora", "Aquilegia viridiflora", - "Aquilegia viridiflora var. atropurpurea", "Aquilegia vulgaris", "Aquilegia yabeana", "Anemonopsis macrophylla", - "Trollius chinensis", "Trollius yunnanensis", "Helleborus thibetanus", "Clematis 'Rooguchi'", "Clematis acerifolia", - "Clematis apiifolia", "Clematis apiifolia var. argentilucida", "Clematis armandii", "Clematis brevicaudata", "Clematis chinensis", - "Clematis chrysocoma", "Clematis courtoisii", "Clematis crassifolia", "Clematis finetiana", "Clematis florida", - "Clematis fruticosa", "Clematis fusca var. violacea", "Clematis henryi", "Clematis heracleifolia", "Clematis hexapetala", - "Clematis integrifolia", "Clematis lasiandra", "Clematis macropetala", "Clematis meyeniana", "Clematis montana", - "Clematis montana var. sterilis", "Clematis nannophylla", "Clematis peterae", "Clematis potaninii", "Clematis pseudootophora", - "Clematis pseudopogonandra", "Clematis ranunculoides", "Clematis rehderiana", "Clematis repens", "Clematis sibirica", - "Clematis sibirica var. ochotensis", "Clematis tangutica", "Clematis terniflora", "Clematis terniflora var. mandshurica", - "Clematis uncinata", "Clematis virginiana", "Anemone acutiloba", "Anemone americana", "Anemone coronaria", "Anemone demissa", - "Anemone flaccida", "Anemone geum subsp. ovalifolia", "Anemone hupehensis", "Anemone obtusiloba", "Anemone rivularis", - "Anemone rivularis var. flore-minore", "Anemone rupicola", "Anemone tomentosa&vitifolia", "Consolida ajacis", "Caltha palustris", - "Caltha sinogracilis", "Oxygraphis glacialis", "Souliea vaginata", "Nigella damascena", "Claytonia caroliniana", - "Claytonia perfoliata", "Claytonia virginica", "Lewisia cotyledon", "Burmannia disticha", "Burmannia itoana", - "Burmannia nepalensis", "Egeria densa", "Ottelia acuminata", "Ottelia acuminata var. crispa", "Ottelia alismoides", - "Hydrocharis dubia", "Polypodium virginianum", "Microsorum pustulatum", "Platycerium bifurcatum", "Platycerium wallichii", - "Aletris scopulorum", "Aletris spicata", "Paulownia", "Paulownia tomentosa", "Sagittaria latifolia", "Sagittaria montevidensis", - "Sagittaria pygmaea", "Sagittaria sagittifolia", "Sagittaria trifolia", "Hydrocleys nymphoides", "Alisma canaliculatum", - "Alisma plantago-aquatica", "Echinodorus grisebachii", "Limnocharis flava", "Pittosporum illicioides", "Pittosporum tobira", - "Lygodium japonicum", "Meliosma flexuosa", "Meliosma rigida", "Meliosma rigida var. pannosa", "Meliosma squamulata", - "Sabia campanulata subsp. ritchieae", "Sabia discolor", "Sabia japonica", "Sabia limoniacea", "Sabia swinhoei", "Malosma laurina", - "Choerospondias axillaris", "Mangifera indica", "Toxicodendron diversilobum", "Toxicodendron radicans", "Toxicodendron succedaneum", - "Rhus aromatica", "Rhus chinensis", "Rhus glabra", "Rhus integrifolia", "Rhus ovata", "Rhus typhina", "Anacardium occidentale", - "Cotinus coggygria", "Pistacia vera", "Juncus allioides", "Juncus effusus", "Juncus prismatocarpus", "Barleria cristata", - "Barleria lupulina", "Asystasia gangetica", "Asystasia gangetica subsp. micrantha", "Asystasia neesiana", - "Crossandra infundibuliformis", "Aphelandra sinclairiana", "Aphelandra squarrosa", "Eranthemum pulchellum", "Rungia densiflora", - "Pseuderanthemum carruthersii", "Pseuderanthemum crenulatum", "Pseuderanthemum laxiflorum", - "Pseuderanthemum reticulatum var. ovarifolium", "Thunbergia alata", "Thunbergia coccinea", "Thunbergia erecta", - "Thunbergia fragrans", "Thunbergia grandiflora", "Thunbergia laurifolia", "Thunbergia mysorensis", "Hygrophila ringens", - "Rhinacanthus nasutus", "Justicia adhatoda", "Justicia austrosinensis", "Justicia betonica", "Justicia brandegeeana", - "Justicia brasiliana", "Justicia procumbens", "Justicia quadrifaria", "Dicliptera chinensis", "Cyrtanthera carnea", - "Andrographis paniculata", "Fittonia albivenis", "Acanthus ilicifolius", "Acanthus mollis", "Perilepta dyeriana", "Ruellia elegans", - "Ruellia simplex", "Ruellia venusta", "Peristrophe hyssopifolia 'Aureo-variegata'", "Peristrophe japonica", - "Megaskepasma erythrochlamys", "Brillantaisia owariensis", "Pachystachys lutea", "Codonacanthus pauciflorus", - "Strobilanthes aprica", "Strobilanthes cusia", "Strobilanthes dimorphotricha", "Strobilanthes hamiltoniana", - "Strobilanthes sarcorrhiza", "Strobilanthes schomburgkii", "Strobilanthes tetrasperma", "Clinacanthus nutans", - "Cystacanthus pyramidalis", "Odontonema strictum", "Sanchezia speciosa", "Rourea microphylla", "Pelargonium graveolens", - "Pelargonium hortorum", "Pelargonium peltatum", "Pelargonium zonale", "Erodium cicutarium", "Erodium stephanianum", - "Geranium carolinianum", "Geranium maculatum", "Geranium nepalense", "Geranium pratense", "Geranium pylzowianum", - "Geranium refractum", "Geranium robertianum", "Geranium sibiricum", "Geranium sinense", "Geranium wilfordii", - "Geranium wlassovianum", "Pinguicula alpina", "Utricularia aurea", "Utricularia australis", "Utricularia bifida", - "Utricularia caerulea", "Utricularia striatula", "Utricularia warburgii", "Saurauia tristyla", "Actinidia arguta", - "Actinidia callosa var. discolor", "Actinidia chinensis", "Actinidia eriantha", "Actinidia lanceolata", "Actinidia latifolia", - "Actinidia macrosperma", "Actinidia rubricaulis var. coriacea", "Nepenthes mirabilis", "Diascia barberae", "Verbascum blattaria", - "Verbascum thapsus", "Scrophularia californica", "Scrophularia ningpoensis", "Leucophyllum frutescens", "Buddleja asiatica", - "Buddleja davidii", "Buddleja fallowiana", "Buddleja lindleyana", "Buddleja officinalis", "Nemesia strumosa", - "Couroupita guianensis", "Barringtonia acutangula", "Barringtonia asiatica", "Barringtonia racemosa", "Onoclea sensibilis", - "Matteuccia struthiopteris", "Aquilaria sinensis", "Stellera chamaejasme", "Daphne aurantiaca", "Daphne championii", - "Daphne genkwa", "Daphne giraldii", "Daphne kiusiana var. atrocaulis", "Daphne longilobata", "Daphne odora", "Daphne papyracea", - "Daphne tangutica", "Edgeworthia chrysantha", "Wikstroemia indica", "Wikstroemia monnula", "Wikstroemia nutans", - "Wikstroemia pilosa", "Sarracenia purpurea", "Eriodictyon californicum", "Hydrophyllum virginianum", "Philydrum lanuginosum", - "Carica papaya", "Mesembryanthemum cordifolium", "Mesembryanthemum crystallinum", "Lampranthus spectabilis", "Carpobrotus edulis", - "Lithops pseudotruncatella subsp. archerae", "Fenestraria aurantiaca", "Glottiphyllum longum", "Rhombophyllum nelii", - "Astridia velutina", "Cananga odorata", "Cananga odorata var. fruticosa", "Desmos chinensis", "Asimina triloba", "Polyalthia laui", - "Polyalthia longifolia", "Polyalthia suberosa", "Fissistigma oldhamii", "Fissistigma polyanthum", "Annona glabra", "Annona montana", - "Annona muricata", "Annona squamosa", "Uvaria boniana", "Uvaria grandiflora", "Uvaria macrophylla", "Uvaria tonkinensis", - "Chieniodendron hainanense", "Mitrephora tomentosa", "Artabotrys hainanensis", "Artabotrys hexapetalus", "Artabotrys hongkongensis", - "Peganum harmala", "Armeria maritima", "Plumbago auriculata", "Plumbago indica", "Plumbago zeylanica", "Limonium bicolor", - "Limonium sinense", "Limonium tenellum", "Peritoma arborea", "Tarenaya hassleriana", "Clintonia borealis", "Calochortus albus", - "Calochortus amabilis", "Calochortus leichtlinii", "Calochortus luteus", "Calochortus plummerae", "Calochortus pulchellus", - "Calochortus splendens", "Calochortus tolmiei", "Calochortus venustus", "Notholirion bulbuliferum", "Cardiocrinum cathayanum", - "Cardiocrinum giganteum", "Cardiocrinum giganteum var. yunnanense", "Medeola virginiana", "Streptopus simplex", - "Tricyrtis formosana", "Tricyrtis macropoda", "Tricyrtis pilosa", "Erythronium albidum", "Erythronium americanum", - "Erythronium grandiflorum", "Erythronium japonicum", "Erythronium oregonum", "Erythronium sibiricum", - "Lilium bakerianum var. rubrum", "Lilium brownii", "Lilium canadense", "Lilium columbianum", "Lilium concolor", - "Lilium concolor var. pulchellum", "Lilium dauricum", "Lilium davidii", "Lilium davidii var. unicolor", "Lilium distichum", - "Lilium duchartrei", "Lilium lankongense", "Lilium longiflorum", "Lilium lophophorum", "Lilium martagon", - "Lilium nanum var. flavidum", "Lilium pardalinum", "Lilium parvum", "Lilium philadelphicum", "Lilium primulinum var. ochraceum", - "Lilium pumilum", "Lilium regale", "Lilium rosthornii", "Lilium souliei", "Lilium speciosum var. gloriosoides", "Lilium taliense", - "Lilium tigrinum", "Amana edulis", "Nomocharis aperta", "Nomocharis pardanthina", "Fritillaria affinis", - "Fritillaria camschatcensis", "Fritillaria imperialis", "Fritillaria maximowiczii", "Fritillaria meleagris", "Fritillaria persica", - "Fritillaria thunbergii", "Fritillaria ussuriensis", "Tulipa gesneriana", "Tulipa iliensis", "Welwitschia mirabilis", - "Stemona japonica", "Stemona mairei", "Stemona tuberosa", "Turpinia arguta", "Euscaphis japonica", "Potamogeton crispus", - "Potamogeton distinctus", "Victoria amazonica", "Victoria cruziana", "Nymphaea", "Nymphaea alba", "Nymphaea nouchali", - "Nymphaea odorata", "Euryale ferox", "Nuphar pumila", "Menyanthes trifoliata", "Nymphoides coreana", "Nymphoides cristata", - "Nymphoides indica", "Nymphoides peltata", "Palhinhaea cernua", "Diphasiastrum digitatum", "Dendrolycopodium obscurum", - "Lycopodiastrum casuarinoides", "Lychnis chalcedonica", "Lychnis fulgens", "Lychnis senno", "Cerastium glomeratum", - "Arenaria smithiana", "Sagina japonica", "Gypsophila oldhamiana", "Gypsophila paniculata", "Dianthus armeria", "Dianthus barbatus", - "Dianthus caryophyllus", "Dianthus chinensis", "Dianthus superbus", "Stellaria alsine", "Stellaria chinensis", "Stellaria media", - "Saponaria officinalis", "Silene armeria", "Silene baccifera", "Silene conoidea", "Silene davidii", "Silene gallica", - "Silene latifolia", "Silene vulgaris", "Myosoton aquaticum", "Agrostemma githago", "Vaccaria hispanica", "Nothoscordum bivalve", - "Boophone disticha", "Eucharis amazonica", "Clivia miniata", "Clivia nobilis", "Clivia × hybrida", "Amaryllis belladonna", - "Crinum amabile", "Crinum asiaticum var. sinicum", "Crinum moorei", "Ipheion uniflorum", "Polianthes tuberosa", - "Cyrtanthus mackenii", "Hippeastrum reticulatum", "Hippeastrum rutilum", "Narcissus bulbocodium", "Narcissus poeticus", - "Narcissus pseudonarcissus", "Narcissus tazetta var. chinensis", "Narcissus triandrus", "Hymenocallis speciosa&littoralis", - "Agapanthus africanus", "Agapanthus praecox", "Lycoris aurea", "Lycoris chinensis", "Lycoris haywardii", "Lycoris incarnata", - "Lycoris longituba", "Lycoris radiata", "Lycoris sprengeri", "Lycoris squamigera", "Lycoris straminea", "Lycoris × rosea", - "Tulbaghia violacea", "Allium carolinianum", "Allium cepa", "Allium chinense", "Allium fistulosum", "Allium giganteum", - "Allium prattii", "Allium sativum", "Allium senescens", "Allium sikkimense", "Allium triquetrum", "Allium tuberosum", - "Allium wallichii", "Zephyranthes candida", "Zephyranthes carinata", "Zephyranthes citrina", "Haemanthus albiflos", - "Haemanthus multiflorus", "Galanthus elwesii", "Leucojum aestivum", "Leucojum vernum", "Eucrosia bicolor", "Histiopteris incisa", - "Pteridium aquilinum", "Lagurus ovatus", "Phyllostachys nigra", "Hordeum jubatum", "Bothriochloa ischaemum", - "Chasmanthium latifolium", "Triticum aestivum", "Poa annua", "Phaenosperma globosa", "Isachne globosa", "Polypogon monspeliensis", - "Oplismenus undulatifolius", "Avena fatua", "Setaria italica var. germanica", "Setaria palmifolia", "Setaria pumila", - "Setaria viridis", "Cynodon dactylon", "Pennisetum alopecuroides", "Pennisetum glaucum", "Pennisetum orientale", - "Pennisetum setaceum 'Rubrum'", "Zea mays", "Saccharum officinarum", "Imperata cylindrica", "Alopecurus aequalis", - "Echinochloa caudata", "Echinochloa crus-galli", "Oryza sativa", "Eleusine indica", "Bambusoideae", "Indocalamus latifolius", - "Bambusa ventricosa", "Miscanthus sinensis 'Gracillimus'", "Miscanthus sinensis 'Zebrinus'", "Arundo donax", "Phragmites australis", - "Microstegium vimineum", "Zizania latifolia", "Cortaderia selloana", "Coix lacryma-jobi", "Phalaris arundinacea", - "Paspalum dilatatum", "Sorghum bicolor", "Sorghum halepense", "Dactylis glomerata", "Panicum virgatum", "Lolium perenne", - "Disporum cantoniense", "Disporum longistylum", "Disporum megalanthum", "Disporum uniflorum", "Disporum viridescens", - "Gloriosa superba", "Sandersonia aurantiaca", "Colchicum autumnale", "Begonia boliviensis", "Begonia circumlobata", - "Begonia cucullata", "Begonia fimbristipula", "Begonia grandis subsp. sinensis", "Begonia leprosa", "Begonia maculata", - "Begonia masoniana", "Begonia palmata", "Begonia soli-mutata", "Begonia × hiemalis", "Ctenanthe setosa", "Thalia dealbata", - "Thalia geniculata", "Maranta leuconeura", "Maranta&Calathea", "Stromanthe sanguinea", "Calathea warscewiczii", "Calathea zebrina", - "Bougainvillea spectabilis&glabra", "Mirabilis jalapa", "Boerhavia diffusa", "Myosotis alpestris", "Ehretia acuminata", - "Ehretia longiflora", "Carmona microphylla", "Heliotropium arborescens", "Heliotropium curassavicum", "Heliotropium indicum", - "Microula sikkimensis", "Bothriospermum chinense", "Bothriospermum zeylanicum", "Onosma hookeri var. longiflorum", - "Mertensia virginica", "Borago officinalis", "Cynoglossum amabile", "Cynoglossum grande", "Cynoglossum lanceolatum", - "Thyrocarpus sampsonii", "Cordia dichotoma", "Cordia subcordata", "Nemophila maculata", "Nemophila menziesii", - "Tournefortia montana", "Tournefortia sibirica", "Stenosolenium saxatile", "Lithospermum incisum", "Lithospermum zollingeri", - "Symphytum officinale", "Echium vulgare", "Echium wildpretii", "Trigonotis peduncularis", "Osmundastrum cinnamomeum", - "Osmunda claytoniana", "Campsis grandiflora", "Campsis radicans", "Kigelia africana", "Catalpa bungei", "Catalpa fargesii", - "Catalpa ovata", "Catalpa speciosa", "Mayodendron igneum", "Spathodea campanulata", "Pyrostegia venusta", - "Markhamia stipulata var. kerrii", "Macfadyena unguis-cati", "Pandorea jasminoides", "Tabebuia impetiginosa", "Tabebuia rosea", - "Radermachera sinica&hainanensis", "Crescentia alata", "Mansoa alliacea", "Jacaranda mimosifolia", "Incarvillea arguta", - "Incarvillea mairei var. multifoliolata", "Incarvillea sinensis", "Clytostoma callistegioides", "Podranea ricasoliana", - "Handroanthus chrysanthus", "Tecoma capensis", "Tecoma stans", "Calophyllum inophyllum", "Calophyllum membranaceum", "Mesua ferrea", - "Bixa orellana", "Bruguiera gymnorhiza", "Kandelia obovata", "Cephalotaxus sinensis", "Torreya grandis 'Merrillii'", - "Taxus baccata", "Taxus wallichiana var. chinensis", "Philadelphus laxiflorus", "Philadelphus pekinensis", - "Philadelphus zhejiangensis", "Dichroa febrifuga", "Deutzia baroniana", "Deutzia crenata", "Deutzia glauca", - "Deutzia glomeruliflora", "Deutzia gracilis", "Deutzia longifolia", "Deutzia ningpoensis", "Deutzia scabra", - "Deutzia scabra var. plena", "Hydrangea", "Hydrangea chinensis", "Hydrangea lingii", "Hydrangea paniculata", - "Hydrangea quercifolia", "Hydrangea strigosa", "Platycrater arguta", "Macleaya cordata", "Chelidonium majus", - "Dicranostigma leptopodum", "Corydalis bungeana", "Corydalis caudata", "Corydalis curviflora", "Corydalis decumbens", - "Corydalis edulis", "Corydalis fangshanensis", "Corydalis flexuosa", "Corydalis hamata", "Corydalis hemidicentra", - "Corydalis incisa", "Corydalis linarioides", "Corydalis melanochlora", "Corydalis mucronata", "Corydalis pachycentra", - "Corydalis pallida", "Corydalis pseudobarbisepala", "Corydalis racemosa", "Corydalis repens", "Corydalis sheareri", - "Corydalis speciosa", "Corydalis turtschaninovii", "Corydalis yanhusuo", "Meconopsis", "Meconopsis balangensis", - "Meconopsis betonicifolia", "Meconopsis chelidoniifolia", "Meconopsis delavayi", "Meconopsis henrici", "Meconopsis horridula", - "Meconopsis impedita", "Meconopsis integrifolia", "Meconopsis lancifolia", "Meconopsis paniculata", "Meconopsis pseudointegrifolia", - "Meconopsis punicea", "Meconopsis quintuplinervia", "Meconopsis racemosa", "Meconopsis simplicifolia", "Meconopsis speciosa", - "Meconopsis sulphurea", "Meconopsis venusta", "Meconopsis wilsonii", "Papaver orientale", "Papaver radicatum var. pseudoradicatum", - "Papaver rhoeas", "Papaver somniferum", "Eschscholzia californica", "Lamprocapnos spectabilis", "Lamprocapnos spectabilis f. alba", - "Hylomecon japonica", "Argemone mexicana", "Sanguinaria canadensis", "Eomecon chionantha", "Dicentra cucullaria", - "Dicentra formosa", "Nageia nagi", "Podocarpus macrophyllus", "Canna", "Canna generalis", "Canna glauca", "Canna indica", - "Canna indica var. flava", "Canna orchioides", "Canna warscewiezii", "Astelia fragrans", "Nephrolepis cordifolia", - "Platycarya strobilacea", "Carya illinoinensis", "Pterocarya stenoptera", "Engelhardia roxburghiana", "Juglans mandshurica", - "Juglans nigra", "Juglans regia", "Cyclocarya paliurus", "Piper aduncum", "Piper hancei", "Piper kadsura", "Piper nigrum", - "Piper sarmentosum", "Peperomia argyreia", "Peperomia caperata", "Peperomia pellucida", "Peperomia polybotrya", - "Peperomia tetraphylla", "Hippophae rhamnoides", "Elaeagnus angustifolia", "Elaeagnus argyi", "Elaeagnus conferta", - "Elaeagnus glabra", "Elaeagnus lanceolata", "Elaeagnus mollis", "Elaeagnus multiflora", "Elaeagnus pungens", - "Elaeagnus Pungens 'Aurea'", "Elaeagnus umbellata", "Paeonia delavayi", "Paeonia lactiflora", "Paeonia obovata", - "Paeonia suffruticosa", "Sesamum indicum", "Uncarina grandidieri", "Musella lasiocarpa", "Musa nana", "Ensete glaucum", - "Stylidium uliginosum", "Cobaea scandens", "Phlox", "Phlox drummondii", "Phlox paniculata", "Phlox subulata", "Ipomopsis aggregata", - "Polemonium caeruleum", "Polemonium chinense", "Butomus umbellatus", "Murraya exotica", "Tetradium austrosinense", - "Tetradium glabrifolium", "Tetradium ruticarpum", "Glycosmis pentaphylla", "Acronychia pedunculata", "Citrus australasica", - "Citrus japonica", "Citrus maxima", "Citrus medica 'Fingered'", "Citrus reticulata", "Citrus reticulata", "Citrus sinensis", - "Citrus trifoliata", "Citrus × limon", "Ptelea trifoliata", "Dictamnus dasycarpus", "Boenninghausenia albiflora", - "Zanthoxylum ailanthoides", "Zanthoxylum bungeanum", "Zanthoxylum nitidum", "Zanthoxylum piperitum", "Zanthoxylum scandens", - "Zanthoxylum simulans", "Skimmia reevesiana", "Melicope pteleifolia", "Toddalia asiatica", "Clausena excavata", "Clausena lansium", - "Gomphrena globosa", "Kochia scoparia", "Cyathula prostrata", "Achyranthes bidentata", "Beta vulgaris", "Salsola tragus", - "Amaranthus caudatus", "Amaranthus hypochondriacus", "Amaranthus spinosus", "Amaranthus tricolor", "Alternanthera bettzickiana", - "Alternanthera philoxeroides", "Spinacia oleracea", "Chenopodium album", "Celosia argentea", "Celosia cristata", "Cycas revoluta", - "Ailanthus altissima", "Brucea javanica", "Hemiboea cavaleriei", "Hemiboea subcapitata", "Didymostigma obtusum", - "Titanotrichum oldhamii", "Lysionotus pauciflorus", "Lysionotus serratus", "Chirita eburnea", "Chirita fimbrisepala", - "Chirita lutea", "Chirita pinnatifida", "Chirita pumila", "Episcia cupreata", "Gyrocheilos chorisepalus", "Sinningia leucotricha", - "Sinningia speciosa", "Gloxinia sylvatica", "Primulina xiziae", "Streptocarpus hybrids", "Streptocarpus saxorum", - "Briggsia chienii", "Rhynchotechum ellipticum", "Didissandra sesquifolia", "Aeschynanthus acuminatus", "Aeschynanthus buxifolius", - "Aeschynanthus sp", "Aeschynanthus speciosus", "Aeschynanthus superbus", "Paraboea sinensis", "Nematanthus wettsteinii", - "Saintpaulia ionantha", "Oreocharis auricula", "Oreocharis benthamii var. reticulata", "Oreocharis maximowiczii", - "Nicandra physalodes", "Cestrum aurantiacum", "Cestrum nocturnum", "Hyoscyamus niger", "Anisodus tanguticus", "Datura inoxia", - "Datura stramonium", "Datura wrightii", "Brugmansia arborea", "Brugmansia aurea", "Brugmansia suaveolens", "Lycium chinense", - "Cyphomandra betacea", "Juanulloa aurantiaca", "Nicotiana alata", "Nicotiana glauca", "Nicotiana tabacum", - "Lycopersicon esculentum", "Petunia × hybrida", "Lycianthes biflora", "Calibrachoa hybrids", "Mandragora caulescens", - "Solanum aculeatissimum", "Solanum capsicoides", "Solanum carolinense", "Solanum dulcamara", "Solanum elaeagnifolium", - "Solanum erianthum", "Solanum jasminoides", "Solanum laciniatum", "Solanum lyratum", "Solanum mammosum", "Solanum melongena", - "Solanum muricatum", "Solanum nigrum&americanum", "Solanum pseudocapsicum", "Solanum pseudocapsicum var. diflorum", - "Solanum rantonnetii", "Solanum rostratum", "Solanum septemlobum", "Solanum texanum", "Solanum torvum", "Solanum tuberosum", - "Solanum virginianum", "Solanum wrightii", "Schizanthus pinnatus", "Capsicum annuum", "Capsicum annuum subsp. cerasiforme", - "Capsicum annuum var. conoides", "Physalis", "Physalis minima", "Physalis philadelphica", "Solandra longiflora", "Solandra maxima", - "Brunfelsia brasiliensis", "Brunfelsia calycina", "Dionaea muscipula", "Drosera burmanni", "Drosera peltata", - "Drosera rotundifolia", "Drosera spatulata", "Psychotria serpens", "Pentas lanceolata", "Coffea", "Pavetta hongkongensis", - "Bouvardia ternifolia", "Morinda citrifolia", "Morinda parvifolia", "Galium aparine", "Galium spurium", "Galium verum", - "Gardenia jasminoides", "Gardenia scabrella", "Adina pilulifera", "Adina rubella", "Coptosapelta diffusa", "Luculia pinceana", - "Diplospora dubia", "Canthium horridum", "Mussaenda 'Alicia'", "Mussaenda erosa", "Mussaenda erythrophylla", "Mussaenda parviflora", - "Mussaenda pubescens", "Mussaenda shikokiana", "Sherardia arvensis", "Serissa japonica", "Serissa japonica 'Variegata'", - "Serissa serissoides", "Neohymenopogon parasiticus", "Lasianthus chinensis", "Houstonia caerulea", "Hedyotis caudatifolia", - "Hedyotis chrysotricha", "Hedyotis diffusa", "Hedyotis hedyotidea", "Hedyotis tenuipes", "Mycetia sinensis", "Coprosma robusta", - "Mitchella repens", "Damnacanthus giganteus", "Ophiorrhiza japonica", "Ophiorrhiza pumila", "Rondeletia leucophylla", - "Rondeletia odorata", "Leptodermis oblonga", "Uncaria hirsuta", "Spermacoce alata", "Hamelia patens", "Cephalanthus occidentalis", - "Cephalanthus tetrandrus", "Paederia foetida", "Ixora chinensis", "Ixora coccinea f. lutea", "Ixora finlaysoniana", - "Ixora paraopaca", "Mappianthus iodoides", "Ribes burejense", "Ribes himalense var. verruculosum", "Ribes nigrum", "Ribes odoratum", - "Ribes reclinatum", "Ribes rubrum", "Ribes rubrum", "Scaevola aemula", "Scaevola taccada", "Goodenia pilosa subsp. chinensis", - "Pilea aquarum", "Pilea cadierei", "Pilea microphylla", "Pilea notata", "Pilea pumila", "Cecropia peltata", "Elatostema cuspidatum", - "Debregeasia orientalis", "Gonostegia hirta", "Oreocnide frutescens", "Nanocnide lobata", "Boehmeria japonica", "Boehmeria nivea", - "Boehmeria tricuspis", "Urtica dioica", "Girardinia diversifolia subsp. suborbiculata", "Pellionia repens", "Pouzolzia zeylanica", - "Calceolaria crenatiflora", "Rhynchospora colorata", "Schoenoplectus tabernaemontani", "Kyllinga brevifolia", "Kyllinga polyphylla", - "Eleocharis dulcis", "Cyperus difformis", "Cyperus glomeratus", "Cyperus involucratus", "Cyperus prolifer", "Cyperus rotundus", - "Trichophorum subcapitatum", "Carex baccans", "Carex scaposa", "Fimbristylis dichotoma", "Illigera celebica", "Illigera rhodantha", - "Nelumbo nucifera", "Brasenia schreberi", "Mycelis muralis", "Solidago canadensis", "Emilia prenanthoidea", "Emilia sonchifolia", - "Tagetes erecta", "Calyptocarpus vialis", "Parasyncalathium souliei", "Mikania micrantha", "Paraprenanthes sororia", - "Praxelis clematidea", "Crepidiastrum lanceolatum", "Crepidiastrum sonchifolium", "Heterotheca subaxillaris", - "Syneilesis aconitifolia", "Ainsliaea fragrans", "Ainsliaea kawakamii", "Gazania rigens", "Smallanthus sonchifolius", - "Senecio analogus", "Senecio cineraria", "Senecio faberi", "Senecio haworthii", "Senecio rowleyanus", "Senecio scandens", - "Senecio serpens", "Senecio vulgaris", "Helianthus annuus", "Helianthus decapetalus", "Helianthus maxillianii", - "Helianthus tuberosus", "Cremanthodium campanulatum", "Helenium amarum", "Helenium autumnale", "Dahlia pinnata", - "Farfugium japonicum", "Gaillardia pulchella&aristata", "Carpesium abrotanoides", "Tragopogon dubius", "Tragopogon porrifolius", - "Tragopogon pratensis", "Wollastonia biflora", "Ixeridium dentatum", "Hieracium aurantiacum", "Dolomiaea souliei", - "Pseudognaphalium hypoleucum", "Inula helenium", "Inula helianthusaquatilis", "Inula japonica", "Argyranthemum frutescens", - "Echinacea purpurea", "Silphium laciniatum", "Silphium perfoliatum", "Nouelia insignis", "Engelmannia peristenia", - "Ligularia sibirica", "Tussilago farfara", "Matricaria chamomilla", "Matricaria discoidea", "Melanoseris atropurpurea", - "Silybum marianum", "Hemisteptia lyrata", "Eupatorium fortunei", "Eupatorium perfoliatum", "Eupatorium serotinum", - "Leucanthemum maximum", "Leucanthemum vulgare", "Rhaponticum chinense", "Rhaponticum uniflorum", "Gerbera jamesonii", - "Leontopodium japonicum", "Leontopodium leontopodioides", "Galinsoga parviflora", "Galinsoga quadriradiata", - "Helminthotheca echioides", "Arctium lappa", "Hypochaeris radicata", "Pericallis hybrida", "Stevia rebaudiana", - "Centaurea solstitialis", "Zinnia elegans", "Cyanus segetum", "Cosmos bipinnatus", "Cosmos sulphureus", "Lapsanastrum apogonoides", - "Ageratina adenophora", "Ageratina altissima", "Aster altaicus", "Aster baccharoides", "Aster hispidus", "Aster indicus", - "Aster likiangensis", "Aster novi-belgii", "Aster pekinensis", "Aster scaber", "Aster trinervius subsp. ageratoides", - "Aster turbinatus", "Carthamus tinctorius", "Eriophyllum confertiflorum", "Eriophyllum staechadifolium", "Thelesperma filifolium", - "Callistephus chinensis", "Symphyotrichum novae-angliae", "Symphyotrichum subulatum", "Tithonia diversifolia", - "Encelia californica", "Blumea megacephala", "Crossostephium chinensis", "Xanthium strumarium", "Sonchus asper", - "Sonchus oleraceus", "Ixeris chinensis", "Glebionis coronaria", "Glebionis segetum", "Ratibida columnifera", "Lactuca indica", - "Lactuca sativa", "Lactuca sativa var. ramosa", "Lactuca serriola", "Lactuca sibirica", "Gynura aurantiaca", "Gynura bicolor", - "Gynura divaricata", "Chrysanthemum multicaule", "Chrysanthemum × morifolium", "Cichorium endivia", "Cichorium intybus", - "Tanacetum vulgare", "Cynara cardunculus", "Cynara scolymus", "Sinosenecio oldhamianus", "Taraxacum mongolicum", - "Taraxacum officinale", "Artemisia argyi", "Artemisia californica", "Artemisia caruifolia", "Artemisia douglasiana", - "Artemisia lactiflora", "Artemisia selengensis", "Achillea millefolium", "Centratherum punctatum", "Echinops gmelinii", - "Cirsium arvense", "Cirsium arvense var. integrifolium", "Cirsium japonicum", "Cirsium leo", "Cirsium souliei", "Cirsium vulgare", - "Ageratum conyzoides", "Ageratum houstonianum", "Myripnois dioica", "Liatris spicata", "Petasites japonicus", - "Xerochrysum bracteatum", "Sphagneticola calendulacea", "Sphagneticola trilobata", "Ambrosia artemisiifolia", "Ambrosia trifida", - "Sigesbeckia orientalis", "Heliopsis helianthoides", "Heliopsis helianthoides var. scabra", "Baccharis halimifolia", - "Baccharis pilularis", "Baccharis salicifolia", "Crassocephalum crepidioides", "Crassocephalum rubens", "Rudbeckia bicolor", - "Rudbeckia fulgida", "Rudbeckia fulgida 'Goldsturm'", "Rudbeckia hirta", "Rudbeckia laciniata", - "Rudbeckia laciniata var. hortensia", "Calendula officinalis", "Synedrella nodiflora", "Acmella paniculata", "Coreopsis basalis", - "Coreopsis lanceolata", "Coreopsis tinctoria", "Coreopsis verticillata", "Vernonia baldwinii", "Vernonia gratiosa", - "Vernonia volkameriifolia", "Parthenium hysterophorus", "Conoclinium coelestinum", "Bellis perennis", "Saussurea involucrata", - "Saussurea medusa", "Saussurea przewalskii", "Saussurea stella", "Saussurea tibetica", "Saussurea velutina", "Carduus crispus", - "Carduus nutans", "Carduus pycnocephalus", "Erigeron annuus", "Erigeron canadensis", "Erigeron glaucus", "Erigeron philadelphicus", - "Erigeron sumatrensis", "Anaphalis margaritacea", "Anaphalis nepalensis", "Anaphalis nepalensis var. monocephala", - "Verbesina virginica", "Osteospermum ecklonis", "Bidens biternata", "Bidens cernua", "Bidens frondosa", "Bidens pilosa", - "Eclipta prostrata", "Brachyscome angustifolia", "Brachyscome iberidifolia", "Euryops pectinatus", "Flaveria bidentis", - "Youngia heterophylla", "Youngia japonica", "Gnaphalium", "Gnaphalium japonicum", "Acorus calamus", "Smilax bona-nox", - "Smilax china", "Smilax davidiana", "Smilax riparia", "Biondia microcentra", "Basella alba", "Anredera cordifolia", - "Cayratia albifolia", "Cayratia japonica", "Yua austro-orientalis", "Parthenocissus laetevirens", "Parthenocissus quinquefolia", - "Parthenocissus tricuspidata", "Tetrastigma hemsleyanum", "Tetrastigma planicaule", "Cissus hexangularis", "Vitis bryoniifolia", - "Vitis flexuosa", "Vitis vinifera", "Ampelopsis aconitifolia", "Ampelopsis delavayana", "Ampelopsis glandulosa", - "Ampelopsis glandulosa var. heterophylla", "Marah fabacea", "Marah macrocarpa", "Luffa aegyptiaca", "Sechium edule", - "Benincasa hispida", "Cucurbita foetidissima", "Cucurbita moschata", "Cucurbita pepo", "Trichosanthes anguina", - "Trichosanthes cucumeroides", "Trichosanthes kirilowii", "Trichosanthes rubriflos", "Diplocyclos palmatus", "Melothria pendula", - "Melothria scabra", "Actinostemma tenerum", "Coccinia grandis", "Gynostemma pentaphyllum", "Momordica charantia", - "Momordica cochinchinensis", "Lagenaria siceraria", "Lagenaria siceraria ‘Hispida’", "Citrullus lanatus", "Thladiantha dubia", - "Thladiantha longifolia", "Thladiantha nudiflora", "Gymnopetalum chinense", "Zehneria japonica", "Cucumis melo", "Cucumis melo", - "Cucumis melo", "Cucumis melo subsp. agrestis", "Cucumis metuliferus", "Cucumis sativus", "Rivina humilis", "Larrea tridentata", - "Tribulus terrestris", "Zygophyllum mucronatum", "Camptotheca acuminata", "Davidia involucrata", "Nyssa sinensis", - "Fallopia multiflora", "Muehlenbeckia complexa", "Rheum alexandrae", "Rheum nobile", "Rheum rhabarbarum", "Oxyria sinensis", - "Coccoloba uvifera", "Antigonon leptopus", "Eriogonum fasciculatum", "Eriogonum latifolium", "Fagopyrum dibotrys", - "Fagopyrum esculentum", "Polygonum aviculare", "Polygonum capitatum", "Polygonum chinense", "Polygonum coriaceum", - "Polygonum japonicum", "Polygonum longisetum", "Polygonum macrophyllum", "Polygonum muricatum", "Polygonum orientale", - "Polygonum perfoliatum", "Polygonum plebeium", "Polygonum pubescens", "Polygonum runcinatum", "Polygonum senticosum", - "Polygonum thunbergii", "Polygonum viscosum", "Persicaria virginiana", "Reynoutria japonica", "Rumex acetosa", "Rumex acetosella", - "Rumex crispus", "Rumex hastatus", "Rumex japonicus", "Rumex obtusifolius", "Antenoron filiforme", - "Antenoron filiforme var. neofiliforme", "Dryas octopetala", "Aruncus sylvester", "Amelanchier canadensis", - "Sanguisorba officinalis", "Potentilla anserina", "Potentilla discolor", "Potentilla fragarioides", "Potentilla freyniana", - "Potentilla fruticosa", "Potentilla glabra", "Potentilla kleiniana", "Potentilla recta", "Potentilla supina", - "Stephanandra chinensis", "Crataegus cuneata", "Crataegus maximowiczii", "Crataegus monogyna", "Crataegus pinnatifida", - "Rubus alceifolius", "Rubus armeniacus", "Rubus buergeri", "Rubus chingii", "Rubus corchorifolius", "Rubus coreanus", - "Rubus crataegifolius", "Rubus fockeanus", "Rubus fruticosus", "Rubus idaeus&hirsutus", "Rubus lambertianus", "Rubus odoratus", - "Rubus pacificus", "Rubus parviflorus", "Rubus parvifolius", "Rubus phoenicolasius", "Rubus pirifolius", "Rubus rosifolius", - "Rubus setchuenensis", "Rubus spectabilis", "Rubus sumatranus", "Rubus swinhoei", "Rubus trianthus", "Rubus ursinus", - "Prinsepia utilis", "Chaenomeles cathayensis", "Chaenomeles sinensis", "Chaenomeles speciosa", "Prunus cerasifera f. atropurpurea", - "Prunus laurocerasus", "Prunus salicina", "Prunus serotina", "Prunus spinosa", "Prunus virginiana", "Armeniaca mume", - "Armeniaca mume var. mume f. alphandii", "Armeniaca mume var. mume f. purpurea", "Armeniaca mume var. mume f. viridicalyx", - "Armeniaca vulgaris", "Eriobotrya japonica", "Adenostoma fasciculatum", "Heteromeles arbutifolia", "Cotoneaster adpressus", - "Cotoneaster horizontalis", "Cotoneaster microphyllus", "Cotoneaster multiflorus", "Amygdalus communis", "Amygdalus persica", - "Amygdalus persica 'Compressa'", "Amygdalus persica 'Juhuatao'", "Amygdalus triloba", "Pyrus", "Pyrus betulifolia", - "Pyrus calleryana", "Pyrus phaeocarpa", "Pyrus sinkiangensis", "Kerria japonica", "Kerria japonica f. pleniflora", - "Cydonia oblonga", "Cerasus campanulata", "Cerasus cerasoides", "Cerasus dielsiana", "Cerasus glandulosa", "Cerasus japonica", - "Cerasus pseudocerasus", "Cerasus serrulata var. lannesiana", "Cerasus tomentosa", "Pyracantha angustifolia", - "Pyracantha fortuneana", "Pyracantha fortuneana 'Harlequin'", "Sorbaria sorbifolia", "Exochorda racemosa", "Rhaphiolepis indica", - "Rhaphiolepis umbellata", "Photinia beauverdiana", "Photinia bodinieri", "Photinia glomerata", "Photinia komarovii", - "Photinia serratifolia", "Photinia × fraseri", "Padus avium", "Padus buergeriana", "Holodiscus discolor", "Neillia sinensis", - "Spiraea alpina", "Spiraea blumei", "Spiraea cantoniensis", "Spiraea fritschiana", "Spiraea japonica", "Spiraea mongolica", - "Spiraea myrtilloides", "Spiraea prunifolia", "Spiraea prunifolia var. simpliciflora", "Spiraea pubescens", "Spiraea thunbergii", - "Spiraea trilobata", "Spiraea × bumalda 'coldfiame'", "Spiraea × bumalda 'Goalden Mound'", "Spiraea × vanhouttei", - "Potaninia mongolica", "Sorbus alnifolia", "Sorbus folgneri", "Sorbus pohuashanensis", "Malus 'American'", "Malus baccata", - "Malus halliana", "Malus hupehensis", "Malus pumila", "Malus × micromalus", "Malus × robusta", "Fragaria orientalis", - "Fragaria vesca", "Fragaria virginiana", "Fragaria × ananassa", "Rosa banksiae", "Rosa banksiae f. lutea", "Rosa bracteata", - "Rosa californica", "Rosa chinensis", "Rosa cymosa", "Rosa davurica", "Rosa henryi", "Rosa laevigata", "Rosa multiflora", - "Rosa multiflora var. carnea", "Rosa multiflora var. cathayensis", "Rosa omeiensis", "Rosa roxburghii", - "Rosa roxburghii f. normalis", "Rosa rugosa", "Rosa rugosa f. albo-plena", "Rosa xanthina", "Rosa xanthina var. normalis", - "Filipendula palmata", "Duchesnea indica", "Geum aleppicum", "Geum canadense", "Geum japonicum var. chinense", - "Physocarpus amurensis", "Spenceria ramalana", "Agrimonia pilosa", "Liquidambar formosana", "Liquidambar styraciflua", - "Altingia chinensis", "Tacca chantrieri", "Tacca plantaginea", "Dioscorea bulbifera", "Dioscorea cirrhosa", - "Dioscorea elephantipes", "Dioscorea japonica", "Dioscorea polystachya", "Ypsilandra thibetica", "Trillium cernuum", - "Trillium chloropetalum", "Trillium cuneatum", "Trillium erectum", "Trillium grandiflorum", "Trillium luteum", "Trillium ovatum", - "Trillium recurvatum", "Trillium undulatum", "Toxicoscordion fremontii", "Chionographis chinensis", "Veratrum californicum", - "Veratrum nigrum", "Veratrum schindleri", "Veratrum viride", "Paris", "Paris luquanensis", "Paris polyphylla", - "Paris polyphylla var. chinensis", "Paris verticillata", "Garcinia cowa", "Garcinia mangostana", "Garcinia multiflora", - "Garcinia oblongifolia", "Garcinia subelliptica", "Garcinia xanthochymus", "Daphniphyllum calycinum", "Daphniphyllum macropodum", - "Mukdenia rossii", "Oresitrophe rupifraga", "Heuchera", "Astilbe chinensis", "Saxifraga egregia", "Saxifraga przewalskii", - "Saxifraga stolonifera", "Tiarella cordifolia", "Tiarella polyphylla", "Balanophora harlandii", "Balanophora laxiflora", - "Calycanthus chinensis", "Calycanthus floridus", "Chimonanthus nitens", "Chimonanthus praecox", "Heliconia latispatha", - "Heliconia metallica", "Heliconia rostrata", "Turnera subulata", "Turnera ulmifolia", "Passiflora alata", "Passiflora amethystina", - "Passiflora caerulea", "Passiflora coccinea", "Passiflora edulis", "Passiflora foetida", "Passiflora incarnata", "Passiflora lutea", - "Passiflora suberosa", "Passiflora yucatanensis", "Eriocaulon buergerianum", "Eriocaulon sexangulare", "Acmispon glaber", - "Amphicarpaea edgeworthii", "Caesalpinia bonduc", "Caesalpinia decapetala", "Caesalpinia minax", "Caesalpinia pulcherrima", - "Caesalpinia pulcherrima 'Flava'", "Caesalpinia sappan", "Lysidice brevicalyx", "Lysidice rhodostegia", "Dendrolobium triangulare", - "Senna alata", "Senna bicapsularis", "Senna occidentalis", "Senna sophera", "Senna spectabilis", "Senna surattensis", - "Delonix regia", "Canavalia gladiata", "Canavalia rosea", "Erythrina corallodendron", "Erythrina crista-galli", - "Erythrina variegata", "Robinia pseudoacacia", "Robinia pseudoacacia f. decaisneana", "Albizia julibrissin", "Albizia kalkora", - "Albizia lebbeck", "Aeschynomene indica", "Mimosa bimucronata", "Mimosa pudica", "Apios carnea", "Apios fortunei", "Glycine max", - "Glycine soja", "Coronilla varia", "Chamaecrista fasciculata", "Chamaecrista mimosoides", "Desmodium heterocarpon", - "Desmodium microphyllum", "Desmodium triflorum", "Lathyrus latifolius", "Lathyrus odoratus", "Fordia cauliflora", - "Lablab purpureus", "Phyllodium pulchellum", "Saraca dives", "Indigofera bungeana", "Indigofera decora", "Indigofera hendecaphylla", - "Indigofera kirilowii", "Cajanus cajan", "Calliandra haematocephala", "Calliandra tergemina var. emarginata", - "Campylotropis macrocarpa", "Campylotropis polyantha", "Castanospermum australe", "Erythrophleum fordii", "Oxytropis aciphylla", - "Oxytropis caerulea", "Oxytropis myriophylla", "Styphnolobium japonicum", "Ammopiptanthus mongolicus", "Sindora glabra", - "Mucuna bennettii", "Mucuna birdwoodiana", "Mucuna lamellata", "Mucuna macrocarpa", "Mucuna sempervirens", - "Adenanthera microsperma", "Prosopis glandulosa", "Uraria crinita", "Uraria picta", "Crotalaria assamica", "Crotalaria pallida", - "Crotalaria sessiliflora", "Crotalaria spectabilis", "Crotalaria trichotoma", "Archidendron clypearia", "Glycyrrhiza uralensis", - "Sesbania cannabina", "Sesbania grandiflora", "Lotus corniculatus", "Gleditsia japonica", "Gleditsia triacanthos", - "Abrus precatorius", "Acacia auriculiformis", "Acacia catechu", "Acacia confusa", "Acacia farnesiana", "Acacia podalyriifolia", - "Peltophorum pterocarpum", "Butea monosperma", "Amorpha fruticosa", "Cercis canadensis", "Cercis chinensis", "Cercis chingii", - "Cercis chuniana", "Cercis glabra", "Wisteria sinensis&villosa", "Ormosia henryi", "Corethrodendron scoparium", - "Bauhinia acuminata", "Bauhinia brachycarpa", "Bauhinia championii", "Bauhinia corymbosa", "Bauhinia didyma", "Bauhinia galpinii", - "Bauhinia glauca", "Bauhinia glauca subsp. tenuiflora", "Bauhinia kockiana", "Bauhinia tomentosa", "Bauhinia touranensis", - "Bauhinia variegata", "Bauhinia variegata var. candida", "Bauhinia × blakeana", "Lupinus arboreus", - "Lupinus micranthus&polyphyllus", "Lupinus texensis", "Strongylodon macrobotrys", "Lespedeza bicolor", "Lespedeza buergeri", - "Lespedeza chinensis", "Lespedeza cuneata", "Lespedeza davidii", "Lespedeza dunnii", "Lespedeza floribunda", "Lespedeza pilosa", - "Lespedeza thunbergii subsp. formosa", "Lespedeza tomentosa", "Lespedeza virgata", "Cassia fistula", "Codoriocalyx motorius", - "Medicago lupulina", "Medicago polymorpha", "Medicago sativa", "Sophora davidii", "Sophora flavescens", "Sphaerophysa salsula", - "Ulex europaeus", "Melilotus albus", "Melilotus indicus", "Melilotus officinalis", "Phaseolus coccineus", "Phaseolus vulgaris", - "Arachis duranensis", "Arachis hypogaea", "Pueraria montana", "Pueraria wallichii", "Bowringia callicarpa", "Clitoria ternatea", - "Cullen corylifolium", "Pachyrhizus erosus", "Vigna radiata", "Vigna umbellata", "Vigna unguiculata", "Vigna vexillata", - "Pisum sativum", "Baptisia australis", "Centrosema pubescens", "Trifolium pratense", "Trifolium repens", "Tamarindus indica", - "Thermopsis barbata", "Thermopsis lanceolata", "Vicia amoena", "Vicia cracca", "Vicia faba", "Vicia sativa", "Vicia sepium", - "Vicia tetrasperma", "Vicia villosa", "Cytisus scoparius", "Leucaena leucocephala", "Caragana jubata", "Caragana rosea", - "Caragana sinica", "Caragana tibetica", "Hylodesmum podocarpum", "Hylodesmum podocarpum subsp. fallax", - "Hylodesmum podocarpum subsp. oxyphyllum", "Chesneya polystichoides", "Tibetia yunnanensis", "Derris alborubra", "Derris fordii", - "Colutea arborescens", "Kummerowia striata", "Callerya dielsiana", "Callerya nitida", "Callerya reticulata", "Callerya speciosa", - "Spartium junceum", "Rhynchosia volubilis", "Dalbergia assamica", "Dalbergia hupeana", "Astragalus sinicus", - "Athyrium filix-femina", "Bacopa diffusa", "Pseudolysimachion longifolium", "Pseudolysimachion spicatum", "Lagotis brevituba", - "Veronica anagallis-aquatica", "Veronica arvensis", "Veronica henryi", "Veronica persica", "Veronica undulata", "Linaria maroccana", - "Linaria vulgaris", "Linaria vulgaris subsp. chinensis", "Digitalis purpurea", "Adenosma glutinosum", "Russelia equisetiformis", - "Veronicastrum axillare", "Otacanthus azureus", "Cymbalaria muralis", "Plantago asiatica", "Plantago depressa", - "Plantago lanceolata", "Plantago major", "Plantago virginica", "Antirrhinum majus", "Penstemon", "Penstemon barbatus", - "Penstemon digitalis", "Collinsia heterophylla", "Hemiphragma heterophyllum", "Angelonia angustifolia", "Chelone glabra", - "Moringa drouhardii", "Moringa oleifera", "Polygala arillata", "Polygala fallax", "Polygala hongkongensis", - "Polygala hongkongensis var. stenophylla", "Polygala japonica", "Polygala latouchei", "Polygala myrtifolia", "Polygala sibirica", - "Polygala tenuifolia", "Salomonia cantoniensis", "Cercidiphyllum japonicum", "Mimulus aurantiacus", "Mimulus guttatus", - "Mimulus szechuanensis", "Lancea tibetica", "Mazus caducifer", "Mazus pumilus", "Oxalis", "Oxalis articulata", "Oxalis barrelieri", - "Oxalis corniculata", "Oxalis corymbosa", "Oxalis griffithii", "Oxalis oregana", "Oxalis palmifrons", "Oxalis pes-caprae", - "Oxalis purpurea", "Oxalis stricta", "Oxalis triangularis 'Urpurea'", "Oxalis violacea", "Averrhoa carambola", - "Oxyspora paniculata", "Blastus cochinchinensis", "Blastus pauciflorus", "Fordiophyton faberi", "Tibouchina semidecandra", - "Tigridiopalma exalata", "Tigridiopalma magnifica", "Sonerila cantonensis", "Memecylon ligustrifolium", "Memecylon octocostatum", - "Medinilla formosana", "Medinilla magnifica", "Bredia fordii", "Bredia quadrangularis", "Melastoma dodecandrum", - "Melastoma malabathricum", "Melastoma malabathricum var. alba", "Melastoma sanguineum", "Osbeckia chinensis", "Osbeckia stellata", - "Phyllagathis cavaleriei", "Hypericum 'Excellent Flair'", "Hypericum androsaemum", "Hypericum faberi", "Hypericum japonicum", - "Hypericum monogynum", "Hypericum patulum", "Hypericum perforatum", "Hypericum sampsonii", "Cratoxylum cochinchinense", - "Phegopteris connectilis", "Sarcandra glabra", "Chloranthus fortunei", "Chloranthus henryi", "Chloranthus japonicus", - "Chloranthus serratus", "Chloranthus spicatus", "Mytilaria laosensis", "Loropetalum chinense", "Loropetalum chinense var. rubrum", - "Loropetalum subcordatum", "Sycopsis sinensis", "Fortunearia sinensis", "Eustigma oblongifolium", "Rhodoleia championii", - "Distylium buxifolium", "Distylium racemosum", "Corylopsis multiflora var. nivea", "Corylopsis sinensis", "Hamamelis mollis", - "Hamamelis virginiana", "Hamamelis × intermedia", "Ochna integerrima", "Ochna serrulata", "Ochna thomasiana", - "Tristellateia australasiae", "Heteropterys glabra", "Thryallis gracilis", "Malpighia glabra", "Hiptage benghalensis", - "Ceratophyllum demersum", "Gelsemium elegans", "Gelsemium sempervirens", "Ancistrocladus tectorius", "Asplenium bulbiferum", - "Asplenium nidus", "Asplenium oblongifolium", "Asplenium platyneuron", "Asplenium trichomanes", "Erythropalum scandens", - "Ginkgo biloba", "", "Byttneria grandifolia", "Triumfetta annua", "Triumfetta cana", "Triumfetta rhomboidea", - "Pentapetes phoenicea", "Anisodontea capensis", "Theobroma cacao", "Ceiba pentandra", "Ceiba speciosa", "Helicteres angustifolia", - "Helicteres hirsuta", "Malvaviscus arboreus", "Malvaviscus arboreus var. mexicanus", "Malvaviscus penduliflorus", "Grewia biloba", - "Grewia biloba var. parviflora", "Grewia occidentalis", "Ambroma augustum", "Bombax ceiba", "Hibiscus acetosella", - "Hibiscus aridicola", "Hibiscus coccineus", "Hibiscus grandiflorus", "Hibiscus grewiifolius", "Hibiscus hamabo", - "Hibiscus moscheutos", "Hibiscus mutabilis", "Hibiscus rosa-sinensis", "Hibiscus sabdariffa", "Hibiscus schizopetalus", - "Hibiscus syriacus", "Hibiscus syriacus var. syriacus f. totus-albus", "Hibiscus tiliaceus", "Hibiscus trionum", - "Firmiana kwangsiensis", "Firmiana simplex", "Reevesia pubescens", "Reevesia thyrsoidea", "Urena lobata", "Urena procumbens", - "Urena procumbens var. microphylla", "Gossypium", "Sidalcea malviflora", "Tilia americana", "Durio zibethinus", - "Diplodiscus trichospermus", "Adansonia digitata", "Pachira glabra", "Corchoropsis crenata", "Microcos paniculata", - "Abelmoschus esculentus", "Abelmoschus manihot", "Abelmoschus sagittifolius", "Pavonia hastata", "Callirhoe involucrata", - "Pterygota alata", "Scaphium wallichii", "Abutilon indicum", "Abutilon megapotamicum", "Abutilon pictum", "Abutilon theophrasti", - "Sterculia lanceolata", "Sterculia monosperma", "Althaea officinalis", "Waltheria indica", "Alcea rosea", - "Malvastrum coromandelianum", "Brachychiton acerifolius", "Brachychiton rupestris", "Heritiera littoralis", "Heritiera parvifolia", - "Malva cathayensis", "Malva pusilla", "Malva verticillata var. crispa", "Dombeya wallichii", "Melochia corchorifolia", - "Kleinhovia hospita", "Sida subcordata", "Corchorus aestuans", "Costus barbatus", "Costus lucanusianus", "Costus woodsonii", - "Stephania cephalantha", "Stephania epigaea&cephalantha", "Stephania longa", "Stephania tetrandra", "Cocculus orbiculatus", - "Diploclisia affinis", "Diploclisia glaucescens", "Menispermum dauricum", "Cyclea racemosa", "Sinomenium acutum", - "Haworthia cooperi var. pilifera", "Haworthia fasciata", "Haworthia truncata", "Dianella ensifolia", "Stypandra glauca", - "Asphodeline lutea", "Kniphofia uvaria", "Geitonoplesium cymosum", "Aloe arborescens", "Aloe ferox", "Aloe mitriformis", - "Aloe vera", "Hemerocallis citrina", "Hemerocallis fulva", "Hemerocallis fulva 'Golden Doll'", "Hemerocallis hybridus", - "Asphodelus fistulosus", "Asphodelus ramosus", "Bulbine bulbosa", "Tricoryne elatior", "Gasteria gracilis var. minima", - "Phormium tenax", "Eichhornia crassipes", "Pontederia cordata", "Pontederia cordata var. alba", "Monochoria korsakowii", - "Monochoria vaginalis", "Sciaphila secundiflora", "Pandanus tectorius", "Schoepfia chinensis", "Helwingia chinensis", - "Helwingia japonica", "Helwingia omeiensis", "Hydnocarpus anthelminthicus", "Hydnocarpus hainanensis", "Typha", - "Typha angustifolia", "Typha latifolia", "Typha orientalis", "Sparganium stoloniferum", "Asarum canadense", "Asarum caudigerum", - "Asarum forbesii", "Asarum heterotropoides", "Aristolochia arborea", "Aristolochia contorta", "Aristolochia debilis", - "Aristolochia elegans", "Aristolochia gentilis", "Aristolochia gibertii", "Aristolochia grandiflora", "Aristolochia griffithii", - "Aristolochia hainanensis", "Aristolochia kwangsiensis", "Aristolochia manshuriensis", "Aristolochia mollissima", - "Aristolochia ringens", "Aristolochia tagala", "Aristolochia tubiflora", "Aristolochia westlandii", "Coriaria nepalensis", - "Mitrasacme pygmaea", "Gardneria multiflora", "Strychnos angustiflora", "Duranta erecta", "Duranta erecta 'Alba'", - "Glandularia bipinnatifida", "Glandularia tenera", "Glandularia × hybrida", "Petrea volubilis", "Phyla canescens", - "Phyla nodiflora", "Lantana camara", "Lantana fucata", "Lantana montevidensis", "Verbena bonariensis", "Verbena brasiliensis", - "Verbena halei", "Verbena hastata", "Verbena officinalis", "Verbena stricta", "Portulaca gilliesii", "Portulaca grandiflora", - "Portulaca molokiniensis", "Portulaca oleracea", "Portulaca pilosa", "Portulaca umbraticola", "", "", "Polystichum acrostichoides", - "Polystichum munitum", "Polystichum vestitum", "Gladiolus communis", "Gladiolus dalenii", "Gladiolus gandavensis", - "Gladiolus imbricatus", "Belamcanda chinensis", "Neomarica gracilis", "Sisyrinchium albidum", "Sisyrinchium angustifolium", - "Sisyrinchium bellum", "Sisyrinchium campestre", "Sisyrinchium micranthum", "Sisyrinchium montanum", "sisyrinchium rosulatum", - "Alophia drummondii", "Olsynium douglasii", "Romulea columnae", "Romulea rosea", "Herbertia lahue", "Crocus biflorus", - "Crocus nudiflorus", "Crocus sativus", "Crocus tommasinianus", "Crocus vernus", "Dietes bicolor", "Nemastylis geminiflora", - "Tigridia pavonia", "Ixia viridiflora", "Trimezia martinicensis", "Crocosmia × crocosmiiflora", "Freesia refracta", - "Sparaxis tricolor", "Iris bulleyana", "Iris chrysographes", "Iris confusa", "Iris cristata", "Iris douglasiana", "Iris ensata", - "Iris foetidissima", "Iris fulva 'Louisiana Hybrids'", "Iris germanica", "Iris hartwegii", "Iris japonica", "Iris lactea", - "Iris lutescens", "Iris macrosiphon", "Iris missouriensis", "Iris pseudacorus", "Iris pumila", "Iris ruthenica", "Iris sanguinea", - "Iris setosa", "Iris sibirica", "Iris speculatrix", "Iris tectorum", "Iris tenax", "Iris verna", "Iris versicolor", - "Iris virginica", "Tinantia anomala", "Tinantia erecta", "Pollia japonica", "Murdannia loriformis", "Murdannia nudiflora", - "Murdannia triquetra", "Amischotolype hispida", "Tradescantia cerinthoides 'Nanouk'", "Tradescantia fluminensis", - "Tradescantia ohiensis", "Tradescantia pallida", "Tradescantia sillamontana", "Tradescantia spathacea", "Tradescantia virginiana", - "Tradescantia zanonia", "Tradescantia zebrina", "Floscopa scandens", "Cyanotis arachnoidea", "Commelina benghalensis", - "Commelina communis", "Commelina diffusa", "Commelina erecta", "Strelitzia nicolai", "Strelitzia reginae", "Ephedra aspera", - "Ephedra californica", "Ephedra distachya", "Ephedra trifurca", "Ephedra viridis", "Pachysandra terminalis", - "Sarcococca hookeriana", "Sarcococca ruscifolia", "Buxus harlandii", "Buxus sinica", "Itea omeiensis", "Berchemia floribunda", - "Berchemia lineata", "Berchemia sinica", "Ziziphus jujuba", "Ziziphus mauritiana", "Hovenia acerba", "Ceanothus", - "Ventilago leiocarpa", "Frangula californica", "Sageretia thea", "Paliurus hemsleyanus", "Paliurus ramosissimus", - "Rhamnus cathartica", "Rhamnus crenata", "Rhamnus davurica", "Rhamnus utilis", "Gentianella azurea", "Latouchea fokienensis", - "Tripterospermum chinense", "Tripterospermum nienkui", "Comastoma pulmonarium", "Megacodon stylophorus", "Gentianopsis barbata", - "Cotylanthera paucisquama", "Eustoma grandiflorum", "Fagraea ceilanica", "Fagraea ceilanica 'Variegata'", "Swertia bimaculata", - "Swertia decora", "Swertia hickinii", "Swertia pseudochinensis", "Centaurium pulchellum var. altaicum", "Canscora lucidissima", - "Sabatia campestris", "Halenia elliptica", "Exacum affine", "Gentiana arethusae var. delicatula", "Gentiana aristata", - "Gentiana dahurica", "Gentiana davidii", "Gentiana lawrencei var. farreri", "Gentiana loureiroi", "Gentiana panthaica", - "Gentiana pseudoaquatica", "Gentiana pudica", "Gentiana rubicunda", "Gentiana squarrosa", "Gentiana straminea", "Gentiana striata", - "Gentiana tatsienensis", "Gentiana urnula", "Gentiana veitchiorum", "Gentiana zollingeri", "Hopea chinensis", "Hopea hainanensis", - "Vatica mangachapoi", "Marsilea quadrifolia" - }; - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_PLANTID_H diff --git a/lite/mnn/cv/mnn_portrait_seg_extremec3net.cpp b/lite/mnn/cv/mnn_portrait_seg_extremec3net.cpp deleted file mode 100644 index 041d0e9c..00000000 --- a/lite/mnn/cv/mnn_portrait_seg_extremec3net.cpp +++ /dev/null @@ -1,135 +0,0 @@ -// -// Created by DefTruth on 2022/6/19. -// - -#include "mnn_portrait_seg_extremec3net.h" -#include "lite/utils.h" - -using mnncv::MNNPortraitSegExtremeC3Net; - -MNNPortraitSegExtremeC3Net::MNNPortraitSegExtremeC3Net( - const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ initialize_pretreat(); } - -void MNNPortraitSegExtremeC3Net::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::BGR, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNPortraitSegExtremeC3Net::transform(const cv::Mat &mat_rs) -{ - pretreat->convert(mat_rs.data, input_width, input_height, - mat_rs.step[0], input_tensor); -} - -void MNNPortraitSegExtremeC3Net::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - PortraitSegExtremeC3NetScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(0, 0, 0)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void MNNPortraitSegExtremeC3Net::detect(const cv::Mat &mat, types::PortraitSegContent &content, - float score_threshold, bool remove_noise) -{ - if (mat.empty()) return; - - // resize & unscale - cv::Mat mat_rs; - PortraitSegExtremeC3NetScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - this->transform(mat_rs); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. generate mask - this->generate_mask(scale_params, output_tensors, mat, content, score_threshold, remove_noise); -} - -static inline void decode_and_zero_if_small_inplace(float *mutable_ptr, float &score) -{ - // ref: https://github.com/clovaai/ext_portrait_segmentation/blob/master/etc/lovasz_losses.py#L143 - const float sign = (1.f / (*mutable_ptr)) <= score ? -1.f : 1.f; - *mutable_ptr = (sign + 1.f) / 2.f; // 0. or 1. -} - -void MNNPortraitSegExtremeC3Net::generate_mask(const PortraitSegExtremeC3NetScaleParams &scale_params, - const std::map &output_tensors, - const cv::Mat &mat, types::PortraitSegContent &content, - float score_threshold, bool remove_noise) -{ - auto device_output_ptr = output_tensors.at("output"); // e.g (1,2,224,224) - MNN::Tensor host_output_tensor(device_output_ptr, device_output_ptr->getDimensionType()); - device_output_ptr->copyToHostTensor(&host_output_tensor); - const unsigned int h = mat.rows; - const unsigned int w = mat.cols; - auto output_dims = host_output_tensor.shape(); - const unsigned int out_h = output_dims.at(2); // e.g 224 - const unsigned int out_w = output_dims.at(3); // e.g 224 - const unsigned int element_size = out_h * out_w; - - float *output_ptr = host_output_tensor.host(); - - // remove small values - for (unsigned int i = 0; i < element_size; ++i) - decode_and_zero_if_small_inplace(output_ptr + i, score_threshold); - - // fetch foreground score - const int dw = scale_params.dw; - const int dh = scale_params.dh; - const int nw = scale_params.new_unpad_w; - const int nh = scale_params.new_unpad_h; - - cv::Mat alpha_pred(out_h, out_w, CV_32FC1, output_ptr); - cv::Mat mask = alpha_pred(cv::Rect(dw, dh, nw, nh)); // 0. ~ 1. - if (remove_noise) lite::utils::remove_small_connected_area(mask, 0.05f); - // already allocated a new continuous memory after resize. - if (nh != h || nw != w) cv::resize(mask, mask, cv::Size(w, h)); - // need clone to allocate a new continuous memory if not performed resize. - // The memory elements point to will release after return. - else mask = mask.clone(); - - content.mask = mask; // auto handle the memory inside ocv with smart ref. - content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_portrait_seg_extremec3net.h b/lite/mnn/cv/mnn_portrait_seg_extremec3net.h deleted file mode 100644 index cb87675b..00000000 --- a/lite/mnn/cv/mnn_portrait_seg_extremec3net.h +++ /dev/null @@ -1,55 +0,0 @@ -// -// Created by DefTruth on 2022/6/19. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_PORTRAIT_SEG_EXTREMEC3NET_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_PORTRAIT_SEG_EXTREMEC3NET_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNPortraitSegExtremeC3Net : public BasicMNNHandler - { - public: - explicit MNNPortraitSegExtremeC3Net(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNPortraitSegExtremeC3Net() override = default; - - private: - // nested classes - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } PortraitSegExtremeC3NetScaleParams; - - private: - const float mean_vals[3] = {107.304565f, 115.69884f, 132.35703f}; // BGR - const float norm_vals[3] = {1.f / (63.97182f * 255.f), 1.f / (65.1337f * 255.f), - 1.f / (68.29726f * 255.f)}; - private: - void initialize_pretreat(); - - void transform(const cv::Mat &mat_rs) override; - - void resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - PortraitSegExtremeC3NetScaleParams &scale_params); - - void generate_mask(const PortraitSegExtremeC3NetScaleParams &scale_params, - const std::map &output_tensors, - const cv::Mat &mat, types::PortraitSegContent &content, - float score_threshold = 0.0f, bool remove_noise = false); - - public: - void detect(const cv::Mat &mat, types::PortraitSegContent &content, - float score_threshold = 0.0f, bool remove_noise = false); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_PORTRAIT_SEG_EXTREMEC3NET_H diff --git a/lite/mnn/cv/mnn_portrait_seg_sinet.cpp b/lite/mnn/cv/mnn_portrait_seg_sinet.cpp deleted file mode 100644 index 26c88d62..00000000 --- a/lite/mnn/cv/mnn_portrait_seg_sinet.cpp +++ /dev/null @@ -1,142 +0,0 @@ -// -// Created by DefTruth on 2022/6/19. -// - -#include "mnn_portrait_seg_sinet.h" -#include "lite/utils.h" - -using mnncv::MNNPortraitSegSINet; - -MNNPortraitSegSINet::MNNPortraitSegSINet(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ initialize_pretreat(); } - -void MNNPortraitSegSINet::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::BGR, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNPortraitSegSINet::transform(const cv::Mat &mat_rs) -{ - pretreat->convert(mat_rs.data, input_width, input_height, - mat_rs.step[0], input_tensor); -} - -void MNNPortraitSegSINet::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - PortraitSegSINetScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(0, 0, 0)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void MNNPortraitSegSINet::detect(const cv::Mat &mat, types::PortraitSegContent &content, - float score_threshold, bool remove_noise) -{ - if (mat.empty()) return; - - // resize & unscale - cv::Mat mat_rs; - PortraitSegSINetScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - this->transform(mat_rs); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. generate mask - this->generate_mask(scale_params, output_tensors, mat, content, score_threshold, remove_noise); -} - -static inline void softmax_inplace(float *mutable_ptr_bgr, float *mutable_ptr_fgr) -{ - const float bgr_exp = std::exp(*mutable_ptr_bgr); - const float fgr_exp = std::exp(*mutable_ptr_fgr); - *mutable_ptr_bgr = bgr_exp / (bgr_exp + fgr_exp + 1e-10f); - *mutable_ptr_fgr = 1.f - *mutable_ptr_bgr; -} - -static inline void zero_if_small_inplace(float *mutable_ptr, float &score) -{ if (*(mutable_ptr) < score) *(mutable_ptr) = 0.f; } - -void MNNPortraitSegSINet::generate_mask(const PortraitSegSINetScaleParams &scale_params, - const std::map &output_tensors, - const cv::Mat &mat, types::PortraitSegContent &content, - float score_threshold, bool remove_noise) -{ - auto device_output_ptr = output_tensors.at("output"); // e.g (1,2,224,224) - MNN::Tensor host_output_tensor(device_output_ptr, device_output_ptr->getDimensionType()); - device_output_ptr->copyToHostTensor(&host_output_tensor); - const unsigned int h = mat.rows; - const unsigned int w = mat.cols; - auto output_dims = host_output_tensor.shape(); - const unsigned int out_h = output_dims.at(2); // e.g 224 - const unsigned int out_w = output_dims.at(3); // e.g 224 - const unsigned int channel_step = out_h * out_w; - - float *output_ptr = host_output_tensor.host(); - - // softmax - for (unsigned int i = 0; i < channel_step; ++i) - softmax_inplace(output_ptr + i, output_ptr + i + channel_step); // bgr & fgr - - // remove small values - for (unsigned int i = 0; i < channel_step; ++i) - zero_if_small_inplace(output_ptr + channel_step + i, score_threshold); - - // fetch foreground score - const int dw = scale_params.dw; - const int dh = scale_params.dh; - const int nw = scale_params.new_unpad_w; - const int nh = scale_params.new_unpad_h; - - cv::Mat alpha_pred(out_h, out_w, CV_32FC1, output_ptr + channel_step); // only need prob of fgr - cv::Mat mask = alpha_pred(cv::Rect(dw, dh, nw, nh)); // 0. ~ 1. - if (remove_noise) lite::utils::remove_small_connected_area(mask, 0.05f); - // already allocated a new continuous memory after resize. - if (nh != h || nw != w) cv::resize(mask, mask, cv::Size(w, h)); - // need clone to allocate a new continuous memory if not performed resize. - // The memory elements point to will release after return. - else mask = mask.clone(); - - content.mask = mask; // auto handle the memory inside ocv with smart ref. - content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_portrait_seg_sinet.h b/lite/mnn/cv/mnn_portrait_seg_sinet.h deleted file mode 100644 index bdecf498..00000000 --- a/lite/mnn/cv/mnn_portrait_seg_sinet.h +++ /dev/null @@ -1,55 +0,0 @@ -// -// Created by DefTruth on 2022/6/19. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_PORTRAIT_SEG_SINET_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_PORTRAIT_SEG_SINET_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNPortraitSegSINet : public BasicMNNHandler - { - public: - explicit MNNPortraitSegSINet(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNPortraitSegSINet() override = default; - - private: - // nested classes - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } PortraitSegSINetScaleParams; - - private: - const float mean_vals[3] = {107.304565f, 115.69884f, 132.35703f}; // BGR - const float norm_vals[3] = {1.f / (63.97182f * 255.f), 1.f / (65.1337f * 255.f), - 1.f / (68.29726f * 255.f)}; - private: - void initialize_pretreat(); - - void transform(const cv::Mat &mat_rs) override; - - void resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - PortraitSegSINetScaleParams &scale_params); - - void generate_mask(const PortraitSegSINetScaleParams &scale_params, - const std::map &output_tensors, - const cv::Mat &mat, types::PortraitSegContent &content, - float score_threshold = 0.02f, bool remove_noise = false); - - public: - void detect(const cv::Mat &mat, types::PortraitSegContent &content, - float score_threshold = 0.02f, bool remove_noise = false); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_PORTRAIT_SEG_SINET_H diff --git a/lite/mnn/cv/mnn_resnet.cpp b/lite/mnn/cv/mnn_resnet.cpp deleted file mode 100644 index beb79202..00000000 --- a/lite/mnn/cv/mnn_resnet.cpp +++ /dev/null @@ -1,68 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "mnn_resnet.h" -#include "lite/utils.h" - -using mnncv::MNNResNet; - -MNNResNet::MNNResNet(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNResNet::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNResNet::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,3,224,224) - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNResNet::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch. - auto device_logits_ptr = output_tensors.at("logits"); - MNN::Tensor host_logits_tensor(device_logits_ptr, device_logits_ptr->getDimensionType()); - device_logits_ptr->copyToHostTensor(&host_logits_tensor); - - auto logits_dims = host_logits_tensor.shape(); - const unsigned int num_classes = logits_dims.at(1); // 1000 - const float *logits = host_logits_tensor.host(); - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_resnet.h b/lite/mnn/cv/mnn_resnet.h deleted file mode 100644 index c8230eda..00000000 --- a/lite/mnn/cv/mnn_resnet.h +++ /dev/null @@ -1,409 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_RESNET_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_RESNET_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNResNet : public BasicMNNHandler - { - public: - explicit MNNResNet(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNResNet() override = default; - - private: - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // resize & normalize. - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_RESNET_H diff --git a/lite/mnn/cv/mnn_resnext.cpp b/lite/mnn/cv/mnn_resnext.cpp deleted file mode 100644 index e484e75e..00000000 --- a/lite/mnn/cv/mnn_resnext.cpp +++ /dev/null @@ -1,68 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "mnn_resnext.h" -#include "lite/utils.h" - -using mnncv::MNNResNeXt; - -MNNResNeXt::MNNResNeXt(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNResNeXt::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNResNeXt::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,3,224,224) - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNResNeXt::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch. - auto device_logits_ptr = output_tensors.at("logits"); - MNN::Tensor host_logits_tensor(device_logits_ptr, device_logits_ptr->getDimensionType()); - device_logits_ptr->copyToHostTensor(&host_logits_tensor); - - auto logits_dims = host_logits_tensor.shape(); - const unsigned int num_classes = logits_dims.at(1); // 1000 - const float *logits = host_logits_tensor.host(); - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_resnext.h b/lite/mnn/cv/mnn_resnext.h deleted file mode 100644 index 089dd51b..00000000 --- a/lite/mnn/cv/mnn_resnext.h +++ /dev/null @@ -1,409 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_RESNEXT_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_RESNEXT_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNResNeXt : public BasicMNNHandler - { - public: - explicit MNNResNeXt(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNResNeXt() override = default; - - private: - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // resize & normalize. - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_RESNEXT_H diff --git a/lite/mnn/cv/mnn_retinaface.cpp b/lite/mnn/cv/mnn_retinaface.cpp deleted file mode 100644 index 547614ff..00000000 --- a/lite/mnn/cv/mnn_retinaface.cpp +++ /dev/null @@ -1,179 +0,0 @@ -// -// Created by DefTruth on 2021/11/20. -// - -#include "mnn_retinaface.h" -#include "lite/utils.h" - -using mnncv::MNNRetinaFace; - -MNNRetinaFace::MNNRetinaFace(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNRetinaFace::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::BGR, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNRetinaFace::transform(const cv::Mat &mat) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - // normalize & HWC -> CHW & BGR -> BGR - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNRetinaFace::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - this->transform(mat); - // 2. inference scores & boxes. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(bbox_collection, output_tensors, score_threshold, img_height, img_width); - // 4. hard|blend nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void MNNRetinaFace::generate_anchors(const int target_height, const int target_width, - std::vector &anchors) -{ - std::vector> feature_maps; - for (auto step: steps) - { - feature_maps.push_back( - { - (int) std::ceil((float) target_height / (float) step), - (int) std::ceil((float) target_width / (float) step) - } // ceil - ); - } - - anchors.clear(); - const int num_feature_map = feature_maps.size(); - - for (int k = 0; k < num_feature_map; ++k) - { - auto f_map = feature_maps.at(k); // e.g [640//8,640//8] - auto tmp_min_sizes = min_sizes.at(k); // e.g [8,16] - int f_h = f_map.at(0); - int f_w = f_map.at(1); - - for (int i = 0; i < f_h; ++i) - { - for (int j = 0; j < f_w; ++j) - { - for (auto min_size: tmp_min_sizes) - { - float s_kx = (float) min_size / (float) target_width; // e.g 16/w - float s_ky = (float) min_size / (float) target_height; // e.g 16/h - // (x + 0.5) * step / w normalized loc mapping to input width - // (y + 0.5) * step / h normalized loc mapping to input height - float cx = ((float) j + 0.5f) * (float) steps.at(k) / (float) target_width; - float cy = ((float) i + 0.5f) * (float) steps.at(k) / (float) target_height; - - anchors.push_back(RetinaAnchor{cx, cy, s_kx, s_ky}); // without clip - } - } - } - } -} - -void MNNRetinaFace::generate_bboxes(std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, float img_height, - float img_width) -{ - auto device_bboxes_ptr = output_tensors.at("bbox"); // e.g (1,16800,4) - auto device_probs_ptr = output_tensors.at("conf"); // e.g (1,16800,2) after softmax - MNN::Tensor host_bboxes_tensor(device_bboxes_ptr, device_bboxes_ptr->getDimensionType()); - MNN::Tensor host_probs_tensor(device_probs_ptr, device_probs_ptr->getDimensionType()); - device_bboxes_ptr->copyToHostTensor(&host_bboxes_tensor); - device_probs_ptr->copyToHostTensor(&host_probs_tensor); - - auto bbox_dims = host_bboxes_tensor.shape(); - const unsigned int bbox_num = bbox_dims.at(1); // n = ? - - std::vector anchors; - this->generate_anchors(input_height, input_width, anchors); - - const unsigned int num_anchors = anchors.size(); - if (num_anchors != bbox_num) - throw std::runtime_error("mismatch num_anchors != bbox_num"); - - const float *bboxes_ptr = host_bboxes_tensor.host(); - const float *probs_ptr = host_probs_tensor.host(); - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - float conf = probs_ptr[2 * i + 1]; - if (conf < score_threshold) continue; // filter first. - - float prior_cx = anchors.at(i).cx; - float prior_cy = anchors.at(i).cy; - float prior_s_kx = anchors.at(i).s_kx; - float prior_s_ky = anchors.at(i).s_ky; - - float dx = bboxes_ptr[4 * i + 0]; - float dy = bboxes_ptr[4 * i + 1]; - float dw = bboxes_ptr[4 * i + 2]; - float dh = bboxes_ptr[4 * i + 3]; - // ref: https://github.com/biubug6/Pytorch_Retinaface/blob/master/utils/box_utils.py - float cx = prior_cx + dx * variance[0] * prior_s_kx; - float cy = prior_cy + dy * variance[0] * prior_s_ky; - float w = prior_s_kx * std::exp(dw * variance[1]); - float h = prior_s_ky * std::exp(dh * variance[1]); // norm coor (0.,1.) - - types::Boxf box; - box.x1 = (cx - w / 2.f) * img_width; - box.y1 = (cy - h / 2.f) * img_height; - box.x2 = (cx + w / 2.f) * img_width; - box.y2 = (cy + h / 2.f) * img_height; - box.score = conf; - box.label = 1; - box.label_text = "face"; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITEMNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void MNNRetinaFace::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - - - - diff --git a/lite/mnn/cv/mnn_retinaface.h b/lite/mnn/cv/mnn_retinaface.h deleted file mode 100644 index 6e6bb2f6..00000000 --- a/lite/mnn/cv/mnn_retinaface.h +++ /dev/null @@ -1,70 +0,0 @@ -// -// Created by DefTruth on 2021/11/20. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_RETINAFACE_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_RETINAFACE_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNRetinaFace : public BasicMNNHandler - { - public: - explicit MNNRetinaFace(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNRetinaFace() override = default; - - private: - // nested classes - struct RetinaAnchor - { - float cx; - float cy; - float s_kx; - float s_ky; - }; - - private: - const float mean_vals[3] = {104.f, 117.f, 123.f}; // bgr order - const float norm_vals[3] = {1.f, 1.f, 1.f}; - const float variance[2] = {0.1f, 0.2f}; - std::vector steps = {8, 16, 32}; - std::vector> min_sizes = { - {16, 32}, - {64, 128}, - {256, 512} - }; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // - - void generate_anchors(const int target_height, - const int target_width, - std::vector &anchors); - - void generate_bboxes(std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.7f, float iou_threshold = 0.3f, - unsigned int topk = 300, unsigned int nms_type = 0); - - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_RETINAFACE_H diff --git a/lite/mnn/cv/mnn_rexnet_emotion7.cpp b/lite/mnn/cv/mnn_rexnet_emotion7.cpp deleted file mode 100644 index ffa3df53..00000000 --- a/lite/mnn/cv/mnn_rexnet_emotion7.cpp +++ /dev/null @@ -1,60 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#include "mnn_rexnet_emotion7.h" -#include "lite/utils.h" - -using mnncv::MNNReXNetEmotion7; - -MNNReXNetEmotion7::MNNReXNetEmotion7(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNReXNetEmotion7::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNReXNetEmotion7::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,3,224,224) - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNReXNetEmotion7::detect(const cv::Mat &mat, types::Emotions &emotions) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch. - auto device_emotion_logits_ptr = output_tensors.at("logits"); // (1,7) - MNN::Tensor host_emotion_logits_tensor(device_emotion_logits_ptr, device_emotion_logits_ptr->getDimensionType()); - device_emotion_logits_ptr->copyToHostTensor(&host_emotion_logits_tensor); - - auto emotion_dims = host_emotion_logits_tensor.shape(); - const unsigned int num_emotions = emotion_dims.at(1); // 7 - - unsigned int pred_label = 0; - const float *pred_logits_ptr = host_emotion_logits_tensor.host(); - - auto softmax_probs = lite::utils::math::softmax(pred_logits_ptr, num_emotions, pred_label); - emotions.label = pred_label; - emotions.score = softmax_probs[pred_label]; - emotions.text = emotion_texts[pred_label]; - emotions.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_rexnet_emotion7.h b/lite/mnn/cv/mnn_rexnet_emotion7.h deleted file mode 100644 index d6276d71..00000000 --- a/lite/mnn/cv/mnn_rexnet_emotion7.h +++ /dev/null @@ -1,35 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_REXNET_EMOTION7_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_REXNET_EMOTION7_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNReXNetEmotion7 : public BasicMNNHandler - { - public: - explicit MNNReXNetEmotion7(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNReXNetEmotion7() override = default; - - private: - const float mean_vals[3] = {255.f * 0.485f, 255.f * 0.456, 255.f * 0.406f}; - const float norm_vals[3] = {1 / (255.f * 0.229f), 1 / (255.f * 0.224f), 1 / (255.f * 0.225f)}; - const char *emotion_texts[7] = { - "angry", "disgust", "fear", "happiness", "neutral", "sadness", "surprise" - }; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // padding & resize & normalize. - - public: - void detect(const cv::Mat &mat, types::Emotions &emotions); - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_REXNET_EMOTION7_H diff --git a/lite/mnn/cv/mnn_rvm.cpp b/lite/mnn/cv/mnn_rvm.cpp deleted file mode 100644 index 6c8de9b6..00000000 --- a/lite/mnn/cv/mnn_rvm.cpp +++ /dev/null @@ -1,346 +0,0 @@ -// -// Created by DefTruth on 2021/10/10. -// - -#include "mnn_rvm.h" -#include "lite/utils.h" - -using mnncv::MNNRobustVideoMatting; - -MNNRobustVideoMatting::MNNRobustVideoMatting( - const std::string &_mnn_path, - unsigned int _num_threads, - unsigned int _variant_type -) : log_id(_mnn_path.data()), - mnn_path(_mnn_path.data()), - num_threads(_num_threads), - variant_type(_variant_type) -{ - initialize_interpreter(); - initialize_context(); - initialize_pretreat(); -} - -MNNRobustVideoMatting::~MNNRobustVideoMatting() -{ - mnn_interpreter->releaseModel(); - if (mnn_session) - mnn_interpreter->releaseSession(mnn_session); -} - -void MNNRobustVideoMatting::initialize_interpreter() -{ - // 1. init interpreter - mnn_interpreter = std::shared_ptr(MNN::Interpreter::createFromFile(mnn_path)); - // 2. init schedule_config - schedule_config.numThread = (int) num_threads; - MNN::BackendConfig backend_config; - backend_config.precision = MNN::BackendConfig::Precision_High; // default Precision_High - schedule_config.backendConfig = &backend_config; - // 3. create session - mnn_session = mnn_interpreter->createSession(schedule_config); - // 4. init input tensor - src_tensor = mnn_interpreter->getSessionInput(mnn_session, "src"); - // 5. init input dims - input_height = src_tensor->height(); - input_width = src_tensor->width(); - dimension_type = src_tensor->getDimensionType(); // CAFFE - mnn_interpreter->resizeTensor(src_tensor, {1, 3, input_height, input_width}); - mnn_interpreter->resizeSession(mnn_session); - src_size = 1 * 3 * input_height * input_width; - // 6. rxi - r1i_tensor = mnn_interpreter->getSessionInput(mnn_session, "r1i"); - r2i_tensor = mnn_interpreter->getSessionInput(mnn_session, "r2i"); - r3i_tensor = mnn_interpreter->getSessionInput(mnn_session, "r3i"); - r4i_tensor = mnn_interpreter->getSessionInput(mnn_session, "r4i"); -#ifdef LITEMNN_DEBUG - this->print_debug_string(); -#endif -} - -void MNNRobustVideoMatting::print_debug_string() -{ - std::cout << "LITEMNN_DEBUG LogId: " << log_id << "\n"; - std::cout << "=============== Input-Dims ==============\n"; - if (src_tensor) src_tensor->printShape(); - if (r1i_tensor) r1i_tensor->printShape(); - if (r2i_tensor) r2i_tensor->printShape(); - if (r3i_tensor) r3i_tensor->printShape(); - if (r4i_tensor) r4i_tensor->printShape(); - if (dimension_type == MNN::Tensor::CAFFE) - std::cout << "Dimension Type: (CAFFE/PyTorch/ONNX)NCHW" << "\n"; - else if (dimension_type == MNN::Tensor::TENSORFLOW) - std::cout << "Dimension Type: (TENSORFLOW)NHWC" << "\n"; - else if (dimension_type == MNN::Tensor::CAFFE_C4) - std::cout << "Dimension Type: (CAFFE_C4)NC4HW4" << "\n"; - std::cout << "=============== Output-Dims ==============\n"; - auto tmp_output_map = mnn_interpreter->getSessionOutputAll(mnn_session); - std::cout << "getSessionOutputAll done!\n"; - for (auto it = tmp_output_map.cbegin(); it != tmp_output_map.cend(); ++it) - { - std::cout << "Output: " << it->first << ": "; - it->second->printShape(); - } - std::cout << "========================================\n"; -} - -void MNNRobustVideoMatting::initialize_context() -{ - if (variant_type == VARIANT::MOBILENETV3) - { - if (input_width == 1920 && input_height == 1080) - { - mnn_interpreter->resizeTensor(r1i_tensor, {1, 16, 135, 240}); - mnn_interpreter->resizeTensor(r2i_tensor, {1, 20, 68, 120}); - mnn_interpreter->resizeTensor(r3i_tensor, {1, 40, 34, 60}); - mnn_interpreter->resizeTensor(r4i_tensor, {1, 64, 17, 30}); - r1i_size = 1 * 16 * 135 * 240; - r2i_size = 1 * 20 * 68 * 120; - r3i_size = 1 * 40 * 34 * 60; - r4i_size = 1 * 64 * 17 * 30; - } // hxw 480x640 480x480 640x480 - else - { - mnn_interpreter->resizeTensor(r1i_tensor, {1, 16, input_height / 2, input_width / 2}); - mnn_interpreter->resizeTensor(r2i_tensor, {1, 20, input_height / 4, input_width / 4}); - mnn_interpreter->resizeTensor(r3i_tensor, {1, 40, input_height / 8, input_width / 8}); - mnn_interpreter->resizeTensor(r4i_tensor, {1, 64, input_height / 16, input_width / 16}); - r1i_size = 1 * 16 * (input_height / 2) * (input_width / 2); - r2i_size = 1 * 20 * (input_height / 4) * (input_width / 4); - r3i_size = 1 * 40 * (input_height / 8) * (input_width / 8); - r4i_size = 1 * 64 * (input_height / 16) * (input_width / 16); - } - }// RESNET50 - else - { - if (input_width == 1920 && input_height == 1080) - { - mnn_interpreter->resizeTensor(r1i_tensor, {1, 16, 135, 240}); - mnn_interpreter->resizeTensor(r2i_tensor, {1, 32, 68, 120}); - mnn_interpreter->resizeTensor(r3i_tensor, {1, 64, 34, 60}); - mnn_interpreter->resizeTensor(r4i_tensor, {1, 128, 17, 30}); - r1i_size = 1 * 16 * 135 * 240; - r2i_size = 1 * 32 * 68 * 120; - r3i_size = 1 * 64 * 34 * 60; - r4i_size = 1 * 128 * 17 * 30; - } // hxw 480x640 480x480 640x480 - else - { - mnn_interpreter->resizeTensor(r1i_tensor, {1, 16, input_height / 2, input_width / 2}); - mnn_interpreter->resizeTensor(r2i_tensor, {1, 32, input_height / 4, input_width / 4}); - mnn_interpreter->resizeTensor(r3i_tensor, {1, 64, input_height / 8, input_width / 8}); - mnn_interpreter->resizeTensor(r4i_tensor, {1, 128, input_height / 16, input_width / 16}); - r1i_size = 1 * 16 * (input_height / 2) * (input_width / 2); - r2i_size = 1 * 32 * (input_height / 4) * (input_width / 4); - r3i_size = 1 * 64 * (input_height / 8) * (input_width / 8); - r4i_size = 1 * 128 * (input_height / 16) * (input_width / 16); - } - } - // resize session - mnn_interpreter->resizeSession(mnn_session); - // init 0. - std::fill_n(r1i_tensor->host(), r1i_size, 0.f); - std::fill_n(r2i_tensor->host(), r2i_size, 0.f); - std::fill_n(r3i_tensor->host(), r3i_size, 0.f); - std::fill_n(r4i_tensor->host(), r4i_size, 0.f); - - context_is_initialized = true; -} - -inline void MNNRobustVideoMatting::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -inline void MNNRobustVideoMatting::transform(const cv::Mat &mat_rs) -{ - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], src_tensor); -} - -void MNNRobustVideoMatting::detect(const cv::Mat &mat, types::MattingContent &content, bool video_mode, - bool remove_noise, bool minimum_post_process) -{ - if (mat.empty()) return; - int img_h = mat.rows; - int img_w = mat.cols; - if (!context_is_initialized) return; - - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - // 1. make input tensor - this->transform(mat_rs); - - // 2. inference & run session - mnn_interpreter->runSession(mnn_session); - - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. generate matting - this->generate_matting(output_tensors, content, img_h, img_w, remove_noise, minimum_post_process); - // 4. update context (needed for video matting) - if (video_mode) - { - context_is_update = false; // init state. - this->update_context(output_tensors); - } -} - -void MNNRobustVideoMatting::detect_video( - const std::string &video_path, const std::string &output_path, - std::vector &contents, bool save_contents, - unsigned int writer_fps, bool remove_noise, bool minimum_post_process, - const cv::Mat &background) -{ - // 0. init video capture - cv::VideoCapture video_capture(video_path); - const unsigned int width = video_capture.get(cv::CAP_PROP_FRAME_WIDTH); - const unsigned int height = video_capture.get(cv::CAP_PROP_FRAME_HEIGHT); - const unsigned int frame_count = video_capture.get(cv::CAP_PROP_FRAME_COUNT); - if (!video_capture.isOpened()) - { - std::cout << "Can not open video: " << video_path << "\n"; - return; - } - // 1. init video writer - cv::VideoWriter video_writer(output_path, cv::VideoWriter::fourcc('m', 'p', '4', 'v'), - writer_fps, cv::Size(width, height)); - if (!video_writer.isOpened()) - { - std::cout << "Can not open writer: " << output_path << "\n"; - return; - } - - // 2. matting loop - cv::Mat mat; - unsigned int i = 0; - while (video_capture.read(mat)) - { - i += 1; - types::MattingContent content; - this->detect(mat, content, true, remove_noise, minimum_post_process); // video_mode true - // 3. save contents and writing out. - if (content.flag) - { -// if (save_contents) contents.push_back(content); -// if (!content.merge_mat.empty()) video_writer.write(content.merge_mat); - - if (save_contents) contents.push_back(content); - // 3.1 do nothing if set minimum_post_process as true - if (background.empty()) - { - if (!content.merge_mat.empty() && !minimum_post_process) - video_writer.write(content.merge_mat); - else if (!content.fgr_mat.empty()) - video_writer.write(content.fgr_mat); - } // - else - { - cv::Mat out_mat; - // 3.2 merge user custom background - if (!content.pha_mat.empty()) - { - if (!content.fgr_mat.empty()) - lite::utils::swap_background(content.fgr_mat, content.pha_mat, - background, out_mat, false); - else - lite::utils::swap_background(mat, content.pha_mat, - background, out_mat, false); - } - if (!out_mat.empty()) video_writer.write(out_mat); - - } - - } - // 4. check context states. - if (!context_is_update) break; -#ifdef LITEMNN_DEBUG - std::cout << i << "/" << frame_count << " done!" << "\n"; -#endif - } - - // 5. release - video_capture.release(); - video_writer.release(); -} - -void MNNRobustVideoMatting::generate_matting( - const std::map &output_tensors, - types::MattingContent &content, int img_h, int img_w, - bool remove_noise, bool minimum_post_process) -{ - auto device_fgr_ptr = output_tensors.at("fgr"); - auto device_pha_ptr = output_tensors.at("pha"); - MNN::Tensor host_fgr_tensor(device_fgr_ptr, device_fgr_ptr->getDimensionType()); // NCHW - MNN::Tensor host_pha_tensor(device_pha_ptr, device_pha_ptr->getDimensionType()); // NCHW - device_fgr_ptr->copyToHostTensor(&host_fgr_tensor); - device_pha_ptr->copyToHostTensor(&host_pha_tensor); - - float *fgr_ptr = host_fgr_tensor.host(); - float *pha_ptr = host_pha_tensor.host(); - const unsigned int channel_step = input_height * input_width; - - // fast assign & channel transpose(CHW->HWC). - cv::Mat rmat(input_height, input_width, CV_32FC1, fgr_ptr); - cv::Mat gmat(input_height, input_width, CV_32FC1, fgr_ptr + channel_step); - cv::Mat bmat(input_height, input_width, CV_32FC1, fgr_ptr + 2 * channel_step); - cv::Mat pmat(input_height, input_width, CV_32FC1, pha_ptr); // ref only, zero-copy. - if (remove_noise) lite::utils::remove_small_connected_area(pmat, 0.05f); - - rmat *= 255.f; - bmat *= 255.f; - gmat *= 255.f; - std::vector fgr_channel_mats; - fgr_channel_mats.push_back(bmat); - fgr_channel_mats.push_back(gmat); - fgr_channel_mats.push_back(rmat); - - // need clone to allocate a new continuous memory. - content.pha_mat = pmat.clone(); // allocated - cv::merge(fgr_channel_mats, content.fgr_mat); - content.fgr_mat.convertTo(content.fgr_mat, CV_8UC3); - - if (!minimum_post_process) - { - cv::Mat rest = 1.f - pmat; - cv::Mat mbmat = bmat.mul(pmat) + rest * 153.f; - cv::Mat mgmat = gmat.mul(pmat) + rest * 255.f; - cv::Mat mrmat = rmat.mul(pmat) + rest * 120.f; - std::vector merge_channel_mats; - merge_channel_mats.push_back(mbmat); - merge_channel_mats.push_back(mgmat); - merge_channel_mats.push_back(mrmat); - cv::merge(merge_channel_mats, content.merge_mat); - content.merge_mat.convertTo(content.merge_mat, CV_8UC3); - } - - if (img_w != input_width || img_h != input_height) - { - cv::resize(content.pha_mat, content.pha_mat, cv::Size(img_w, img_h)); - cv::resize(content.fgr_mat, content.fgr_mat, cv::Size(img_w, img_h)); - if (!minimum_post_process) - cv::resize(content.merge_mat, content.merge_mat, cv::Size(img_w, img_h)); - } - - content.flag = true; -} - -void MNNRobustVideoMatting::update_context(const std::map &output_tensors) -{ - auto device_r1o_ptr = output_tensors.at("r1o"); - auto device_r2o_ptr = output_tensors.at("r2o"); - auto device_r3o_ptr = output_tensors.at("r3o"); - auto device_r4o_ptr = output_tensors.at("r4o"); - - device_r1o_ptr->copyToHostTensor(r1i_tensor); - device_r2o_ptr->copyToHostTensor(r2i_tensor); - device_r3o_ptr->copyToHostTensor(r3i_tensor); - device_r4o_ptr->copyToHostTensor(r4i_tensor); - - context_is_update = true; -} diff --git a/lite/mnn/cv/mnn_rvm.h b/lite/mnn/cv/mnn_rvm.h deleted file mode 100644 index 6add8427..00000000 --- a/lite/mnn/cv/mnn_rvm.h +++ /dev/null @@ -1,201 +0,0 @@ -// -// Created by DefTruth on 2021/10/10. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_RVM_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_RVM_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNRobustVideoMatting - { - public: - explicit MNNRobustVideoMatting(const std::string &_mnn_path, - unsigned int _num_threads = 1, - unsigned int _variant_type = 0); // - ~MNNRobustVideoMatting(); - - private: - std::shared_ptr mnn_interpreter; - MNN::Session *mnn_session = nullptr; - MNN::ScheduleConfig schedule_config; - std::shared_ptr pretreat; // init at runtime - const char *log_id = nullptr; - const char *mnn_path = nullptr; - - private: - const float mean_vals[3] = {0.f, 0.f, 0.f}; // RGB - const float norm_vals[3] = {1.f / 255.f, 1.f / 255.f, 1.f / 255.f}; - // hardcode input node names, hint only. - // downsample_ratio has been freeze while onnx exported - // and, the input size of each input has been freeze, also. - std::vector input_node_names = { - "src", - "r1i", - "r2i", - "r3i", - "r4i" - }; - // hardcode output node names, hint only. - std::vector output_node_names = { - "fgr", - "pha", - "r1o", - "r2o", - "r3o", - "r4o" - }; - bool context_is_update = false; - bool context_is_initialized = false; - - private: - enum VARIANT - { - MOBILENETV3 = 0, - RESNET50 = 1 - }; - - const unsigned int num_threads; // initialize at runtime. - // multi inputs, rxi will be update inner video matting process. - MNN::Tensor *src_tensor = nullptr; - MNN::Tensor *r1i_tensor = nullptr; - MNN::Tensor *r2i_tensor = nullptr; - MNN::Tensor *r3i_tensor = nullptr; - MNN::Tensor *r4i_tensor = nullptr; - // input size & variant_type, initialize at runtime. - const unsigned int variant_type; - int input_height; - int input_width; - int dimension_type; // hint only - unsigned int src_size; - unsigned int r1i_size; - unsigned int r2i_size; - unsigned int r3i_size; - unsigned int r4i_size; - - // un-copyable - protected: - MNNRobustVideoMatting(const MNNRobustVideoMatting &) = delete; // - MNNRobustVideoMatting(MNNRobustVideoMatting &&) = delete; // - MNNRobustVideoMatting &operator=(const MNNRobustVideoMatting &) = delete; // - MNNRobustVideoMatting &operator=(MNNRobustVideoMatting &&) = delete; // - - private: - void print_debug_string(); - - private: - void transform(const cv::Mat &mat_rs); // without resize - - void initialize_pretreat(); // - - void initialize_interpreter(); - - void initialize_context(); - - void generate_matting(const std::map &output_tensors, - types::MattingContent &content, int img_h, int img_w, - bool remove_noise = false, bool minimum_post_process = false); - - void update_context(const std::map &output_tensors); - - public: - /** - * Image Matting Using RVM(https://github.com/PeterL1n/RobustVideoMatting) - * @param mat: cv::Mat BGR HWC - * @param content: types::MattingContent to catch the detected results. - * @param video_mode: false by default. - * See https://github.com/PeterL1n/RobustVideoMatting/blob/master/documentation/inference_zh_Hans.md - * @param remove_noise: remove small connected area or not - * @param minimum_post_process: if True, will run matting with minimum post process - * in order to speed up the matting processes. - */ - void detect(const cv::Mat &mat, types::MattingContent &content, bool video_mode = false, - bool remove_noise = false, bool minimum_post_process = false); - - /** - * Video Matting Using RVM(https://github.com/PeterL1n/RobustVideoMatting) - * @param video_path: eg. xxx/xxx/input.mp4 - * @param output_path: eg. xxx/xxx/output.mp4 - * @param contents: vector of MattingContent to catch the detected results. - * @param save_contents: false by default, whether to save MattingContent. - * See https://github.com/PeterL1n/RobustVideoMatting/blob/master/documentation/inference_zh_Hans.md - * @param writer_fps: FPS for VideoWriter, 20 by default. - * @param remove_noise: remove small connected area or not - * @param minimum_post_process: if True, will run matting with minimum post process - * in order to speed up the matting processes. - * @param background: user's custom background setting, will return with this target - * background if background Mat is not empty instead of green background. - */ - void detect_video(const std::string &video_path, - const std::string &output_path, - std::vector &contents, - bool save_contents = false, - unsigned int writer_fps = 20, - bool remove_noise = false, - bool minimum_post_process = false, - const cv::Mat &background = cv::Mat()); - - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_RVM_H - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lite/mnn/cv/mnn_scrfd.cpp b/lite/mnn/cv/mnn_scrfd.cpp deleted file mode 100644 index 44d91141..00000000 --- a/lite/mnn/cv/mnn_scrfd.cpp +++ /dev/null @@ -1,415 +0,0 @@ -// -// Created by DefTruth on 2021/12/30. -// - -#include "mnn_scrfd.h" - -using mnncv::MNNSCRFD; - -MNNSCRFD::MNNSCRFD(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); - initial_context(); -} - -inline void MNNSCRFD::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNSCRFD::initial_context() -{ - if (num_outputs == 6) - { - fmc = 3; - feat_stride_fpn = {8, 16, 32}; - num_anchors = 2; - use_kps = false; - } // kps - else if (num_outputs == 9) - { - fmc = 3; - feat_stride_fpn = {8, 16, 32}; - num_anchors = 2; - use_kps = true; - } -} - -inline void MNNSCRFD::transform(const cv::Mat &mat_rs) -{ - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNSCRFD::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - SCRFDScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(0, 0, 0)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.ratio = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.flag = true; -} - -void MNNSCRFD::detect(const cv::Mat &mat, std::vector &detected_boxes_kps, - float score_threshold, float iou_threshold, unsigned int topk) -{ - if (mat.empty()) return; - auto img_height = static_cast(mat.rows); - auto img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - SCRFDScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - this->transform(mat_rs); - - // 2. inference scores & boxes. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - - // 3. rescale & exclude. - std::vector bbox_kps_collection; - this->generate_bboxes_kps(scale_params, bbox_kps_collection, output_tensors, - score_threshold, img_height, img_width); - // 4. hard nms with topk. - this->nms_bboxes_kps(bbox_kps_collection, detected_boxes_kps, iou_threshold, topk); -} - -void MNNSCRFD::generate_points(const int target_height, const int target_width) -{ - if (center_points_is_update) return; - // 8, 16, 32 - for (auto stride : feat_stride_fpn) - { - unsigned int num_grid_w = target_width / stride; - unsigned int num_grid_h = target_height / stride; - // y - for (unsigned int i = 0; i < num_grid_h; ++i) - { - // x - for (unsigned int j = 0; j < num_grid_w; ++j) - { - // num_anchors, col major - for (unsigned int k = 0; k < num_anchors; ++k) - { - SCRFDPoint point; - point.cx = (float) j; - point.cy = (float) i; - point.stride = (float) stride; - center_points[stride].push_back(point); - } - - } - } - } - - center_points_is_update = true; -} - -void MNNSCRFD::generate_bboxes_kps(const SCRFDScaleParams &scale_params, - std::vector &bbox_kps_collection, - const std::map &output_tensors, - float score_threshold, float img_height, - float img_width) -{ - // score_8,score_16,score_32,bbox_8,bbox_16,bbox_32 - auto device_score_8 = output_tensors.at("score_8"); - auto device_score_16 = output_tensors.at("score_16"); - auto device_score_32 = output_tensors.at("score_32"); - auto device_bbox_8 = output_tensors.at("bbox_8"); - auto device_bbox_16 = output_tensors.at("bbox_16"); - auto device_bbox_32 = output_tensors.at("bbox_32"); - this->generate_points(input_height, input_width); - - MNN::Tensor host_score_8(device_score_8, device_score_8->getDimensionType()); - MNN::Tensor host_score_16(device_score_16, device_score_16->getDimensionType()); - MNN::Tensor host_score_32(device_score_32, device_score_32->getDimensionType()); - MNN::Tensor host_bbox_8(device_bbox_8, device_bbox_8->getDimensionType()); - MNN::Tensor host_bbox_16(device_bbox_16, device_bbox_16->getDimensionType()); - MNN::Tensor host_bbox_32(device_bbox_32, device_bbox_32->getDimensionType()); - - device_score_8->copyToHostTensor(&host_score_8); - device_score_16->copyToHostTensor(&host_score_16); - device_score_32->copyToHostTensor(&host_score_32); - device_bbox_8->copyToHostTensor(&host_bbox_8); - device_bbox_16->copyToHostTensor(&host_bbox_16); - device_bbox_32->copyToHostTensor(&host_bbox_32); - - bbox_kps_collection.clear(); - - if (use_kps) - { - auto device_kps_8 = output_tensors.at("kps_8"); - auto device_kps_16 = output_tensors.at("kps_16"); - auto device_kps_32 = output_tensors.at("kps_32"); - - MNN::Tensor host_kps_8(device_kps_8, device_kps_8->getDimensionType()); - MNN::Tensor host_kps_16(device_kps_16, device_kps_16->getDimensionType()); - MNN::Tensor host_kps_32(device_kps_32, device_kps_32->getDimensionType()); - - device_kps_8->copyToHostTensor(&host_kps_8); - device_kps_16->copyToHostTensor(&host_kps_16); - device_kps_32->copyToHostTensor(&host_kps_32); - - // level 8 & 16 & 32 with kps - this->generate_bboxes_kps_single_stride(scale_params, host_score_8, host_bbox_8, host_kps_8, 8, score_threshold, - img_height, img_width, bbox_kps_collection); - this->generate_bboxes_kps_single_stride(scale_params, host_score_16, host_bbox_16, host_kps_16, 16, score_threshold, - img_height, img_width, bbox_kps_collection); - this->generate_bboxes_kps_single_stride(scale_params, host_score_32, host_bbox_32, host_kps_32, 32, score_threshold, - img_height, img_width, bbox_kps_collection); - - } // no kps - else - { - // level 8 & 16 & 32 - this->generate_bboxes_single_stride(scale_params, host_score_8, host_bbox_8, 8, score_threshold, - img_height, img_width, bbox_kps_collection); - this->generate_bboxes_single_stride(scale_params, host_score_16, host_bbox_16, 16, score_threshold, - img_height, img_width, bbox_kps_collection); - this->generate_bboxes_single_stride(scale_params, host_score_32, host_bbox_32, 32, score_threshold, - img_height, img_width, bbox_kps_collection); - } - -#if LITEMNN_DEBUG - std::cout << "generate_bboxes_kps num: " << bbox_kps_collection.size() << "\n"; -#endif -} - -void MNNSCRFD::generate_bboxes_single_stride( - const SCRFDScaleParams &scale_params, MNN::Tensor &score_pred, MNN::Tensor &bbox_pred, - unsigned int stride, float score_threshold, float img_height, float img_width, - std::vector &bbox_kps_collection) -{ - unsigned int nms_pre_ = (stride / 8) * nms_pre; // 1 * 1000,2*1000,... - nms_pre_ = nms_pre_ >= nms_pre ? nms_pre_ : nms_pre; - - auto stride_dims = score_pred.shape(); - const unsigned int num_points = stride_dims.at(1); // 12800 - const float *score_ptr = score_pred.host(); // [1,12800,1] - const float *bbox_ptr = bbox_pred.host(); // [1,12800,4] - - float ratio = scale_params.ratio; - int dw = scale_params.dw; - int dh = scale_params.dh; - - unsigned int count = 0; - auto &stride_points = center_points[stride]; - - for (unsigned int i = 0; i < num_points; ++i) - { - const float cls_conf = score_ptr[i]; - if (cls_conf < score_threshold) continue; // filter - auto &point = stride_points.at(i); - const float cx = point.cx; // cx - const float cy = point.cy; // cy - const float s = point.stride; // stride - - // bbox - const float *offsets = bbox_ptr + i * 4; - float l = offsets[0]; // left - float t = offsets[1]; // top - float r = offsets[2]; // right - float b = offsets[3]; // bottom - - types::BoxfWithLandmarks box_kps; - float x1 = ((cx - l) * s - (float) dw) / ratio; // cx - l x1 - float y1 = ((cy - t) * s - (float) dh) / ratio; // cy - t y1 - float x2 = ((cx + r) * s - (float) dw) / ratio; // cx + r x2 - float y2 = ((cy + b) * s - (float) dh) / ratio; // cy + b y2 - box_kps.box.x1 = std::max(0.f, x1); - box_kps.box.y1 = std::max(0.f, y1); - box_kps.box.x2 = std::min(img_width - 1.f, x2); - box_kps.box.y2 = std::min(img_height - 1.f, y2); - box_kps.box.score = cls_conf; - box_kps.box.label = 1; - box_kps.box.label_text = "face"; - box_kps.box.flag = true; - box_kps.flag = true; - - bbox_kps_collection.push_back(box_kps); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - - if (bbox_kps_collection.size() > nms_pre_) - { - std::sort( - bbox_kps_collection.begin(), bbox_kps_collection.end(), - [](const types::BoxfWithLandmarks &a, const types::BoxfWithLandmarks &b) - { return a.box.score > b.box.score; } - ); // sort inplace - // trunc - bbox_kps_collection.resize(nms_pre_); - } - -} - -void MNNSCRFD::generate_bboxes_kps_single_stride( - const SCRFDScaleParams &scale_params, MNN::Tensor &score_pred, MNN::Tensor &bbox_pred, - MNN::Tensor &kps_pred, unsigned int stride, float score_threshold, float img_height, - float img_width, std::vector &bbox_kps_collection) -{ - unsigned int nms_pre_ = (stride / 8) * nms_pre; // 1 * 1000,2*1000,... - nms_pre_ = nms_pre_ >= nms_pre ? nms_pre_ : nms_pre; - - auto stride_dims = score_pred.shape(); - const unsigned int num_points = stride_dims.at(1); // 12800 - const float *score_ptr = score_pred.host(); // [1,12800,1] - const float *bbox_ptr = bbox_pred.host(); // [1,12800,4] - const float *kps_ptr = kps_pred.host(); // [1,12800,10] - - float ratio = scale_params.ratio; - int dw = scale_params.dw; - int dh = scale_params.dh; - - unsigned int count = 0; - auto &stride_points = center_points[stride]; - - for (unsigned int i = 0; i < num_points; ++i) - { - const float cls_conf = score_ptr[i]; - if (cls_conf < score_threshold) continue; // filter - auto &point = stride_points.at(i); - const float cx = point.cx; // cx - const float cy = point.cy; // cy - const float s = point.stride; // stride - - // bbox - const float *offsets = bbox_ptr + i * 4; - float l = offsets[0]; // left - float t = offsets[1]; // top - float r = offsets[2]; // right - float b = offsets[3]; // bottom - - types::BoxfWithLandmarks box_kps; - float x1 = ((cx - l) * s - (float) dw) / ratio; // cx - l x1 - float y1 = ((cy - t) * s - (float) dh) / ratio; // cy - t y1 - float x2 = ((cx + r) * s - (float) dw) / ratio; // cx + r x2 - float y2 = ((cy + b) * s - (float) dh) / ratio; // cy + b y2 - box_kps.box.x1 = std::max(0.f, x1); - box_kps.box.y1 = std::max(0.f, y1); - box_kps.box.x2 = std::min(img_width - 1.f, x2); - box_kps.box.y2 = std::min(img_height - 1.f, y2); - box_kps.box.score = cls_conf; - box_kps.box.label = 1; - box_kps.box.label_text = "face"; - box_kps.box.flag = true; - - // landmarks - const float *kps_offsets = kps_ptr + i * 10; - for (unsigned int j = 0; j < 10; j += 2) - { - cv::Point2f kps; - float kps_l = kps_offsets[j]; - float kps_t = kps_offsets[j + 1]; - float kps_x = ((cx + kps_l) * s - (float) dw) / ratio; // cx - l x - float kps_y = ((cy + kps_t) * s - (float) dh) / ratio; // cy - t y - kps.x = std::min(std::max(0.f, kps_x), img_width - 1.f); - kps.y = std::min(std::max(0.f, kps_y), img_height - 1.f); - box_kps.landmarks.points.push_back(kps); - } - box_kps.landmarks.flag = true; - box_kps.flag = true; - - bbox_kps_collection.push_back(box_kps); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - - if (bbox_kps_collection.size() > nms_pre_) - { - std::sort( - bbox_kps_collection.begin(), bbox_kps_collection.end(), - [](const types::BoxfWithLandmarks &a, const types::BoxfWithLandmarks &b) - { return a.box.score > b.box.score; } - ); // sort inplace - // trunc - bbox_kps_collection.resize(nms_pre_); - } - -} - -void MNNSCRFD::nms_bboxes_kps(std::vector &input, - std::vector &output, - float iou_threshold, unsigned int topk) -{ - if (input.empty()) return; - std::sort( - input.begin(), input.end(), - [](const types::BoxfWithLandmarks &a, const types::BoxfWithLandmarks &b) - { return a.box.score > b.box.score; } - ); - const unsigned int box_num = input.size(); - std::vector merged(box_num, 0); - - unsigned int count = 0; - for (unsigned int i = 0; i < box_num; ++i) - { - if (merged[i]) continue; - std::vector buf; - - buf.push_back(input[i]); - merged[i] = 1; - - for (unsigned int j = i + 1; j < box_num; ++j) - { - if (merged[j]) continue; - - float iou = static_cast(input[i].box.iou_of(input[j].box)); - - if (iou > iou_threshold) - { - merged[j] = 1; - buf.push_back(input[j]); - } - - } - output.push_back(buf[0]); - - // keep top k - count += 1; - if (count >= topk) - break; - } -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_scrfd.h b/lite/mnn/cv/mnn_scrfd.h deleted file mode 100644 index c95c6172..00000000 --- a/lite/mnn/cv/mnn_scrfd.h +++ /dev/null @@ -1,106 +0,0 @@ -// -// Created by DefTruth on 2021/12/30. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_SCRFD_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_SCRFD_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNSCRFD : public BasicMNNHandler - { - public: - explicit MNNSCRFD(const std::string &_mnn_path, unsigned int _num_threads = 1); - - ~MNNSCRFD() override = default; - - private: - // nested classes - typedef struct - { - float cx; - float cy; - float stride; - } SCRFDPoint; - typedef struct - { - float ratio; - int dw; - int dh; - bool flag; - } SCRFDScaleParams; - - private: - // blob = cv2.dnn.blobFromImage(img, 1.0/128, input_size, (127.5, 127.5, 127.5), swapRB=True) - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 128.f, 1.f / 128.f, 1.f / 128.f}; - unsigned int fmc = 3; // feature map count - bool use_kps = false; - unsigned int num_anchors = 2; - std::vector feat_stride_fpn = {8, 16, 32}; // steps, may [8, 16, 32, 64, 128] - // if num_anchors>1, then stack points in col major -> (height*num_anchor*width,2) - // anchor_centers = np.stack([anchor_centers]*self._num_anchors, axis=1).reshape( (-1,2) ) - std::unordered_map> center_points; - bool center_points_is_update = false; - static constexpr const unsigned int nms_pre = 1000; - static constexpr const unsigned int max_nms = 30000; - - private: - void transform(const cv::Mat &mat_rs) override; // without resize - - // initial steps and num_anchors - // https://github.com/deepinsight/insightface/blob/master/detection/scrfd/tools/scrfd.py - void initial_context(); - - void initialize_pretreat(); // - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - SCRFDScaleParams &scale_params); - - // generate once. - void generate_points(const int target_height, const int target_width); - - void generate_bboxes_single_stride(const SCRFDScaleParams &scale_params, - MNN::Tensor &score_pred, - MNN::Tensor &bbox_pred, - unsigned int stride, - float score_threshold, - float img_height, - float img_width, - std::vector &bbox_kps_collection); - - void generate_bboxes_kps_single_stride(const SCRFDScaleParams &scale_params, - MNN::Tensor &score_pred, - MNN::Tensor &bbox_pred, - MNN::Tensor &kps_pred, - unsigned int stride, - float score_threshold, - float img_height, - float img_width, - std::vector &bbox_kps_collection); - - void generate_bboxes_kps(const SCRFDScaleParams &scale_params, - std::vector &bbox_kps_collection, - const std::map &output_tensors, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms_bboxes_kps(std::vector &input, - std::vector &output, - float iou_threshold, unsigned int topk); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes_kps, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 400); - - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_SCRFD_H diff --git a/lite/mnn/cv/mnn_shufflenetv2.cpp b/lite/mnn/cv/mnn_shufflenetv2.cpp deleted file mode 100644 index 538f0bcd..00000000 --- a/lite/mnn/cv/mnn_shufflenetv2.cpp +++ /dev/null @@ -1,68 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "mnn_shufflenetv2.h" -#include "lite/utils.h" - -using mnncv::MNNShuffleNetV2; - -MNNShuffleNetV2::MNNShuffleNetV2(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNShuffleNetV2::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNShuffleNetV2::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // (1,3,224,224) - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNShuffleNetV2::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch. - auto device_logits_ptr = output_tensors.at("output"); - MNN::Tensor host_logits_tensor(device_logits_ptr, device_logits_ptr->getDimensionType()); - device_logits_ptr->copyToHostTensor(&host_logits_tensor); - - auto logits_dims = host_logits_tensor.shape(); - const unsigned int num_classes = logits_dims.at(1); // 1000 - const float *logits = host_logits_tensor.host(); - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_shufflenetv2.h b/lite/mnn/cv/mnn_shufflenetv2.h deleted file mode 100644 index 8ea5f070..00000000 --- a/lite/mnn/cv/mnn_shufflenetv2.h +++ /dev/null @@ -1,410 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_SHUFFLENETV2_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_SHUFFLENETV2_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNShuffleNetV2 : public BasicMNNHandler - { - public: - explicit MNNShuffleNetV2(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNShuffleNetV2() override = default; - - private: - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // resize & normalize. - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_SHUFFLENETV2_H diff --git a/lite/mnn/cv/mnn_sphere_face.cpp b/lite/mnn/cv/mnn_sphere_face.cpp deleted file mode 100644 index 3375811e..00000000 --- a/lite/mnn/cv/mnn_sphere_face.cpp +++ /dev/null @@ -1,58 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "mnn_sphere_face.h" - -using mnncv::MNNSphereFace; - -MNNSphereFace::MNNSphereFace(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - - -inline void MNNSphereFace::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::BGR, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNSphereFace::transform(const cv::Mat &mat) -{ - // normalize & HWC -> CHW & BGR -> RGB - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNSphereFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - - auto device_embedding_ptr = output_tensors.at("embedding"); - MNN::Tensor host_embedding_tensor(device_embedding_ptr, device_embedding_ptr->getDimensionType()); // NCHW - device_embedding_ptr->copyToHostTensor(&host_embedding_tensor); - - auto embedding_dims = host_embedding_tensor.shape(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = host_embedding_tensor.host(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_sphere_face.h b/lite/mnn/cv/mnn_sphere_face.h deleted file mode 100644 index be3b07cf..00000000 --- a/lite/mnn/cv/mnn_sphere_face.h +++ /dev/null @@ -1,33 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_SPHERE_FACE_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_SPHERE_FACE_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNSphereFace : public BasicMNNHandler - { - public: - explicit MNNSphereFace(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNSphereFace() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 128.0f, 1.f / 128.0f, 1.f / 128.0f}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_SPHERE_FACE_H diff --git a/lite/mnn/cv/mnn_ssrnet.cpp b/lite/mnn/cv/mnn_ssrnet.cpp deleted file mode 100644 index 5fa5e480..00000000 --- a/lite/mnn/cv/mnn_ssrnet.cpp +++ /dev/null @@ -1,59 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#include "mnn_ssrnet.h" - -using mnncv::MNNSSRNet; - -MNNSSRNet::MNNSSRNet(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNSSRNet::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::BGR, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNSSRNet::transform(const cv::Mat &mat) -{ - cv::Mat canvas; - // (1,3,64,64) - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - pretreat->convert(canvas.data, input_width, input_height, canvas.step[0], input_tensor); -} - -void MNNSSRNet::detect(const cv::Mat &mat, types::Age &age) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. fetch. - auto device_age_ptr = output_tensors.at("age"); - MNN::Tensor host_age_tensor(device_age_ptr, device_age_ptr->getDimensionType()); - device_age_ptr->copyToHostTensor(&host_age_tensor); - - const float *age_ptr = host_age_tensor.host(); - const float pred_age = age_ptr[0]; - - const unsigned int interval_min = static_cast(pred_age - 2.f > 0.f ? pred_age - 2.f : 0.f); - const unsigned int interval_max = static_cast(pred_age + 3.f < 100.f ? pred_age + 3.f : 100.f); - - age.age = pred_age; - age.age_interval[0] = interval_min; - age.age_interval[1] = interval_max; - age.interval_prob = 1.0f; - age.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_ssrnet.h b/lite/mnn/cv/mnn_ssrnet.h deleted file mode 100644 index c112e9ec..00000000 --- a/lite/mnn/cv/mnn_ssrnet.h +++ /dev/null @@ -1,35 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_SSRNET_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_SSRNET_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNSSRNet : public BasicMNNHandler - { - public: - explicit MNNSSRNet(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNSSRNet() override = default; - - private: - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0f / 255.0f), - (1.0f / 0.224f) * (1.0f / 255.0f), - (1.0f / 0.225f) * (1.0f / 255.0f)}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // padding & resize & normalize. - - public: - void detect(const cv::Mat &mat, types::Age &age); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_SSRNET_H diff --git a/lite/mnn/cv/mnn_subpixel_cnn.cpp b/lite/mnn/cv/mnn_subpixel_cnn.cpp deleted file mode 100644 index 8d93614a..00000000 --- a/lite/mnn/cv/mnn_subpixel_cnn.cpp +++ /dev/null @@ -1,87 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "mnn_subpixel_cnn.h" - -using mnncv::MNNSubPixelCNN; - -MNNSubPixelCNN::MNNSubPixelCNN(const std::string &_mnn_path, unsigned int _num_threads) - : BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNSubPixelCNN::initialize_pretreat() -{ - pretreat = nullptr; // no use -} - -void MNNSubPixelCNN::transform(const cv::Mat &mat) -{ - cv::Mat mat_y; // assume that input mat is Y of YCrCb - mat.convertTo(mat_y, CV_32FC1, 1.0f / 255.0f, 0.f); // (224,224,1) range (0.,1.0) - - auto tmp_host_nchw_tensor = new MNN::Tensor(input_tensor, MNN::Tensor::CAFFE); // tmp - std::memcpy(tmp_host_nchw_tensor->host(), mat_y.data, - input_height * input_width * sizeof(float)); - input_tensor->copyFromHostTensor(tmp_host_nchw_tensor); - - delete tmp_host_nchw_tensor; -} - -void MNNSubPixelCNN::detect(const cv::Mat &mat, types::SuperResolutionContent &super_resolution_content) -{ - if (mat.empty()) return; - cv::Mat mat_copy = mat.clone(); - cv::resize(mat_copy, mat_copy, cv::Size(input_width, input_height)); // (224,224,3) - cv::Mat mat_ycrcb, mat_y, mat_cr, mat_cb; - cv::cvtColor(mat_copy, mat_ycrcb, cv::COLOR_BGR2YCrCb); - - // 0. split - std::vector split_mats; - cv::split(mat_ycrcb, split_mats); - mat_y = split_mats.at(0); // (224,224,1) uchar CV_8UC1 - mat_cr = split_mats.at(1); - mat_cb = split_mats.at(2); - - // 1. make input tensor - this->transform(mat_y); // (1,1,224,224) - // 2. inference - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - - auto device_pred_ptr = output_tensors.at("output"); // (1,1,672,672) - MNN::Tensor host_pred_tensor(device_pred_ptr, device_pred_ptr->getDimensionType()); - device_pred_ptr->copyToHostTensor(&host_pred_tensor); - - auto pred_dims = host_pred_tensor.shape(); - const unsigned int rows = pred_dims.at(2); // H - const unsigned int cols = pred_dims.at(3); // W - - float *pred_ptr = host_pred_tensor.host(); - - mat_y = cv::Mat(rows, cols, CV_32FC1, pred_ptr); // release & create - - mat_y *= 255.0f; - - mat_y.convertTo(mat_y, CV_8UC1); - - cv::resize(mat_cr, mat_cr, cv::Size(cols, rows)); - cv::resize(mat_cb, mat_cb, cv::Size(cols, rows)); - - std::vector out_mats; - out_mats.push_back(mat_y); - out_mats.push_back(mat_cr); - out_mats.push_back(mat_cb); - - // 3. merge - cv::merge(out_mats, super_resolution_content.mat); - if (super_resolution_content.mat.empty()) - { - super_resolution_content.flag = false; - return; - } - cv::cvtColor(super_resolution_content.mat, super_resolution_content.mat, cv::COLOR_YCrCb2BGR); - super_resolution_content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_subpixel_cnn.h b/lite/mnn/cv/mnn_subpixel_cnn.h deleted file mode 100644 index 67ef0bdd..00000000 --- a/lite/mnn/cv/mnn_subpixel_cnn.h +++ /dev/null @@ -1,29 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_SUBPIXEL_CNN_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_SUBPIXEL_CNN_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNSubPixelCNN : public BasicMNNHandler - { - public: - explicit MNNSubPixelCNN(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNSubPixelCNN() override = default; - - private: - void initialize_pretreat(); // no use - - void transform(const cv::Mat &mat) override; // resize & normalize. - - public: - void detect(const cv::Mat &mat, types::SuperResolutionContent &super_resolution_content); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_SUBPIXEL_CNN_H diff --git a/lite/mnn/cv/mnn_tencent_cifp_face.cpp b/lite/mnn/cv/mnn_tencent_cifp_face.cpp deleted file mode 100644 index 762a2048..00000000 --- a/lite/mnn/cv/mnn_tencent_cifp_face.cpp +++ /dev/null @@ -1,58 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "mnn_tencent_cifp_face.h" - -using mnncv::MNNTencentCifpFace; - -MNNTencentCifpFace::MNNTencentCifpFace(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - - -inline void MNNTencentCifpFace::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNTencentCifpFace::transform(const cv::Mat &mat) -{ - // normalize & HWC -> CHW & BGR -> RGB - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNTencentCifpFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - - auto device_embedding_ptr = output_tensors.at("embedding"); - MNN::Tensor host_embedding_tensor(device_embedding_ptr, device_embedding_ptr->getDimensionType()); // NCHW - device_embedding_ptr->copyToHostTensor(&host_embedding_tensor); - - auto embedding_dims = host_embedding_tensor.shape(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = host_embedding_tensor.host(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_tencent_cifp_face.h b/lite/mnn/cv/mnn_tencent_cifp_face.h deleted file mode 100644 index c63cbc0a..00000000 --- a/lite/mnn/cv/mnn_tencent_cifp_face.h +++ /dev/null @@ -1,33 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_TENCENT_CIFP_FACE_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_TENCENT_CIFP_FACE_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNTencentCifpFace : public BasicMNNHandler - { - public: - explicit MNNTencentCifpFace(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNTencentCifpFace() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_TENCENT_CIFP_FACE_H diff --git a/lite/mnn/cv/mnn_tencent_curricular_face.cpp b/lite/mnn/cv/mnn_tencent_curricular_face.cpp deleted file mode 100644 index ecb03e25..00000000 --- a/lite/mnn/cv/mnn_tencent_curricular_face.cpp +++ /dev/null @@ -1,58 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "mnn_tencent_curricular_face.h" - -using mnncv::MNNTencentCurricularFace; - -MNNTencentCurricularFace::MNNTencentCurricularFace(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - - -inline void MNNTencentCurricularFace::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNTencentCurricularFace::transform(const cv::Mat &mat) -{ - // normalize & HWC -> CHW & BGR -> RGB - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNTencentCurricularFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - this->transform(mat); - // 2. inference. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - - auto device_embedding_ptr = output_tensors.at("embedding"); - MNN::Tensor host_embedding_tensor(device_embedding_ptr, device_embedding_ptr->getDimensionType()); // NCHW - device_embedding_ptr->copyToHostTensor(&host_embedding_tensor); - - auto embedding_dims = host_embedding_tensor.shape(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = host_embedding_tensor.host(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_tencent_curricular_face.h b/lite/mnn/cv/mnn_tencent_curricular_face.h deleted file mode 100644 index 8f226099..00000000 --- a/lite/mnn/cv/mnn_tencent_curricular_face.h +++ /dev/null @@ -1,33 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_TENCENT_CURRICULAR_FACE_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_TENCENT_CURRICULAR_FACE_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNTencentCurricularFace : public BasicMNNHandler - { - public: - explicit MNNTencentCurricularFace(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNTencentCurricularFace() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_TENCENT_CURRICULAR_FACE_H diff --git a/lite/mnn/cv/mnn_ultraface.cpp b/lite/mnn/cv/mnn_ultraface.cpp deleted file mode 100644 index 7da43920..00000000 --- a/lite/mnn/cv/mnn_ultraface.cpp +++ /dev/null @@ -1,136 +0,0 @@ -// -// Created by DefTruth on 2021/11/20. -// - -#include "mnn_ultraface.h" -#include "lite/utils.h" - -using mnncv::MNNUltraFace; - -MNNUltraFace::MNNUltraFace(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNUltraFace::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNUltraFace::transform(const cv::Mat &mat) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - // normalize & HWC -> CHW & BGR -> RGB - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNUltraFace::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - // this->transform(mat); - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - this->transform(mat); - // 2. inference scores & boxes. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(bbox_collection, output_tensors, score_threshold, img_height, img_width); - // 4. hard|blend nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void MNNUltraFace::generate_bboxes(std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, float img_height, - float img_width) -{ - auto device_scores_ptr = output_tensors.at("scores"); // (1,n,2) - auto device_boxes_ptr = output_tensors.at("boxes"); // (1,n,4) - MNN::Tensor host_scores_tensor(device_scores_ptr, device_scores_ptr->getDimensionType()); - MNN::Tensor host_boxes_tensor(device_boxes_ptr, device_boxes_ptr->getDimensionType()); - device_scores_ptr->copyToHostTensor(&host_scores_tensor); - device_boxes_ptr->copyToHostTensor(&host_boxes_tensor); - - auto scores_dims = host_scores_tensor.shape(); // (1,n,2) - const unsigned int num_anchors = scores_dims.at(1); // n = 17640 (640x480) - const float *scores_ptr = host_scores_tensor.host(); - const float *boxes_ptr = host_boxes_tensor.host(); - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - float confidence = scores_ptr[2 * i + 1]; - if (confidence < score_threshold) continue; - types::Boxf box; - box.x1 = boxes_ptr[4 * i + 0] * img_width; - box.y1 = boxes_ptr[4 * i + 1] * img_height; - box.x2 = boxes_ptr[4 * i + 2] * img_width; - box.y2 = boxes_ptr[4 * i + 3] * img_height; - box.score = confidence; - box.label_text = "face"; - box.label = 1; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITEMNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void MNNUltraFace::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lite/mnn/cv/mnn_ultraface.h b/lite/mnn/cv/mnn_ultraface.h deleted file mode 100644 index 9ec895f8..00000000 --- a/lite/mnn/cv/mnn_ultraface.h +++ /dev/null @@ -1,48 +0,0 @@ -// -// Created by DefTruth on 2021/11/20. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_ULTRAFACE_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_ULTRAFACE_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNUltraFace : public BasicMNNHandler - { - public: - explicit MNNUltraFace(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNUltraFace() override = default; - - private: - const float mean_vals[3] = {127.0f, 127.0f, 127.0f}; - const float norm_vals[3] = {1.0f / 128.0f, 1.0f / 128.0f, 1.0f / 128.0f}; - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat) override; // - - void generate_bboxes(std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.7f, float iou_threshold = 0.3f, - unsigned int topk = 300, unsigned int nms_type = 0); - - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_ULTRAFACE_H diff --git a/lite/mnn/cv/mnn_yolo5face.cpp b/lite/mnn/cv/mnn_yolo5face.cpp deleted file mode 100644 index aae8f543..00000000 --- a/lite/mnn/cv/mnn_yolo5face.cpp +++ /dev/null @@ -1,213 +0,0 @@ -// -// Created by DefTruth on 2022/1/16. -// - -#include "mnn_yolo5face.h" - -using mnncv::MNNYOLO5Face; - -MNNYOLO5Face::MNNYOLO5Face(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNYOLO5Face::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -inline void MNNYOLO5Face::transform(const cv::Mat &mat_rs) -{ - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNYOLO5Face::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YOLO5FaceScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(0, 0, 0)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.ratio = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.flag = true; -} - -void MNNYOLO5Face::detect(const cv::Mat &mat, std::vector &detected_boxes_kps, - float score_threshold, float iou_threshold, unsigned int topk) -{ - if (mat.empty()) return; - auto img_height = static_cast(mat.rows); - auto img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - YOLO5FaceScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - this->transform(mat_rs); - - // 2. inference scores & boxes. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - - // 3. rescale & exclude. - std::vector bbox_kps_collection; - this->generate_bboxes_kps(scale_params, bbox_kps_collection, output_tensors, - score_threshold, img_height, img_width); - // 4. hard nms with topk. - this->nms_bboxes_kps(bbox_kps_collection, detected_boxes_kps, iou_threshold, topk); - -} - -void MNNYOLO5Face::generate_bboxes_kps(const YOLO5FaceScaleParams &scale_params, - std::vector &bbox_kps_collection, - const std::map &output_tensors, - float score_threshold, float img_height, float img_width) -{ - auto device_output_pred = output_tensors.at("output"); - MNN::Tensor host_output_pred(device_output_pred, device_output_pred->getDimensionType()); - device_output_pred->copyToHostTensor(&host_output_pred); - - auto output_dims = host_output_pred.shape(); - const unsigned int num_anchors = output_dims.at(1); // n = ? - const float *output_ptr = host_output_pred.host(); - - float r_ = scale_params.ratio; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - bbox_kps_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *row_ptr = output_ptr + i * 16; - float obj_conf = row_ptr[4]; - if (obj_conf < score_threshold) continue; // filter first. - float cls_conf = row_ptr[15]; - if (cls_conf < score_threshold) continue; // face score. - - // bounding box - const float *offsets = row_ptr; - float cx = offsets[0]; - float cy = offsets[1]; - float w = offsets[2]; - float h = offsets[3]; - - types::BoxfWithLandmarks box_kps; - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - box_kps.box.x1 = std::max(0.f, x1); - box_kps.box.y1 = std::max(0.f, y1); - box_kps.box.x2 = std::min(img_width - 1.f, x2); - box_kps.box.y2 = std::min(img_height - 1.f, y2); - box_kps.box.score = cls_conf; - box_kps.box.label = 1; - box_kps.box.label_text = "face"; - box_kps.box.flag = true; - - // landmarks - const float *kps_offsets = row_ptr + 5; - for (unsigned int j = 0; j < 10; j += 2) - { - cv::Point2f kps; - float kps_x = (kps_offsets[j] - (float) dw_) / r_; - float kps_y = (kps_offsets[j + 1] - (float) dh_) / r_; - kps.x = std::min(std::max(0.f, kps_x), img_width - 1.f); - kps.y = std::min(std::max(0.f, kps_y), img_height - 1.f); - box_kps.landmarks.points.push_back(kps); - } - box_kps.landmarks.flag = true; - box_kps.flag = true; - - bbox_kps_collection.push_back(box_kps); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - -#if LITEMNN_DEBUG - std::cout << "generate_bboxes_kps num: " << bbox_kps_collection.size() << "\n"; -#endif - -} - -void MNNYOLO5Face::nms_bboxes_kps(std::vector &input, - std::vector &output, - float iou_threshold, unsigned int topk) -{ - if (input.empty()) return; - std::sort( - input.begin(), input.end(), - [](const types::BoxfWithLandmarks &a, const types::BoxfWithLandmarks &b) - { return a.box.score > b.box.score; } - ); - const unsigned int box_num = input.size(); - std::vector merged(box_num, 0); - - unsigned int count = 0; - for (unsigned int i = 0; i < box_num; ++i) - { - if (merged[i]) continue; - std::vector buf; - - buf.push_back(input[i]); - merged[i] = 1; - - for (unsigned int j = i + 1; j < box_num; ++j) - { - if (merged[j]) continue; - - float iou = static_cast(input[i].box.iou_of(input[j].box)); - - if (iou > iou_threshold) - { - merged[j] = 1; - buf.push_back(input[j]); - } - - } - output.push_back(buf[0]); - - // keep top k - count += 1; - if (count >= topk) - break; - } -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_yolo5face.h b/lite/mnn/cv/mnn_yolo5face.h deleted file mode 100644 index 9ae038f1..00000000 --- a/lite/mnn/cv/mnn_yolo5face.h +++ /dev/null @@ -1,64 +0,0 @@ -// -// Created by DefTruth on 2022/1/16. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_YOLO5FACE_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_YOLO5FACE_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNYOLO5Face : public BasicMNNHandler - { - public: - explicit MNNYOLO5Face(const std::string &_mnn_path, unsigned int _num_threads = 1); - - ~MNNYOLO5Face() override = default; - - private: - // nested classes - typedef struct - { - float ratio; - int dw; - int dh; - bool flag; - } YOLO5FaceScaleParams; - - private: - const float mean_vals[3] = {0.f, 0.f, 0.f}; // RGB - const float norm_vals[3] = {1.f / 255.f, 1.f / 255.f, 1.f / 255.f}; - static constexpr const unsigned int max_nms = 30000; - - private: - void transform(const cv::Mat &mat_rs) override; // without resize - - void initialize_pretreat(); - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YOLO5FaceScaleParams &scale_params); - - void generate_bboxes_kps(const YOLO5FaceScaleParams &scale_params, - std::vector &bbox_kps_collection, - const std::map &output_tensors, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms_bboxes_kps(std::vector &input, - std::vector &output, - float iou_threshold, unsigned int topk); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes_kps, - float score_threshold = 0.3f, float iou_threshold = 0.45f, - unsigned int topk = 400); - - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_YOLO5FACE_H diff --git a/lite/mnn/cv/mnn_yolop.cpp b/lite/mnn/cv/mnn_yolop.cpp deleted file mode 100644 index 076c9628..00000000 --- a/lite/mnn/cv/mnn_yolop.cpp +++ /dev/null @@ -1,272 +0,0 @@ -// -// Created by DefTruth on 2021/10/18. -// - -#include "mnn_yolop.h" -#include "lite/utils.h" - -using mnncv::MNNYOLOP; - -MNNYOLOP::MNNYOLOP(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNYOLOP::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -inline void MNNYOLOP::transform(const cv::Mat &mat_rs) -{ - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNYOLOP::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YOLOPScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat = mat.clone(); - cv::resize(new_unpad_mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void MNNYOLOP::detect(const cv::Mat &mat, - std::vector &detected_boxes, - types::SegmentContent &da_seg_content, - types::SegmentContent &ll_seg_content, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - auto img_height = static_cast(mat.rows); - auto img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - YOLOPScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - this->transform(mat_rs); - - // 2. inference scores & boxes. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes_da_ll(scale_params, output_tensors, bbox_collection, - da_seg_content, ll_seg_content, score_threshold, - img_height, img_width); - // 4. hard|blend nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void MNNYOLOP::generate_bboxes_da_ll(const YOLOPScaleParams &scale_params, - const std::map &output_tensors, - std::vector &bbox_collection, - types::SegmentContent &da_seg_content, - types::SegmentContent &ll_seg_content, - float score_threshold, float img_height, - float img_width) -{ - auto device_det_out_ptr = output_tensors.at("det_out"); - auto device_da_seg_out_ptr = output_tensors.at("drive_area_seg"); - auto device_ll_seg_out_ptr = output_tensors.at("lane_line_seg"); - // copy to host - MNN::Tensor host_det_out(device_det_out_ptr, device_det_out_ptr->getDimensionType()); - MNN::Tensor host_da_seg_out(device_da_seg_out_ptr, device_da_seg_out_ptr->getDimensionType()); - MNN::Tensor host_ll_seg_out(device_ll_seg_out_ptr, device_ll_seg_out_ptr->getDimensionType()); - device_det_out_ptr->copyToHostTensor(&host_det_out); - device_da_seg_out_ptr->copyToHostTensor(&host_da_seg_out); - device_ll_seg_out_ptr->copyToHostTensor(&host_ll_seg_out); - - auto det_dims = host_det_out.shape(); - const unsigned int num_anchors = det_dims.at(1); // n = ? - - float r = scale_params.r; - int dw = scale_params.dw; - int dh = scale_params.dh; - int new_unpad_w = scale_params.new_unpad_w; - int new_unpad_h = scale_params.new_unpad_h; - - // generate bounding boxes. - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *offset_obj_cls_ptr = host_det_out.host() + (i * 6); - float obj_conf = offset_obj_cls_ptr[4]; - if (obj_conf < score_threshold) continue; // filter first. - - unsigned int label = 1; // 1 class only - float cls_conf = offset_obj_cls_ptr[5]; - float conf = obj_conf * cls_conf; // cls_conf (0.,1.) - if (conf < score_threshold) continue; // filter - - float cx = offset_obj_cls_ptr[0]; - float cy = offset_obj_cls_ptr[1]; - float w = offset_obj_cls_ptr[2]; - float h = offset_obj_cls_ptr[3]; - float x1 = ((cx - w / 2.f) - (float) dw) / r; - float y1 = ((cy - h / 2.f) - (float) dh) / r; - float x2 = ((cx + w / 2.f) - (float) dw) / r; - float y2 = ((cy + h / 2.f) - (float) dh) / r; - - types::Boxf box; - // de-padding & rescaling - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(x2, (float) img_width); - box.y2 = std::min(y2, (float) img_height); - box.score = conf; - box.label = label; - box.label_text = "traffic car"; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITEMNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif - - // generate da && ll seg. - da_seg_content.names_map.clear(); - da_seg_content.class_mat = cv::Mat(new_unpad_h, new_unpad_w, CV_8UC1, cv::Scalar(0)); - da_seg_content.color_mat = cv::Mat(new_unpad_h, new_unpad_w, CV_8UC3, cv::Scalar(0, 0, 0)); - ll_seg_content.names_map.clear(); - ll_seg_content.class_mat = cv::Mat(new_unpad_h, new_unpad_w, CV_8UC1, cv::Scalar(0)); - ll_seg_content.color_mat = cv::Mat(new_unpad_h, new_unpad_w, CV_8UC3, cv::Scalar(0, 0, 0)); - - const unsigned int channel_step = input_height * input_width; - const float *da_seg_bg_ptr = host_da_seg_out.host(); // background - const float *da_seg_fg_ptr = host_da_seg_out.host() + channel_step; // foreground - const float *ll_seg_bg_ptr = host_ll_seg_out.host(); // background - const float *ll_seg_fg_ptr = host_ll_seg_out.host() + channel_step; // foreground - - for (int i = dh; i < dh + new_unpad_h; ++i) - { - // row ptr. - uchar *da_p_class = da_seg_content.class_mat.ptr(i - dh); - uchar *ll_p_class = ll_seg_content.class_mat.ptr(i - dh); - cv::Vec3b *da_p_color = da_seg_content.color_mat.ptr(i - dh); - cv::Vec3b *ll_p_color = ll_seg_content.color_mat.ptr(i - dh); - - for (int j = dw; j < dw + new_unpad_w; ++j) - { - // argmax - float da_bg_prob = da_seg_bg_ptr[i * input_height + j]; - float da_fg_prob = da_seg_fg_ptr[i * input_height + j]; - float ll_bg_prob = ll_seg_bg_ptr[i * input_height + j]; - float ll_fg_prob = ll_seg_fg_ptr[i * input_height + j]; - unsigned int da_label = da_bg_prob < da_fg_prob ? 1 : 0; - unsigned int ll_label = ll_bg_prob < ll_fg_prob ? 1 : 0; - - if (da_label == 1) - { - // assign label for pixel(i,j) - da_p_class[j - dw] = 1 * 255; // 255 indicate drivable area, for post resize - // assign color for detected class at pixel(i,j). - da_p_color[j - dw][0] = 0; - da_p_color[j - dw][1] = 255; // green - da_p_color[j - dw][2] = 0; - // assign names map - da_seg_content.names_map[255] = "drivable area"; - } - - if (ll_label == 1) - { - // assign label for pixel(i,j) - ll_p_class[j - dw] = 1 * 255; // 255 indicate lane line, for post resize - // assign color for detected class at pixel(i,j). - ll_p_color[j - dw][0] = 0; - ll_p_color[j - dw][1] = 0; - ll_p_color[j - dw][2] = 255; // red - // assign names map - ll_seg_content.names_map[255] = "lane line"; - } - - } - } - // resize to original size. - const unsigned int img_h = static_cast(img_height); - const unsigned int img_w = static_cast(img_width); - // da_seg_mask 255 or 0 - cv::resize(da_seg_content.class_mat, da_seg_content.class_mat, - cv::Size(img_w, img_h), cv::INTER_LINEAR); - cv::resize(da_seg_content.color_mat, da_seg_content.color_mat, - cv::Size(img_w, img_h), cv::INTER_LINEAR); - // ll_seg_mask 255 or 0 - cv::resize(ll_seg_content.class_mat, ll_seg_content.class_mat, - cv::Size(img_w, img_h), cv::INTER_LINEAR); - cv::resize(ll_seg_content.color_mat, ll_seg_content.color_mat, - cv::Size(img_w, img_h), cv::INTER_LINEAR); - - da_seg_content.flag = true; - ll_seg_content.flag = true; -} - -void MNNYOLOP::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - - - - - - - - - - - - - - - - - diff --git a/lite/mnn/cv/mnn_yolop.h b/lite/mnn/cv/mnn_yolop.h deleted file mode 100644 index 905bb0eb..00000000 --- a/lite/mnn/cv/mnn_yolop.h +++ /dev/null @@ -1,73 +0,0 @@ -// -// Created by DefTruth on 2021/10/18. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOP_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOP_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNYOLOP : public BasicMNNHandler - { - public: - explicit MNNYOLOP(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNYOLOP() override = default; - - private: - // nested classes - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } YOLOPScaleParams; - - private: - const float mean_vals[3] = {255.f * 0.485f, 255.f * 0.456, 255.f * 0.406f}; // RGB - const float norm_vals[3] = {1.f / (255.f * 0.229f), 1.f / (255.f * 0.224f), 1.f / (255.f * 0.225f)}; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - void transform(const cv::Mat &mat_rs) override; // without resize - - void initialize_pretreat(); // - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YOLOPScaleParams &scale_params); - - void generate_bboxes_da_ll(const YOLOPScaleParams &scale_params, - const std::map &output_tensors, - std::vector &bbox_collection, - types::SegmentContent &da_seg_content, - types::SegmentContent &ll_seg_content, - float score_threshold, float img_height, - float img_width); // det,da_seg,ll_seg - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, - std::vector &detected_boxes, - types::SegmentContent &da_seg_content, - types::SegmentContent &ll_seg_content, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - - }; - -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOP_H diff --git a/lite/mnn/cv/mnn_yolor.cpp b/lite/mnn/cv/mnn_yolor.cpp deleted file mode 100644 index 00db4919..00000000 --- a/lite/mnn/cv/mnn_yolor.cpp +++ /dev/null @@ -1,179 +0,0 @@ -// -// Created by DefTruth on 2021/11/7. -// - -#include "mnn_yolor.h" -#include "lite/utils.h" - -using mnncv::MNNYoloR; - -MNNYoloR::MNNYoloR(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -void MNNYoloR::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -inline void MNNYoloR::transform(const cv::Mat &mat_rs) -{ - // normalize & HWC -> CHW & BGR -> RGB - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNYoloR::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YoloRScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void MNNYoloR::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - YoloRScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - this->transform(mat_rs); - // 2. inference scores & boxes. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, output_tensors, score_threshold, img_height, img_width); - // 4. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void MNNYoloR::generate_bboxes(const YoloRScaleParams &scale_params, - std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, int img_height, - int img_width) -{ - // device tensors - auto device_pred_ptr = output_tensors.at("output"); - // (1,n,85=5+80=cxcy+cwch+obj_conf+cls_conf) - MNN::Tensor host_pred_tensor(device_pred_ptr, device_pred_ptr->getDimensionType()); // NCHW - device_pred_ptr->copyToHostTensor(&host_pred_tensor); - - auto pred_dims = host_pred_tensor.shape(); - const unsigned int num_anchors = pred_dims.at(1); // n = ? - const unsigned int num_classes = pred_dims.at(2) - 5; // 80 - - float r_ = scale_params.r; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *offset_obj_cls_ptr = - host_pred_tensor.host() + (i * (num_classes + 5)); // row ptr - float obj_conf = offset_obj_cls_ptr[4]; - if (obj_conf < score_threshold) continue; // filter first. - - float cls_conf = offset_obj_cls_ptr[5]; - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = offset_obj_cls_ptr[j + 5]; - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - - float conf = obj_conf * cls_conf; // cls_conf (0.,1.) - if (conf < score_threshold) continue; // filter - - float cx = offset_obj_cls_ptr[0]; - float cy = offset_obj_cls_ptr[1]; - float w = offset_obj_cls_ptr[2]; - float h = offset_obj_cls_ptr[3]; - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - - types::Boxf box; - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(x2, (float) img_width - 1.f); - box.y2 = std::min(y2, (float) img_height - 1.f); - box.score = conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITEMNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void MNNYoloR::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - diff --git a/lite/mnn/cv/mnn_yolor.h b/lite/mnn/cv/mnn_yolor.h deleted file mode 100644 index 7a509953..00000000 --- a/lite/mnn/cv/mnn_yolor.h +++ /dev/null @@ -1,78 +0,0 @@ -// -// Created by DefTruth on 2021/11/7. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOR_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOR_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNYoloR : public BasicMNNHandler - { - public: - explicit MNNYoloR(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNYoloR() override = default; - - private: - // nested classes - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } YoloRScaleParams; - - private: - const float mean_vals[3] = {0.f, 0.f, 0.f}; // RGB - const float norm_vals[3] = {1.0 / 255.f, 1.0 / 255.f, 1.0 / 255.f}; - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat_rs) override; // without resize - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YoloRScaleParams &scale_params); - - void generate_bboxes(const YoloRScaleParams &scale_params, - std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, int img_height, - int img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOR_H diff --git a/lite/mnn/cv/mnn_yolov5.cpp b/lite/mnn/cv/mnn_yolov5.cpp deleted file mode 100644 index a9418572..00000000 --- a/lite/mnn/cv/mnn_yolov5.cpp +++ /dev/null @@ -1,198 +0,0 @@ -// -// Created by DefTruth on 2021/11/6. -// - -#include "mnn_yolov5.h" -#include "lite/utils.h" - -using mnncv::MNNYoloV5; - -MNNYoloV5::MNNYoloV5(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -void MNNYoloV5::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -inline void MNNYoloV5::transform(const cv::Mat &mat_rs) -{ - // normalize & HWC -> CHW & BGR -> RGB - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNYoloV5::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YoloV5ScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void MNNYoloV5::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - YoloV5ScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - this->transform(mat_rs); - // 2. inference scores & boxes. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, output_tensors, score_threshold, img_height, img_width); - // 4. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void MNNYoloV5::generate_bboxes(const YoloV5ScaleParams &scale_params, - std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, int img_height, - int img_width) -{ - // device tensors - auto device_pred_ptr = output_tensors.at("pred"); - // (1,n,85=5+80=cxcy+cwch+obj_conf+cls_conf) - MNN::Tensor host_pred_tensor(device_pred_ptr, device_pred_ptr->getDimensionType()); // NCHW - device_pred_ptr->copyToHostTensor(&host_pred_tensor); - - auto pred_dims = host_pred_tensor.shape(); - const unsigned int num_anchors = pred_dims.at(1); // n = ? - const unsigned int num_classes = pred_dims.at(2) - 5; // 80 - - float r_ = scale_params.r; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *offset_obj_cls_ptr = - host_pred_tensor.host() + (i * (num_classes + 5)); // row ptr - float obj_conf = offset_obj_cls_ptr[4]; - if (obj_conf < score_threshold) continue; // filter first. - - float cls_conf = offset_obj_cls_ptr[5]; - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = offset_obj_cls_ptr[j + 5]; - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - - float conf = obj_conf * cls_conf; // cls_conf (0.,1.) - if (conf < score_threshold) continue; // filter - - float cx = offset_obj_cls_ptr[0]; - float cy = offset_obj_cls_ptr[1]; - float w = offset_obj_cls_ptr[2]; - float h = offset_obj_cls_ptr[3]; - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - - types::Boxf box; - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(x2, (float) img_width - 1.f); - box.y2 = std::min(y2, (float) img_height - 1.f); - box.score = conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITEMNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void MNNYoloV5::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - - - - - - - - - - - - - - - - - - - - diff --git a/lite/mnn/cv/mnn_yolov5.h b/lite/mnn/cv/mnn_yolov5.h deleted file mode 100644 index f2cce467..00000000 --- a/lite/mnn/cv/mnn_yolov5.h +++ /dev/null @@ -1,78 +0,0 @@ -// -// Created by DefTruth on 2021/11/6. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOV5_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOV5_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNYoloV5 : public BasicMNNHandler - { - public: - explicit MNNYoloV5(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNYoloV5() override = default; - - private: - // nested classes - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } YoloV5ScaleParams; - - private: - const float mean_vals[3] = {0.f, 0.f, 0.f}; // RGB - const float norm_vals[3] = {1.0 / 255.f, 1.0 / 255.f, 1.0 / 255.f}; - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat_rs) override; // without resize - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YoloV5ScaleParams &scale_params); - - void generate_bboxes(const YoloV5ScaleParams &scale_params, - std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, int img_height, - int img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOV5_H diff --git a/lite/mnn/cv/mnn_yolov5_blazeface.cpp b/lite/mnn/cv/mnn_yolov5_blazeface.cpp deleted file mode 100644 index f63ad09a..00000000 --- a/lite/mnn/cv/mnn_yolov5_blazeface.cpp +++ /dev/null @@ -1,213 +0,0 @@ -// -// Created by DefTruth on 2022/5/8. -// - -#include "mnn_yolov5_blazeface.h" - -using mnncv::MNNYOLOv5BlazeFace; - -MNNYOLOv5BlazeFace::MNNYOLOv5BlazeFace(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNYOLOv5BlazeFace::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -inline void MNNYOLOv5BlazeFace::transform(const cv::Mat &mat_rs) -{ - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNYOLOv5BlazeFace::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YOLOv5BlazeFaceScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(0, 0, 0)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.ratio = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.flag = true; -} - -void MNNYOLOv5BlazeFace::detect(const cv::Mat &mat, std::vector &detected_boxes_kps, - float score_threshold, float iou_threshold, unsigned int topk) -{ - if (mat.empty()) return; - auto img_height = static_cast(mat.rows); - auto img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - YOLOv5BlazeFaceScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - this->transform(mat_rs); - - // 2. inference scores & boxes. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - - // 3. rescale & exclude. - std::vector bbox_kps_collection; - this->generate_bboxes_kps(scale_params, bbox_kps_collection, output_tensors, - score_threshold, img_height, img_width); - // 4. hard nms with topk. - this->nms_bboxes_kps(bbox_kps_collection, detected_boxes_kps, iou_threshold, topk); - -} - -void MNNYOLOv5BlazeFace::generate_bboxes_kps(const YOLOv5BlazeFaceScaleParams &scale_params, - std::vector &bbox_kps_collection, - const std::map &output_tensors, - float score_threshold, float img_height, float img_width) -{ - auto device_output_pred = output_tensors.at("output"); - MNN::Tensor host_output_pred(device_output_pred, device_output_pred->getDimensionType()); - device_output_pred->copyToHostTensor(&host_output_pred); - - auto output_dims = host_output_pred.shape(); - const unsigned int num_anchors = output_dims.at(1); // n = ? - const float *output_ptr = host_output_pred.host(); - - float r_ = scale_params.ratio; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - bbox_kps_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *row_ptr = output_ptr + i * 16; - float obj_conf = row_ptr[4]; - if (obj_conf < score_threshold) continue; // filter first. - float cls_conf = row_ptr[15]; - if (cls_conf < score_threshold) continue; // face score. - - // bounding box - const float *offsets = row_ptr; - float cx = offsets[0]; - float cy = offsets[1]; - float w = offsets[2]; - float h = offsets[3]; - - types::BoxfWithLandmarks box_kps; - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - box_kps.box.x1 = std::max(0.f, x1); - box_kps.box.y1 = std::max(0.f, y1); - box_kps.box.x2 = std::min(img_width - 1.f, x2); - box_kps.box.y2 = std::min(img_height - 1.f, y2); - box_kps.box.score = cls_conf; - box_kps.box.label = 1; - box_kps.box.label_text = "face"; - box_kps.box.flag = true; - - // landmarks - const float *kps_offsets = row_ptr + 5; - for (unsigned int j = 0; j < 10; j += 2) - { - cv::Point2f kps; - float kps_x = (kps_offsets[j] - (float) dw_) / r_; - float kps_y = (kps_offsets[j + 1] - (float) dh_) / r_; - kps.x = std::min(std::max(0.f, kps_x), img_width - 1.f); - kps.y = std::min(std::max(0.f, kps_y), img_height - 1.f); - box_kps.landmarks.points.push_back(kps); - } - box_kps.landmarks.flag = true; - box_kps.flag = true; - - bbox_kps_collection.push_back(box_kps); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - -#if LITEMNN_DEBUG - std::cout << "generate_bboxes_kps num: " << bbox_kps_collection.size() << "\n"; -#endif - -} - -void MNNYOLOv5BlazeFace::nms_bboxes_kps(std::vector &input, - std::vector &output, - float iou_threshold, unsigned int topk) -{ - if (input.empty()) return; - std::sort( - input.begin(), input.end(), - [](const types::BoxfWithLandmarks &a, const types::BoxfWithLandmarks &b) - { return a.box.score > b.box.score; } - ); - const unsigned int box_num = input.size(); - std::vector merged(box_num, 0); - - unsigned int count = 0; - for (unsigned int i = 0; i < box_num; ++i) - { - if (merged[i]) continue; - std::vector buf; - - buf.push_back(input[i]); - merged[i] = 1; - - for (unsigned int j = i + 1; j < box_num; ++j) - { - if (merged[j]) continue; - - float iou = static_cast(input[i].box.iou_of(input[j].box)); - - if (iou > iou_threshold) - { - merged[j] = 1; - buf.push_back(input[j]); - } - - } - output.push_back(buf[0]); - - // keep top k - count += 1; - if (count >= topk) - break; - } -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_yolov5_blazeface.h b/lite/mnn/cv/mnn_yolov5_blazeface.h deleted file mode 100644 index 58536d6c..00000000 --- a/lite/mnn/cv/mnn_yolov5_blazeface.h +++ /dev/null @@ -1,64 +0,0 @@ -// -// Created by DefTruth on 2022/5/8. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOV5_BLAZEFACE_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOV5_BLAZEFACE_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNYOLOv5BlazeFace : public BasicMNNHandler - { - public: - explicit MNNYOLOv5BlazeFace(const std::string &_mnn_path, unsigned int _num_threads = 1); - - ~MNNYOLOv5BlazeFace() override = default; - - private: - // nested classes - typedef struct - { - float ratio; - int dw; - int dh; - bool flag; - } YOLOv5BlazeFaceScaleParams; - - private: - const float mean_vals[3] = {0.f, 0.f, 0.f}; // RGB - const float norm_vals[3] = {1.f / 255.f, 1.f / 255.f, 1.f / 255.f}; - static constexpr const unsigned int max_nms = 30000; - - private: - void transform(const cv::Mat &mat_rs) override; // without resize - - void initialize_pretreat(); - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YOLOv5BlazeFaceScaleParams &scale_params); - - void generate_bboxes_kps(const YOLOv5BlazeFaceScaleParams &scale_params, - std::vector &bbox_kps_collection, - const std::map &output_tensors, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms_bboxes_kps(std::vector &input, - std::vector &output, - float iou_threshold, unsigned int topk); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes_kps, - float score_threshold = 0.3f, float iou_threshold = 0.45f, - unsigned int topk = 400); - - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOV5_BLAZEFACE_H diff --git a/lite/mnn/cv/mnn_yolov5_v6.0.cpp b/lite/mnn/cv/mnn_yolov5_v6.0.cpp deleted file mode 100644 index d99a2e52..00000000 --- a/lite/mnn/cv/mnn_yolov5_v6.0.cpp +++ /dev/null @@ -1,179 +0,0 @@ -// -// Created by DefTruth on 2021/11/10. -// - -#include "mnn_yolov5_v6.0.h" -#include "lite/utils.h" - -using mnncv::MNNYoloV5_V_6_0; - -MNNYoloV5_V_6_0::MNNYoloV5_V_6_0(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -void MNNYoloV5_V_6_0::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -inline void MNNYoloV5_V_6_0::transform(const cv::Mat &mat_rs) -{ - // normalize & HWC -> CHW & BGR -> RGB - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNYoloV5_V_6_0::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YoloV5ScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void MNNYoloV5_V_6_0::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - YoloV5ScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - this->transform(mat_rs); - // 2. inference scores & boxes. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, output_tensors, score_threshold, img_height, img_width); - // 4. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void MNNYoloV5_V_6_0::generate_bboxes(const YoloV5ScaleParams &scale_params, - std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, int img_height, - int img_width) -{ - // device tensors - auto device_pred_ptr = output_tensors.at("output"); - // (1,n,85=5+80=cxcy+cwch+obj_conf+cls_conf) - MNN::Tensor host_pred_tensor(device_pred_ptr, device_pred_ptr->getDimensionType()); // NCHW - device_pred_ptr->copyToHostTensor(&host_pred_tensor); - - auto pred_dims = host_pred_tensor.shape(); - const unsigned int num_anchors = pred_dims.at(1); // n = ? - const unsigned int num_classes = pred_dims.at(2) - 5; // 80 - - float r_ = scale_params.r; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *offset_obj_cls_ptr = - host_pred_tensor.host() + (i * (num_classes + 5)); // row ptr - float obj_conf = offset_obj_cls_ptr[4]; - if (obj_conf < score_threshold) continue; // filter first. - - float cls_conf = offset_obj_cls_ptr[5]; - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = offset_obj_cls_ptr[j + 5]; - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - - float conf = obj_conf * cls_conf; // cls_conf (0.,1.) - if (conf < score_threshold) continue; // filter - - float cx = offset_obj_cls_ptr[0]; - float cy = offset_obj_cls_ptr[1]; - float w = offset_obj_cls_ptr[2]; - float h = offset_obj_cls_ptr[3]; - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - - types::Boxf box; - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(x2, (float) img_width - 1.f); - box.y2 = std::min(y2, (float) img_height - 1.f); - box.score = conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITEMNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void MNNYoloV5_V_6_0::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - diff --git a/lite/mnn/cv/mnn_yolov5_v6.0.h b/lite/mnn/cv/mnn_yolov5_v6.0.h deleted file mode 100644 index ea0d686c..00000000 --- a/lite/mnn/cv/mnn_yolov5_v6.0.h +++ /dev/null @@ -1,78 +0,0 @@ -// -// Created by DefTruth on 2021/11/10. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOV5_V6_0_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOV5_V6_0_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNYoloV5_V_6_0 : public BasicMNNHandler - { - public: - explicit MNNYoloV5_V_6_0(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNYoloV5_V_6_0() override = default; - - private: - // nested classes - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } YoloV5ScaleParams; - - private: - const float mean_vals[3] = {0.f, 0.f, 0.f}; // RGB - const float norm_vals[3] = {1.0 / 255.f, 1.0 / 255.f, 1.0 / 255.f}; - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat_rs) override; // without resize - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YoloV5ScaleParams &scale_params); - - void generate_bboxes(const YoloV5ScaleParams &scale_params, - std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, int img_height, - int img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOV5_V6_0_H diff --git a/lite/mnn/cv/mnn_yolov5_v6.1.cpp b/lite/mnn/cv/mnn_yolov5_v6.1.cpp deleted file mode 100644 index 025d8453..00000000 --- a/lite/mnn/cv/mnn_yolov5_v6.1.cpp +++ /dev/null @@ -1,179 +0,0 @@ -// -// Created by DefTruth on 2022/5/8. -// - -#include "mnn_yolov5_v6.1.h" -#include "lite/utils.h" - -using mnncv::MNNYoloV5_V_6_1; - -MNNYoloV5_V_6_1::MNNYoloV5_V_6_1(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -void MNNYoloV5_V_6_1::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -inline void MNNYoloV5_V_6_1::transform(const cv::Mat &mat_rs) -{ - // normalize & HWC -> CHW & BGR -> RGB - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNYoloV5_V_6_1::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YoloV5ScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void MNNYoloV5_V_6_1::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - YoloV5ScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - this->transform(mat_rs); - // 2. inference scores & boxes. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, output_tensors, score_threshold, img_height, img_width); - // 4. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void MNNYoloV5_V_6_1::generate_bboxes(const YoloV5ScaleParams &scale_params, - std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, int img_height, - int img_width) -{ - // device tensors - auto device_pred_ptr = output_tensors.at("output"); - // (1,n,85=5+80=cxcy+cwch+obj_conf+cls_conf) - MNN::Tensor host_pred_tensor(device_pred_ptr, device_pred_ptr->getDimensionType()); // NCHW - device_pred_ptr->copyToHostTensor(&host_pred_tensor); - - auto pred_dims = host_pred_tensor.shape(); - const unsigned int num_anchors = pred_dims.at(1); // n = ? - const unsigned int num_classes = pred_dims.at(2) - 5; // 80 - - float r_ = scale_params.r; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *offset_obj_cls_ptr = - host_pred_tensor.host() + (i * (num_classes + 5)); // row ptr - float obj_conf = offset_obj_cls_ptr[4]; - if (obj_conf < score_threshold) continue; // filter first. - - float cls_conf = offset_obj_cls_ptr[5]; - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = offset_obj_cls_ptr[j + 5]; - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - - float conf = obj_conf * cls_conf; // cls_conf (0.,1.) - if (conf < score_threshold) continue; // filter - - float cx = offset_obj_cls_ptr[0]; - float cy = offset_obj_cls_ptr[1]; - float w = offset_obj_cls_ptr[2]; - float h = offset_obj_cls_ptr[3]; - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - - types::Boxf box; - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(x2, (float) img_width - 1.f); - box.y2 = std::min(y2, (float) img_height - 1.f); - box.score = conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITEMNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void MNNYoloV5_V_6_1::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - diff --git a/lite/mnn/cv/mnn_yolov5_v6.1.h b/lite/mnn/cv/mnn_yolov5_v6.1.h deleted file mode 100644 index e1378e09..00000000 --- a/lite/mnn/cv/mnn_yolov5_v6.1.h +++ /dev/null @@ -1,79 +0,0 @@ -// -// Created by DefTruth on 2022/5/8. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOV5_V6_1_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOV5_V6_1_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNYoloV5_V_6_1 : public BasicMNNHandler - { - public: - explicit MNNYoloV5_V_6_1(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNYoloV5_V_6_1() override = default; - - private: - // nested classes - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } YoloV5ScaleParams; - - private: - const float mean_vals[3] = {0.f, 0.f, 0.f}; // RGB - const float norm_vals[3] = {1.0 / 255.f, 1.0 / 255.f, 1.0 / 255.f}; - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat_rs) override; // without resize - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YoloV5ScaleParams &scale_params); - - void generate_bboxes(const YoloV5ScaleParams &scale_params, - std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, int img_height, - int img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - }; -} - - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOV5_V6_1_H diff --git a/lite/mnn/cv/mnn_yolov6.cpp b/lite/mnn/cv/mnn_yolov6.cpp deleted file mode 100644 index 1c5abd72..00000000 --- a/lite/mnn/cv/mnn_yolov6.cpp +++ /dev/null @@ -1,221 +0,0 @@ -// -// Created by DefTruth on 2022/6/25. -// - -#include "mnn_yolov6.h" -#include "lite/utils.h" - -using mnncv::MNNYOLOv6; - -MNNYOLOv6::MNNYOLOv6(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -void MNNYOLOv6::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -void MNNYOLOv6::transform(const cv::Mat &mat_rs) -{ - // normalize & HWC -> CHW & BGR -> RGB - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -// letterbox -void MNNYOLOv6::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YOLOv6ScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void MNNYOLOv6::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - YOLOv6ScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - this->transform(mat_rs); - // 2. inference scores & boxes. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, output_tensors, score_threshold, img_height, img_width); - // 4. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void MNNYOLOv6::generate_anchors(const int target_height, - const int target_width, - std::vector &strides, - std::vector &anchors) -{ - for (auto stride: strides) - { - int num_grid_w = target_width / stride; - int num_grid_h = target_height / stride; - for (int g1 = 0; g1 < num_grid_h; ++g1) - { - for (int g0 = 0; g0 < num_grid_w; ++g0) - { - YOLOv6Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchors.push_back(anchor); - } - } - } -} - -static inline float sigmoid(float x) -{ - return static_cast(1.f / (1.f + std::exp(-x))); -} - -void MNNYOLOv6::generate_bboxes(const YOLOv6ScaleParams &scale_params, - std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, int img_height, - int img_width) -{ - // device tensors - auto device_pred_ptr = output_tensors.at("outputs"); - // (1,n,85=5+80=cxcy+cwch+obj_conf+cls_conf) - MNN::Tensor host_pred_tensor(device_pred_ptr, device_pred_ptr->getDimensionType()); // NCHW - device_pred_ptr->copyToHostTensor(&host_pred_tensor); - - auto pred_dims = host_pred_tensor.shape(); - const unsigned int num_anchors = pred_dims.at(1); // n = ? - const unsigned int num_classes = pred_dims.at(2) - 5; // 80 - - std::vector anchors; - std::vector strides = {8, 16, 32}; // might have stride=64 - this->generate_anchors(input_height, input_width, strides, anchors); - - float r_ = scale_params.r; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *offset_obj_cls_ptr = - host_pred_tensor.host() + (i * (num_classes + 5)); // row ptr - float obj_conf = sigmoid(offset_obj_cls_ptr[4]); - if (obj_conf < score_threshold) continue; // filter first. - - float cls_conf = sigmoid(offset_obj_cls_ptr[5]); - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = sigmoid(offset_obj_cls_ptr[j + 5]); - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - - float conf = obj_conf * cls_conf; // cls_conf (0.,1.) - if (conf < score_threshold) continue; // filter - - const int grid0 = anchors.at(i).grid0; - const int grid1 = anchors.at(i).grid1; - const int stride = anchors.at(i).stride; - - float dx = offset_obj_cls_ptr[0]; - float dy = offset_obj_cls_ptr[1]; - float dw = offset_obj_cls_ptr[2]; - float dh = offset_obj_cls_ptr[3]; - - float cx = (dx + (float) grid0) * (float) stride; - float cy = (dy + (float) grid1) * (float) stride; - float w = std::exp(dw) * (float) stride; - float h = std::exp(dh) * (float) stride; - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - - types::Boxf box; - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(x2, (float) img_width - 1.f); - box.y2 = std::min(y2, (float) img_height - 1.f); - box.score = conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITEMNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void MNNYOLOv6::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - diff --git a/lite/mnn/cv/mnn_yolov6.h b/lite/mnn/cv/mnn_yolov6.h deleted file mode 100644 index c7543208..00000000 --- a/lite/mnn/cv/mnn_yolov6.h +++ /dev/null @@ -1,91 +0,0 @@ -// -// Created by DefTruth on 2022/6/25. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOV6_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOV6_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNYOLOv6 : public BasicMNNHandler - { - public: - explicit MNNYOLOv6(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNYOLOv6() override = default; - - private: - // nested classes - typedef struct GridAndStride - { - int grid0; - int grid1; - int stride; - } YOLOv6Anchor; - - // nested classes - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } YOLOv6ScaleParams; - - private: - const float mean_vals[3] = {0.f, 0.f, 0.f}; // RGB - const float norm_vals[3] = {1.0 / 255.f, 1.0 / 255.f, 1.0 / 255.f}; - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat_rs) override; // without resize - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YOLOv6ScaleParams &scale_params); - - void generate_anchors(const int target_height, - const int target_width, - std::vector &strides, - std::vector &anchors); - - void generate_bboxes(const YOLOv6ScaleParams &scale_params, - std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, int img_height, - int img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOV6_H diff --git a/lite/mnn/cv/mnn_yolox.cpp b/lite/mnn/cv/mnn_yolox.cpp deleted file mode 100644 index 6e6cfaf1..00000000 --- a/lite/mnn/cv/mnn_yolox.cpp +++ /dev/null @@ -1,218 +0,0 @@ -// -// Created by DefTruth on 2021/10/14. -// - -#include "mnn_yolox.h" -#include "lite/utils.h" - -using mnncv::MNNYoloX; - -MNNYoloX::MNNYoloX(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNYoloX::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::RGB, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -inline void MNNYoloX::transform(const cv::Mat &mat_rs) -{ - // normalize & HWC -> CHW & BGR -> RGB - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNYoloX::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YoloXScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void MNNYoloX::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - YoloXScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - this->transform(mat_rs); - // 2. inference scores & boxes. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, output_tensors, score_threshold, img_height, img_width); - // 4. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void MNNYoloX::generate_anchors(const int target_height, - const int target_width, - std::vector &strides, - std::vector &anchors) -{ - for (auto stride : strides) - { - int num_grid_w = target_width / stride; - int num_grid_h = target_height / stride; - for (int g1 = 0; g1 < num_grid_h; ++g1) - { - for (int g0 = 0; g0 < num_grid_w; ++g0) - { -#ifdef LITE_WIN32 - YoloXAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchors.push_back(anchor); -#else - anchors.push_back((YoloXAnchor) {g0, g1, stride}); -#endif - } - } - } -} - -void MNNYoloX::generate_bboxes(const YoloXScaleParams &scale_params, - std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, int img_height, - int img_width) -{ - // device tensors - auto device_pred_ptr = output_tensors.at("outputs"); - // (1,n,85=5+80=cxcy+cwch+obj_conf+cls_conf) - MNN::Tensor host_pred_tensor(device_pred_ptr, device_pred_ptr->getDimensionType()); // NCHW - device_pred_ptr->copyToHostTensor(&host_pred_tensor); - - auto pred_dims = host_pred_tensor.shape(); - const unsigned int num_anchors = pred_dims.at(1); // n = ? - const unsigned int num_classes = pred_dims.at(2) - 5; - - std::vector anchors; - std::vector strides = {8, 16, 32}; // might have stride=64 - this->generate_anchors(input_height, input_width, strides, anchors); - - float r_ = scale_params.r; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *offset_obj_cls_ptr = - host_pred_tensor.host() + (i * (num_classes + 5)); // row ptr - float obj_conf = offset_obj_cls_ptr[4]; - if (obj_conf < score_threshold) continue; // filter first. - - float cls_conf = offset_obj_cls_ptr[5]; - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = offset_obj_cls_ptr[j + 5]; - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - - float conf = obj_conf * cls_conf; // cls_conf (0.,1.) - if (conf < score_threshold) continue; // filter - - const int grid0 = anchors.at(i).grid0; - const int grid1 = anchors.at(i).grid1; - const int stride = anchors.at(i).stride; - - float dx = offset_obj_cls_ptr[0]; - float dy = offset_obj_cls_ptr[1]; - float dw = offset_obj_cls_ptr[2]; - float dh = offset_obj_cls_ptr[3]; - - float cx = (dx + (float) grid0) * (float) stride; - float cy = (dy + (float) grid1) * (float) stride; - float w = std::exp(dw) * (float) stride; - float h = std::exp(dh) * (float) stride; - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - - types::Boxf box; - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(x2, (float) img_width - 1.f); - box.y2 = std::min(y2, (float) img_height - 1.f); - box.score = conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITEMNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void MNNYoloX::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_yolox.h b/lite/mnn/cv/mnn_yolox.h deleted file mode 100644 index e8aa8881..00000000 --- a/lite/mnn/cv/mnn_yolox.h +++ /dev/null @@ -1,91 +0,0 @@ -// -// Created by DefTruth on 2021/10/14. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOX_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOX_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNYoloX : public BasicMNNHandler - { - public: - explicit MNNYoloX(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNYoloX() override = default; - - private: - // nested classes - typedef struct GridAndStride - { - int grid0; - int grid1; - int stride; - } YoloXAnchor; - - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } YoloXScaleParams; - - private: - const float mean_vals[3] = {255.f * 0.485f, 255.f * 0.456, 255.f * 0.406f}; - const float norm_vals[3] = {1.f / (255.f * 0.229f), 1.f / (255.f * 0.224f), 1.f / (255.f * 0.225f)}; - - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat_rs) override; // - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YoloXScaleParams &scale_params); - - void generate_anchors(const int target_height, - const int target_width, - std::vector &strides, - std::vector &anchors); - - void generate_bboxes(const YoloXScaleParams &scale_params, - std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, int img_height, - int img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOX_H diff --git a/lite/mnn/cv/mnn_yolox_v0.1.1.cpp b/lite/mnn/cv/mnn_yolox_v0.1.1.cpp deleted file mode 100644 index d480fc14..00000000 --- a/lite/mnn/cv/mnn_yolox_v0.1.1.cpp +++ /dev/null @@ -1,218 +0,0 @@ -// -// Created by DefTruth on 2021/11/6. -// - -#include "mnn_yolox_v0.1.1.h" -#include "lite/utils.h" - -using mnncv::MNNYoloX_V_0_1_1; - -MNNYoloX_V_0_1_1::MNNYoloX_V_0_1_1(const std::string &_mnn_path, unsigned int _num_threads) : - BasicMNNHandler(_mnn_path, _num_threads) -{ - initialize_pretreat(); -} - -inline void MNNYoloX_V_0_1_1::initialize_pretreat() -{ - pretreat = std::shared_ptr( - MNN::CV::ImageProcess::create( - MNN::CV::BGR, - MNN::CV::BGR, - mean_vals, 3, - norm_vals, 3 - ) - ); -} - -inline void MNNYoloX_V_0_1_1::transform(const cv::Mat &mat_rs) -{ - // normalize & HWC -> CHW & BGR -> RGB - pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor); -} - -void MNNYoloX_V_0_1_1::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YoloXScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void MNNYoloX_V_0_1_1::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - YoloXScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - this->transform(mat_rs); - // 2. inference scores & boxes. - mnn_interpreter->runSession(mnn_session); - auto output_tensors = mnn_interpreter->getSessionOutputAll(mnn_session); - // 3. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, output_tensors, score_threshold, img_height, img_width); - // 4. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void MNNYoloX_V_0_1_1::generate_anchors(const int target_height, - const int target_width, - std::vector &strides, - std::vector &anchors) -{ - for (auto stride : strides) - { - int num_grid_w = target_width / stride; - int num_grid_h = target_height / stride; - for (int g1 = 0; g1 < num_grid_h; ++g1) - { - for (int g0 = 0; g0 < num_grid_w; ++g0) - { -#ifdef LITE_WIN32 - YoloXAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchors.push_back(anchor); -#else - anchors.push_back((YoloXAnchor) {g0, g1, stride}); -#endif - } - } - } -} - -void MNNYoloX_V_0_1_1::generate_bboxes(const YoloXScaleParams &scale_params, - std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, int img_height, - int img_width) -{ - // device tensors - auto device_pred_ptr = output_tensors.at("output"); - // (1,n,85=5+80=cxcy+cwch+obj_conf+cls_conf) - MNN::Tensor host_pred_tensor(device_pred_ptr, device_pred_ptr->getDimensionType()); // NCHW - device_pred_ptr->copyToHostTensor(&host_pred_tensor); - - auto pred_dims = host_pred_tensor.shape(); - const unsigned int num_anchors = pred_dims.at(1); // n = ? - const unsigned int num_classes = pred_dims.at(2) - 5; - - std::vector anchors; - std::vector strides = {8, 16, 32}; // might have stride=64 - this->generate_anchors(input_height, input_width, strides, anchors); - - float r_ = scale_params.r; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *offset_obj_cls_ptr = - host_pred_tensor.host() + (i * (num_classes + 5)); // row ptr - float obj_conf = offset_obj_cls_ptr[4]; - if (obj_conf < score_threshold) continue; // filter first. - - float cls_conf = offset_obj_cls_ptr[5]; - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = offset_obj_cls_ptr[j + 5]; - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - - float conf = obj_conf * cls_conf; // cls_conf (0.,1.) - if (conf < score_threshold) continue; // filter - - const int grid0 = anchors.at(i).grid0; - const int grid1 = anchors.at(i).grid1; - const int stride = anchors.at(i).stride; - - float dx = offset_obj_cls_ptr[0]; - float dy = offset_obj_cls_ptr[1]; - float dw = offset_obj_cls_ptr[2]; - float dh = offset_obj_cls_ptr[3]; - - float cx = (dx + (float) grid0) * (float) stride; - float cy = (dy + (float) grid1) * (float) stride; - float w = std::exp(dw) * (float) stride; - float h = std::exp(dh) * (float) stride; - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - - types::Boxf box; - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(x2, (float) img_width - 1.f); - box.y2 = std::min(y2, (float) img_height - 1.f); - box.score = conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITEMNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void MNNYoloX_V_0_1_1::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} \ No newline at end of file diff --git a/lite/mnn/cv/mnn_yolox_v0.1.1.h b/lite/mnn/cv/mnn_yolox_v0.1.1.h deleted file mode 100644 index 66fc935b..00000000 --- a/lite/mnn/cv/mnn_yolox_v0.1.1.h +++ /dev/null @@ -1,91 +0,0 @@ -// -// Created by DefTruth on 2021/11/6. -// - -#ifndef LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOX_V0_1_1_H -#define LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOX_V0_1_1_H - -#include "lite/mnn/core/mnn_core.h" - -namespace mnncv -{ - class LITE_EXPORTS MNNYoloX_V_0_1_1 : public BasicMNNHandler - { - public: - explicit MNNYoloX_V_0_1_1(const std::string &_mnn_path, unsigned int _num_threads = 1); // - ~MNNYoloX_V_0_1_1() override = default; - - private: - // nested classes - typedef struct GridAndStride - { - int grid0; - int grid1; - int stride; - } YoloXAnchor; - - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } YoloXScaleParams; - - private: - const float mean_vals[3] = {0.f, 0.f, 0.f}; - const float norm_vals[3] = {1.f, 1.f, 1.f}; - - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void initialize_pretreat(); // - - void transform(const cv::Mat &mat_rs) override; // - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YoloXScaleParams &scale_params); - - void generate_anchors(const int target_height, - const int target_width, - std::vector &strides, - std::vector &anchors); - - void generate_bboxes(const YoloXScaleParams &scale_params, - std::vector &bbox_collection, - const std::map &output_tensors, - float score_threshold, int img_height, - int img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - - }; -} - -#endif //LITE_AI_TOOLKIT_MNN_CV_MNN_YOLOX_V0_1_1_H diff --git a/lite/models.h b/lite/models.h index 10e7be71..85f6f193 100644 --- a/lite/models.h +++ b/lite/models.h @@ -154,255 +154,6 @@ #include "lite/trt/sd/trt_pipeline.h" #endif -// ENABLE_MNN -#ifdef ENABLE_MNN - -#include "lite/mnn/core/mnn_core.h" -#include "lite/mnn/core/mnn_utils.h" -#include "lite/mnn/cv/mnn_nanodet.h" -#include "lite/mnn/cv/mnn_nanodet_efficientnet_lite.h" -#include "lite/mnn/cv/mnn_rvm.h" -#include "lite/mnn/cv/mnn_yolox.h" -#include "lite/mnn/cv/mnn_yolop.h" -#include "lite/mnn/cv/mnn_yolov5.h" -#include "lite/mnn/cv/mnn_yolox_v0.1.1.h" -#include "lite/mnn/cv/mnn_yolor.h" -#include "lite/mnn/cv/mnn_yolov5_v6.0.h" -#include "lite/mnn/cv/mnn_glint_arcface.h" -#include "lite/mnn/cv/mnn_glint_cosface.h" -#include "lite/mnn/cv/mnn_glint_partial_fc.h" -#include "lite/mnn/cv/mnn_facenet.h" -#include "lite/mnn/cv/mnn_focal_arcface.h" -#include "lite/mnn/cv/mnn_focal_asia_arcface.h" -#include "lite/mnn/cv/mnn_tencent_curricular_face.h" -#include "lite/mnn/cv/mnn_tencent_cifp_face.h" -#include "lite/mnn/cv/mnn_center_loss_face.h" -#include "lite/mnn/cv/mnn_sphere_face.h" -#include "lite/mnn/cv/mnn_mobile_facenet.h" -#include "lite/mnn/cv/mnn_cava_ghost_arcface.h" -#include "lite/mnn/cv/mnn_cava_combined_face.h" -#include "lite/mnn/cv/mnn_mobilese_focal_face.h" -#include "lite/mnn/cv/mnn_ultraface.h" -#include "lite/mnn/cv/mnn_retinaface.h" -#include "lite/mnn/cv/mnn_faceboxes.h" -#include "lite/mnn/cv/mnn_face_landmarks_1000.h" -#include "lite/mnn/cv/mnn_pfld.h" -#include "lite/mnn/cv/mnn_pfld68.h" -#include "lite/mnn/cv/mnn_pfld98.h" -#include "lite/mnn/cv/mnn_mobilenetv2_68.h" -#include "lite/mnn/cv/mnn_mobilenetv2_se_68.h" -#include "lite/mnn/cv/mnn_fsanet.h" -#include "lite/mnn/cv/mnn_age_googlenet.h" -#include "lite/mnn/cv/mnn_gender_googlenet.h" -#include "lite/mnn/cv/mnn_emotion_ferplus.h" -#include "lite/mnn/cv/mnn_efficient_emotion7.h" -#include "lite/mnn/cv/mnn_efficient_emotion8.h" -#include "lite/mnn/cv/mnn_ssrnet.h" -#include "lite/mnn/cv/mnn_mobile_emotion7.h" -#include "lite/mnn/cv/mnn_rexnet_emotion7.h" -#include "lite/mnn/cv/mnn_efficientnet_lite4.h" -#include "lite/mnn/cv/mnn_shufflenetv2.h" -#include "lite/mnn/cv/mnn_densenet.h" -#include "lite/mnn/cv/mnn_ghostnet.h" -#include "lite/mnn/cv/mnn_hdrdnet.h" -#include "lite/mnn/cv/mnn_ibnnet.h" -#include "lite/mnn/cv/mnn_mobilenetv2.h" -#include "lite/mnn/cv/mnn_resnet.h" -#include "lite/mnn/cv/mnn_resnext.h" -#include "lite/mnn/cv/mnn_deeplabv3_resnet101.h" -#include "lite/mnn/cv/mnn_fcn_resnet101.h" -#include "lite/mnn/cv/mnn_colorizer.h" -#include "lite/mnn/cv/mnn_fast_style_transfer.h" -#include "lite/mnn/cv/mnn_subpixel_cnn.h" -#include "lite/mnn/cv/mnn_mg_matting.h" -#include "lite/mnn/cv/mnn_nanodet_plus.h" -#include "lite/mnn/cv/mnn_scrfd.h" -#include "lite/mnn/cv/mnn_yolo5face.h" -#include "lite/mnn/cv/mnn_faceboxesv2.h" -#include "lite/mnn/cv/mnn_pipnet98.h" -#include "lite/mnn/cv/mnn_pipnet68.h" -#include "lite/mnn/cv/mnn_pipnet29.h" -#include "lite/mnn/cv/mnn_pipnet19.h" -#include "lite/mnn/cv/mnn_insectdet.h" -#include "lite/mnn/cv/mnn_insectid.h" -#include "lite/mnn/cv/mnn_plantid.h" -#include "lite/mnn/cv/mnn_modnet.h" -#include "lite/mnn/cv/mnn_backgroundmattingv2.h" -#include "lite/mnn/cv/mnn_yolov5_blazeface.h" -#include "lite/mnn/cv/mnn_yolov5_v6.1.h" -#include "lite/mnn/cv/mnn_head_seg.h" -#include "lite/mnn/cv/mnn_female_photo2cartoon.h" -#include "lite/mnn/cv/mnn_fast_portrait_seg.h" -#include "lite/mnn/cv/mnn_portrait_seg_sinet.h" -#include "lite/mnn/cv/mnn_portrait_seg_extremec3net.h" -#include "lite/mnn/cv/mnn_hair_seg.h" -#include "lite/mnn/cv/mnn_face_hair_seg.h" -#include "lite/mnn/cv/mnn_mobile_human_matting.h" -#include "lite/mnn/cv/mnn_mobile_hair_seg.h" -#include "lite/mnn/cv/mnn_yolov6.h" -#include "lite/mnn/cv/mnn_face_parsing_bisenet.h" - -#endif - -// ENABLE_NCNN -#ifdef ENABLE_NCNN - -#include "lite/ncnn/core/ncnn_core.h" -#include "lite/ncnn/core/ncnn_utils.h" -#include "lite/ncnn/cv/ncnn_nanodet.h" -#include "lite/ncnn/cv/ncnn_nanodet_efficientnet_lite.h" -#include "lite/ncnn/cv/ncnn_nanodet_depreciated.h" -#include "lite/ncnn/cv/ncnn_nanodet_efficientdet_lite_depreciated.h" -#include "lite/ncnn/cv/ncnn_rvm.h" -#include "lite/ncnn/cv/ncnn_yolox.h" -#include "lite/ncnn/cv/ncnn_yolop.h" -#include "lite/ncnn/cv/ncnn_yolov5.h" -#include "lite/ncnn/cv/ncnn_yolox_v0.1.1.h" -#include "lite/ncnn/cv/ncnn_yolor.h" -#include "lite/ncnn/cv/ncnn_yolor_ssss.h" -#include "lite/ncnn/cv/ncnn_yolov5_v6.0.h" -#include "lite/ncnn/cv/ncnn_yolov5_v6.0_p6.h" -#include "lite/ncnn/cv/ncnn_glint_arcface.h" -#include "lite/ncnn/cv/ncnn_glint_cosface.h" -#include "lite/ncnn/cv/ncnn_glint_partial_fc.h" -#include "lite/ncnn/cv/ncnn_facenet.h" -#include "lite/ncnn/cv/ncnn_focal_arcface.h" -#include "lite/ncnn/cv/ncnn_focal_asia_arcface.h" -#include "lite/ncnn/cv/ncnn_tencent_curricular_face.h" -#include "lite/ncnn/cv/ncnn_tencent_cifp_face.h" -#include "lite/ncnn/cv/ncnn_center_loss_face.h" -#include "lite/ncnn/cv/ncnn_sphere_face.h" -#include "lite/ncnn/cv/ncnn_mobile_facenet.h" -#include "lite/ncnn/cv/ncnn_cava_ghost_arcface.h" -#include "lite/ncnn/cv/ncnn_cava_combined_face.h" -#include "lite/ncnn/cv/ncnn_mobilese_focal_face.h" -#include "lite/ncnn/cv/ncnn_ultraface.h" -#include "lite/ncnn/cv/ncnn_retinaface.h" -#include "lite/ncnn/cv/ncnn_faceboxes.h" -#include "lite/ncnn/cv/ncnn_face_landmarks_1000.h" -#include "lite/ncnn/cv/ncnn_pfld.h" -#include "lite/ncnn/cv/ncnn_pfld68.h" -#include "lite/ncnn/cv/ncnn_pfld98.h" -#include "lite/ncnn/cv/ncnn_mobilenetv2_68.h" -#include "lite/ncnn/cv/ncnn_mobilenetv2_se_68.h" -#include "lite/ncnn/cv/ncnn_age_googlenet.h" -#include "lite/ncnn/cv/ncnn_gender_googlenet.h" -#include "lite/ncnn/cv/ncnn_emotion_ferplus.h" -#include "lite/ncnn/cv/ncnn_efficient_emotion7.h" -#include "lite/ncnn/cv/ncnn_efficient_emotion8.h" -#include "lite/ncnn/cv/ncnn_mobile_emotion7.h" -#include "lite/ncnn/cv/ncnn_efficientnet_lite4.h" -#include "lite/ncnn/cv/ncnn_shufflenetv2.h" -#include "lite/ncnn/cv/ncnn_densenet.h" -#include "lite/ncnn/cv/ncnn_ghostnet.h" -#include "lite/ncnn/cv/ncnn_hdrdnet.h" -#include "lite/ncnn/cv/ncnn_ibnnet.h" -#include "lite/ncnn/cv/ncnn_mobilenetv2.h" -#include "lite/ncnn/cv/ncnn_resnet.h" -#include "lite/ncnn/cv/ncnn_resnext.h" -#include "lite/ncnn/cv/ncnn_deeplabv3_resnet101.h" -#include "lite/ncnn/cv/ncnn_fcn_resnet101.h" -#include "lite/ncnn/cv/ncnn_colorizer.h" -#include "lite/ncnn/cv/ncnn_fast_style_transfer.h" -#include "lite/ncnn/cv/ncnn_subpixel_cnn.h" -#include "lite/ncnn/cv/ncnn_nanodet_plus.h" -#include "lite/ncnn/cv/ncnn_scrfd.h" -#include "lite/ncnn/cv/ncnn_yolo5face.h" -#include "lite/ncnn/cv/ncnn_faceboxesv2.h" -#include "lite/ncnn/cv/ncnn_pipnet98.h" -#include "lite/ncnn/cv/ncnn_pipnet68.h" -#include "lite/ncnn/cv/ncnn_pipnet29.h" -#include "lite/ncnn/cv/ncnn_pipnet19.h" -#include "lite/ncnn/cv/ncnn_insectid.h" -#include "lite/ncnn/cv/ncnn_plantid.h" -#include "lite/ncnn/cv/ncnn_modnet.h" -#include "lite/ncnn/cv/ncnn_female_photo2cartoon.h" -#include "lite/ncnn/cv/ncnn_yolov6.h" -#include "lite/ncnn/cv/ncnn_face_parsing_bisenet.h" - -#endif - -// ENABLE_TNN -#ifdef ENABLE_TNN - -#include "lite/tnn/core/tnn_core.h" -#include "lite/tnn/core/tnn_utils.h" -#include "lite/tnn/cv/tnn_yolox.h" -#include "lite/tnn/cv/tnn_rvm.h" -#include "lite/tnn/cv/tnn_yolop.h" -#include "lite/tnn/cv/tnn_nanodet.h" -#include "lite/tnn/cv/tnn_nanodet_efficientnet_lite.h" -#include "lite/tnn/cv/tnn_yolov5.h" -#include "lite/tnn/cv/tnn_yolox_v0.1.1.h" -#include "lite/tnn/cv/tnn_yolor.h" -#include "lite/tnn/cv/tnn_yolov5_v6.0.h" -#include "lite/tnn/cv/tnn_glint_arcface.h" -#include "lite/tnn/cv/tnn_glint_cosface.h" -#include "lite/tnn/cv/tnn_glint_partial_fc.h" -#include "lite/tnn/cv/tnn_facenet.h" -#include "lite/tnn/cv/tnn_focal_arcface.h" -#include "lite/tnn/cv/tnn_focal_asia_arcface.h" -#include "lite/tnn/cv/tnn_tencent_curricular_face.h" -#include "lite/tnn/cv/tnn_tencent_cifp_face.h" -#include "lite/tnn/cv/tnn_center_loss_face.h" -#include "lite/tnn/cv/tnn_sphere_face.h" -#include "lite/tnn/cv/tnn_mobile_facenet.h" -#include "lite/tnn/cv/tnn_cava_ghost_arcface.h" -#include "lite/tnn/cv/tnn_cava_combined_face.h" -#include "lite/tnn/cv/tnn_mobilese_focal_face.h" -#include "lite/tnn/cv/tnn_ultraface.h" -#include "lite/tnn/cv/tnn_retinaface.h" -#include "lite/tnn/cv/tnn_faceboxes.h" -#include "lite/tnn/cv/tnn_face_landmarks_1000.h" -#include "lite/tnn/cv/tnn_pfld.h" -#include "lite/tnn/cv/tnn_pfld68.h" -#include "lite/tnn/cv/tnn_pfld98.h" -#include "lite/tnn/cv/tnn_mobilenetv2_68.h" -#include "lite/tnn/cv/tnn_mobilenetv2_se_68.h" -#include "lite/tnn/cv/tnn_fsanet.h" -#include "lite/tnn/cv/tnn_age_googlenet.h" -#include "lite/tnn/cv/tnn_gender_googlenet.h" -#include "lite/tnn/cv/tnn_emotion_ferplus.h" -#include "lite/tnn/cv/tnn_efficient_emotion7.h" -#include "lite/tnn/cv/tnn_efficient_emotion8.h" -#include "lite/tnn/cv/tnn_ssrnet.h" -#include "lite/tnn/cv/tnn_mobile_emotion7.h" -#include "lite/tnn/cv/tnn_rexnet_emotion7.h" -#include "lite/tnn/cv/tnn_efficientnet_lite4.h" -#include "lite/tnn/cv/tnn_shufflenetv2.h" -#include "lite/tnn/cv/tnn_densenet.h" -#include "lite/tnn/cv/tnn_ghostnet.h" -#include "lite/tnn/cv/tnn_hdrdnet.h" -#include "lite/tnn/cv/tnn_ibnnet.h" -#include "lite/tnn/cv/tnn_mobilenetv2.h" -#include "lite/tnn/cv/tnn_resnet.h" -#include "lite/tnn/cv/tnn_resnext.h" -#include "lite/tnn/cv/tnn_deeplabv3_resnet101.h" -#include "lite/tnn/cv/tnn_fcn_resnet101.h" -#include "lite/tnn/cv/tnn_colorizer.h" -#include "lite/tnn/cv/tnn_fast_style_transfer.h" -#include "lite/tnn/cv/tnn_subpixel_cnn.h" -#include "lite/tnn/cv/tnn_mg_matting.h" -#include "lite/tnn/cv/tnn_nanodet_plus.h" -#include "lite/tnn/cv/tnn_scrfd.h" -#include "lite/tnn/cv/tnn_yolo5face.h" -#include "lite/tnn/cv/tnn_faceboxesv2.h" -#include "lite/tnn/cv/tnn_pipnet98.h" -#include "lite/tnn/cv/tnn_pipnet68.h" -#include "lite/tnn/cv/tnn_pipnet29.h" -#include "lite/tnn/cv/tnn_pipnet19.h" -#include "lite/tnn/cv/tnn_insectdet.h" -#include "lite/tnn/cv/tnn_insectid.h" -#include "lite/tnn/cv/tnn_plantid.h" -#include "lite/tnn/cv/tnn_modnet.h" -#include "lite/tnn/cv/tnn_backgroundmattingv2.h" -#include "lite/tnn/cv/tnn_head_seg.h" -#include "lite/tnn/cv/tnn_female_photo2cartoon.h" -#include "lite/tnn/cv/tnn_yolov6.h" -#include "lite/tnn/cv/tnn_face_parsing_bisenet.h" - -#endif - // ONNXRuntime version namespace lite { @@ -854,485 +605,11 @@ namespace lite{ -// MNN version -namespace lite -{ -#ifdef ENABLE_MNN - namespace mnn - { - namespace cv - { - // classification - namespace classification - { - typedef mnncv::MNNEfficientNetLite4 EfficientNetLite4; - typedef mnncv::MNNShuffleNetV2 ShuffleNetV2; - typedef mnncv::MNNDenseNet DenseNet; - typedef mnncv::MNNGhostNet GhostNet; - typedef mnncv::MNNHdrDNet HdrDNet; - typedef mnncv::MNNIBNNet IBNNet; - typedef mnncv::MNNMobileNetV2 MobileNetV2; - typedef mnncv::MNNResNet ResNet; - typedef mnncv::MNNResNeXt ResNeXt; - typedef mnncv::MNNInsectID InsectID; - typedef mnncv::MNNPlantID PlantID; - } - // object detection - namespace detection - { - typedef mnncv::MNNNanoDet NanoDet; - typedef mnncv::MNNNanoDetEfficientNetLite NanoDetEfficientNetLite; - typedef mnncv::MNNYoloX YoloX; - typedef mnncv::MNNYOLOP YOLOP; - typedef mnncv::MNNYoloV5 YoloV5; - typedef mnncv::MNNYoloX_V_0_1_1 YoloX_V_0_1_1; - typedef mnncv::MNNYoloR YoloR; - typedef mnncv::MNNYoloV5_V_6_0 YoloV5_V_6_0; - typedef mnncv::MNNNanoDetPlus NanoDetPlus; - typedef mnncv::MNNInsectDet InsectDet; - typedef mnncv::MNNYoloV5_V_6_1 YoloV5_V_6_1; - typedef mnncv::MNNYOLOv6 YOLOv6; - } - // face etc. - namespace face - { - namespace detect - { - typedef mnncv::MNNUltraFace UltraFace; - typedef mnncv::MNNRetinaFace RetinaFace; - typedef mnncv::MNNFaceBoxes FaceBoxes; - typedef mnncv::MNNSCRFD SCRFD; - typedef mnncv::MNNYOLO5Face YOLO5Face; - typedef mnncv::MNNFaceBoxesV2 FaceBoxesV2; - typedef mnncv::MNNYOLOv5BlazeFace YOLOv5BlazeFace; - } - namespace align - { - typedef mnncv::MNNFaceLandmark1000 FaceLandmark1000; - typedef mnncv::MNNPFLD PFLD; - typedef mnncv::MNNPFLD68 PFLD68; - typedef mnncv::MNNPFLD98 PFLD98; - typedef mnncv::MNNMobileNetV268 MobileNetV268; - typedef mnncv::MNNMobileNetV2SE68 MobileNetV2SE68; - typedef mnncv::MNNPIPNet98 PIPNet98; - typedef mnncv::MNNPIPNet68 PIPNet68; - typedef mnncv::MNNPIPNet29 PIPNet29; - typedef mnncv::MNNPIPNet19 PIPNet19; - } - - namespace align3d - { - - } - - namespace pose - { - typedef mnncv::MNNFSANet FSANet; - } - namespace attr - { - typedef mnncv::MNNAgeGoogleNet AgeGoogleNet; - typedef mnncv::MNNGenderGoogleNet GenderGoogleNet; - typedef mnncv::MNNEmotionFerPlus EmotionFerPlus; - typedef mnncv::MNNSSRNet SSRNet; - typedef mnncv::MNNEfficientEmotion7 EfficientEmotion7; - typedef mnncv::MNNEfficientEmotion8 EfficientEmotion8; - typedef mnncv::MNNMobileEmotion7 MobileEmotion7; - typedef mnncv::MNNReXNetEmotion7 ReXNetEmotion7; - } - } - // face recognition - namespace faceid - { - typedef mnncv::MNNGlintArcFace GlintArcFace; - typedef mnncv::MNNGlintCosFace GlintCosFace; - typedef mnncv::MNNGlintPartialFC GlintPartialFC; - typedef mnncv::MNNFaceNet FaceNet; - typedef mnncv::MNNFocalArcFace FocalArcFace; - typedef mnncv::MNNFocalAsiaArcFace FocalAsiaArcFace; - typedef mnncv::MNNTencentCurricularFace TencentCurricularFace; - typedef mnncv::MNNTencentCifpFace TencentCifpFace; - typedef mnncv::MNNCenterLossFace CenterLossFace; - typedef mnncv::MNNSphereFace SphereFace; - typedef mnncv::MNNMobileFaceNet MobileFaceNet; - typedef mnncv::MNNCavaGhostArcFace CavaGhostArcFace; - typedef mnncv::MNNCavaCombinedFace CavaCombinedFace; - typedef mnncv::MNNMobileSEFocalFace MobileSEFocalFace; - } - // segmentation - namespace segmentation - { - typedef mnncv::MNNDeepLabV3ResNet101 DeepLabV3ResNet101; - typedef mnncv::MNNFCNResNet101 FCNResNet101; - typedef mnncv::MNNHeadSeg HeadSeg; - typedef mnncv::MNNFastPortraitSeg FastPortraitSeg; - typedef mnncv::MNNPortraitSegSINet PortraitSegSINet; - typedef mnncv::MNNPortraitSegExtremeC3Net PortraitSegExtremeC3Net; - typedef mnncv::MNNHairSeg HairSeg; - typedef mnncv::MNNFaceHairSeg FaceHairSeg; - typedef mnncv::MNNMobileHairSeg MobileHairSeg; - typedef mnncv::MNNFaceParsingBiSeNet FaceParsingBiSeNet; - } - // reid - namespace reid - { - } - // ocr - namespace ocr - { - } - // matting - namespace matting - { - typedef mnncv::MNNRobustVideoMatting RobustVideoMatting; - typedef mnncv::MNNMGMatting MGMatting; - typedef mnncv::MNNMODNet MODNet; - typedef mnncv::MNNBackgroundMattingV2 BackgroundMattingV2; - typedef mnncv::MNNMobileHumanMatting MobileHumanMatting; - } - - // style transfer - namespace style - { - typedef mnncv::MNNFastStyleTransfer FastStyleTransfer; - typedef mnncv::MNNFemalePhoto2Cartoon FemalePhoto2Cartoon; - } - - // colorization - namespace colorization - { - typedef mnncv::MNNColorizer Colorizer; - } - // super resolution - namespace resolution - { - typedef mnncv::MNNSubPixelCNN SubPixelCNN; - } - // mediapipe - namespace mediapipe - { - } - - } // namespace cv - - } -#endif -} - -// NCNN version -namespace lite -{ -#ifdef ENABLE_NCNN - namespace ncnn - { - // mediapipe - namespace mediapipe - { - } - - namespace cv - { - // classification - namespace classification - { - typedef ncnncv::NCNNEfficientNetLite4 EfficientNetLite4; - typedef ncnncv::NCNNShuffleNetV2 ShuffleNetV2; - typedef ncnncv::NCNNDenseNet DenseNet; - typedef ncnncv::NCNNGhostNet GhostNet; - typedef ncnncv::NCNNHdrDNet HdrDNet; - typedef ncnncv::NCNNIBNNet IBNNet; - typedef ncnncv::NCNNMobileNetV2 MobileNetV2; - typedef ncnncv::NCNNResNet ResNet; - typedef ncnncv::NCNNResNeXt ResNeXt; - typedef ncnncv::NCNNInsectID InsectID; - typedef ncnncv::NCNNPlantID PlantID; - } - // object detection - namespace detection - { - typedef ncnncv::NCNNNanoDet NanoDet; - typedef ncnncv::NCNNNanoDetEfficientNetLite NanoDetEfficientNetLite; - typedef ncnncv::NCNNNanoDetDepreciated NanoDetDepreciated; - typedef ncnncv::NCNNNanoDetEfficientNetLiteDepreciated NanoDetEfficientNetLiteDepreciated; - typedef ncnncv::NCNNYoloX YoloX; - typedef ncnncv::NCNNYOLOP YOLOP; - typedef ncnncv::NCNNYoloV5 YoloV5; - typedef ncnncv::NCNNYoloX_V_0_1_1 YoloX_V_0_1_1; - typedef ncnncv::NCNNYoloR YoloR; - typedef ncnncv::NCNNYoloRssss YoloRssss; - typedef ncnncv::NCNNYoloV5_V_6_0 YoloV5_V_6_0; - typedef ncnncv::NCNNYoloV5_V_6_0_P6 YoloV5_V_6_0_P6; - typedef ncnncv::NCNNNanoDetPlus NanoDetPlus; - typedef ncnncv::NCNNYOLOv6 YOLOv6; - } - // face etc. - namespace face - { - namespace detect - { - typedef ncnncv::NCNNUltraFace UltraFace; - typedef ncnncv::NCNNRetinaFace RetinaFace; - typedef ncnncv::NCNNFaceBoxes FaceBoxes; - typedef ncnncv::NCNNSCRFD SCRFD; - typedef ncnncv::NCNNYOLO5Face YOLO5Face; - typedef ncnncv::NCNNFaceBoxesV2 FaceBoxesV2; - } - namespace align - { - typedef ncnncv::NCNNFaceLandmark1000 FaceLandmark1000; - typedef ncnncv::NCNNPFLD PFLD; - typedef ncnncv::NCNNPFLD68 PFLD68; - typedef ncnncv::NCNNPFLD98 PFLD98; - typedef ncnncv::NCNNMobileNetV268 MobileNetV268; - typedef ncnncv::NCNNMobileNetV2SE68 MobileNetV2SE68; - typedef ncnncv::NCNNPIPNet98 PIPNet98; - typedef ncnncv::NCNNPIPNet68 PIPNet68; - typedef ncnncv::NCNNPIPNet29 PIPNet29; - typedef ncnncv::NCNNPIPNet19 PIPNet19; - } - - namespace align3d - { - } - - namespace pose - { - } - namespace attr - { - typedef ncnncv::NCNNAgeGoogleNet AgeGoogleNet; - typedef ncnncv::NCNNGenderGoogleNet GenderGoogleNet; - typedef ncnncv::NCNNEmotionFerPlus EmotionFerPlus; - typedef ncnncv::NCNNEfficientEmotion7 EfficientEmotion7; - typedef ncnncv::NCNNEfficientEmotion8 EfficientEmotion8; - typedef ncnncv::NCNNMobileEmotion7 MobileEmotion7; - } - } - // face recognition - namespace faceid - { - typedef ncnncv::NCNNGlintArcFace GlintArcFace; - typedef ncnncv::NCNNGlintCosFace GlintCosFace; - typedef ncnncv::NCNNGlintPartialFC GlintPartialFC; - typedef ncnncv::NCNNFaceNet FaceNet; - typedef ncnncv::NCNNFocalArcFace FocalArcFace; - typedef ncnncv::NCNNFocalAsiaArcFace FocalAsiaArcFace; - typedef ncnncv::NCNNTencentCurricularFace TencentCurricularFace; - typedef ncnncv::NCNNTencentCifpFace TencentCifpFace; - typedef ncnncv::NCNNCenterLossFace CenterLossFace; - typedef ncnncv::NCNNSphereFace SphereFace; - typedef ncnncv::NCNNMobileFaceNet MobileFaceNet; - typedef ncnncv::NCNNCavaGhostArcFace CavaGhostArcFace; - typedef ncnncv::NCNNCavaCombinedFace CavaCombinedFace; - typedef ncnncv::NCNNMobileSEFocalFace MobileSEFocalFace; - } - // segmentation - namespace segmentation - { - typedef ncnncv::NCNNDeepLabV3ResNet101 DeepLabV3ResNet101; - typedef ncnncv::NCNNFCNResNet101 FCNResNet101; - typedef ncnncv::NCNNFaceParsingBiSeNet FaceParsingBiSeNet; - } - // reid - namespace reid - { - } - // ocr - namespace ocr - { - } - // matting - namespace matting - { - typedef ncnncv::NCNNRobustVideoMatting RobustVideoMatting; - typedef ncnncv::NCNNMODNet MODNet; - } - // style transfer - namespace style - { - typedef ncnncv::NCNNFastStyleTransfer FastStyleTransfer; - typedef ncnncv::NCNNFemalePhoto2Cartoon FemalePhoto2Cartoon; - } - - // colorization - namespace colorization - { - typedef ncnncv::NCNNColorizer Colorizer; - } - // super resolution - namespace resolution - { - typedef ncnncv::NCNNSubPixelCNN SubPixelCNN; - } - - } // namespace cv - - } -#endif -} - -// TNN version -namespace lite -{ -#ifdef ENABLE_TNN - namespace tnn - { - // mediapipe - namespace mediapipe - { - } - - namespace cv - { - // classification - namespace classification - { - typedef tnncv::TNNEfficientNetLite4 EfficientNetLite4; - typedef tnncv::TNNShuffleNetV2 ShuffleNetV2; - typedef tnncv::TNNDenseNet DenseNet; - typedef tnncv::TNNGhostNet GhostNet; - typedef tnncv::TNNHdrDNet HdrDNet; - typedef tnncv::TNNIBNNet IBNNet; - typedef tnncv::TNNMobileNetV2 MobileNetV2; - typedef tnncv::TNNResNet ResNet; - typedef tnncv::TNNResNeXt ResNeXt; - typedef tnncv::TNNInsectID InsectID; - typedef tnncv::TNNPlantID PlantID; - } - // object detection - namespace detection - { - typedef tnncv::TNNYoloX YoloX; - typedef tnncv::TNNYOLOP YOLOP; - typedef tnncv::TNNNanoDet NanoDet; - typedef tnncv::TNNNanoDetEfficientNetLite NanoDetEfficientNetLite; - typedef tnncv::TNNYoloV5 YoloV5; - typedef tnncv::TNNYoloX_V_0_1_1 YoloX_V_0_1_1; - typedef tnncv::TNNYoloR YoloR; - typedef tnncv::TNNYoloV5_V_6_0 YoloV5_V_6_0; - typedef tnncv::TNNNanoDetPlus NanoDetPlus; - typedef tnncv::TNNInsectDet InsectDet; - typedef tnncv::TNNYOLOv6 YOLOv6; - } - // face etc. - namespace face - { - namespace detect - { - typedef tnncv::TNNUltraFace UltraFace; - typedef tnncv::TNNRetinaFace RetinaFace; - typedef tnncv::TNNFaceBoxes FaceBoxes; - typedef tnncv::TNNSCRFD SCRFD; - typedef tnncv::TNNYOLO5Face YOLO5Face; - typedef tnncv::TNNFaceBoxesV2 FaceBoxesV2; - } - namespace align - { - typedef tnncv::TNNFaceLandmark1000 FaceLandmark1000; - typedef tnncv::TNNPFLD PFLD; - typedef tnncv::TNNPFLD68 PFLD68; - typedef tnncv::TNNPFLD98 PFLD98; - typedef tnncv::TNNMobileNetV268 MobileNetV268; - typedef tnncv::TNNMobileNetV2SE68 MobileNetV2SE68; - typedef tnncv::TNNPIPNet98 PIPNet98; - typedef tnncv::TNNPIPNet68 PIPNet68; - typedef tnncv::TNNPIPNet29 PIPNet29; - typedef tnncv::TNNPIPNet19 PIPNet19; - } - namespace align3d - { - } - namespace pose - { - typedef tnncv::TNNFSANet FSANet; - } - namespace attr - { - typedef tnncv::TNNAgeGoogleNet AgeGoogleNet; - typedef tnncv::TNNGenderGoogleNet GenderGoogleNet; - typedef tnncv::TNNEmotionFerPlus EmotionFerPlus; - typedef tnncv::TNNSSRNet SSRNet; - typedef tnncv::TNNEfficientEmotion7 EfficientEmotion7; - typedef tnncv::TNNEfficientEmotion8 EfficientEmotion8; - typedef tnncv::TNNMobileEmotion7 MobileEmotion7; - typedef tnncv::TNNReXNetEmotion7 ReXNetEmotion7; - } - } - // face recognition - namespace faceid - { - typedef tnncv::TNNGlintArcFace GlintArcFace; - typedef tnncv::TNNGlintCosFace GlintCosFace; - typedef tnncv::TNNGlintPartialFC GlintPartialFC; - typedef tnncv::TNNFaceNet FaceNet; - typedef tnncv::TNNFocalArcFace FocalArcFace; - typedef tnncv::TNNFocalAsiaArcFace FocalAsiaArcFace; - typedef tnncv::TNNTencentCurricularFace TencentCurricularFace; - typedef tnncv::TNNTencentCifpFace TencentCifpFace; - typedef tnncv::TNNCenterLossFace CenterLossFace; - typedef tnncv::TNNSphereFace SphereFace; - typedef tnncv::TNNMobileFaceNet MobileFaceNet; - typedef tnncv::TNNCavaGhostArcFace CavaGhostArcFace; - typedef tnncv::TNNCavaCombinedFace CavaCombinedFace; - typedef tnncv::TNNMobileSEFocalFace MobileSEFocalFace; - } - // segmentation - namespace segmentation - { - typedef tnncv::TNNDeepLabV3ResNet101 DeepLabV3ResNet101; - typedef tnncv::TNNFCNResNet101 FCNResNet101; - typedef tnncv::TNNHeadSeg HeadSeg; - typedef tnncv::TNNFaceParsingBiSeNet FaceParsingBiSeNet; - } - // reid - namespace reid - { - } - // ocr - namespace ocr - { - } - // matting - namespace matting - { - typedef tnncv::TNNRobustVideoMatting RobustVideoMatting; - typedef tnncv::TNNMGMatting MGMatting; - typedef tnncv::TNNMODNet MODNet; - typedef tnncv::TNNBackgroundMattingV2 BackgroundMattingV2; - } - // style transfer - namespace style - { - typedef tnncv::TNNFastStyleTransfer FastStyleTransfer; - typedef tnncv::TNNFemalePhoto2Cartoon FemalePhoto2Cartoon; - } - // colorization - namespace colorization - { - typedef tnncv::TNNColorizer Colorizer; - } - // super resolution - namespace resolution - { - typedef tnncv::TNNSubPixelCNN SubPixelCNN; - } - - } // namespace cv - } -#endif -} - // Default Engine ONNXRuntime namespace lite { #if defined(ENABLE_ONNXRUNTIME) namespace cv = lite::onnxruntime::cv; -#elif defined(ENABLE_MNN) - namespace cv = lite::mnn::cv; -#elif defined(ENABLE_NCNN) - namespace cv = lite::ncnn::cv; -#elif defined(ENABLE_TNN) - namespace cv = lite::tnn::cv; #endif } diff --git a/lite/ncnn/.gitignore b/lite/ncnn/.gitignore deleted file mode 100644 index e69de29b..00000000 diff --git a/lite/ncnn/core/ncnn_config.h b/lite/ncnn/core/ncnn_config.h deleted file mode 100644 index 9e75f658..00000000 --- a/lite/ncnn/core/ncnn_config.h +++ /dev/null @@ -1,18 +0,0 @@ -// -// Created by DefTruth on 2021/10/7. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CORE_NCNN_CONFIG_H -#define LITE_AI_TOOLKIT_NCNN_CORE_NCNN_CONFIG_H - -#include "ncnn_defs.h" -#include "lite/lite.ai.headers.h" - -#ifdef ENABLE_NCNN -#include "ncnn/net.h" -#include "ncnn/layer.h" -#endif - -namespace ncnncore {} - -#endif //LITE_AI_TOOLKIT_NCNN_CORE_NCNN_CONFIG_H diff --git a/lite/ncnn/core/ncnn_core.h b/lite/ncnn/core/ncnn_core.h deleted file mode 100644 index 50a8d156..00000000 --- a/lite/ncnn/core/ncnn_core.h +++ /dev/null @@ -1,102 +0,0 @@ -// -// Created by DefTruth on 2021/10/7. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CORE_NCNN_CORE_H -#define LITE_AI_TOOLKIT_NCNN_CORE_NCNN_CORE_H - -#include "ncnn_config.h" -#include "ncnn_handler.h" -#include "ncnn_types.h" -#include "ncnn_custom.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNNanoDet; // [0] * reference: https://github.com/RangiLyu/nanodet - class LITE_EXPORTS NCNNNanoDetEfficientNetLite; // [1] * reference: https://github.com/RangiLyu/nanodet - class LITE_EXPORTS NCNNNanoDetDepreciated; // [2] * reference: https://github.com/RangiLyu/nanodet - class LITE_EXPORTS NCNNNanoDetEfficientNetLiteDepreciated; // [3] * reference: https://github.com/RangiLyu/nanodet - class LITE_EXPORTS NCNNRobustVideoMatting; // [4] * reference: https://github.com/PeterL1n/RobustVideoMatting - class LITE_EXPORTS NCNNYoloX; // [5] * reference: https://github.com/Megvii-BaseDetection/YOLOX - class LITE_EXPORTS NCNNYOLOP; // [6] * reference: https://github.com/hustvl/YOLOP - class LITE_EXPORTS NCNNYoloV5; // [7] * reference: https://github.com/ultralytics/yolov5 - class LITE_EXPORTS NCNNYoloX_V_0_1_1; // [8] * reference: https://github.com/Megvii-BaseDetection/YOLOX - class LITE_EXPORTS NCNNYoloR; // [9] * reference: https://github.com/WongKinYiu/yolor - class LITE_EXPORTS NCNNYoloRssss; // [10] * reference: https://github.com/WongKinYiu/yolor - class LITE_EXPORTS NCNNYoloV5_V_6_0; // [11] * reference: https://github.com/ultralytics/yolov5 - class LITE_EXPORTS NCNNYoloV5_V_6_0_P6; // [12] * reference: https://github.com/ultralytics/yolov5 - class LITE_EXPORTS NCNNGlintArcFace; // [13] * reference: https://github.com/deepinsight/insightface/tree/master/recognition/arcface_torch - class LITE_EXPORTS NCNNGlintCosFace; // [14] * reference: https://github.com/deepinsight/insightface/tree/master/recognition/arcface_torch - class LITE_EXPORTS NCNNGlintPartialFC; // [15] * reference: https://github.com/deepinsight/insightface/tree/master/recognition/partial_fc - class LITE_EXPORTS NCNNFaceNet; // [16] * reference: https://github.com/timesler/facenet-pytorch - class LITE_EXPORTS NCNNFocalArcFace; // [17] * reference: https://github.com/ZhaoJ9014/face.evoLVe.PyTorch - class LITE_EXPORTS NCNNFocalAsiaArcFace; // [18] * reference: https://github.com/ZhaoJ9014/face.evoLVe.PyTorch - class LITE_EXPORTS NCNNTencentCurricularFace; // [19] * reference: https://github.com/Tencent/TFace/tree/master/tasks/distfc - class LITE_EXPORTS NCNNTencentCifpFace; // [20] * reference: https://github.com/Tencent/TFace/tree/master/tasks/cifp - class LITE_EXPORTS NCNNCenterLossFace; // [21] * reference: https://github.com/louis-she/center-loss.pytorch - class LITE_EXPORTS NCNNSphereFace; // [22] * reference: https://github.com/clcarwin/sphereface_pytorch - class LITE_EXPORTS NCNNMobileFaceNet; // [23] * reference: https://github.com/Xiaoccer/MobileFaceNet_Pytorch - class LITE_EXPORTS NCNNCavaGhostArcFace; // [24] * reference: https://github.com/cavalleria/cavaface.pytorch - class LITE_EXPORTS NCNNCavaCombinedFace; // [25] * reference: https://github.com/cavalleria/cavaface.pytorch - class LITE_EXPORTS NCNNMobileSEFocalFace; // [26] * reference: https://github.com/grib0ed0v/face_recognition.pytorch - class LITE_EXPORTS NCNNUltraFace; // [27] * reference: https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB - class LITE_EXPORTS NCNNRetinaFace; // [28] * reference: https://github.com/biubug6/Pytorch_Retinaface - class LITE_EXPORTS NCNNFaceBoxes; // [29] * reference: https://github.com/zisianw/FaceBoxes.PyTorch - class LITE_EXPORTS NCNNPFLD; // [30] * reference: https://github.com/Hsintao/pfld_106_face_landmarks - class LITE_EXPORTS NCNNPFLD98; // [31] * reference: https://github.com/polarisZhao/PFLD-pytorch - class LITE_EXPORTS NCNNMobileNetV268; // [32] * reference: https://github.com/cunjian/pytorch_face_landmark - class LITE_EXPORTS NCNNMobileNetV2SE68; // [33] * reference: https://github.com/cunjian/pytorch_face_landmark - class LITE_EXPORTS NCNNPFLD68; // [34] * reference: https://github.com/cunjian/pytorch_face_landmark - class LITE_EXPORTS NCNNFaceLandmark1000; // [35] * reference: https://github.com/Single430/FaceLandmark1000 - class LITE_EXPORTS NCNNAgeGoogleNet; // [36] * reference: https://github.com/onnx/models/tree/master/vision/body_analysis/age_gender - class LITE_EXPORTS NCNNGenderGoogleNet; // [37] * reference: https://github.com/onnx/models/tree/master/vision/body_analysis/age_gender - class LITE_EXPORTS NCNNEmotionFerPlus; // [38] * reference: https://github.com/onnx/models/blob/master/vision/body_analysis/emotion_ferplus - class LITE_EXPORTS NCNNEfficientEmotion7; // [39] * reference: https://github.com/HSE-asavchenko/face-emotion-recognition - class LITE_EXPORTS NCNNEfficientEmotion8; // [40] * reference: https://github.com/HSE-asavchenko/face-emotion-recognition - class LITE_EXPORTS NCNNMobileEmotion7; // [41] * reference: https://github.com/HSE-asavchenko/face-emotion-recognition - class LITE_EXPORTS NCNNEfficientNetLite4; // [42] * reference: https://github.com/onnx/models/blob/master/vision/classification/efficientnet-lite4 - class LITE_EXPORTS NCNNShuffleNetV2; // [43] * reference: https://github.com/onnx/models/blob/master/vision/classification/shufflenet - class LITE_EXPORTS NCNNDenseNet; // [44] * reference: https://pytorch.org/hub/pytorch_vision_densenet/ - class LITE_EXPORTS NCNNGhostNet; // [45] * reference：https://pytorch.org/hub/pytorch_vision_ghostnet/ - class LITE_EXPORTS NCNNHdrDNet; // [46] * reference: https://pytorch.org/hub/pytorch_vision_hardnet/ - class LITE_EXPORTS NCNNIBNNet; // [47] * reference: https://pytorch.org/hub/pytorch_vision_ibnnet/ - class LITE_EXPORTS NCNNMobileNetV2; // [48] * reference: https://pytorch.org/hub/pytorch_vision_mobilenet_v2/ - class LITE_EXPORTS NCNNResNet; // [49] * reference: https://pytorch.org/hub/pytorch_vision_resnet/ - class LITE_EXPORTS NCNNResNeXt; // [50] * reference: https://pytorch.org/hub/pytorch_vision_resnext/ - class LITE_EXPORTS NCNNFastStyleTransfer; // [51] * reference: https://github.com/onnx/models/blob/master/vision/style_transfer/fast_neural_style - class LITE_EXPORTS NCNNColorizer; // [52] * reference: https://github.com/richzhang/colorization - class LITE_EXPORTS NCNNSubPixelCNN; // [53] * reference: https://github.com/niazwazir/SUB_PIXEL_CNN - class LITE_EXPORTS NCNNDeepLabV3ResNet101; // [54] * reference: https://pytorch.org/hub/pytorch_vision_deeplabv3_resnet101/ - class LITE_EXPORTS NCNNFCNResNet101; // [55] * reference: https://pytorch.org/hub/pytorch_vision_fcn_resnet101/ - class LITE_EXPORTS NCNNNanoDetPlus; // [56] * reference: https://github.com/RangiLyu/nanodet - class LITE_EXPORTS NCNNSCRFD; // [57] * reference: https://github.com/deepinsight/insightface/tree/master/detection/scrfd - class LITE_EXPORTS NCNNYOLO5Face; // [58] * reference: https://github.com/deepcam-cn/yolov5-face - class LITE_EXPORTS NCNNFaceBoxesV2; // [59] * reference: https://github.com/jhb86253817/FaceBoxesV2 - class LITE_EXPORTS NCNNPIPNet19; // [60] * reference: https://github.com/jhb86253817/PIPNet - class LITE_EXPORTS NCNNPIPNet29; // [61] * reference: https://github.com/jhb86253817/PIPNet - class LITE_EXPORTS NCNNPIPNet68; // [62] * reference: https://github.com/jhb86253817/PIPNet - class LITE_EXPORTS NCNNPIPNet98; // [63] * reference: https://github.com/jhb86253817/PIPNet - class LITE_EXPORTS NCNNInsectID; // [64] * reference: https://github.com/quarrying/quarrying-insect-id - class LITE_EXPORTS NCNNPlantID; // [65] * reference: https://github.com/quarrying/quarrying-plant-id - class LITE_EXPORTS NCNNMODNet; // [66] * reference: https://github.com/ZHKKKe/MODNet - class LITE_EXPORTS NCNNFemalePhoto2Cartoon; // [67] * reference: https://github.com/minivision-ai/photo2cartoon - class LITE_EXPORTS NCNNYOLOv6; // [68] * reference: https://github.com/meituan/YOLOv6 - class LITE_EXPORTS NCNNFaceParsingBiSeNet; // [69] * reference: https://github.com/zllrunning/face-parsing.PyTorch -} - -namespace ncnncv -{ - using ncnncore::BasicNCNNHandler; -} - -namespace ncnnnlp -{ - using ncnncore::BasicNCNNHandler; -} - -namespace ncnnasr -{ - using ncnncore::BasicNCNNHandler; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CORE_NCNN_CORE_H diff --git a/lite/ncnn/core/ncnn_custom.cpp b/lite/ncnn/core/ncnn_custom.cpp deleted file mode 100644 index 9441ca30..00000000 --- a/lite/ncnn/core/ncnn_custom.cpp +++ /dev/null @@ -1,47 +0,0 @@ -// -// Created by DefTruth on 2021/10/31. -// - -#include "ncnn_custom.h" - -int YoloV5Focus::forward(const ncnn::Mat &bottom_blob, ncnn::Mat &top_blob, const ncnn::Option &opt) const -{ - int w = bottom_blob.w; - int h = bottom_blob.h; - int channels = bottom_blob.c; - - int outw = w / 2; - int outh = h / 2; - int outc = channels * 4; - - top_blob.create(outw, outh, outc, 4u, 1, opt.blob_allocator); - if (top_blob.empty()) - return -100; - - // TODO: add omp support - for (int p = 0; p < outc; p++) - { - const float *ptr = bottom_blob.channel(p % channels).row((p / channels) % 2) + ((p / channels) / 2); - float *outptr = top_blob.channel(p); - - for (int i = 0; i < outh; i++) - { - for (int j = 0; j < outw; j++) - { - *outptr = *ptr; - - outptr += 1; - ptr += 2; - } - - ptr += w; - } - } - - return 0; -} - -ncnn::Layer *YoloV5Focus_layer_creator(void * /*userdata*/) -{ - return new YoloV5Focus; -} \ No newline at end of file diff --git a/lite/ncnn/core/ncnn_custom.h b/lite/ncnn/core/ncnn_custom.h deleted file mode 100644 index 743991db..00000000 --- a/lite/ncnn/core/ncnn_custom.h +++ /dev/null @@ -1,24 +0,0 @@ -// -// Created by DefTruth on 2021/10/31. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CORE_NCNN_CUSTOM_H -#define LITE_AI_TOOLKIT_NCNN_CORE_NCNN_CUSTOM_H - -#include "ncnn_config.h" - -// YOLOX|YOLOP|YOLOR ... use the same focus in yolov5 -class YoloV5Focus : public ncnn::Layer -{ -public: - YoloV5Focus() - { - one_blob_only = true; - } - - virtual int forward(const ncnn::Mat &bottom_blob, ncnn::Mat &top_blob, const ncnn::Option &opt) const; -}; - -ncnn::Layer* YoloV5Focus_layer_creator(void * /*userdata*/); - -#endif //LITE_AI_TOOLKIT_NCNN_CORE_NCNN_CUSTOM_H diff --git a/lite/ncnn/core/ncnn_defs.h b/lite/ncnn/core/ncnn_defs.h deleted file mode 100644 index 4936ee8a..00000000 --- a/lite/ncnn/core/ncnn_defs.h +++ /dev/null @@ -1,25 +0,0 @@ -// -// Created by DefTruth on 2021/10/7. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CORE_NCNN_DEFS_H -#define LITE_AI_TOOLKIT_NCNN_CORE_NCNN_DEFS_H - -#include "lite/config.h" -#include "lite/lite.ai.defs.h" - -#ifdef ENABLE_DEBUG_STRING -# define LITENCNN_DEBUG 1 -#else -# define LITENCNN_DEBUG 0 -#endif - - -#ifdef LITE_WIN32 -# ifndef NOMINMAX -# define NOMINMAX -# endif -#endif - - -#endif //LITE_AI_TOOLKIT_NCNN_CORE_NCNN_DEFS_H diff --git a/lite/ncnn/core/ncnn_handler.cpp b/lite/ncnn/core/ncnn_handler.cpp deleted file mode 100644 index 59ec2a64..00000000 --- a/lite/ncnn/core/ncnn_handler.cpp +++ /dev/null @@ -1,76 +0,0 @@ -// -// Created by DefTruth on 2021/10/7. -// - -#include "ncnn_handler.h" - -using ncnncore::BasicNCNNHandler; - -BasicNCNNHandler::BasicNCNNHandler( - const std::string &_param_path, const std::string &_bin_path, unsigned int _num_threads) : - log_id(_param_path.data()), param_path(_param_path.data()), - bin_path(_bin_path.data()), num_threads(_num_threads) -{ - initialize_handler(); -} - -void BasicNCNNHandler::initialize_handler() -{ - // init net, change this setting for better performance. - net = new ncnn::Net(); - net->opt.use_vulkan_compute = false; // default - net->opt.use_fp16_arithmetic = false; - net->load_param(param_path); - net->load_model(bin_path); - input_indexes = net->input_indexes(); - output_indexes = net->output_indexes(); -#ifdef NCNN_STRING - input_names = net->input_names(); - output_names = net->output_names(); -#endif - num_outputs = output_indexes.size(); -#ifdef LITENCNN_DEBUG - this->print_debug_string(); -#endif -} - -BasicNCNNHandler::~BasicNCNNHandler() -{ - if (net) delete net; - net = nullptr; -} - -void BasicNCNNHandler::print_debug_string() -{ - std::cout << "LITENCNN_DEBUG LogId: " << log_id << "\n"; - std::cout << "=============== Input-Dims ==============\n"; - for (int i = 0; i < input_indexes.size(); ++i) - { - std::cout << "Input: "; - auto tmp_in_blob = net->blobs().at(input_indexes.at(i)); -#ifdef NCNN_STRING - std::cout << input_names.at(i) << ": "; -#endif - std::cout << "shape: c=" << tmp_in_blob.shape.c - << " h=" << tmp_in_blob.shape.h << " w=" << tmp_in_blob.shape.w << "\n"; - } - - std::cout << "=============== Output-Dims ==============\n"; - for (int i = 0; i < output_indexes.size(); ++i) - { - auto tmp_out_blob = net->blobs().at(output_indexes.at(i)); - std::cout << "Output: "; -#ifdef NCNN_STRING - std::cout << output_names.at(i) << ": "; -#endif - std::cout << "shape: c=" << tmp_out_blob.shape.c - << " h=" << tmp_out_blob.shape.h << " w=" << tmp_out_blob.shape.w << "\n"; - } - std::cout << "========================================\n"; -} - -// static method -void BasicNCNNHandler::print_shape(const ncnn::Mat &mat, const std::string name) -{ - std::cout << name << ": " << "c=" << mat.c << ",h=" << mat.h << ",w=" << mat.w << "\n"; -} \ No newline at end of file diff --git a/lite/ncnn/core/ncnn_handler.h b/lite/ncnn/core/ncnn_handler.h deleted file mode 100644 index 2ad507ef..00000000 --- a/lite/ncnn/core/ncnn_handler.h +++ /dev/null @@ -1,56 +0,0 @@ -// -// Created by DefTruth on 2021/10/7. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CORE_NCNN_HANDLER_H -#define LITE_AI_TOOLKIT_NCNN_CORE_NCNN_HANDLER_H - -#include "ncnn_config.h" - -namespace ncnncore -{ - class LITE_EXPORTS BasicNCNNHandler - { - protected: - ncnn::Net *net = nullptr; - const char *log_id = nullptr; - const char *param_path = nullptr; - const char *bin_path = nullptr; - std::vector input_names; - std::vector output_names; - std::vector input_indexes; - std::vector output_indexes; - int num_outputs = 1; - - protected: - const unsigned int num_threads; // initialize at runtime. - - protected: - explicit BasicNCNNHandler(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - virtual ~BasicNCNNHandler(); - - // un-copyable - protected: - BasicNCNNHandler(const BasicNCNNHandler &) = delete; // - BasicNCNNHandler(BasicNCNNHandler &&) = delete; // - BasicNCNNHandler &operator=(const BasicNCNNHandler &) = delete; // - BasicNCNNHandler &operator=(BasicNCNNHandler &&) = delete; // - - private: - virtual void transform(const cv::Mat &mat, ncnn::Mat &in) = 0; - - private: - void initialize_handler(); - - void print_debug_string(); - - public: - static void print_shape(const ncnn::Mat &mat, const std::string name = ""); - - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CORE_NCNN_HANDLER_H diff --git a/lite/ncnn/core/ncnn_types.h b/lite/ncnn/core/ncnn_types.h deleted file mode 100644 index 20ffda34..00000000 --- a/lite/ncnn/core/ncnn_types.h +++ /dev/null @@ -1,15 +0,0 @@ -// -// Created by DefTruth on 2021/10/7. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CORE_NCNN_TYPES_H -#define LITE_AI_TOOLKIT_NCNN_CORE_NCNN_TYPES_H - -#include "lite/types.h" - -namespace ncnncv -{ - namespace types = lite::types; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CORE_NCNN_TYPES_H diff --git a/lite/ncnn/core/ncnn_utils.cpp b/lite/ncnn/core/ncnn_utils.cpp deleted file mode 100644 index ac8570ee..00000000 --- a/lite/ncnn/core/ncnn_utils.cpp +++ /dev/null @@ -1,5 +0,0 @@ -// -// Created by DefTruth on 2021/10/7. -// - -#include "ncnn_utils.h" \ No newline at end of file diff --git a/lite/ncnn/core/ncnn_utils.h b/lite/ncnn/core/ncnn_utils.h deleted file mode 100644 index 11ea31ad..00000000 --- a/lite/ncnn/core/ncnn_utils.h +++ /dev/null @@ -1,13 +0,0 @@ -// -// Created by DefTruth on 2021/10/7. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CORE_NCNN_UTILS_H -#define LITE_AI_TOOLKIT_NCNN_CORE_NCNN_UTILS_H - -namespace ncnncv -{ - // no specific utils for NCNN now. -} - -#endif //LITE_AI_TOOLKIT_NCNN_CORE_NCNN_UTILS_H diff --git a/lite/ncnn/cv/ncnn_age_googlenet.cpp b/lite/ncnn/cv/ncnn_age_googlenet.cpp deleted file mode 100644 index 64c129fc..00000000 --- a/lite/ncnn/cv/ncnn_age_googlenet.cpp +++ /dev/null @@ -1,57 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#include "ncnn_age_googlenet.h" -#include "lite/utils.h" - -using ncnncv::NCNNAgeGoogleNet; - -NCNNAgeGoogleNet::NCNNAgeGoogleNet(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNAgeGoogleNet::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - // (1,3,224,224) - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNAgeGoogleNet::detect(const cv::Mat &mat, types::Age &age) -{ - if (mat.empty()) return; - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - // 3. fetch. - ncnn::Mat age_logits; - extractor.extract("loss3/loss3_Y", age_logits); // c=1,h=1,w=8 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(age_logits, "loss3/loss3_Y"); -#endif - - unsigned int interval = 0; - const unsigned int num_intervals = age_logits.w; // 8 - const float *pred_logits_ptr = (float *) age_logits.data; - - auto softmax_probs = lite::utils::math::softmax(pred_logits_ptr, num_intervals, interval); - const float pred_age = static_cast(age_intervals[interval][0] + age_intervals[interval][1]) / 2.0f; - - age.age = pred_age; - age.age_interval[0] = age_intervals[interval][0]; - age.age_interval[1] = age_intervals[interval][1]; - age.interval_prob = softmax_probs[interval]; - age.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_age_googlenet.h b/lite/ncnn/cv/ncnn_age_googlenet.h deleted file mode 100644 index 738e7018..00000000 --- a/lite/ncnn/cv/ncnn_age_googlenet.h +++ /dev/null @@ -1,46 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_AGE_GOOGLENET_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_AGE_GOOGLENET_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNAgeGoogleNet : public BasicNCNNHandler - { - public: - explicit NCNNAgeGoogleNet(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNAgeGoogleNet() override = default; - - private: - const int input_height = 224; - const int input_width = 224; - const float mean_vals[3] = {104.0f, 117.0f, 123.0f}; - const float norm_vals[3] = {1.0f, 1.0f, 1.0f}; - - const unsigned int age_intervals[8][2] = { - {0, 2}, - {4, 6}, - {8, 12}, - {15, 20}, - {25, 32}, - {38, 43}, - {48, 53}, - {60, 100} - }; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::Age &age); - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_AGE_GOOGLENET_H diff --git a/lite/ncnn/cv/ncnn_backgroundmattingv2.cpp b/lite/ncnn/cv/ncnn_backgroundmattingv2.cpp deleted file mode 100644 index 261fd18a..00000000 --- a/lite/ncnn/cv/ncnn_backgroundmattingv2.cpp +++ /dev/null @@ -1,4 +0,0 @@ -// -// Created by DefTruth on 2022/4/9. -// - diff --git a/lite/ncnn/cv/ncnn_backgroundmattingv2.h b/lite/ncnn/cv/ncnn_backgroundmattingv2.h deleted file mode 100644 index 69a0e864..00000000 --- a/lite/ncnn/cv/ncnn_backgroundmattingv2.h +++ /dev/null @@ -1,8 +0,0 @@ -// -// Created by DefTruth on 2022/4/9. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_BACKGROUNDMATTINGV2_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_BACKGROUNDMATTINGV2_H - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_BACKGROUNDMATTINGV2_H diff --git a/lite/ncnn/cv/ncnn_cava_combined_face.cpp b/lite/ncnn/cv/ncnn_cava_combined_face.cpp deleted file mode 100644 index f620c66d..00000000 --- a/lite/ncnn/cv/ncnn_cava_combined_face.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "ncnn_cava_combined_face.h" - -using ncnncv::NCNNCavaCombinedFace; - -void NCNNCavaCombinedFace::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW - int h = mat.rows; - int w = mat.cols; - in = ncnn::Mat::from_pixels_resize( - mat.data, ncnn::Mat::PIXEL_BGR2RGB, - w, h, input_width, input_height - ); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNCavaCombinedFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - ncnn::Mat embedding; - extractor.extract("embedding", embedding); - - const unsigned int hidden_dim = embedding.w; // 512 - const float *embedding_values = (float *) embedding.data; - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_cava_combined_face.h b/lite/ncnn/cv/ncnn_cava_combined_face.h deleted file mode 100644 index 1772938e..00000000 --- a/lite/ncnn/cv/ncnn_cava_combined_face.h +++ /dev/null @@ -1,37 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_CAVA_COMBINED_FACE_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_CAVA_COMBINED_FACE_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNCavaCombinedFace : public BasicNCNNHandler - { - public: - explicit NCNNCavaCombinedFace(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) - {}; - - ~NCNNCavaCombinedFace() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 128.0f, 1.f / 128.0f, 1.f / 128.0f}; - static constexpr const int input_width = 112; - static constexpr const int input_height = 112; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_CAVA_COMBINED_FACE_H diff --git a/lite/ncnn/cv/ncnn_cava_ghost_arcface.cpp b/lite/ncnn/cv/ncnn_cava_ghost_arcface.cpp deleted file mode 100644 index 0396b719..00000000 --- a/lite/ncnn/cv/ncnn_cava_ghost_arcface.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "ncnn_cava_ghost_arcface.h" - -using ncnncv::NCNNCavaGhostArcFace; - -void NCNNCavaGhostArcFace::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW - int h = mat.rows; - int w = mat.cols; - in = ncnn::Mat::from_pixels_resize( - mat.data, ncnn::Mat::PIXEL_BGR2RGB, - w, h, input_width, input_height - ); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNCavaGhostArcFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - ncnn::Mat embedding; - extractor.extract("embedding", embedding); - - const unsigned int hidden_dim = embedding.w; // 512 - const float *embedding_values = (float *) embedding.data; - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_cava_ghost_arcface.h b/lite/ncnn/cv/ncnn_cava_ghost_arcface.h deleted file mode 100644 index 3a79f263..00000000 --- a/lite/ncnn/cv/ncnn_cava_ghost_arcface.h +++ /dev/null @@ -1,38 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_CAVA_GHOST_ARCFACE_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_CAVA_GHOST_ARCFACE_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNCavaGhostArcFace : public BasicNCNNHandler - { - public: - explicit NCNNCavaGhostArcFace(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) - {}; - - ~NCNNCavaGhostArcFace() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 128.0f, 1.f / 128.0f, 1.f / 128.0f}; - static constexpr const int input_width = 112; - static constexpr const int input_height = 112; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_CAVA_GHOST_ARCFACE_H diff --git a/lite/ncnn/cv/ncnn_center_loss_face.cpp b/lite/ncnn/cv/ncnn_center_loss_face.cpp deleted file mode 100644 index ab3fb9c2..00000000 --- a/lite/ncnn/cv/ncnn_center_loss_face.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "ncnn_center_loss_face.h" - -using ncnncv::NCNNCenterLossFace; - -void NCNNCenterLossFace::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW - int h = mat.rows; - int w = mat.cols; - in = ncnn::Mat::from_pixels_resize( - mat.data, ncnn::Mat::PIXEL_BGR2RGB, - w, h, input_width, input_height - ); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNCenterLossFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - ncnn::Mat embedding; - extractor.extract("embedding", embedding); - - const unsigned int hidden_dim = embedding.w; // 512 - const float *embedding_values = (float *) embedding.data; - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_center_loss_face.h b/lite/ncnn/cv/ncnn_center_loss_face.h deleted file mode 100644 index 3e1f3e73..00000000 --- a/lite/ncnn/cv/ncnn_center_loss_face.h +++ /dev/null @@ -1,37 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_CENTER_LOSS_FACE_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_CENTER_LOSS_FACE_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNCenterLossFace : public BasicNCNNHandler - { - public: - explicit NCNNCenterLossFace(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) - {}; - - ~NCNNCenterLossFace() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - static constexpr const int input_width = 128; - static constexpr const int input_height = 96; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_CENTER_LOSS_FACE_H diff --git a/lite/ncnn/cv/ncnn_colorizer.cpp b/lite/ncnn/cv/ncnn_colorizer.cpp deleted file mode 100644 index 83e1c459..00000000 --- a/lite/ncnn/cv/ncnn_colorizer.cpp +++ /dev/null @@ -1,106 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "ncnn_colorizer.h" - -using ncnncv::NCNNColorizer; - -NCNNColorizer::NCNNColorizer( - const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNColorizer::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_l; // assume that input mat is L of Lab - mat.convertTo(mat_l, CV_32FC1, 1.0f, 0.f); // (256,256,1) range (0.,100.) - - in = ncnn::Mat(input_width, input_height, mat_l.data); -} - -void NCNNColorizer::detect(const cv::Mat &mat, types::ColorizeContent &colorize_content) -{ - if (mat.empty()) return; - const unsigned int height = mat.rows; - const unsigned int width = mat.cols; - - cv::Mat mat_rs = mat.clone(); - cv::resize(mat_rs, mat_rs, cv::Size(input_width, input_height)); // (256,256,3) - cv::Mat mat_rs_norm, mat_orig_norm; - mat_rs.convertTo(mat_rs_norm, CV_32FC3, 1.0f / 255.0f, 0.f); // (0.,1.) BGR - mat.convertTo(mat_orig_norm, CV_32FC3, 1.0f / 255.0f, 0.f); // (0.,1.) BGR - if (mat_rs_norm.empty() || mat_orig_norm.empty()) return; - - cv::Mat mat_lab_orig, mat_lab_rs; - cv::cvtColor(mat_rs_norm, mat_lab_rs, cv::COLOR_BGR2Lab); - cv::cvtColor(mat_orig_norm, mat_lab_orig, cv::COLOR_BGR2Lab); - - cv::Mat mat_rs_l, mat_orig_l; - std::vector mats_rs_lab, mats_orig_lab; - cv::split(mat_lab_rs, mats_rs_lab); - cv::split(mat_lab_orig, mats_orig_lab); - - mat_rs_l = mats_rs_lab.at(0); - mat_orig_l = mats_orig_lab.at(0); - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat_rs_l, input); // (1,1,256,256) - - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - // 3. fetch. - ncnn::Mat pred_ab; - extractor.extract("out_ab", pred_ab); // (1,2,256,256) -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(pred_ab, "out_ab"); -#endif - - const unsigned int rows = pred_ab.h; // H - const unsigned int cols = pred_ab.w; // W - const unsigned int pred_step = rows * cols; - - float *pred_ab_ptr = (float *) pred_ab.data; - - cv::Mat out_a_orig(rows, cols, CV_32FC1); - cv::Mat out_b_orig(rows, cols, CV_32FC1); - - for (unsigned int i = 0; i < rows; ++i) - { - float *pa = out_a_orig.ptr(i); - float *pb = out_b_orig.ptr(i); - for (unsigned int j = 0; j < cols; ++j) - { - pa[j] = pred_ab_ptr[0 * pred_step + i * cols + j]; - pb[j] = pred_ab_ptr[1 * pred_step + i * cols + j]; - } // CHW->HWC - } - - if (rows != height || cols != width) - { - cv::resize(out_a_orig, out_a_orig, cv::Size(width, height)); - cv::resize(out_b_orig, out_b_orig, cv::Size(width, height)); - } - - std::vector out_mats_lab; - out_mats_lab.push_back(mat_orig_l); - out_mats_lab.push_back(out_a_orig); - out_mats_lab.push_back(out_b_orig); - - cv::Mat merge_mat_lab, mat_bgr_norm; - cv::merge(out_mats_lab, merge_mat_lab); - if (merge_mat_lab.empty()) return; - cv::cvtColor(merge_mat_lab, mat_bgr_norm, cv::COLOR_Lab2BGR); // CV_32FC3 - mat_bgr_norm *= 255.0f; - - mat_bgr_norm.convertTo(colorize_content.mat, CV_8UC3); // uint8 - - colorize_content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_colorizer.h b/lite/ncnn/cv/ncnn_colorizer.h deleted file mode 100644 index 2a40f1e4..00000000 --- a/lite/ncnn/cv/ncnn_colorizer.h +++ /dev/null @@ -1,35 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_COLORIZER_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_COLORIZER_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNColorizer : public BasicNCNNHandler - { - public: - explicit NCNNColorizer(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); // - ~NCNNColorizer() override = default; - - private: - int input_height = 256; - int input_width = 256; - - private: - - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::ColorizeContent &colorize_content); - - }; -} - - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_COLORIZER_H diff --git a/lite/ncnn/cv/ncnn_deeplabv3_resnet101.cpp b/lite/ncnn/cv/ncnn_deeplabv3_resnet101.cpp deleted file mode 100644 index f6e672e8..00000000 --- a/lite/ncnn/cv/ncnn_deeplabv3_resnet101.cpp +++ /dev/null @@ -1,106 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "ncnn_deeplabv3_resnet101.h" - -using ncnncv::NCNNDeepLabV3ResNet101; - -NCNNDeepLabV3ResNet101::NCNNDeepLabV3ResNet101( - const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNDeepLabV3ResNet101::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - const int img_width = mat.cols; - const int img_height = mat.rows; - in = ncnn::Mat::from_pixels_resize(mat.data, - ncnn::Mat::PIXEL_BGR2RGB, - img_width, - img_height, - input_width, - input_height); - - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNDeepLabV3ResNet101::detect(const cv::Mat &mat, types::SegmentContent &content) -{ - if (mat.empty()) return; - const int img_width = mat.cols; - const int img_height = mat.rows; - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - - // 3. fetch. - ncnn::Mat scores; - extractor.extract("out", scores); // (1,21,h,w) c=21,h,w -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(scores, "out"); -#endif - - const unsigned int output_classes = scores.c; - const unsigned int output_height = scores.h; - const unsigned int output_width = scores.w; - - const float *scores_ptr = (float *) scores.data; - - // time cost! - content.names_map.clear(); - content.class_mat = cv::Mat(output_height, output_width, CV_8UC1, cv::Scalar(0)); - cv::resize(mat, content.color_mat, cv::Size(output_width, output_height)); // init color mat - - const unsigned int scores_step = output_height * output_width; // h x w - - for (unsigned int i = 0; i < output_height; ++i) - { - - uchar *p_class = content.class_mat.ptr(i); - cv::Vec3b *p_color = content.color_mat.ptr(i); - - for (unsigned int j = 0; j < output_width; ++j) - { - // argmax - unsigned int max_label = 0; - float max_conf = scores_ptr[0 * scores_step + i * output_width + j]; - - for (unsigned int l = 0; l < output_classes; ++l) - { - float conf = scores_ptr[l * scores_step + i * output_width + j]; - if (conf > max_conf) - { - max_conf = conf; - max_label = l; - } - } - - if (max_label == 0) continue; - - // assign label for pixel(i,j) - p_class[j] = cv::saturate_cast(max_label); - // assign color for detected class at pixel(i,j). - p_color[j][0] = cv::saturate_cast((max_label % 10) * 20); - p_color[j][1] = cv::saturate_cast((max_label % 5) * 40); - p_color[j][2] = cv::saturate_cast((max_label % 10) * 20); - // assign names map - content.names_map[max_label] = class_names[max_label - 1]; // max_label >= 1 - } - - } - - cv::resize(content.class_mat, content.class_mat, cv::Size(img_width, img_height)); - cv::resize(content.color_mat, content.color_mat, cv::Size(img_width, img_height)); - - content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_deeplabv3_resnet101.h b/lite/ncnn/cv/ncnn_deeplabv3_resnet101.h deleted file mode 100644 index 64e63883..00000000 --- a/lite/ncnn/cv/ncnn_deeplabv3_resnet101.h +++ /dev/null @@ -1,46 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_DEEPLABV3_RESNET101_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_DEEPLABV3_RESNET101_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNDeepLabV3ResNet101 : public BasicNCNNHandler - { - public: - explicit NCNNDeepLabV3ResNet101(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); // - ~NCNNDeepLabV3ResNet101() override = default; - - private: - const float norm_vals[3] = {(1.f / 0.229f) * (1.f / 255.f), - (1.f / 0.224f) * (1.f / 255.f), - (1.f / 0.225f) * (1.f / 255.f)}; - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; // RGB - - private: - int input_height = 512; // init only, will change according to input mat. - int input_width = 512; // init only, will change according to input mat. - - const char *class_names[20] = { - "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", - "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", - "train", "tvmonitor" - }; // 20 classes - - private: - - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::SegmentContent &content); - - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_DEEPLABV3_RESNET101_H diff --git a/lite/ncnn/cv/ncnn_densenet.h b/lite/ncnn/cv/ncnn_densenet.h deleted file mode 100644 index 7aa2f23b..00000000 --- a/lite/ncnn/cv/ncnn_densenet.h +++ /dev/null @@ -1,413 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_DENSENET_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_DENSENET_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNDenseNet : public BasicNCNNHandler - { - public: - explicit NCNNDenseNet(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNDenseNet() override = default; - - private: - const int input_height = 224; - const int input_width = 224; - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_DENSENET_H diff --git a/lite/ncnn/cv/ncnn_densent.cpp b/lite/ncnn/cv/ncnn_densent.cpp deleted file mode 100644 index c781beb7..00000000 --- a/lite/ncnn/cv/ncnn_densent.cpp +++ /dev/null @@ -1,62 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "ncnn_densenet.h" -#include "lite/utils.h" - -using ncnncv::NCNNDenseNet; - -NCNNDenseNet::NCNNDenseNet(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNDenseNet::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNDenseNet::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - // 3. fetch. - ncnn::Mat logits_mat; - extractor.extract("logits", logits_mat); // c=1,h=1,w=1000 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(logits_mat, "logits"); -#endif - - const unsigned int num_classes = logits_mat.w; - const float *logits = (float *) logits_mat.data; - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_efficient_emotion7.cpp b/lite/ncnn/cv/ncnn_efficient_emotion7.cpp deleted file mode 100644 index 5124fcea..00000000 --- a/lite/ncnn/cv/ncnn_efficient_emotion7.cpp +++ /dev/null @@ -1,54 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#include "ncnn_efficient_emotion7.h" -#include "lite/utils.h" - -using ncnncv::NCNNEfficientEmotion7; - -NCNNEfficientEmotion7::NCNNEfficientEmotion7(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNEfficientEmotion7::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNEfficientEmotion7::detect(const cv::Mat &mat, types::Emotions &emotions) -{ - if (mat.empty()) return; - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - // 3. fetch. - ncnn::Mat emotion_logits; - extractor.extract("logits", emotion_logits); // c=1,h=1,w=7 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(emotion_logits, "logits"); -#endif - - const unsigned int num_emotions = emotion_logits.w; - - unsigned int pred_label = 0; - const float *pred_logits_ptr = (float *) emotion_logits.data; - - auto softmax_probs = lite::utils::math::softmax(pred_logits_ptr, num_emotions, pred_label); - emotions.label = pred_label; - emotions.score = softmax_probs[pred_label]; - emotions.text = emotion_texts[pred_label]; - emotions.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_efficient_emotion7.h b/lite/ncnn/cv/ncnn_efficient_emotion7.h deleted file mode 100644 index fa734d05..00000000 --- a/lite/ncnn/cv/ncnn_efficient_emotion7.h +++ /dev/null @@ -1,38 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_EFFICIENT_EMOTION7_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_EFFICIENT_EMOTION7_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNEfficientEmotion7 : public BasicNCNNHandler - { - public: - explicit NCNNEfficientEmotion7(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNEfficientEmotion7() override = default; - - private: - const int input_height = 224; - const int input_width = 224; - const float mean_vals[3] = {255.f * 0.485f, 255.f * 0.456, 255.f * 0.406f}; - const float norm_vals[3] = {1.f / (255.f * 0.229f), 1.f / (255.f * 0.224f), 1.f / (255.f * 0.225f)}; - const char *emotion_texts[7] = { - "angry", "disgust", "fear", "happiness", "neutral", "sadness", "surprise" - }; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::Emotions &emotions); - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_EFFICIENT_EMOTION7_H diff --git a/lite/ncnn/cv/ncnn_efficient_emotion8.cpp b/lite/ncnn/cv/ncnn_efficient_emotion8.cpp deleted file mode 100644 index 9c5605b8..00000000 --- a/lite/ncnn/cv/ncnn_efficient_emotion8.cpp +++ /dev/null @@ -1,54 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#include "ncnn_efficient_emotion8.h" -#include "lite/utils.h" - -using ncnncv::NCNNEfficientEmotion8; - -NCNNEfficientEmotion8::NCNNEfficientEmotion8(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNEfficientEmotion8::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNEfficientEmotion8::detect(const cv::Mat &mat, types::Emotions &emotions) -{ - if (mat.empty()) return; - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - // 3. fetch. - ncnn::Mat emotion_logits; - extractor.extract("logits", emotion_logits); // c=1,h=1,w=8 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(emotion_logits, "logits"); -#endif - - const unsigned int num_emotions = emotion_logits.w; - - unsigned int pred_label = 0; - const float *pred_logits_ptr = (float *) emotion_logits.data; - - auto softmax_probs = lite::utils::math::softmax(pred_logits_ptr, num_emotions, pred_label); - emotions.label = pred_label; - emotions.score = softmax_probs[pred_label]; - emotions.text = emotion_texts[pred_label]; - emotions.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_efficient_emotion8.h b/lite/ncnn/cv/ncnn_efficient_emotion8.h deleted file mode 100644 index 08d38a6a..00000000 --- a/lite/ncnn/cv/ncnn_efficient_emotion8.h +++ /dev/null @@ -1,39 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_EFFICIENT_EMOTION8_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_EFFICIENT_EMOTION8_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNEfficientEmotion8 : public BasicNCNNHandler - { - public: - explicit NCNNEfficientEmotion8(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNEfficientEmotion8() override = default; - - private: - const int input_height = 224; - const int input_width = 224; - const float mean_vals[3] = {255.f * 0.485f, 255.f * 0.456, 255.f * 0.406f}; - const float norm_vals[3] = {1.f / (255.f * 0.229f), 1.f / (255.f * 0.224f), 1.f / (255.f * 0.225f)}; - const char *emotion_texts[8] = { - "angry", "contempt", "disgust", "fear", "happiness", "neutral", "sadness", "surprise" - }; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::Emotions &emotions); - }; -} - - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_EFFICIENT_EMOTION8_H diff --git a/lite/ncnn/cv/ncnn_efficientnet_lite4.cpp b/lite/ncnn/cv/ncnn_efficientnet_lite4.cpp deleted file mode 100644 index d9bccd53..00000000 --- a/lite/ncnn/cv/ncnn_efficientnet_lite4.cpp +++ /dev/null @@ -1,60 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "ncnn_efficientnet_lite4.h" -#include "lite/utils.h" - -using ncnncv::NCNNEfficientNetLite4; - -NCNNEfficientNetLite4::NCNNEfficientNetLite4(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNEfficientNetLite4::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNEfficientNetLite4::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("images:0", input); - // 3. fetch. - ncnn::Mat scores_mat; - extractor.extract("Softmax:0", scores_mat); // c=1,h=1,w=1000 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(scores_mat, "Softmax:0"); -#endif - - const unsigned int num_classes = scores_mat.w; - const float *scores = (float *) scores_mat.data; - - std::vector sorted_indices = lite::utils::math::argsort(scores, num_classes); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_efficientnet_lite4.h b/lite/ncnn/cv/ncnn_efficientnet_lite4.h deleted file mode 100644 index 3ee15ff9..00000000 --- a/lite/ncnn/cv/ncnn_efficientnet_lite4.h +++ /dev/null @@ -1,412 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_EFFICIENTNET_LITE4_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_EFFICIENTNET_LITE4_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNEfficientNetLite4 : public BasicNCNNHandler - { - public: - explicit NCNNEfficientNetLite4(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNEfficientNetLite4() override = default; - - private: - const int input_height = 224; - const int input_width = 224; - const float mean_vals[3] = {127.f, 127.f, 127.f}; - const float norm_vals[3] = {1.f / 128.f, 1.f / 128.f, 1.f / 128.f}; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - - }; -} - - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_EFFICIENTNET_LITE4_H diff --git a/lite/ncnn/cv/ncnn_emotion_ferplus.cpp b/lite/ncnn/cv/ncnn_emotion_ferplus.cpp deleted file mode 100644 index 00cb18fd..00000000 --- a/lite/ncnn/cv/ncnn_emotion_ferplus.cpp +++ /dev/null @@ -1,54 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#include "ncnn_emotion_ferplus.h" -#include "lite/utils.h" - -using ncnncv::NCNNEmotionFerPlus; - -NCNNEmotionFerPlus::NCNNEmotionFerPlus(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNEmotionFerPlus::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2GRAY, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNEmotionFerPlus::detect(const cv::Mat &mat, types::Emotions &emotions) -{ - if (mat.empty()) return; - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("Input3", input); - // 3. fetch. - ncnn::Mat emotion_logits; - extractor.extract("Plus692_Output_0", emotion_logits); // c=1,h=1,w=8 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(emotion_logits, "Plus692_Output_0"); -#endif - - const unsigned int num_emotions = emotion_logits.w; - - unsigned int pred_label = 0; - const float *pred_logits_ptr = (float *) emotion_logits.data; - - auto softmax_probs = lite::utils::math::softmax(pred_logits_ptr, num_emotions, pred_label); - emotions.label = pred_label; - emotions.score = softmax_probs[pred_label]; - emotions.text = emotion_texts[pred_label]; - emotions.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_emotion_ferplus.h b/lite/ncnn/cv/ncnn_emotion_ferplus.h deleted file mode 100644 index d304352f..00000000 --- a/lite/ncnn/cv/ncnn_emotion_ferplus.h +++ /dev/null @@ -1,40 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_EMOTION_FERPLUS_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_EMOTION_FERPLUS_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNEmotionFerPlus : public BasicNCNNHandler - { - public: - explicit NCNNEmotionFerPlus(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNEmotionFerPlus() override = default; - - private: - const int input_height = 64; - const int input_width = 64; - const float mean_vals[1] = {0.f}; - const float norm_vals[1] = {1.0f}; - const char *emotion_texts[8] = { - "neutral", "happiness", "surprise", "sadness", "anger", - "disgust", "fear", "contempt" - }; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::Emotions &emotions); - }; -} - - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_EMOTION_FERPLUS_H diff --git a/lite/ncnn/cv/ncnn_face_landmarks_1000.cpp b/lite/ncnn/cv/ncnn_face_landmarks_1000.cpp deleted file mode 100644 index b5e2b05f..00000000 --- a/lite/ncnn/cv/ncnn_face_landmarks_1000.cpp +++ /dev/null @@ -1,60 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#include "ncnn_face_landmarks_1000.h" - -using ncnncv::NCNNFaceLandmark1000; - -NCNNFaceLandmark1000::NCNNFaceLandmark1000(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNFaceLandmark1000::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2GRAY, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNFaceLandmark1000::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input0", input); - // 3. fetch landmarks. - ncnn::Mat landmarks_norm; - extractor.extract("output0", landmarks_norm); // c=1,h=1,w=1953 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(landmarks_norm, "output0"); -#endif - unsigned int num_landmarks = landmarks_norm.w; - if (num_landmarks > 1946) num_landmarks = 1946; - - const float *landmarks_ptr = (float *) landmarks_norm.data; - - for (unsigned int i = 0; i < num_landmarks; i += 2) - { - float x = landmarks_ptr[i]; - float y = landmarks_ptr[i + 1]; - - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - landmarks.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_face_landmarks_1000.h b/lite/ncnn/cv/ncnn_face_landmarks_1000.h deleted file mode 100644 index 83fd2ae8..00000000 --- a/lite/ncnn/cv/ncnn_face_landmarks_1000.h +++ /dev/null @@ -1,36 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_FACE_LANDMARKS_1000_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_FACE_LANDMARKS_1000_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNFaceLandmark1000 : public BasicNCNNHandler - { - public: - explicit NCNNFaceLandmark1000(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNFaceLandmark1000() override = default; - - private: - const int input_height = 128; - const int input_width = 128; - const float mean_vals[1] = {0.0f}; - const float norm_vals[1] = {1.0f}; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - }; -} - - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_FACE_LANDMARKS_1000_H diff --git a/lite/ncnn/cv/ncnn_face_parsing_bisenet.cpp b/lite/ncnn/cv/ncnn_face_parsing_bisenet.cpp deleted file mode 100644 index 3f4c8c83..00000000 --- a/lite/ncnn/cv/ncnn_face_parsing_bisenet.cpp +++ /dev/null @@ -1,190 +0,0 @@ -// -// Created by DefTruth on 2022/7/2. -// - -#include "ncnn_face_parsing_bisenet.h" - -using ncnncv::NCNNFaceParsingBiSeNet; - -NCNNFaceParsingBiSeNet::NCNNFaceParsingBiSeNet(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads, - unsigned int _input_height, - unsigned int _input_width) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads), - input_height(_input_height), input_width(_input_width) -{ -} - -void NCNNFaceParsingBiSeNet::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - // will do deepcopy inside ncnn - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNFaceParsingBiSeNet::detect(const cv::Mat &mat, types::FaceParsingContent &content, - bool minimum_post_process) -{ - if (mat.empty()) return; - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - // 3. generate mask - this->generate_mask(extractor, mat, content, minimum_post_process); -} - -static inline uchar argmax(float *mutable_ptr, const unsigned int &step) -{ - std::vector logits(19, 0.f); - for (unsigned int i = 0; i < 19; ++i) - logits[i] = *(mutable_ptr + i * step); - uchar label = 0; - float max_logit = logits[0]; - for (unsigned int i = 1; i < 19; ++i) - { - if (logits[i] > max_logit) - { - max_logit = logits[i]; - label = (uchar) i; - } - } - return label; -} - -static const uchar part_colors[20][3] = { - {255, 0, 0}, - {255, 85, 0}, - {255, 170, 0}, - {255, 0, 85}, - {255, 0, 170}, - {0, 255, 0}, - {85, 255, 0}, - {170, 255, 0}, - {0, 255, 85}, - {0, 255, 170}, - {0, 0, 255}, - {85, 0, 255}, - {170, 0, 255}, - {0, 85, 255}, - {0, 170, 255}, - {255, 255, 0}, - {255, 255, 85}, - {255, 255, 170}, - {255, 0, 255}, - {255, 85, 255} -}; - -void NCNNFaceParsingBiSeNet::generate_mask(ncnn::Extractor &extractor, const cv::Mat &mat, - types::FaceParsingContent &content, - bool minimum_post_process) -{ - ncnn::Mat output; - extractor.extract("out", output); -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(output, "out"); -#endif - const unsigned int h = mat.rows; - const unsigned int w = mat.cols; - - const unsigned int out_h = output.h; - const unsigned int out_w = output.w; - const unsigned int channel_step = out_h * out_w; - - float *output_ptr = (float *) output.data; - std::vector elements(channel_step, 0); // allocate - for (unsigned int i = 0; i < channel_step; ++i) - elements[i] = argmax(output_ptr + i, channel_step); - - cv::Mat label(out_h, out_w, CV_8UC1, elements.data()); - - if (!minimum_post_process) - { - // FaceParsingBiSeNet only predict integer label mask, - // no fgr. So, the fake fgr and merge mat may not need, - // let the fgr mat and merge mat empty to - // Speed up the post processes. - const uchar *label_ptr = label.data; - cv::Mat color_mat(out_h, out_w, CV_8UC3, cv::Scalar(255, 255, 255)); - for (unsigned int i = 0; i < color_mat.rows; ++i) - { - cv::Vec3b *p = color_mat.ptr(i); - for (unsigned int j = 0; j < color_mat.cols; ++j) - { - if (label_ptr[i * out_w + j] == 0) continue; - p[j][0] = part_colors[label_ptr[i * out_w + j]][0]; - p[j][1] = part_colors[label_ptr[i * out_w + j]][1]; - p[j][2] = part_colors[label_ptr[i * out_w + j]][2]; - } - } - if (out_h != h || out_w != w) - cv::resize(color_mat, color_mat, cv::Size(w, h)); - cv::addWeighted(mat, 0.4, color_mat, 0.6, 0., content.merge); - } - // already allocated a new continuous memory after resize. - if (out_h != h || out_w != w) cv::resize(label, label, cv::Size(w, h)); - // need clone to allocate a new continuous memory if not performed resize. - // The memory elements point to will release after return. - else label = label.clone(); - - content.label = label; // auto handle the memory inside ocv with smart ref. - content.flag = true; -} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lite/ncnn/cv/ncnn_face_parsing_bisenet.h b/lite/ncnn/cv/ncnn_face_parsing_bisenet.h deleted file mode 100644 index f1c82ffe..00000000 --- a/lite/ncnn/cv/ncnn_face_parsing_bisenet.h +++ /dev/null @@ -1,43 +0,0 @@ -// -// Created by DefTruth on 2022/7/2. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_FACE_PARSING_BISENET_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_FACE_PARSING_BISENET_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNFaceParsingBiSeNet : public BasicNCNNHandler - { - public: - explicit NCNNFaceParsingBiSeNet(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1, - unsigned int _input_height = 512, - unsigned int _input_width = 512); - - ~NCNNFaceParsingBiSeNet() override = default; - - private: - const int input_height; - const int input_width; - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; // RGB - const float norm_vals[3] = {1.f / (0.229f * 255.f), 1.f / (0.224f * 255.f), 1.f / (0.225f * 255.f)}; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - void generate_mask(ncnn::Extractor &extractor, - const cv::Mat &mat, types::FaceParsingContent &content, - bool minimum_post_process = false); - - public: - void detect(const cv::Mat &mat, types::FaceParsingContent &content, - bool minimum_post_process = false); - - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_FACE_PARSING_BISENET_H diff --git a/lite/ncnn/cv/ncnn_faceboxes.cpp b/lite/ncnn/cv/ncnn_faceboxes.cpp deleted file mode 100644 index 431418d8..00000000 --- a/lite/ncnn/cv/ncnn_faceboxes.cpp +++ /dev/null @@ -1,203 +0,0 @@ -// -// Created by DefTruth on 2021/11/20. -// - -#include "ncnn_faceboxes.h" -#include "lite/utils.h" - -using ncnncv::NCNNFaceBoxes; - -NCNNFaceBoxes::NCNNFaceBoxes(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads, - int _input_height, - int _input_width) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads), - input_height(_input_height), input_width(_input_width) -{ -} - -void NCNNFaceBoxes::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNFaceBoxes::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - // 3.rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(bbox_collection, extractor, score_threshold, img_height, img_width); - // 4. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void NCNNFaceBoxes::generate_anchors(const int target_height, const int target_width, - std::vector &anchors) -{ - std::vector> feature_maps; - for (auto step: steps) - { - feature_maps.push_back( - { - (int) std::ceil((float) target_height / (float) step), - (int) std::ceil((float) target_width / (float) step) - } // ceil - ); - } - - anchors.clear(); - const int num_feature_map = feature_maps.size(); - - for (int k = 0; k < num_feature_map; ++k) - { - auto f_map = feature_maps.at(k); // e.g [640//32,640/32] - auto tmp_min_sizes = min_sizes.at(k); // e.g [32,64,128] - int f_h = f_map.at(0); - int f_w = f_map.at(1); - std::vector offset_32 = {0.f, 0.25f, 0.5f, 0.75f}; - std::vector offset_64 = {0.f, 0.5f}; - - for (int i = 0; i < f_h; ++i) - { - for (int j = 0; j < f_w; ++j) - { - for (auto min_size: tmp_min_sizes) - { - float s_kx = (float) min_size / (float) target_width; // e.g 32/w - float s_ky = (float) min_size / (float) target_height; // e.g 32/h - - // 32 anchor size - if (min_size == 32) - { - // range y offsets first and then x - for (auto offset_y: offset_32) - { - for (auto offset_x: offset_32) - { - // (x or y + offset) * step / w or h normalized loc mapping to input size. - float cx = ((float) j + offset_x) * (float) steps.at(k) / (float) target_width; - float cy = ((float) i + offset_y) * (float) steps.at(k) / (float) target_height; - anchors.push_back(FaceBoxesAnchor{cx, cy, s_kx, s_ky}); // without clip - } - } - - } // 64 anchor size - else if (min_size == 64) - { - // range y offsets first and then x - for (auto offset_y: offset_64) - { - for (auto offset_x: offset_64) - { - float cx = ((float) j + offset_x) * (float) steps.at(k) / (float) target_width; - float cy = ((float) i + offset_y) * (float) steps.at(k) / (float) target_height; - anchors.push_back(FaceBoxesAnchor{cx, cy, s_kx, s_ky}); // without clip - } - } - - } // other anchor size - else - { - float cx = ((float) j + 0.5f) * (float) steps.at(k) / (float) target_width; - float cy = ((float) i + 0.5f) * (float) steps.at(k) / (float) target_height; - anchors.push_back(FaceBoxesAnchor{cx, cy, s_kx, s_ky}); // without clip - } - } - } - } - } -} - -void NCNNFaceBoxes::generate_bboxes(std::vector &bbox_collection, - ncnn::Extractor &extractor, float score_threshold, - float img_height, float img_width) -{ - ncnn::Mat bboxes, probs; - extractor.extract("bbox", bboxes); // c=1 h=? w=4 - extractor.extract("conf", probs); // c=1 h=? w=2 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(bboxes, "bbox"); - BasicNCNNHandler::print_shape(probs, "conf"); -#endif - const unsigned int bbox_num = bboxes.h; // n = ? - - std::vector anchors; - this->generate_anchors(input_height, input_width, anchors); - - const unsigned int num_anchors = anchors.size(); - if (num_anchors != bbox_num) - throw std::runtime_error("mismatch num_anchors != bbox_num"); - - const float *bboxes_ptr = (float *) bboxes.data; - const float *probs_ptr = (float *) probs.data; - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - float conf = probs_ptr[2 * i + 1]; - if (conf < score_threshold) continue; // filter first. - - float prior_cx = anchors.at(i).cx; - float prior_cy = anchors.at(i).cy; - float prior_s_kx = anchors.at(i).s_kx; - float prior_s_ky = anchors.at(i).s_ky; - - float dx = bboxes_ptr[4 * i + 0]; - float dy = bboxes_ptr[4 * i + 1]; - float dw = bboxes_ptr[4 * i + 2]; - float dh = bboxes_ptr[4 * i + 3]; - // ref: https://github.com/biubug6/Pytorch_Retinaface/blob/master/utils/box_utils.py - float cx = prior_cx + dx * variance[0] * prior_s_kx; - float cy = prior_cy + dy * variance[0] * prior_s_ky; - float w = prior_s_kx * std::exp(dw * variance[1]); - float h = prior_s_ky * std::exp(dh * variance[1]); // norm coor (0.,1.) - - types::Boxf box; - box.x1 = (cx - w / 2.f) * img_width; - box.y1 = (cy - h / 2.f) * img_height; - box.x2 = (cx + w / 2.f) * img_width; - box.y2 = (cy + h / 2.f) * img_height; - box.score = conf; - box.label = 1; - box.label_text = "face"; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - -#if LITENCNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void NCNNFaceBoxes::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - diff --git a/lite/ncnn/cv/ncnn_faceboxes.h b/lite/ncnn/cv/ncnn_faceboxes.h deleted file mode 100644 index 1f6aa62b..00000000 --- a/lite/ncnn/cv/ncnn_faceboxes.h +++ /dev/null @@ -1,79 +0,0 @@ -// -// Created by DefTruth on 2021/11/20. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_FACEBOXES_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_FACEBOXES_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNFaceBoxes : public BasicNCNNHandler - { - public: - explicit NCNNFaceBoxes(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1, - int _input_height = 640, - int _input_width = 640); - - ~NCNNFaceBoxes() override = default; - - private: - // nested classes - struct FaceBoxesAnchor - { - float cx; - float cy; - float s_kx; - float s_ky; - }; - - private: - const int input_height; // 640/320 - const int input_width; // 640/320 - - const float mean_vals[3] = {104.f, 117.f, 123.f}; // bgr order - const float norm_vals[3] = {1.f, 1.f, 1.f}; - const float variance[2] = {0.1f, 0.2f}; - std::vector steps = {32, 64, 128}; - std::vector> min_sizes = { - {32, 64, 128}, - {256}, - {512} - }; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - void generate_anchors(const int target_height, - const int target_width, - std::vector &anchors); - - - void generate_bboxes(std::vector &bbox_collection, - ncnn::Extractor &extractor, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.7f, float iou_threshold = 0.3f, - unsigned int topk = 300, unsigned int nms_type = 0); - - - }; -} - - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_FACEBOXES_H diff --git a/lite/ncnn/cv/ncnn_faceboxesv2.cpp b/lite/ncnn/cv/ncnn_faceboxesv2.cpp deleted file mode 100644 index f0b33014..00000000 --- a/lite/ncnn/cv/ncnn_faceboxesv2.cpp +++ /dev/null @@ -1,203 +0,0 @@ -// -// Created by DefTruth on 2022/3/19. -// - -#include "ncnn_faceboxesv2.h" -#include "lite/utils.h" - -using ncnncv::NCNNFaceBoxesV2; - -NCNNFaceBoxesV2::NCNNFaceBoxesV2(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads, - int _input_height, - int _input_width) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads), - input_height(_input_height), input_width(_input_width) -{ -} - -void NCNNFaceBoxesV2::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNFaceBoxesV2::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("img", input); - // 3.rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(bbox_collection, extractor, score_threshold, img_height, img_width); - // 4. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void NCNNFaceBoxesV2::generate_anchors(const int target_height, const int target_width, - std::vector &anchors) -{ - std::vector> feature_maps; - for (auto step: steps) - { - feature_maps.push_back( - { - (int) std::ceil((float) target_height / (float) step), - (int) std::ceil((float) target_width / (float) step) - } // ceil - ); - } - - anchors.clear(); - const int num_feature_map = feature_maps.size(); - - for (int k = 0; k < num_feature_map; ++k) - { - auto f_map = feature_maps.at(k); // e.g [640//32,640/32] - auto tmp_min_sizes = min_sizes.at(k); // e.g [32,64,128] - int f_h = f_map.at(0); - int f_w = f_map.at(1); - std::vector offset_32 = {0.f, 0.25f, 0.5f, 0.75f}; - std::vector offset_64 = {0.f, 0.5f}; - - for (int i = 0; i < f_h; ++i) - { - for (int j = 0; j < f_w; ++j) - { - for (auto min_size: tmp_min_sizes) - { - float s_kx = (float) min_size / (float) target_width; // e.g 32/w - float s_ky = (float) min_size / (float) target_height; // e.g 32/h - - // 32 anchor size - if (min_size == 32) - { - // range y offsets first and then x - for (auto offset_y: offset_32) - { - for (auto offset_x: offset_32) - { - // (x or y + offset) * step / w or h normalized loc mapping to input size. - float cx = ((float) j + offset_x) * (float) steps.at(k) / (float) target_width; - float cy = ((float) i + offset_y) * (float) steps.at(k) / (float) target_height; - anchors.push_back(FaceBoxesAnchorV2{cx, cy, s_kx, s_ky}); // without clip - } - } - - } // 64 anchor size - else if (min_size == 64) - { - // range y offsets first and then x - for (auto offset_y: offset_64) - { - for (auto offset_x: offset_64) - { - float cx = ((float) j + offset_x) * (float) steps.at(k) / (float) target_width; - float cy = ((float) i + offset_y) * (float) steps.at(k) / (float) target_height; - anchors.push_back(FaceBoxesAnchorV2{cx, cy, s_kx, s_ky}); // without clip - } - } - - } // other anchor size - else - { - float cx = ((float) j + 0.5f) * (float) steps.at(k) / (float) target_width; - float cy = ((float) i + 0.5f) * (float) steps.at(k) / (float) target_height; - anchors.push_back(FaceBoxesAnchorV2{cx, cy, s_kx, s_ky}); // without clip - } - } - } - } - } -} - -void NCNNFaceBoxesV2::generate_bboxes(std::vector &bbox_collection, - ncnn::Extractor &extractor, float score_threshold, - float img_height, float img_width) -{ - ncnn::Mat bboxes, probs; - extractor.extract("loc", bboxes); // c=1 h=? w=4 - extractor.extract("conf", probs); // c=1 h=? w=2 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(bboxes, "loc"); - BasicNCNNHandler::print_shape(probs, "conf"); -#endif - const unsigned int bbox_num = bboxes.h; // n = ? - - std::vector anchors; - this->generate_anchors(input_height, input_width, anchors); - - const unsigned int num_anchors = anchors.size(); - if (num_anchors != bbox_num) - throw std::runtime_error("mismatch num_anchors != bbox_num"); - - const float *bboxes_ptr = (float *) bboxes.data; - const float *probs_ptr = (float *) probs.data; - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - float conf = probs_ptr[2 * i + 1]; - if (conf < score_threshold) continue; // filter first. - - float prior_cx = anchors.at(i).cx; - float prior_cy = anchors.at(i).cy; - float prior_s_kx = anchors.at(i).s_kx; - float prior_s_ky = anchors.at(i).s_ky; - - float dx = bboxes_ptr[4 * i + 0]; - float dy = bboxes_ptr[4 * i + 1]; - float dw = bboxes_ptr[4 * i + 2]; - float dh = bboxes_ptr[4 * i + 3]; - // ref: https://github.com/biubug6/Pytorch_Retinaface/blob/master/utils/box_utils.py - float cx = prior_cx + dx * variance[0] * prior_s_kx; - float cy = prior_cy + dy * variance[0] * prior_s_ky; - float w = prior_s_kx * std::exp(dw * variance[1]); - float h = prior_s_ky * std::exp(dh * variance[1]); // norm coor (0.,1.) - - types::Boxf box; - box.x1 = (cx - w / 2.f) * img_width; - box.y1 = (cy - h / 2.f) * img_height; - box.x2 = (cx + w / 2.f) * img_width; - box.y2 = (cy + h / 2.f) * img_height; - box.score = conf; - box.label = 1; - box.label_text = "face"; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - -#if LITENCNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void NCNNFaceBoxesV2::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - diff --git a/lite/ncnn/cv/ncnn_faceboxesv2.h b/lite/ncnn/cv/ncnn_faceboxesv2.h deleted file mode 100644 index 09042bf2..00000000 --- a/lite/ncnn/cv/ncnn_faceboxesv2.h +++ /dev/null @@ -1,79 +0,0 @@ -// -// Created by DefTruth on 2022/3/19. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_FACEBOXESV2_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_FACEBOXESV2_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNFaceBoxesV2 : public BasicNCNNHandler - { - public: - explicit NCNNFaceBoxesV2(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1, - int _input_height = 640, - int _input_width = 640); - - ~NCNNFaceBoxesV2() override = default; - - private: - // nested classes - struct FaceBoxesAnchorV2 - { - float cx; - float cy; - float s_kx; - float s_ky; - }; - - private: - const int input_height; // 640/320 - const int input_width; // 640/320 - - const float mean_vals[3] = {104.f, 117.f, 123.f}; // bgr order - const float norm_vals[3] = {1.f, 1.f, 1.f}; - const float variance[2] = {0.1f, 0.2f}; - std::vector steps = {32, 64, 128}; - std::vector> min_sizes = { - {32, 64, 128}, - {256}, - {512} - }; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - void generate_anchors(const int target_height, - const int target_width, - std::vector &anchors); - - - void generate_bboxes(std::vector &bbox_collection, - ncnn::Extractor &extractor, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.35f, float iou_threshold = 0.3f, - unsigned int topk = 300, unsigned int nms_type = 0); - - - }; -} - - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_FACEBOXESV2_H diff --git a/lite/ncnn/cv/ncnn_facenet.cpp b/lite/ncnn/cv/ncnn_facenet.cpp deleted file mode 100644 index 8be9ff1b..00000000 --- a/lite/ncnn/cv/ncnn_facenet.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "ncnn_facenet.h" - -using ncnncv::NCNNFaceNet; - -void NCNNFaceNet::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW - int h = mat.rows; - int w = mat.cols; - in = ncnn::Mat::from_pixels_resize( - mat.data, ncnn::Mat::PIXEL_BGR2RGB, - w, h, input_width, input_height - ); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNFaceNet::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - ncnn::Mat embedding; - extractor.extract("embedding", embedding); - - const unsigned int hidden_dim = embedding.w; // 512 - const float *embedding_values = (float *) embedding.data; - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_facenet.h b/lite/ncnn/cv/ncnn_facenet.h deleted file mode 100644 index 824dfd03..00000000 --- a/lite/ncnn/cv/ncnn_facenet.h +++ /dev/null @@ -1,37 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_FACENET_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_FACENET_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNFaceNet : public BasicNCNNHandler - { - public: - explicit NCNNFaceNet(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) - {}; - - ~NCNNFaceNet() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 128.0f, 1.f / 128.0f, 1.f / 128.0f}; - static constexpr const int input_width = 160; - static constexpr const int input_height = 160; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_FACENET_H diff --git a/lite/ncnn/cv/ncnn_fast_style_transfer.cpp b/lite/ncnn/cv/ncnn_fast_style_transfer.cpp deleted file mode 100644 index 52286f43..00000000 --- a/lite/ncnn/cv/ncnn_fast_style_transfer.cpp +++ /dev/null @@ -1,67 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "ncnn_fast_style_transfer.h" - -using ncnncv::NCNNFastStyleTransfer; - -NCNNFastStyleTransfer::NCNNFastStyleTransfer( - const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNFastStyleTransfer::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat canvas; - cv::resize(mat, canvas, cv::Size(input_width, input_height)); - in = ncnn::Mat::from_pixels(canvas.data, - ncnn::Mat::PIXEL_BGR2RGB, - input_width, - input_height); - - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNFastStyleTransfer::detect(const cv::Mat &mat, types::StyleContent &style_content) -{ - if (mat.empty()) return; - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input1", input); - // 3. fetch. - ncnn::Mat pred; - extractor.extract("output1", pred); // (1,3,224,224) -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(pred, "output1"); -#endif - - const unsigned int rows = pred.h; // H - const unsigned int cols = pred.w; // W - const unsigned int channel_step = rows * cols; - - float *pred_ptr = (float *) pred.data; - - // fast copy & channel transpose(CHW->HWC). - cv::Mat rmat(rows, cols, CV_32FC1, pred_ptr); // ref only, zero-copy. - cv::Mat gmat(rows, cols, CV_32FC1, pred_ptr + channel_step); - cv::Mat bmat(rows, cols, CV_32FC1, pred_ptr + 2 * channel_step); - std::vector channel_mats; - channel_mats.push_back(bmat); - channel_mats.push_back(gmat); - channel_mats.push_back(rmat); - - cv::merge(channel_mats, style_content.mat); // BGR - - style_content.mat.convertTo(style_content.mat, CV_8UC3); - - style_content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_fast_style_transfer.h b/lite/ncnn/cv/ncnn_fast_style_transfer.h deleted file mode 100644 index 60cd74b6..00000000 --- a/lite/ncnn/cv/ncnn_fast_style_transfer.h +++ /dev/null @@ -1,39 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_FAST_STYLE_TRANSFER_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_FAST_STYLE_TRANSFER_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNFastStyleTransfer : public BasicNCNNHandler - { - public: - explicit NCNNFastStyleTransfer(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); // - ~NCNNFastStyleTransfer() override = default; - - private: - const float mean_vals[3] = {0.f, 0.f, 0.f}; - const float norm_vals[3] = {1.f, 1.f, 1.f}; - - private: - int input_height = 224; - int input_width = 224; - - private: - - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::StyleContent &style_content); - - }; -} - - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_FAST_STYLE_TRANSFER_H diff --git a/lite/ncnn/cv/ncnn_fcn_resnet101.cpp b/lite/ncnn/cv/ncnn_fcn_resnet101.cpp deleted file mode 100644 index 1860da77..00000000 --- a/lite/ncnn/cv/ncnn_fcn_resnet101.cpp +++ /dev/null @@ -1,107 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "ncnn_fcn_resnet101.h" - -using ncnncv::NCNNFCNResNet101; - -NCNNFCNResNet101::NCNNFCNResNet101( - const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNFCNResNet101::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - const int img_width = mat.cols; - const int img_height = mat.rows; - - in = ncnn::Mat::from_pixels_resize(mat.data, - ncnn::Mat::PIXEL_BGR2RGB, - img_width, - img_height, - input_width, - input_height); - - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNFCNResNet101::detect(const cv::Mat &mat, types::SegmentContent &content) -{ - if (mat.empty()) return; - const int img_width = mat.cols; - const int img_height = mat.rows; - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - - // 3. fetch. - ncnn::Mat scores; - extractor.extract("out", scores); // (1,21,h,w) c=21,h,w -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(scores, "out"); -#endif - - const unsigned int output_classes = scores.c; - const unsigned int output_height = scores.h; - const unsigned int output_width = scores.w; - - const float *scores_ptr = (float *) scores.data; - - // time cost! - content.names_map.clear(); - content.class_mat = cv::Mat(output_height, output_width, CV_8UC1, cv::Scalar(0)); - cv::resize(mat, content.color_mat, cv::Size(output_width, output_height)); // init color mat - - const unsigned int scores_step = output_height * output_width; // h x w - - for (unsigned int i = 0; i < output_height; ++i) - { - - uchar *p_class = content.class_mat.ptr(i); - cv::Vec3b *p_color = content.color_mat.ptr(i); - - for (unsigned int j = 0; j < output_width; ++j) - { - // argmax - unsigned int max_label = 0; - float max_conf = scores_ptr[0 * scores_step + i * output_width + j]; - - for (unsigned int l = 0; l < output_classes; ++l) - { - float conf = scores_ptr[l * scores_step + i * output_width + j]; - if (conf > max_conf) - { - max_conf = conf; - max_label = l; - } - } - - if (max_label == 0) continue; - - // assign label for pixel(i,j) - p_class[j] = cv::saturate_cast(max_label); - // assign color for detected class at pixel(i,j). - p_color[j][0] = cv::saturate_cast((max_label % 10) * 20); - p_color[j][1] = cv::saturate_cast((max_label % 5) * 40); - p_color[j][2] = cv::saturate_cast((max_label % 10) * 20); - // assign names map - content.names_map[max_label] = class_names[max_label - 1]; // max_label >= 1 - } - - } - - cv::resize(content.class_mat, content.class_mat, cv::Size(img_width, img_height)); - cv::resize(content.color_mat, content.color_mat, cv::Size(img_width, img_height)); - - content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_fcn_resnet101.h b/lite/ncnn/cv/ncnn_fcn_resnet101.h deleted file mode 100644 index a9c1d6ea..00000000 --- a/lite/ncnn/cv/ncnn_fcn_resnet101.h +++ /dev/null @@ -1,47 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_FCN_RESNET101_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_FCN_RESNET101_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNFCNResNet101 : public BasicNCNNHandler - { - public: - explicit NCNNFCNResNet101(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); // - ~NCNNFCNResNet101() override = default; - - private: - const float norm_vals[3] = {(1.f / 0.229f) * (1.f / 255.f), - (1.f / 0.224f) * (1.f / 255.f), - (1.f / 0.225f) * (1.f / 255.f)}; - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; // RGB - - private: - int input_height = 512; - int input_width = 512; - - const char *class_names[20] = { - "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", - "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", - "train", "tvmonitor" - }; // 20 classes - - private: - - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::SegmentContent &content); - - }; -} - - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_FCN_RESNET101_H diff --git a/lite/ncnn/cv/ncnn_female_photo2cartoon.cpp b/lite/ncnn/cv/ncnn_female_photo2cartoon.cpp deleted file mode 100644 index e643c6b4..00000000 --- a/lite/ncnn/cv/ncnn_female_photo2cartoon.cpp +++ /dev/null @@ -1,99 +0,0 @@ -// -// Created by DefTruth on 2022/6/12. -// - -#include "ncnn_female_photo2cartoon.h" - -using ncnncv::NCNNFemalePhoto2Cartoon; - -NCNNFemalePhoto2Cartoon::NCNNFemalePhoto2Cartoon( - const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads, - unsigned int _input_height, - unsigned int _input_width) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads), - input_height(_input_height), input_width(_input_width) -{ -} - -void NCNNFemalePhoto2Cartoon::transform(const cv::Mat &mat_merged_rs, ncnn::Mat &in) -{ - // will do deepcopy inside ncnn - in = ncnn::Mat::from_pixels(mat_merged_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNFemalePhoto2Cartoon::detect( - const cv::Mat &mat, const cv::Mat &mask, - types::FemalePhoto2CartoonContent &content) -{ - if (mat.empty() || mask.empty()) return; - const unsigned int channels = mat.channels(); - if (channels != 3) return; - const unsigned int mask_channels = mask.channels(); - if (mask_channels != 1 && mask_channels != 3) return; - // model input size - const unsigned int input_h = input_height; // 256 - const unsigned int input_w = input_width; // 256 - - // resize before merging mat and mask - cv::Mat mat_rs, mask_rs; - cv::resize(mat, mat_rs, cv::Size(input_w, input_h)); - cv::resize(mask, mask_rs, cv::Size(input_w, input_h)); // CV_32FC1 - if (mask_channels != 3) cv::cvtColor(mask_rs, mask_rs, cv::COLOR_GRAY2BGR); // CV_32FC3 - mat_rs.convertTo(mat_rs, CV_32FC3, 1.f, 0.f); // CV_32FC3 - // merge mat_rs and mask_rs - cv::Mat mat_merged_rs = mat_rs.mul(mask_rs) + (1.f - mask_rs) * 255.f; - mat_merged_rs.convertTo(mat_merged_rs, CV_8UC3); // keep CV_8UC3 BGR - // 1. make input tensor - ncnn::Mat input; - this->transform(mat_merged_rs, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - // 3. generate cartoon - this->generate_cartoon(extractor, mask_rs, content); -} - -void NCNNFemalePhoto2Cartoon::generate_cartoon( - ncnn::Extractor &extractor, const cv::Mat &mask_rs, - types::FemalePhoto2CartoonContent &content) -{ - ncnn::Mat cartoon_pred; - extractor.extract("output", cartoon_pred); -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(cartoon_pred, "output"); -#endif - - const unsigned int out_h = cartoon_pred.h; - const unsigned int out_w = cartoon_pred.w; - const unsigned int channel_step = out_h * out_w; - const unsigned int mask_h = mask_rs.rows; - const unsigned int mask_w = mask_rs.cols; - // fast assign & channel transpose(CHW->HWC). - float *cartoon_ptr = (float *) cartoon_pred.data; - std::vector cartoon_channel_mats; - cv::Mat rmat(out_h, out_w, CV_32FC1, cartoon_ptr); // R - cv::Mat gmat(out_h, out_w, CV_32FC1, cartoon_ptr + channel_step); // G - cv::Mat bmat(out_h, out_w, CV_32FC1, cartoon_ptr + 2 * channel_step); // B - rmat = (rmat + 1.f) * 127.5f; - gmat = (gmat + 1.f) * 127.5f; - bmat = (bmat + 1.f) * 127.5f; - cartoon_channel_mats.push_back(rmat); - cartoon_channel_mats.push_back(gmat); - cartoon_channel_mats.push_back(bmat); - cv::Mat cartoon; - cv::merge(cartoon_channel_mats, cartoon); // CV_32FC3 - if (out_h != mask_h || out_w != mask_w) - cv::resize(cartoon, cartoon, cv::Size(mask_w, mask_h)); - // combine & RGB -> BGR -> uint8 - cartoon = cartoon.mul(mask_rs) + (1.f - mask_rs) * 255.f; - cv::cvtColor(cartoon, cartoon, cv::COLOR_RGB2BGR); - cartoon.convertTo(cartoon, CV_8UC3); - - content.cartoon = cartoon; - content.flag = true; -} diff --git a/lite/ncnn/cv/ncnn_female_photo2cartoon.h b/lite/ncnn/cv/ncnn_female_photo2cartoon.h deleted file mode 100644 index 5b5c1548..00000000 --- a/lite/ncnn/cv/ncnn_female_photo2cartoon.h +++ /dev/null @@ -1,41 +0,0 @@ -// -// Created by DefTruth on 2022/6/12. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_FEMALE_PHOTO2CARTOON_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_FEMALE_PHOTO2CARTOON_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNFemalePhoto2Cartoon : public BasicNCNNHandler - { - public: - explicit NCNNFemalePhoto2Cartoon(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1, - unsigned int _input_height = 256, - unsigned int _input_width = 256); - - ~NCNNFemalePhoto2Cartoon() override = default; - - private: - const int input_height; - const int input_width; - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; - const float norm_vals[3] = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - - private: - void transform(const cv::Mat &mat_merged_rs /*merged & resized mat*/, ncnn::Mat &in) override; - - void generate_cartoon(ncnn::Extractor &extractor, const cv::Mat &mask_rs, - types::FemalePhoto2CartoonContent &content); - - public: - void detect(const cv::Mat &mat, const cv::Mat &mask, types::FemalePhoto2CartoonContent &content); - }; -} - - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_FEMALE_PHOTO2CARTOON_H diff --git a/lite/ncnn/cv/ncnn_focal_arcface.cpp b/lite/ncnn/cv/ncnn_focal_arcface.cpp deleted file mode 100644 index a0c8af02..00000000 --- a/lite/ncnn/cv/ncnn_focal_arcface.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "ncnn_focal_arcface.h" - -using ncnncv::NCNNFocalArcFace; - -void NCNNFocalArcFace::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW - int h = mat.rows; - int w = mat.cols; - in = ncnn::Mat::from_pixels_resize( - mat.data, ncnn::Mat::PIXEL_BGR2RGB, - w, h, input_width, input_height - ); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNFocalArcFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - ncnn::Mat embedding; - extractor.extract("embedding", embedding); - - const unsigned int hidden_dim = embedding.w; // 512 - const float *embedding_values = (float *) embedding.data; - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_focal_arcface.h b/lite/ncnn/cv/ncnn_focal_arcface.h deleted file mode 100644 index ddba0ab5..00000000 --- a/lite/ncnn/cv/ncnn_focal_arcface.h +++ /dev/null @@ -1,37 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_FOCAL_ARCFACE_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_FOCAL_ARCFACE_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNFocalArcFace : public BasicNCNNHandler - { - public: - explicit NCNNFocalArcFace(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) - {}; - - ~NCNNFocalArcFace() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - static constexpr const int input_width = 112; - static constexpr const int input_height = 112; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_FOCAL_ARCFACE_H diff --git a/lite/ncnn/cv/ncnn_focal_asia_arcface.cpp b/lite/ncnn/cv/ncnn_focal_asia_arcface.cpp deleted file mode 100644 index ec45b38f..00000000 --- a/lite/ncnn/cv/ncnn_focal_asia_arcface.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "ncnn_focal_asia_arcface.h" - -using ncnncv::NCNNFocalAsiaArcFace; - -void NCNNFocalAsiaArcFace::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW - int h = mat.rows; - int w = mat.cols; - in = ncnn::Mat::from_pixels_resize( - mat.data, ncnn::Mat::PIXEL_BGR2RGB, - w, h, input_width, input_height - ); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNFocalAsiaArcFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - ncnn::Mat embedding; - extractor.extract("embedding", embedding); - - const unsigned int hidden_dim = embedding.w; // 512 - const float *embedding_values = (float *) embedding.data; - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_focal_asia_arcface.h b/lite/ncnn/cv/ncnn_focal_asia_arcface.h deleted file mode 100644 index dcd99494..00000000 --- a/lite/ncnn/cv/ncnn_focal_asia_arcface.h +++ /dev/null @@ -1,36 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_FOCAL_ASIA_ARCFACE_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_FOCAL_ASIA_ARCFACE_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNFocalAsiaArcFace : public BasicNCNNHandler - { - public: - explicit NCNNFocalAsiaArcFace(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) - {}; - - ~NCNNFocalAsiaArcFace() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - static constexpr const int input_width = 112; - static constexpr const int input_height = 112; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_FOCAL_ASIA_ARCFACE_H diff --git a/lite/ncnn/cv/ncnn_gender_googlenet.cpp b/lite/ncnn/cv/ncnn_gender_googlenet.cpp deleted file mode 100644 index 55c15565..00000000 --- a/lite/ncnn/cv/ncnn_gender_googlenet.cpp +++ /dev/null @@ -1,55 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#include "ncnn_gender_googlenet.h" -#include "lite/utils.h" - -using ncnncv::NCNNGenderGoogleNet; - -NCNNGenderGoogleNet::NCNNGenderGoogleNet(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNGenderGoogleNet::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - // (1,3,224,224) - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNGenderGoogleNet::detect(const cv::Mat &mat, types::Gender &gender) -{ - if (mat.empty()) return; - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - // 3. fetch. - ncnn::Mat gender_logits; - extractor.extract("loss3/loss3_Y", gender_logits); // c=1,h=1,w=2 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(gender_logits, "loss3/loss3_Y"); -#endif - - const unsigned int num_genders = gender_logits.w; - const float *pred_logits_ptr = (float *) gender_logits.data; - - unsigned int pred_gender = 0; - auto softmax_probs = lite::utils::math::softmax(pred_logits_ptr, num_genders, pred_gender); - unsigned int gender_label = pred_gender == 1 ? 0 : 1; - gender.label = gender_label; - gender.text = gender_texts[gender_label]; - gender.score = softmax_probs[pred_gender]; - gender.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_gender_googlenet.h b/lite/ncnn/cv/ncnn_gender_googlenet.h deleted file mode 100644 index 0fb8bd5b..00000000 --- a/lite/ncnn/cv/ncnn_gender_googlenet.h +++ /dev/null @@ -1,37 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_GENDER_GOOGLENET_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_GENDER_GOOGLENET_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNGenderGoogleNet : public BasicNCNNHandler - { - public: - explicit NCNNGenderGoogleNet(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNGenderGoogleNet() override = default; - - private: - const int input_height = 224; - const int input_width = 224; - const float mean_vals[3] = {104.0f, 117.0f, 123.0f}; - const float norm_vals[3] = {1.0f, 1.0f, 1.0f}; - const char *gender_texts[2] = {"female", "male"}; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::Gender &gender); - }; -} - - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_GENDER_GOOGLENET_H diff --git a/lite/ncnn/cv/ncnn_ghostnet.cpp b/lite/ncnn/cv/ncnn_ghostnet.cpp deleted file mode 100644 index 36121852..00000000 --- a/lite/ncnn/cv/ncnn_ghostnet.cpp +++ /dev/null @@ -1,62 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "ncnn_ghostnet.h" -#include "lite/utils.h" - -using ncnncv::NCNNGhostNet; - -NCNNGhostNet::NCNNGhostNet(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNGhostNet::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNGhostNet::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - // 3. fetch. - ncnn::Mat logits_mat; - extractor.extract("logits", logits_mat); // c=1,h=1,w=1000 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(logits_mat, "logits"); -#endif - - const unsigned int num_classes = logits_mat.w; - const float *logits = (float *) logits_mat.data; - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_ghostnet.h b/lite/ncnn/cv/ncnn_ghostnet.h deleted file mode 100644 index 605d30b1..00000000 --- a/lite/ncnn/cv/ncnn_ghostnet.h +++ /dev/null @@ -1,413 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_GHOSTNET_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_GHOSTNET_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNGhostNet : public BasicNCNNHandler - { - public: - explicit NCNNGhostNet(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNGhostNet() override = default; - - private: - const int input_height = 224; - const int input_width = 224; - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_GHOSTNET_H diff --git a/lite/ncnn/cv/ncnn_glint_arcface.cpp b/lite/ncnn/cv/ncnn_glint_arcface.cpp deleted file mode 100644 index 8660d260..00000000 --- a/lite/ncnn/cv/ncnn_glint_arcface.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// -// Created by DefTruth on 2021/11/13. -// - -#include "ncnn_glint_arcface.h" - -using ncnncv::NCNNGlintArcFace; - -void NCNNGlintArcFace::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW - int h = mat.rows; - int w = mat.cols; - in = ncnn::Mat::from_pixels_resize( - mat.data, ncnn::Mat::PIXEL_BGR2RGB, - w, h, input_width, input_height - ); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNGlintArcFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - ncnn::Mat embedding; - extractor.extract("embedding", embedding); - - const unsigned int hidden_dim = embedding.w; // 512 - const float *embedding_values = (float *) embedding.data; - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_glint_arcface.h b/lite/ncnn/cv/ncnn_glint_arcface.h deleted file mode 100644 index 4952e1c7..00000000 --- a/lite/ncnn/cv/ncnn_glint_arcface.h +++ /dev/null @@ -1,37 +0,0 @@ -// -// Created by DefTruth on 2021/11/13. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_GLINT_ARCFACE_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_GLINT_ARCFACE_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNGlintArcFace : public BasicNCNNHandler - { - public: - explicit NCNNGlintArcFace(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) - {}; - - ~NCNNGlintArcFace() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - static constexpr const int input_width = 112; - static constexpr const int input_height = 112; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_GLINT_ARCFACE_H diff --git a/lite/ncnn/cv/ncnn_glint_cosface.cpp b/lite/ncnn/cv/ncnn_glint_cosface.cpp deleted file mode 100644 index 2d90b788..00000000 --- a/lite/ncnn/cv/ncnn_glint_cosface.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// -// Created by DefTruth on 2021/11/13. -// - -#include "ncnn_glint_cosface.h" - -using ncnncv::NCNNGlintCosFace; - -void NCNNGlintCosFace::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW - int h = mat.rows; - int w = mat.cols; - in = ncnn::Mat::from_pixels_resize( - mat.data, ncnn::Mat::PIXEL_BGR2RGB, - w, h, input_width, input_height - ); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNGlintCosFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - ncnn::Mat embedding; - extractor.extract("embedding", embedding); - - const unsigned int hidden_dim = embedding.w; // 512 - const float *embedding_values = (float *) embedding.data; - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_glint_cosface.h b/lite/ncnn/cv/ncnn_glint_cosface.h deleted file mode 100644 index 3abdb3e4..00000000 --- a/lite/ncnn/cv/ncnn_glint_cosface.h +++ /dev/null @@ -1,36 +0,0 @@ -// -// Created by DefTruth on 2021/11/13. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_GLINT_COSFACE_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_GLINT_COSFACE_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNGlintCosFace : public BasicNCNNHandler - { - public: - explicit NCNNGlintCosFace(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) - {}; - - ~NCNNGlintCosFace() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - static constexpr const int input_width = 112; - static constexpr const int input_height = 112; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_GLINT_COSFACE_H diff --git a/lite/ncnn/cv/ncnn_glint_partial_fc.cpp b/lite/ncnn/cv/ncnn_glint_partial_fc.cpp deleted file mode 100644 index 90cfc441..00000000 --- a/lite/ncnn/cv/ncnn_glint_partial_fc.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// -// Created by DefTruth on 2021/11/13. -// - -#include "ncnn_glint_partial_fc.h" - -using ncnncv::NCNNGlintPartialFC; - -void NCNNGlintPartialFC::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW - int h = mat.rows; - int w = mat.cols; - in = ncnn::Mat::from_pixels_resize( - mat.data, ncnn::Mat::PIXEL_BGR2RGB, - w, h, input_width, input_height - ); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNGlintPartialFC::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - ncnn::Mat embedding; - extractor.extract("embedding", embedding); - - const unsigned int hidden_dim = embedding.w; // 512 - const float *embedding_values = (float *) embedding.data; - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_glint_partial_fc.h b/lite/ncnn/cv/ncnn_glint_partial_fc.h deleted file mode 100644 index 84561e17..00000000 --- a/lite/ncnn/cv/ncnn_glint_partial_fc.h +++ /dev/null @@ -1,37 +0,0 @@ -// -// Created by DefTruth on 2021/11/13. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_GLINT_PARTIAL_FC_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_GLINT_PARTIAL_FC_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNGlintPartialFC : public BasicNCNNHandler - { - public: - explicit NCNNGlintPartialFC(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) - {}; - - ~NCNNGlintPartialFC() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - static constexpr const int input_width = 112; - static constexpr const int input_height = 112; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_GLINT_PARTIAL_FC_H diff --git a/lite/ncnn/cv/ncnn_hdrdnet.cpp b/lite/ncnn/cv/ncnn_hdrdnet.cpp deleted file mode 100644 index 9beb4091..00000000 --- a/lite/ncnn/cv/ncnn_hdrdnet.cpp +++ /dev/null @@ -1,62 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "ncnn_hdrdnet.h" -#include "lite/utils.h" - -using ncnncv::NCNNHdrDNet; - -NCNNHdrDNet::NCNNHdrDNet(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNHdrDNet::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNHdrDNet::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - // 3. fetch. - ncnn::Mat logits_mat; - extractor.extract("logits", logits_mat); // c=1,h=1,w=1000 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(logits_mat, "logits"); -#endif - - const unsigned int num_classes = logits_mat.w; - const float *logits = (float *) logits_mat.data; - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_hdrdnet.h b/lite/ncnn/cv/ncnn_hdrdnet.h deleted file mode 100644 index 827a7778..00000000 --- a/lite/ncnn/cv/ncnn_hdrdnet.h +++ /dev/null @@ -1,413 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_HDRDNET_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_HDRDNET_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNHdrDNet : public BasicNCNNHandler - { - public: - explicit NCNNHdrDNet(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNHdrDNet() override = default; - - private: - const int input_height = 224; - const int input_width = 224; - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_HDRDNET_H diff --git a/lite/ncnn/cv/ncnn_ibnnet.cpp b/lite/ncnn/cv/ncnn_ibnnet.cpp deleted file mode 100644 index d689639c..00000000 --- a/lite/ncnn/cv/ncnn_ibnnet.cpp +++ /dev/null @@ -1,62 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "ncnn_ibnnet.h" -#include "lite/utils.h" - -using ncnncv::NCNNIBNNet; - -NCNNIBNNet::NCNNIBNNet(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNIBNNet::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNIBNNet::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - // 3. fetch. - ncnn::Mat logits_mat; - extractor.extract("logits", logits_mat); // c=1,h=1,w=1000 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(logits_mat, "logits"); -#endif - - const unsigned int num_classes = logits_mat.w; - const float *logits = (float *) logits_mat.data; - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_ibnnet.h b/lite/ncnn/cv/ncnn_ibnnet.h deleted file mode 100644 index ca33ef5f..00000000 --- a/lite/ncnn/cv/ncnn_ibnnet.h +++ /dev/null @@ -1,414 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_IBNNET_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_IBNNET_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNIBNNet : public BasicNCNNHandler - { - public: - explicit NCNNIBNNet(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNIBNNet() override = default; - - private: - const int input_height = 224; - const int input_width = 224; - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - - }; -} - - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_IBNNET_H diff --git a/lite/ncnn/cv/ncnn_insectid.cpp b/lite/ncnn/cv/ncnn_insectid.cpp deleted file mode 100644 index 424a0f8b..00000000 --- a/lite/ncnn/cv/ncnn_insectid.cpp +++ /dev/null @@ -1,63 +0,0 @@ -// -// Created by DefTruth on 2022/3/27. -// - -#include "ncnn_insectid.h" -#include "lite/utils.h" - -using ncnncv::NCNNInsectID; - -NCNNInsectID::NCNNInsectID(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNInsectID::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - // will do deepcopy inside ncnn - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNInsectID::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input.1", input); - // 3. fetch. - ncnn::Mat logits_mat; - extractor.extract("477", logits_mat); // c=1,h=1,w=2037 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(logits_mat, "477"); -#endif - - const unsigned int num_classes = logits_mat.w; - const float *logits = (float *) logits_mat.data; - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} diff --git a/lite/ncnn/cv/ncnn_insectid.h b/lite/ncnn/cv/ncnn_insectid.h deleted file mode 100644 index 0180d97e..00000000 --- a/lite/ncnn/cv/ncnn_insectid.h +++ /dev/null @@ -1,375 +0,0 @@ -// -// Created by DefTruth on 2022/3/27. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_INSECTID_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_INSECTID_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNInsectID : public BasicNCNNHandler - { - public: - explicit NCNNInsectID(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNInsectID() override = default; - - private: - const int input_height = 224; - const int input_width = 224; - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[2037] = { - "Pseudoscorpiones", "Diplopoda", "Megymenum", "Cicadellidae", "Bothrogonia addita", "Bothrogonia ferruginea", "Cicadella viridis", - "Maiestas dorsalis", "Nephotettix cincticeps", "Mileewa", "Ledra", "Olidiana brevis", "Acanthosoma denticaudum", - "Sastragala esakii", "Neolethaeus dallasi", "Metochus uniguttatus", "Metochus abbreviatus", "Horridipamera inconspicua", - "Geocoris pallidipennis", "Geocoris varius", "Clovia", "Omalophora pectoralis", "Ricaniidae", "Ricaniidae", "Ricanula pulverosa", - "Ricania speculum", "Euricania facialis", "Ricania guttata", "Ricanula sublimata", "Euricania ocella", "Ricania taeniata", - "Euricania clara", "Ricania simulans", "Urochela quadrinotata", "Cercopidae", "Cosmoscarta", "Cosmoscarta abdominalis", - "Cosmoscarta exultans", "Cosmoscarta dimidiata", "Cosmoscarta dorsimacula", "Callitettix versicolor", "Reduviidae", - "Haematoloecha nigrorufa", "Platymeris", "Agriosphodrus dohrni", "Euagoras plagiatus", "Yolinus albopustulatus", - "Sycanus croceovittatus", "Sphedanolestes impressicollis", "Epidaus", "Epidaus sexspinus", "Vesbius sanguinosus", "Acanthaspis", - "Isyndus obscurus", "Sirthenea flavipes", "Ectrychotes andreae", "Sclomina erinacea", "Issidae", "Phymatidae", "Miridae", - "Eurystylus coelestialium", "Apolygus lucorum", "Helopeltis cinchonae", "Eucorysses grandis", "Hyperoncus lateritius", - "Poecilocoris nepalensis", "Poecilocoris sanszeusignatus", "Poecilocoris druraei", "Poecilocoris latus", "", "Poecilocoris lewisi", - "", "Tetrarthria variegata", "Sphaerocoris annulus", "Scutellera amethystina(Scutellera fasciata)", "Chrysocoris stollii", - "Lamprocoris lateralis", "Calliphara nobilis", "Cantao ocellatus", "Pyrrhocoridae", "Pyrrhocoris sibiricus", "Macrocheraia grandis", - "Physopelta quadriguttata", "Physopelta gutta", "", "Dysdercus decussatus", "Dysdercus cingulatus", "Dysdercus poecilus", - "Dindymus rubiginosus", "Dindymus brevis", "Antilochus coquebertii", "Coreidae", "Mictis tenebrosa", "Mictis gallina", - "Mictis serina", "Mictis fuscipes", "Paradasynus spinosus", "Homoeocerus unipunctatus", "Homoeocerus dilatatus", - "Homoeocerus striicornis", "Molipteryx", "Molipteryx lunata", "Cletus", "Acanthocoris scaber", "Riptortus", "Riptortus pedestris", - "Plinachtus bicoloripes", "Notobitus meleagris", "Tingidae", "Corythucha ciliata", "Corythucha marmorata", "Anthocoris confusus", - "Eurostus", "", "Tessaratoma papillosa", "", "Borysthenes maculatus", "Flatidae", "Cerynia maria", "Lawana imitata", - "Geisha distinctissima", "Salurnis marginella", "Pyrops", "Pyrops spinolae", "Pyrops watanabei", "Pyrops watanabei", - "Pyrops candelaria", "Penthicodes atomaria", "Lycorma delicatula", "Lycorma delicatula", "Penthicodes pulchella", "Saiva bullata", - "Cicadidae", "Cicadidae", "Talainga chinensis", "Meimuna", "Gaeana maculata", "Hyalessa maculaticollis", "Scieroptera", - "Sulphogaeana sulphurea", "Polymeura chenni", "Chremistica ochracea", "Platypleura kaempferi", "Tacua speciosa", - "Formotosena seebohmi", "Huechys sanguinea", "Cryptotympana atrata", "Nepidae", "Eysarcoris", "Eysarcoris guttigerus", - "Eysarcoris aeneus", "Eysarcoris ventralis", "Metonymia glandulosa", "Palomena viridissima", "Priassus spiniger", "Dalpada", - "Lelia decempunctata", "Dolycoris baccarum", "Eurydema gebleri", "Plautia", "Cazira", "Nezara", "Carpocoris purpureipennis", - "Menida violacea", "Palomena prasina", "Catacanthus incarnatus", "Alcimocoris", "Halyomorpha halys", "Eurydema dominulus", - "Zicrona caerulea", "", "Graphosoma rubrolineatum", "Erthesina fullo", "", "Derbidae", "Diostrombus politus", "Membracidae", - "Dictyopharidae", "Kirkaldyia deyrollei", "Berytidae", "Lygaeus equestris", "Spilostethus hospes", "Tropidothorax elegans", - "Lygaeus hanseni", "Graptostethus servus", "Gerridae", "Plataspidae", "Tipulidae", "", "Tephritidae", "Tachinidae", "Chironomidae", - "Stratiomyidae", "Ptecticus aurifer", "Hermetia illucens", "Liriomyza sativae", "Anthomyia illocata", "Culicidae", "Psychodidae", - "Bombyliidae", "Muscidae", "Asilidae", "Microstylum oberthurii", "Syrphidae", "Eupeodes nitens", "Eupeodes corollae", - "Eristalinus arvorum", "Eristalis cerealis", "Ischiodon scutellaris", "Eristalis arbustorum", "Phytomia zonata", "Phytomia errans", - "Syrphus torvus", "Paragus crenulatus", "Syrphus ribesii", "Eristalinus quinquestriatus", "Episyrphus balteatus", - "Helophilus pendulus", "Corydalidae", "", "Neochauliodes", "", "", "Trichoptera", "Opiliones", "Ornebius kanetataki", - "Eucriotettix oculatus", "Tetrix japonica", "Erianthus dohrni", "Acrida cinerea", "Oedaleus infernalis", "Chondracris rosea", - "Trilophidia annulata", "Xenocatantops brachycerus", "Oxya chinensis", "Shirakiacris", "Stauroderus scalaris", - "Aiolopus thalassinus tamulus", "Pseudoxya diminuta", "Ceracris nigricornis", "Locusta migratoria", "Aularches miliaris", "Patanga", - "", "Tettigoniidae", "Pseudophyllus titan", "", "Ducetia japonica", "Hexacentrus unicolor", "", "", "Conocephalus melaenus", "", - "Gampsocleis sedakovii", "Phaneroptera falcata", "Sanaa intermedia", "Gryllacrididae", "Xenogryllus marmoratus", - "Teleogryllus mitratus", "Gryllus bimaculatus", "Teleogryllus emma", "Atractomorpha sinensis", "", "", "", "Ixodida", "Phasmatodea", - "Porcellio", "Lepismatidae", "Nemopteridae", "Chrysopidae", "Myrmeleontidae", "Psychopsidae", "Ascalaphidae", - "Ascalaphus sibiricus", "Mantispidae", "Hemerobiidae", "Tenthredinidae", "Scolia superciliaris", "Ichneumonidae", "Megarhyssa", - "Xanthopimpla", "Brachymeria minuta", "Liris aurulentus", "", "Ampulex compressa", "Sphex argentatus", "Sceliphron madraspatanum", - "Sphex subtruncatus", "Sceliphron javanum", "Vespidae", "Parapolybia nodosa", "Parapolybia varia", "Polistes snelleni", - "Polistes japonicus", "Polistes gigas", "Polistes jokahamae", "Vespa velutina", "Vespa mandarinia", "Vespa affinis", "Polistinae", - "Vespula flaviceps", "Formicidae", "Pseudoneoponera rufipes", "Oecophylla smaragdina", "Mutillidae", "Pompilidae", "Apidae", - "Xylocopinae", "Bombus", "Bombus pyrosoma", "Bombus picipes", "Amegilla calceifera", "Delta esuriens", "Phimenes flavopictus", - "Oreumenes decoratus", "Delta pyriforme", "Chrysididae", "Scutigeridae", "Scolopendridae", "Ephemeroptera", "Araneae", "Araneidae", - "Araneus diadematus", "Araneus ventricosus", "Macracantha arcuata", "Neoscona mellotteei", "Gasteracantha hasselti", - "Gasteracantha kuhli", "Gasteracantha diadesmia", "Nephila pilipes", "", "Neoscona vigilans", "Argiope", "Argiope amoena", - "Araneus ejusmodi", "Araneus mitificus", "Heteropoda venatoria", "Pholcidae", "Macrothele raveni", "Agelenidae", "Lycosidae", - "Steatoda nobilis", "Latrodectus tredecimguttatus", "Tetragnathidae", "Leucauge tessellata", "", "Ebrechtella tricuspidata", - "Salticidae", "Thiania bhamoensis", "Telamonia caprina", "Plexippoides", "Siler semiglaucus", "Pancorius crassipes", "Epeus", - "Hasarius adansoni", "Phintella bifurcilinea", "Cheliceroides longipalpis", "Plexippus paykulli", "", "Eresidae", "Blattodea", - "Periplaneta australasiae", "Periplaneta americana", "Periplaneta fuliginosa", "Blattella germanica", "Corydidae", - "Indolestes peregrinus", "Indolestes cyaneus", "Chlorogomphus papilio", "", "Platycnemididae", "Copera annulata", - "Coeliccia cyanomelas", "Pseudolestes mirabilis", "Gomphidae", "Sinictinogomphus clavatus", "Ictinogomphus rapax", - "Gomphidia confluens", "", "Philoganga vetusta", "Euphaea decorata", "Calopterygidae", "Calopteryx splendens", - "Neurobasis chinensis", "Matrona basilaris", "Calopteryx virgo", "Mnais", "Mnais mneme", "Archineura incarnata", - "Atrocalopteryx atrata", "Anax guttatus", "Anax parthenope", "Anax immaculifrons", "Anax nigrofasciatus", "Gynacantha japonica", - "Gynacantha subinterrupta", "Aeshna mixta", "Rhyothemis", "Rhyothemis variegata", "Rhyothemis fuliginosa", "Tholymis tillarga", - "Palpopleura sexmaculata", "Tramea virginia", "Deielia phaon", "Tetrathemis platyptera", "Sympetrum vulgatum", - "Indothemis carnatica", "Potamarcha congener", "Orthetrum", "Orthetrum chrysis", "Orthetrum luzonicum", "Orthetrum melania", - "Orthetrum poecilops", "Orthetrum sabina", "Orthetrum albistylum", "Orthetrum cancellatum", "Orthetrum lineostigma", - "Orthetrum pruinosum", "Orthetrum glaucum", "Orthetrum triangulare", "Pseudothemis zonata", "Crocothemis servilia", - "Zyxomma petiolatum", "Neurothemis taiwanensis", "Neurothemis tullia", "Neurothemis fulvia", "Neurothemis intermedia", - "Diplacodes trivialis", "Brachydiplax chalybea", "Trithemis festiva", "Trithemis aurora", "Sympetrum croceolum", - "Sympetrum parvulum", "Sympetrum risi", "Sympetrum eroticum", "Sympetrum pedemontanum", "Sympetrum danae", "Acisoma panorpoides", - "Lyriothemis pachygastra", "Epophthalmia elegans", "Brachythemis contaminata", "Pantala flavescens", "Selysiothemis nigra", - "Pseudagrion rubriceps", "Ceriagrion fallax", "Ischnura asiatica", "Ischnura senegalensis", "Ischnura rufostigma", - "Ischnura aurora", "Agriocnemis femina", "Enallagma cyathigerum", "Paracercion calamorum", "Ceriagrion nipponicum", - "Agriocnemis pygmaea", "Chlorocyphidae", "Heliocypha perforata", "Scorpiones", "Heterometrus petersii", "Mantodea", - "Pseudocreobotra wahlbergi", "Phyllocrania paradoxa", "Acromantis japonica", "Creobroter", "Sibylla pretiosa", - "Hymenopus coronatus", "Tenodera sinensis", "Tenodera aridifolia", "Phyllothelys", "Hierodula patellifera", "Mantis religiosa", - "Statilia maculata", "Plecoptera", "Mecoptera", "", "Trictenotomidae", "Rutelidae", "Anomala", "Popillia", - "Eumorphus quadriguttatus", "Attelabidae", "Byctiscus betulae", "Paratrachelophorus nodicornis", "Tomapoderus ruficollis", - "Apoderus coryli", "Aspidobyctiscus lacunipennis", "Trachelophorus giraffa", "Elateridae", "Campsosternus", "Campsosternus gemma", - "Chrysomelidae", "Gallerucida bifasciata", "Monolepta quadriguttata", "Chrysomela populi", "Chrysomela vigintipunctata", - "Plagiodera versicolora", "Oides decempunctata", "Oides bowringii", "Colasposoma dauricum", "Leptinotarsa decemlineata", - "Sagra femorata", "Agasicles hygrophila", "Criocerinae", "", "Chrysolina polita", "Chaetocnema hortensis", "Aulacophora indica", - "Monolepta signata", "Phyllotreta striolata", "Diabrotica undecimpunctata", "Podontia lutea", "Aulacophora lewisii", - "Gastrolina thoracica", "Aulacophora nigripennis", "Buprestidae", "Chrysochroa fulgidissima", "Agrilus planipennis", "Chalcophora", - "Cerambycidae", "Thysia", "Monochamus saltuarius", "Leptura duodecimguttata", "Lamiomimus gottschei", "Moechotypa diphysis", - "Xystrocera globosa", "Mesosa myops", "Dorysthenes", "Monochamus alternatus", "Polyzonus fasciatus", "Agapanthia amurensis", - "Stenocorus meridianus", "Acanthocinus griseus", "Leptura thoracica", "Apomecyna saltator", "Anoplophora", "Anoplophora horsfieldi", - "Leptura annularis", "Rhytiphora bankii", "Semanotus bifasciatus", "Strangalia attenuata", "Neocerambyx raddei", - "Pterolophia annulata", "Glenea relicta", "Imantocera penicillata", "Eupromus ruber", "Aristobia horridula", - "Dicelosternus corallinus", "Batocera", "", "Batocera rubus", "Glenea cantor", "Oberea", "Olenecamptus", "Apriona rugicollis", - "Apriona swainson", "Purpuricenus temminckii", "Callidium violaceum", "Chlorophorus", "Chlorophorus douei", - "Chlorophorus annularis", "Chlorophorus signaticollis", "Eucomatocera vittata", "Xylotrechus", "Xylotrechus yanoi", - "Xylotrechus rusticus", "Asemum striatum", "Paraglenea fortunei", "Phytoecia rufiventris", "Xylorhiza", "", "Aegosoma", - "Arhopalus rusticus", "Stromatium longicorne", "Macrochenus guerini", "Euryphagus", "Saperda populnea", "Aromia bungii", - "Tetraopes tetrophthalmus", "Thyestilla gebleri", "Psacothea", "Paraleprodera diophthalma", "", "", "Tenebrionidae", "Lagriinae", - "Blaps rynchopetera", "", "", "Carabidae", "Therates fruhstorferi", "Pheropsophus", "Carabus lafossei", "Carabus elysii", - "Carabus smaragdinus", "Scarites", "Dolichus halensis", "Chlaenius", "Carabus brandti", "Dynastidae", "Allomyrina dichotoma", - "Oryctes rhinoceros", "Xylotrupes gideon", "", "Eupatorus gracilicornis", "Trichogomphus mongol", "Oryctes nasicornis", - "Dynastes hercules", "Coccinellidae", "Coccinellidae", "Coccinella septempunctata", "Aiolocaria hexaspilota", - "Cheilomenes sexmaculata", "Oenopia formosana", "Vibidia duodecimguttata", "Coccinula quatuordecimpustulata", - "Coelophora biplagiata", "Calvia muiri", "Propylaea quatuordecimpunctata", "Illeis koebelei", "Henosepilachna vigintioctopunctata", - "Oenopia conglobata", "Halmus chalybeus", "Henosepilachna vigintioctomaculata", "Propylea japonica", "Lasioderma serricorne", - "Geotrupidae", "Eumolpidae", "Platycorynus parryi", "Smaragdina nigrifrons", "Euchiridae", "Cheirotonus gestroi", - "Cheirotonus jansoni", "Meloidae", "Lytta caraganae", "Epicauta", "", "Themus", "Cetoniidae", "Euselates", "Goliathus", - "Gametis jucunda", "Pseudotorynorrhina japonica", "Protaetia", "Clinterocera mandarina", "Dicronorhina derbyana", - "Glycyphana horsfieldi", "Agestrata orichalca", "Rhomborhina", "Campsiura mirabilis", "Dicronocephalus adamsi", - "Dicronocephalus wallichii", "Dicronocephalus bowringi", "Pyrocoelia", "Pyrocoelia analis", "Silphidae", "Collyris", "Tricondyla", - "Cicindela", "Cicindela chinenesis", "Cicindela separata", "Cicindela gemmata", "Cicindela aurulenta", "Aphodius fimetarius", - "Bruchidae", "Curculionidae", "Cryptorhynchus lapathi", "Sipalinus gigas", "Eucryptorrhynchus", "Cylas formicarius", "", - "Sitophilus oryzae", "Rhynchophorus ferrugineus", "Hypomeces pulviger", "Pyrochroidae", "Cleridae", "Trichodes sinae", - "Scarabaeoidea", "Hispidae", "Cassida rubiginosa", "Chiridopsis bowringii", "Aspidimorpha miliaris", "Aspidimorpha furcata", - "Aspidimorpha sanctaecrucis", "Taiwania circumdata", "Laccoptera nepalensis(Laccoptera quadrimaculata)", "Cassida nebulosa", - "Lucanidae", "Dorcus titanus", "Dorcus hopei", "Neolucanus", "Neolucanus swinhoei", "", "Lucanus", "Prosopocoilus confucius", - "Prosopocoilus astacoides", "Prosopocoilus girafa", "Prosopocoilus biplagiatus", "Odontolabis cuvera", "Odontolabis siva", - "Eucorynus crassicornis", "Bolboceratidae", "Staphylinidae", "Melolonthidae", "Polyphylla", "Polyphylla decemlineata", - "Melolontha hippocastani", "Amphimallon solstitiale", "Dytiscidae", "Uropygi", "Heliodinidae", "Epicopeia mencia", - "Epicopeia hainesii", "Papilionidae", "Sericinus montelus", "Papilio krishna", "Papilio glaucus", "", "Papilio multicaudata", - "Papilio hermosanus", "Papilio ulysses", "Papilio nephelus", "Papilio paris", "Papilio dehaanii", "Papilio prexaspes", - "Papilio xuthus", "", "Papilio polytes", "Papilio helenus", "Papilio castor", "Papilio bianor", "Papilio dialis", - "Papilio arcturus", "Papilio alcmenor", "Papilio maackii", "Papilio memnon", "Papilio macilentus", "Papilio cresphontes", - "Papilio protenor", "Papilio demoleus", "Papilio hoppo", "Papilio machaon", "", "Papilio troilus", "Pazala", "Pazala eurous", - "Pazala mullah", "Teinopalpus imperialis", "Teinopalpus aureus", "Agehana elwesi", "Bhutanitis thaidina", "Bhutanitis ludlowi", - "Bhutanitis lidderdalii", "Chilasa clytia", "Chilasa clytia", "Iphiclides podalirius", "Atrophaneura horishana", - "Atrophaneura varuna", "Lamproptera curius", "Lamproptera meges", "Pachliopta aristolochiae", "Trogonoptera brookiana", - "Pathysa agetes", "Pathysa_antiphates", "Luehdorfia chinensis", "Troides magellanus", "Troides helena", "Troides aeacus", - "Meandrusa sciron", "Meandrusa payeni", "Losaria coon", "Graphium", "Graphium cloanthus", "Graphium doson", "Graphium chironides", - "Graphium nomius", "Graphium megarus", "Graphium agamemnon", "Graphium sarpedon", "Graphium leechi", "Eurytides marcellus", "Byasa", - "Byasa confusa", "Byasa hedistus", "Byasa polyeuctes", "Byasa mencius", "Byasa dasarada", "Byasa impediens", "Byasa alcinous", - "Limacodidae", "", "Chalcoscelides castaneipars", "Ceratonema", "Thosea", "Matsumurides", "Iragoides conjuncta", "", - "Narosoideus flavidorsalis", "Iraga rugosa", "Rhamnosa uniformis", "Scopelodes venosa", "Scopelodes contracta", "", "Narosa", - "Phocoderma velutina", "Parasa", "Parasa bicolor", "Parasa bicolor", "Parasa lepida", "", "Parasa darma", "Parasa consocia", "", - "Parasa pastoralis", "", "Belippa horrida", "Demonarosa rufotessellata", "Setora postornata", "", "Setora baibarana", - "Miresa bracteata", "Miresa fulgida", "Hyphorma minax", "Monema flavescens", "Monema flavescens", "Thosea sinensis", - "Thosea sinensis", "Tortricidae", "Gypsonoma minutana", "Loboschiza koenigiana", "Eupoecilia ambiguella", "Epiblema foenella", - "Eucosma campoliliana", "Cerace xanthocosma", "Grapholita delineana", "Libythea lepita", "Libythea myrrha", "Noctuidae", - "Chalciope geometrica", "Chalciope mygdon", "Chalciope hyppasia", "Anomis mesogona", "Hadjina chinensis", - "Thysanoplusia intermixta", "Sphragifera sigillata", "Chytonix segregata", "Anisoneura aluco", "Sarbanissa subflava", - "Daddala lucilla", "Cucullia fraterna", "Pericyma cruegeri", "Acronicta tridens", "Acronicta tridens", "Acronicta cuspis", - "Acronicta euphorbiae", "Acronicta euphorbiae", "Acronicta alni", "Acronicta alni", "Acronicta rumicis", "Acronicta rumicis", - "Acronicta hercules", "Acronicta denticulata", "Acronicta psi", "Acronicta psi", "Acronicta pruinosa", "Acronicta pruinosa", - "Acronicta megacephala", "Acronicta megacephala", "Supersypnoides simplex", "Conservula indica", "Hypopyra vespertilio", - "Mimeusemia vilemani", "Mimeusemia vilemani", "Asota heliconia", "Asota heliconia", "Hylophilodes tsukusensis", "Paracolax fentoni", - "Paracolax sugii", "Corgatha nitens", "Corgatha dictaria", "Ophiusa coronata", "Ophiusa tirhaca", "Protoschinia scutosa", - "Agrotis ipsilon", "Oruza albigutta", "Parallelia arctotaenia", "Parallelia stuposa", "Parallelia maturata", "Phyllodes imperialis", - "Staurophora celsia", "Episteme vetula", "Episteme lectrix", "Episteme adulatrix", "Lopharthrum comprimens", "Asota tortuosa", - "Mimeusemia persimilis", "Tiracola plagiata", "Callopistria nobilior", "Callopistria repleta", "Eligma narcissus", "", - "Spirama retorta", "Sphragifera biplagiata", "Lophoptera squamigera", "Ercheia cyllaria", "Axylia putris", "Ramadasa pavo", - "Adris tyrannus", "Hydrillodes lentalis", "Diarsia canescens", "Diarsia subtincta", "Brithys crini", "", "Mocis frugalis", - "Mocis undata", "Spodoptera depravata", "Macdunnoughia purissima", "Spodoptera picta", "Spodoptera litura", "Spodoptera pecten", - "Narangodes argyrostrigatus", "Athetis lepigone", "Xanthodes transversa", "", "Mamestra brassicae", "Spodoptera exigua", "Bocula", - "Cosmia restituta", "Aedia leucomelas", "Phlogophora albovittata", "Trachea auriplena", "Ctenoplusia albostriata", - "Pangrapta lunulata", "Edessena gentiusalis", "Erebus macrops", "Erebus pilosa", "Erebus albicincta", "Erebus caprimulgus", - "Erebus crepuscularis", "Erebus ephesperis", "Ommatophora luminosa", "Cruriopsis funebris", "Checupa stegeri", - "Ischyja ferrifracta", "Narangodes confluens", "Adris okurai", "Sarcopteron punctimargo", "Catocala fraxini", "Thyas honesta", - "Eudocima salaminia", "", "Eudocima phalonia", "Yepcalphis dilectissima", "Arcte coerula", "", "Spodoptera frugiperda", - "Xylostola indistincta", "Achaea janata", "Ischyja manlia", "Catocala electa", "Heliophobus dissectus", "Baorisa hieroglyphica", - "Scrobigera", "Sinna extrema", "Sinna floralis", "Apsarasa radians", "Thysanoplusia daubei", "Tiracola aureata", - "Anacronicta nitida", "Anacronicta horishana", "Edessena hamada", "Serrodes campana", "Gabala argentata", "Othreis homaena", "", - "Asota plana", "Asota plana", "Daseochaeta pulchra", "Diphtherocome", "Hypena", "Hypena trigonalis", "Hypena vestita", - "Hypena lignealis", "Hypena amica", "Hypena indicatalis", "Hypena albopunctalis", "Hypena strigatus", "Hypena perspicua", - "Hypena obesalis", "Hypena lividalis", "Hypena laceratalis", "Sympis rufibasis", "Saturniidae", "Attacus atlas", - "Graellsia isabellae", "Antheraea yamamai", "Actias sinensis", "Caligula simla", "Antheraea polyphemus", "Actias maenas", - "Cricula andrei", "", "Argema mittrei", "Actias luna", "Antheraea pernyi", "Samia", "", "Automeris io", "", "", "Saturnia thibeta", - "Loepa", "Loepa oberthuri", "Loepa megacore", "Antheraea assamensis", "Dictyoploca japonica(Caligula japonica)", "", "Sphingidae", - "Marumba saishiuana", "Marumba sperchius", "Marumba dyras", "Marumba cristata", "Meganoton analis", "Hayesiana triopus", - "Eupanacra mydon", "Theretra oldenlandiae", "", "Theretra alecto subsp. cretica", "Theretra latreillei", "Theretra silhetensis", "", - "Theretra tibetiana", "Theretra pallicosta", "Theretra japonica", "Theretra nessus", "Hippotion rafflesii", "Hippotion rosetta", - "Hippotion celerio", "Pergesa acteus", "", "Dolbina inexacta", "Dolbina tancrei", "Sphecodina caudata", "Parum colligata", "", - "Cypoides", "Callambulyx tatarinovii", "Agrius convolvuli", "", "Rhagastis", "Daphnis nerii", "", "Daphnis hypothous", - "Smerinthus caecus", "Smerinthus planus", "Phyllosphingia", "Deilephila elpenor", "Angonyx testacea", "Acosmeryx formosana", - "Acosmeryx castanea", "Acosmeryx naga", "Acosmeryx miskini", "Cechenena minor", "Cechenena lineosa", "Cechenena subangustata", - "Amplypterus panopus", "Ampelophaga rubiginosa", "Clanis", "Cephonodes hylas", "Nephele hespera", "Langia zenzeroides", - "Macroglossum", "Macroglossum fritzei", "Macroglossum stellatarum", "Macroglossum passalus", "", "Macroglossum bombylans", - "Macroglossum pyrrhosticta", "", "Psilogramma increta", "Psilogramma menephron", "Acherontia styx", "Acherontia atropos", "", - "Acherontia lachesis", "", "Ambulyx", "Haemorrhagiae", "Ethmia lineatonotella", "Labdia semicoccinea", "Geometridae", - "Mixochlora vittata", "Sarcinodes aequilinearia", "Abraxas suspecta", "Xanthabraxas hemionata", "Plutodes", "Plutodes flavescens", - "Plutodes exquisita", "Plutodes costatus", "Gandaritis fixseni", "Semiothisa emersaria", "Paramaxates", "Biston comitata", - "Megaspilates mundataria", "Neohipparchus vallata", "Cleora cinctaria", "Chlorodontopera discospilata", "Semiothisa intermediaria", - "Dalima patularia", "Terpna subtrita", "Ectropis excellens", "Percnia cordiforma", "Naxa seriaria", "Herochroma cristata", - "Herochroma supraviridaria", "Psyra conferta", "Jankowskia fuscaria", "Idaea muricata", "Hypomecis punctinalis", - "Ourapteryx sambucaria", "Ourapteryx nigrociliaris", "Ourapteryx clara", "Ourapteryx nivea", "Scopula yamanei", "Dindica taiwana", - "Dindica polyphaenaria", "Ophthalmitis cordularia", "Agnibesa pictaria", "Eucyclodes semialba", - "Eucyclodes gavissima(Chloromachia gavissima)", "Antipercnia albinigrata", "Plagodis dolabraria", "Telenomeuta punctimarginaria", - "Hemithea tritonaria", "Oxymacaria temeraria", "Dooabia lunifera", "Biston panterinaria", "Deileptenia ribeata", - "Percnia giraffata", "", "Erebomorpha fulguraria", "Ophthalmitis albosignaria", "Chiasmia hebesata", "Phthonandria atrilineata", - "Apochima excavata", "", "Abraxas sylvata", "Thalassodes antiquadraria", "Inurois membranaria", "Chiasmia defixaria", - "Catoria olivescens", "Myrteta angelica", "Hydrelia bicauliata", "Hydrelia bicolorata", "Hydrelia ulula", "Hydrelia enisaria", - "Hydrelia flammeolaria", "Evecliptopera decurrens", "Biston suppressaria", "Biston marginata", "Uliocnemis castalaria", - "Nycterosea obstipata", "Ninodes splendens", "Tyloptera bella", "Chartographa", "Ectropis bhurmitra", "Biston perclara", - "Myrteta tinagmaria", "Thalassodes immissaria", "Percnia suffusa", "Bizia aexaria", "Electrophaes zaphenges", - "Electrophaes corylata", "Xandrames latiferaria", "Xandrames dholaria", "Cyclothea disjuncta", "Stegania cararia", - "Lophomachia lalashana", "Abraxaphantes perampla", "Operophtera relegata", "Krananda latimarginaria", "Krananda semihyalina", - "Krananda lucidaria", "Colotois pennaria", "Amblychia angeronaria", "Dischidesia cinerea", "Problepsis", "Problepsis vulgaris", - "Problepsis superans", "Problepsis albidior", "Ennomos autumnaria", "Corymica", "Pingasa ruginaria", "Pingasa alba", "Idaea impexa", - "Fascellina chromataria", "", "Palpoctenidia phoenicosoma", "Berta rugosivalva", "Timandra dichela", "Timandra stueningi", - "Timandra convectaria", "Timandra synthaca", "Timandra comptaria", "Timandra recompta", "Comibaena", "Comibaena pictipennis", - "Comostola subtiliaria", "Comibaena nigromacularia", "Comibaena procumbaria", "Hemistola monotona", "Fascellina plagiata", - "Tanaoctenia haliaria", "Episothalma robustaria", "Aporandria specularia", "Hypochrosis hyadaria", "Capasa festivaria", - "Gnamptoloma aventiaria", "", "Timandromorpha discolor", "Laciniodes plurilinearia", "Ascotis selenaria", "Xenoplia trivialis", - "Agathia", "Agathia lycaenaria", "Agathia hilarata", "Agathia arcuata", "Agathia laetata", "Agathia diversiformis", - "Agathia carissima", "Milionia basalis", "Cystidia", "Pseudomiza aurata", "Chorodna creataria", "Hydatocapnia gemina", - "Tephrina inchoata", "Metallolophia arenaria", "Dysphania militaris", "Obeidia tigrata", "Obeidia gigantearia", "Obeidia lucifera", - "Odontopera insulata", "Odontopera bilinearia", "Culpinia diffusa", "Iotaphora", "Spilopera divaricata", "Plesiomorpha flaviceps", - "", "Acolutha pulchella subsp. semifulva", "Hyposidra aquilaria", "Heterolocha aristonaria", "Ophthalmitis herbidaria", - "Auaxa cesadaria", "Tanaorhinus viridiluteata", "Tanaorhinus kina", "Tanaorhinus rafflesii", "Tanaorhinus reciprocata", - "Sibatania arizana", "Eumelea ludovicata", "Alcis angulifera", "Alcis repandata", "Heterolocha coccinea", - "Trichopteryx polycommata", "Opisthograptis moelleri", "Garaeus specularis", "Zanclopera falcata", "Arichanna melanaria", - "Nothomiza flavicosta", "", "Thinopteryx crocoptera", "Eilicrinia flava", "Borbacha pardaria", "Hyposidra infixaria", - "Cleora fraterna", "Medasina corticaria", "Yponomeutidae", "Yponomeuta evonymella", "Yponomeuta padella", "Hesperiidae", - "Burara gomata", "Baoris farri", "Udaspes folus", "Polytremis lubricans", "Badamia exclamationis", "Isoteinon lamprospilus", - "Celaenorrhinus maculosus", "Mooreana trichoneura", "Matapa aria", "Erynnis montanus", "Erynnis tages", "Seseria dohertyi", - "Abraximorpha davidii", "Parnara naso", "Parnara ganga", "Parnara guttata", "Borbo cinnara", "Suastus gremius", "", - "Astictopterus jama", "Erionota torus", "Notocrypta curvifascia", "Tagiades litigiosa", "Tagiades menaka", "Pseudocoladenia dan", - "Odontoptilum angulatum", "Pelopidas", "Pelopidas agna", "Pelopidas conjuncta", "Pelopidas mathias", "Hasora badra", - "Hasora chromus", "Hasora anura", "Hasora vitta", "Halpe porus", "Ancistroides nigrita", "Telicota besta", "Telicota colon", - "Telicota ohara", "Iambrix salsala", "Potanthus confucius", "Potanthus trachala", "Ampittia virgata", "Daimio tethys", "Zygaenidae", - "", "Erasmia pulchella", "", "Pryeria sinica", "Pidorus", "Campylotes", "Phauda flammans", "", "Elcysma westwoodi", - "Thyrassia penangae", "", "Artona hainana", "Trypanophora semihyalina", "", "Eterusia aedea", "", "Clelea sapphirina", - "Cyclosia midama", "Cyclosia papilionaris", "Cyclosia papilionaris", "Cyclosia panthona", "Amesia sanguiflua", "Histia rhodope", - "Gynautocera papilionaria", "Soritia strandi", "Soritia strandi", "Rhodopsona rubiginosa", "Idea leuconoe", "Danaus genutia", - "Danaus chrysippus", "", "Danaus plexippus", "Ideopsis similis", "Ideopsis vulgaris", "Euploea", "Euploea sylvester", - "Euploea tulliolus", "Euploea core", "Euploea mulciber", "Euploea midamus", "Parantica", "Parantica sita", "Parantica swinhoei", - "Parantica aglea", "Parantica melaneus", "Tirumala septentrionis", "Tirumala limniace", "Cossidae", "Zeuzera coffeae", - "Zeuzera multistrigata", "Zeuzera pyrina", "Lasiocampidae", "Gastropacha quercifolia", "Gastropacha populifolia", "Trabala vishnou", - "", "Gastropacha pardale", "Lebeda nobilis", "", "Euthrix laeta", "Metanastria gemella", "", "Odonestis pruni", "Euthrix isocyma", - "Cosmotriche discitincta", "Lymantriidae", "Calliteara pudibunda", "Calliteara horsfieldii", "Calliteara horsfieldii", - "Calliteara grotei", "Calliteara grotei", "Arna bipunctapex", "Orgyia antiqua", "Orgyia antiqua", "Orgyia postica", - "Orgyia postica", "Olene mendosa", "Olene mendosa", "Leucoma salicis", "Lymantria mathura", "Lymantria mathura#幼虫", - "Lymantria concolor", "Lymantria dispar", "Lymantria dispar", "Lymantria marginata", "Dasychira suisharyonis", - "Dasychira suisharyonis", "Arctornis l-nigrum", "Laelia coenosa", "Olene dudgeoni", "Olene dudgeoni", "Cifuna locuples", - "Euproctis similis", "Euproctis similis", "Habrosyne pyritoides", "Parapsestis tomponis", "Thyatira batis", "Tethea consimilis", - "Arctiidae", "Phragmatobia luctifera", "Areas galactina", "Peridrome subfascia", "Phragmatobia fuliginosa", - "Phragmatobia fuliginosa", "Ammatho tairadiata", "Peridrome orbicularis", "Eilema costipuncta", "Nudaria ranruna", - "Aglaomorpha histrio", "Utetheisa lotrix", "Pericallia matronula", "Asota plaginota", "Spilosoma lubricipeda", "Asota ficus", - "Asota egens", "Pelosia muscerda", "Arctia flavia", "Arctia caja", "Eilema griseola", "Creatonotus transiens", "Creatonotos gangis", - "Stictane rectilinea", "Rhyparioides metelkana", "Agrisius fuliginosus", "Stigmatophora palmata", "Stigmatophora flava", - "Vamuna remelana", "Aloa lactinea", "Spilosoma subcarnea", "", "Tyria jacobaeae", "", "Macrobrochis gigas", "", "Hyphantria cunea", - "Hyphantria cunea", "Miltochrista", "Miltochrista sauteri(Barsine sauteri)", "Miltochrista ziczac", "Miltochrista convexa", - "Miltochrista fuscozonata", "Miltochrista miniata", "Mangina argus", "Teulisna tumida", "Eugoa grisea", "", "Nyctemera lacticinia", - "Nyctemera lacticinia", "Nyctemera baulus", "Nyctemera tripunctaria", "Nyctemera adversata", "Euplocia membliaria", - "Amerila astreus", "Chrysaeglia magnifica", "Neochera dominia", "Paraona staudingeri", "Cyana", "Cyana hamata", "Cyana propinqua", - "Spilosoma taiwanensis", "Lycaenidae", "Ticherra acte", "Amblopala_avidiena", "Miletus_chinensis", "Lampides boeticus", - "Creon cleobis", "Tajuria cippus", "Zizeeria karsandra", "Catochrysops strabo", "Catochrysops panormus", "Poritia erycinoides", - "Udara dilectus", "Udara albocaerulea", "Arhopala paramuta", "Arhopala bazala", "Arhopala rama", "Nacaduba kurava", - "Nacaduba berenice", "Plebejus orbitulus", "Ancema blanka", "Iraota timoleon", "Heliophorus", "Heliophorus brahma", - "Heliophorus epicles", "Heliophorus ila", "heliophorus saphir", "Caleta roxus", "Horaga onyx", "Horaga albimacula", - "Yasoda tripunctata", "Zizeeria otis", "Prosotas nora", "Lycaena dispar", "Lycaena phlaeas", "Neopithecops zalmora", "Rapala", - "Rapala suffusa", "Rapala nissa", "Tongeia potanini", "Tongeia filicaudis", "Tongeia fischeri", "Mahathala ameria", - "Deudorix epijarbas", "Pratapa deva", "Zeltus amasa", "Scolitantides orion", "Celastrina argiolus", "Sinthusa chandrana", - "Chilades pandava", "Tarucus plinius", "Artipe eryx", "Megisba malaya", "Remelana jangala", "Everes argiades", "Taraka hamada", - "Plebejus argyrognomon", "Ussuriana michaelis", "Pseudozizeeria maha", "Acytolepis puspa", "Teratozephyrus arisanus", - "Curetis acuta", "Spindasis", "Spindasis syama", "Allotinus_drumila", "Aeromachus pygmaeus", "Aeromachus inachus", "Zizula hylax", - "Jamides alecto", "Jamides celeno", "Jamides bochus", "Spialia galba", "Loxura atymnus", "Niphanda fusca", "Dysaethria erasaria", - "Urapteroides astheniata", "Orudiza protheclaria", "Lyssa zampa", "Acropteris leptaliata", "Acropteris iphiata", - "Warreniplema fumicosta", "Urania leilus", "Chrysiridia rhipheus", "Amathusiidae", "Faunis eumeus", "Faunis aerope", - "Faunis canens", "Thauria lathyi", "Thaumantis diores", "Discophora sondaica", "Stichophthalma howqua", "Aemona amathusia", - "Acraea violae", "Acraea terpsicore", "Acraea issoria", "", "Siglophora sanguinolenta", "Westermannia elliptica", - "Risoba prominens", "Blenina quinaria", "Blenina senex", "Iragaodes nobilis", "Carea varipes", "Satyridae", "Neorina patria", - "Mandarinia regalis", "Penthema formosanum", "Penthema darlisa", "Penthema adelma", "Melanitis leda", "Melanitis phedima", - "Coenonympha amaryllis", "Melanargia", "Melanargia galathea", "Mycalesis intermedia", "Mycalesis sangaica", "Mycalesis anaxias", - "Mycalesis mineus", "Mycalesis zonata", "Mycalesis francisca", "Mycalesis gotama", "Mycalesis perseus", "Ypthima", - "Ypthima motschulskyi", "Ypthima praenubila", "Ypthima baldus", "Callerebia", "Neope", "Neope bremeri", "Neope muirheadii", - "Neope pulaha", "Elymnias hypermnestra", "Aphantopus hyperantus", "Lethe", "Lethe mekara", "Lethe butleri", "Lethe gemina", - "Lethe sinorix", "Lethe vindhya", "Lethe chandica", "Lethe christophi", "Lethe rohria", "Lethe insana", "Lethe verma", - "Lethe confusa", "Lethe lanaris", "Lethe syrcis", "Lethe europa", "Lethe dura", "Brahmaeidae", "Brahmaea wallichii", - "Brahmaea porphyrio", "Brahmaea hearseyi", "Brahmaea certhia", "Pieridae", "Pontia daplidice", "Pontia chloridice", - "Leptidea sinapis", "Leptidea amurensis", "Leptidea morsei", "Appias libythea", "Appias lyncida", "Appias albina", "Appias nero", - "Delias hyparete", "Delias pasithoe", "Delias descombesi", "Delias acalis", "Delias belladonna", "Dercas verhuelli", "Ixias pyrene", - "Gandaca harina", "Pieris canidia", "Pieris napi", "Pieris rapae", "Pieris melete", "Leptosia nina", "Aporia", "Aporia agathon", - "Aporia crataegi", "Anthocharis bambusarum", "Anthocharis scolymus", "Colias erate", "Colias fieldii", "Colias hyale", - "Colias palaeno", "Catopsilia pyranthe", "Catopsilia pomona", "Catopsilia scylla", "Gonepteryx amintha", "Gonepteryx rhamni", - "Prioneris thestylis", "Pareronia valeria", "Hebomoia glaucippe", "Eurema mandarina", "Eurema andersoni", "Eurema hecabe", - "Eurema laeta", "Eurema brigitta", "Eurema blanda", "Cepora nerissa", "Promalactis suzukiella", "Scythris sinensis", - "Eretmocera impactella", "Parnassius", "Parnassius citrinarius", "Parnassius nomion", "Parnassius phoebus", "Parnassius bremeri", - "Parnassius apollonius", "Parnassius apollo", "Thyrididae", "Striglina scitaria", "Thyris fenestrella", "Pyrinioides sinuosa", - "Pterophoridae", "Saptha divitiosa", "Notodontidae", "Gazalina chrysolopha", "Cerura menciana", "Cerura vinula", "", - "Syntypistis subgeneris", "Shachihoka formosana", "Clostera anastomosis", "Formofentonia orbifer", "Quadricalcarifera viridipicta", - "Mimopydna", "Phalera", "Phalera grotei", "Phalera bucephala", "Phalera assimilis", "Phalera flavescens", "Pheosia rimosa", - "Clostera anachoreta", "Fentonia ocypete", "Netria viridescens", "Syntypistis comatus", "Clostera albosigma", "Rachia striata", - "Ptilodon saturata", "Uropyia meticulodina", "Spatalia doerriesi", "Stauropus fagi", "Syntypistis pallidifascia", - "Gonoclostera timoniorum", "Gangarides", "Euhampsonia splendida", "Ginshachia elongata", "Euhampsonia cristata", - "Dudusa sphingiformis", "Patania chlorophanta", "Paracymoriza cataclystalis", "Pycnarmon lactiferalis", "Heterocnephes lymphatalis", - "Pagyda quinquelineata", "Cotachena histricalis", "Anania funebris", "Talanga sexpunctalis", "Agathodes ostentalis", - "Syllepte taiwanalis", "Nagiella quadrimaculalis", "Glyphodes quadrimaculalis", "Cirrhochrista brizoalis", "Polythlipta liquidalis", - "Botyodes principalis", "Eoophyla gibbosalis", "Eoophyla conjunctalis", "Parapediasia teterrellus", "Syllepte iophanes", - "Glyphodes duplicalis", "Pleuroptya balteata", "Glyphodes pyloalis", "Syllepte derogata", "Ramila acciusalis", "Tyspanodes striata", - "Cotachena pubescens", "Herpetogramma licarsisalis", "Pachynoa sabelialis", "Pycnarmon cribrata", "Paracymoriza prodigalis", - "Diaphania indica", "Omphisa anastomosalis", "Botyodes asialis", "Cangetta rectilinea", "Agrioglypta itysalis", - "Cnaphalocrocis medinalis", "Crypsiptya coclesalis", "Parapoynx stagnalis", "Parapoynx fluctuosalis", "Parapoynx vittalis", - "Parapoynx crisonalis", "Parapoynx villidalis", "Parapoynx diminutalis", "Pleuroptya iopasalis", "Palpita", - "Palpita nigropunctalis", "Nevrina procopia", "Nosophora semitritalis", "Loxostege sticticalis", "Poliobotys ablactalis", - "Diplopseustis perieresalis", "Pagyda nebulosa", "Cyrtogramme turbata", "Agrotera scissalis", "Pleuroptya ruralis", - "Maruca vitrata", "Pycnarmon pantherata", "Pseudargyria interruptella", "Eumorphobotys eumorphalis", "Botyodes diniasalis", - "Goniorhynchus butyrosa", "Triuncina brunnea", "Bombyx mandarina", "Bombyx mandarina", "Rondotia menciana", "", "Riodinidae", - "Dodona", "Dodona egeon", "Dodona maculosa", "Dodona durga", "Dodona eugenes", "Zemeros flegyas", "Stiboges nymphidia", - "Abisara saturata", "Abisara fylloides", "Abisara burnii", "Abisara echerius", "Abisara bifasciata", "Abisara neophron", - "Abisara fylla", "Nymphalidae", "银纹红袖蝶 Agraulis vanillae", "Cyrestis cocles", "Cyrestis thyodamas", "Cyrestis nivea", - "Parthenos syvia", "Parasarpa dudu", "Chersonesia risa", "Chalinga", "Abrota ganga", "Siproeta stelenes", "Boloria titania", - "Brenthis daphne", "Polyura narcaea", "Polyura eudamippus", "Polyura nepenthes", "Polyura athamas", "Sephisa chandra", - "Sephisa princeps", "Pararge aegeria", "Terinos atlita", "Athyma", "Athyma cama", "Athyma zeroca", "Athyma selenophora", - "Athyma perius", "Athyma asura", "Athyma nefte", "Athyma ranga", "Athyma opalina", "Vagrans egista", "Lexias pardalis", - "Vindula erota", "Argyreus hyperbius", "Asterocampa celtis", "Hypolimnas bolina", "Hypolimnas missipus", "Kallima inachus", - "Euphaedra themis", "Ariadne ariadne", "Ariadne merione", "Diaethria", "Herona marathus", "Timelaea", "Timelaea albescens", - "Neptis", "Neptis hylas", "Neptis soma", "Neptis namba", "Neptis nata", "Neptis sappho", "Neptis miah", "Neptis sankara", - "Neptis clinia", "Neptis pryeri", "Tanaecia julii", "Tanaecia jahnu", "Clossiana freija", "Clossiana euphrosyne", "Clossiana dia", - "Phalanta phalantha", "Issoria eugenia", "Issoria lathonia", "Kaniska canace", "Prothoe franck", "Dichorragia nesimachus", - "Helcyra subalba", "Symbrenthia lilaea", "Symbrenthia brabira", "Junonia atlites", "Junonia almana", "Junonia orithya", - "Junonia lemonias", "Junonia iphita", "Junonia coenia", "Junonia coenia", "Junonia hierta", "Fabriciana adippe", - "Pseudergolis wedah", "Moduza procris", "Dilipa fenestra", "Sasakia charonda", "Sasakia funebris", "Vanessa atalanta", - "Vanessa indica", "Vanessa cardui", "Vanessa virginiensis", "Limenitis", "Limenitis doerriesi", "Limenitis sulpitia", - "Limenitis populi", "Calinaga buddha", "Dophla evelina", "Melitaea", "Rohana parisatis", "Euthalia", "Euthalia", "Euthalia phemius", - "Euthalia pratti", "Euthalia aconthea", "Euthalia lubentina", "Euthalia niepelti", "Argyronome laodice", "Bhagadatta austenia", - "Hestina persimilis", "Hestina nama", "Hestina assimilis", "Phaedyma columella", "Hamadryas", "Nymphalis xanthomelas", - "Nymphalis vau-album", "Nymphalis antiopa", "", "Araschnia doris", "Araschnia prorsoides", "Araschnia levana", "Charaxes bernardus", - "Charaxes bernardus", "Pantoporia hordonia", "Doleschallia bisaltide", "Heliconius erato", "Heliconius charithonia", - "Cupha erymanthis", "Cupha erymanthis", "Argynnis paphia", "Argynnis aglaja", "Mimathyma schrenckii", "Polygonia c-album", - "Polygonia c-aureum", "Proclossiana eunomia", "Chitoria ulupi", "Cethosia cyane", "Cethosia biblis", "Apatura ilia", "Apatura iris", - "Damora sagana", "Stibochiona nicea", "Aglais io", "Aglais urticae", "Lebadea martha", "Pyralidae", "Mabra charonialis", - "Plodia interpunctella", "Eurrhyparodes bracteolalis", "Aethaloessa calidalis", "Endotricha olivacealis", "Ostrinia palustralis", - "Spoladea recurvalis", "Bocchoris inspersalis", "Arippara indicator", "Ancylolomia japonica", "Circobotys aurealis", - "Oncocera semirubella", "Heortia vitessoides", "Locastra muscosalis", "Nosophora insignis", "Orybina regalis", - "Rhectothyris gratiosalis", "Leucinodes orbonalis", "Herpetogramma luctuosalis", "Conogethes punctiferalis", "Pyralis pictalis", - "Pyralis farinalis", "Pyralis regalis", "Diasemia accalis", "Apomyelois ceratoniae", "Omiodes indicata", "Orybina flaviplaga", - "Lista haraldusalis", "Eurrhyparodes tricoloralis", "Rehimena phrynealis", "Cydalima perspectalis", "", "Tyspanodes hypsalis", - "Lamprosema commixta", "Bocchoris onychinalis", "Ericeia inangulata", "Gesonia obeditalis", "Eublemma anachoresis", - "Nagadeba indecoralis", "Lagoptera juno", "Artena dotata", "Scoliopteryx libatrix", "Eublemma cochylioides", "Oruza glaucotorna", - "Autoba tristalis", "Paracolax pryeri", "Ercheia umbrosa", "Cruxoruza decorata", "Opogona nipponica", "Sesiidae", - "Paranthrene tabaniformis", "Drepanidae", "Drepana pallida", "Pseudalbara parvula", "Canucha miranda", "Callidrepana patrana", - "Oreta insignis", "Cyclidia substigmaria", "Cyclidia orciferaria", "Macrauzata maxima", "Oreta loochooana", "Nordstromia japonica", - "Ditrigona triangularia", "Macrocilix mysticata", "Deroca hidda", "Drepana curvatula", "Agnidra scabiosa", "Macrocilix maia", - "Drapetodes mitaria", "", "Petavia attenuata", "Tetragonus catamitus", "Adelidae", "Lepidotarphius perornatellus", "Ctenuchidae", - "Syntomoides imaon", "Amata sperbius", "Amata germana", "Amata fortunei", "Amata grotei", "Anacampsis populella", - "Dichomeris sandycitis" - }; - - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_INSECTID_H diff --git a/lite/ncnn/cv/ncnn_mobile_emotion7.cpp b/lite/ncnn/cv/ncnn_mobile_emotion7.cpp deleted file mode 100644 index ca6a86c3..00000000 --- a/lite/ncnn/cv/ncnn_mobile_emotion7.cpp +++ /dev/null @@ -1,64 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#include "ncnn_mobile_emotion7.h" -#include "lite/utils.h" - -using ncnncv::NCNNMobileEmotion7; - -NCNNMobileEmotion7::NCNNMobileEmotion7(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNMobileEmotion7::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNMobileEmotion7::detect(const cv::Mat &mat, types::Emotions &emotions) -{ - if (mat.empty()) return; - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input_1", input); - // 3. fetch. - ncnn::Mat emotion_probs; - extractor.extract("emotion_preds", emotion_probs); // c=1,h=1,w=7 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(emotion_probs, "emotion_preds"); -#endif - - const unsigned int num_emotions = emotion_probs.w; - - unsigned int pred_label = 0; - const float *pred_probs_ptr = (float *) emotion_probs.data; - - float pred_score = pred_probs_ptr[0]; - - for (unsigned int i = 0; i < num_emotions; ++i) - { - if (pred_probs_ptr[i] > pred_score) - { - pred_score = pred_probs_ptr[i]; - pred_label = i; - } - } - - emotions.label = pred_label; - emotions.score = pred_score; - emotions.text = emotion_texts[pred_label]; - emotions.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_mobile_emotion7.h b/lite/ncnn/cv/ncnn_mobile_emotion7.h deleted file mode 100644 index ea14298a..00000000 --- a/lite/ncnn/cv/ncnn_mobile_emotion7.h +++ /dev/null @@ -1,38 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_MOBILE_EMOTION7_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_MOBILE_EMOTION7_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNMobileEmotion7 : public BasicNCNNHandler - { - public: - explicit NCNNMobileEmotion7(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNMobileEmotion7() override = default; - - private: - const int input_height = 224; - const int input_width = 224; - const float mean_vals[3] = {103.939f, 116.779f, 123.68f}; - const float norm_vals[3] = {1.f, 1.f, 1.f}; - const char *emotion_texts[7] = { - "angry", "disgust", "fear", "happiness", "neutral", "sadness", "surprise" - }; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::Emotions &emotions); - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_MOBILE_EMOTION7_H diff --git a/lite/ncnn/cv/ncnn_mobile_facenet.cpp b/lite/ncnn/cv/ncnn_mobile_facenet.cpp deleted file mode 100644 index 8072743e..00000000 --- a/lite/ncnn/cv/ncnn_mobile_facenet.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "ncnn_mobile_facenet.h" - -using ncnncv::NCNNMobileFaceNet; - -void NCNNMobileFaceNet::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW - int h = mat.rows; - int w = mat.cols; - in = ncnn::Mat::from_pixels_resize( - mat.data, ncnn::Mat::PIXEL_BGR2RGB, - w, h, input_width, input_height - ); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNMobileFaceNet::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - ncnn::Mat embedding; - extractor.extract("embedding", embedding); - - const unsigned int hidden_dim = embedding.w; // 512 - const float *embedding_values = (float *) embedding.data; - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_mobile_facenet.h b/lite/ncnn/cv/ncnn_mobile_facenet.h deleted file mode 100644 index ae49e74a..00000000 --- a/lite/ncnn/cv/ncnn_mobile_facenet.h +++ /dev/null @@ -1,38 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_MOBILE_FACENET_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_MOBILE_FACENET_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNMobileFaceNet : public BasicNCNNHandler - { - public: - explicit NCNNMobileFaceNet(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) - {}; - - ~NCNNMobileFaceNet() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 128.0f, 1.f / 128.0f, 1.f / 128.0f}; - static constexpr const int input_width = 96; - static constexpr const int input_height = 112; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_MOBILE_FACENET_H diff --git a/lite/ncnn/cv/ncnn_mobilenetv2.cpp b/lite/ncnn/cv/ncnn_mobilenetv2.cpp deleted file mode 100644 index d5741e53..00000000 --- a/lite/ncnn/cv/ncnn_mobilenetv2.cpp +++ /dev/null @@ -1,62 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "ncnn_mobilenetv2.h" -#include "lite/utils.h" - -using ncnncv::NCNNMobileNetV2; - -NCNNMobileNetV2::NCNNMobileNetV2(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNMobileNetV2::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNMobileNetV2::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - // 3. fetch. - ncnn::Mat logits_mat; - extractor.extract("logits", logits_mat); // c=1,h=1,w=1000 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(logits_mat, "logits"); -#endif - - const unsigned int num_classes = logits_mat.w; - const float *logits = (float *) logits_mat.data; - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_mobilenetv2.h b/lite/ncnn/cv/ncnn_mobilenetv2.h deleted file mode 100644 index 9dc160c7..00000000 --- a/lite/ncnn/cv/ncnn_mobilenetv2.h +++ /dev/null @@ -1,414 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_MOBILENETV2_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_MOBILENETV2_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNMobileNetV2 : public BasicNCNNHandler - { - public: - explicit NCNNMobileNetV2(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNMobileNetV2() override = default; - - private: - const int input_height = 224; - const int input_width = 224; - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - - }; -} - - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_MOBILENETV2_H diff --git a/lite/ncnn/cv/ncnn_mobilenetv2_68.cpp b/lite/ncnn/cv/ncnn_mobilenetv2_68.cpp deleted file mode 100644 index 839c5adb..00000000 --- a/lite/ncnn/cv/ncnn_mobilenetv2_68.cpp +++ /dev/null @@ -1,58 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#include "ncnn_mobilenetv2_68.h" - -using ncnncv::NCNNMobileNetV268; - -NCNNMobileNetV268::NCNNMobileNetV268(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNMobileNetV268::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNMobileNetV268::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - // 3. fetch landmarks. - ncnn::Mat landmarks_norm; - extractor.extract("output", landmarks_norm); // c=1,w=68*2,h=1 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(landmarks_norm, "output"); -#endif - const unsigned int num_landmarks = landmarks_norm.w; - const float *landmarks_ptr = (float *) landmarks_norm.data; - - for (unsigned int i = 0; i < num_landmarks; i += 2) - { - float x = landmarks_ptr[i]; - float y = landmarks_ptr[i + 1]; - - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - landmarks.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_mobilenetv2_68.h b/lite/ncnn/cv/ncnn_mobilenetv2_68.h deleted file mode 100644 index a02af0f1..00000000 --- a/lite/ncnn/cv/ncnn_mobilenetv2_68.h +++ /dev/null @@ -1,36 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_MOBILENETV2_68_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_MOBILENETV2_68_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNMobileNetV268 : public BasicNCNNHandler - { - public: - explicit NCNNMobileNetV268(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNMobileNetV268() override = default; - - private: - const int input_height = 56; - const int input_width = 56; - const float mean_vals[3] = {255.f * 0.485f, 255.f * 0.456, 255.f * 0.406f}; - const float norm_vals[3] = {1.0f / (255.f * 0.229f), 1.0f / (255.f * 0.224f), 1.0f / (255.f * 0.225f)}; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - }; -} - - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_MOBILENETV2_68_H diff --git a/lite/ncnn/cv/ncnn_mobilenetv2_se_68.cpp b/lite/ncnn/cv/ncnn_mobilenetv2_se_68.cpp deleted file mode 100644 index c5a13bd5..00000000 --- a/lite/ncnn/cv/ncnn_mobilenetv2_se_68.cpp +++ /dev/null @@ -1,58 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#include "ncnn_mobilenetv2_se_68.h" - -using ncnncv::NCNNMobileNetV2SE68; - -NCNNMobileNetV2SE68::NCNNMobileNetV2SE68(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNMobileNetV2SE68::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNMobileNetV2SE68::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - // 3. fetch landmarks. - ncnn::Mat landmarks_norm; - extractor.extract("output", landmarks_norm); // c=1,w=68*2,h=1 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(landmarks_norm, "output"); -#endif - const unsigned int num_landmarks = landmarks_norm.w; - const float *landmarks_ptr = (float *) landmarks_norm.data; - - for (unsigned int i = 0; i < num_landmarks; i += 2) - { - float x = landmarks_ptr[i]; - float y = landmarks_ptr[i + 1]; - - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - landmarks.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_mobilenetv2_se_68.h b/lite/ncnn/cv/ncnn_mobilenetv2_se_68.h deleted file mode 100644 index 79e95c0e..00000000 --- a/lite/ncnn/cv/ncnn_mobilenetv2_se_68.h +++ /dev/null @@ -1,35 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_MOBILENETV2_SE_68_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_MOBILENETV2_SE_68_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNMobileNetV2SE68 : public BasicNCNNHandler - { - public: - explicit NCNNMobileNetV2SE68(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNMobileNetV2SE68() override = default; - - private: - const int input_height = 56; - const int input_width = 56; - const float mean_vals[3] = {255.f * 0.485f, 255.f * 0.456, 255.f * 0.406f}; - const float norm_vals[3] = {1.0f / (255.f * 0.229f), 1.0f / (255.f * 0.224f), 1.0f / (255.f * 0.225f)}; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_MOBILENETV2_SE_68_H diff --git a/lite/ncnn/cv/ncnn_mobilese_focal_face.cpp b/lite/ncnn/cv/ncnn_mobilese_focal_face.cpp deleted file mode 100644 index 42800bdf..00000000 --- a/lite/ncnn/cv/ncnn_mobilese_focal_face.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "ncnn_mobilese_focal_face.h" - -using ncnncv::NCNNMobileSEFocalFace; - -void NCNNMobileSEFocalFace::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW - int h = mat.rows; - int w = mat.cols; - in = ncnn::Mat::from_pixels_resize( - mat.data, ncnn::Mat::PIXEL_BGR2RGB, - w, h, input_width, input_height - ); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNMobileSEFocalFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - ncnn::Mat embedding; - extractor.extract("embedding", embedding); - - const unsigned int hidden_dim = embedding.c; // 256 - const float *embedding_values = (float *) embedding.data; - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_mobilese_focal_face.h b/lite/ncnn/cv/ncnn_mobilese_focal_face.h deleted file mode 100644 index b1586a5f..00000000 --- a/lite/ncnn/cv/ncnn_mobilese_focal_face.h +++ /dev/null @@ -1,38 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_MOBILESE_FOCAL_FACE_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_MOBILESE_FOCAL_FACE_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNMobileSEFocalFace : public BasicNCNNHandler - { - public: - explicit NCNNMobileSEFocalFace(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) - {}; - - ~NCNNMobileSEFocalFace() override = default; - - private: - const float mean_vals[3] = {0.f, 0.f, 0.f}; // RGB - const float norm_vals[3] = {1.f / 255.0f, 1.f / 255.0f, 1.f / 255.0f}; - static constexpr const int input_width = 128; - static constexpr const int input_height = 128; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_MOBILESE_FOCAL_FACE_H diff --git a/lite/ncnn/cv/ncnn_modnet.cpp b/lite/ncnn/cv/ncnn_modnet.cpp deleted file mode 100644 index 4677d4b3..00000000 --- a/lite/ncnn/cv/ncnn_modnet.cpp +++ /dev/null @@ -1,114 +0,0 @@ -// -// Created by DefTruth on 2022/3/27. -// - -#include "ncnn_modnet.h" -#include "lite/utils.h" - -using ncnncv::NCNNMODNet; - -NCNNMODNet::NCNNMODNet(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads, - unsigned int _input_height, - unsigned int _input_width) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads), - input_height(_input_height), input_width(_input_width) -{ -} - -void NCNNMODNet::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - // will do deepcopy inside ncnn - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNMODNet::detect(const cv::Mat &mat, types::MattingContent &content, bool remove_noise, - bool minimum_post_process) -{ - if (mat.empty()) return; - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - // 3. generate matting - this->generate_matting(extractor, mat, content, remove_noise, minimum_post_process); -} - -void NCNNMODNet::generate_matting(ncnn::Extractor &extractor, - const cv::Mat &mat, types::MattingContent &content, - bool remove_noise, bool minimum_post_process) -{ - ncnn::Mat output; - extractor.extract("output", output); -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(output, "output"); -#endif - const unsigned int h = mat.rows; - const unsigned int w = mat.cols; - - const unsigned int out_h = input_height; - const unsigned int out_w = input_width; - - float *output_ptr = (float *) output.data; - - cv::Mat alpha_pred(out_h, out_w, CV_32FC1, output_ptr); - // post process - if (remove_noise) lite::utils::remove_small_connected_area(alpha_pred, 0.05f); - // resize alpha - if (out_h != h || out_w != w) - // already allocated a new continuous memory after resize. - cv::resize(alpha_pred, alpha_pred, cv::Size(w, h)); - // need clone to allocate a new continuous memory if not performed resize. - // The memory elements point to will release after return. - else alpha_pred = alpha_pred.clone(); - - cv::Mat pmat = alpha_pred; // ref - content.pha_mat = pmat; // auto handle the memory inside ocv with smart ref. - - if (!minimum_post_process) - { - // MODNet only predict Alpha, no fgr. So, - // the fake fgr and merge mat may not need, - // let the fgr mat and merge mat empty to - // Speed up the post processes. - cv::Mat mat_copy; - mat.convertTo(mat_copy, CV_32FC3); - // merge mat and fgr mat may not need - std::vector mat_channels; - cv::split(mat_copy, mat_channels); - cv::Mat bmat = mat_channels.at(0); - cv::Mat gmat = mat_channels.at(1); - cv::Mat rmat = mat_channels.at(2); // ref only, zero-copy. - bmat = bmat.mul(pmat); - gmat = gmat.mul(pmat); - rmat = rmat.mul(pmat); - cv::Mat rest = 1.f - pmat; - cv::Mat mbmat = bmat.mul(pmat) + rest * 153.f; - cv::Mat mgmat = gmat.mul(pmat) + rest * 255.f; - cv::Mat mrmat = rmat.mul(pmat) + rest * 120.f; - std::vector fgr_channel_mats, merge_channel_mats; - fgr_channel_mats.push_back(bmat); - fgr_channel_mats.push_back(gmat); - fgr_channel_mats.push_back(rmat); - merge_channel_mats.push_back(mbmat); - merge_channel_mats.push_back(mgmat); - merge_channel_mats.push_back(mrmat); - - cv::merge(fgr_channel_mats, content.fgr_mat); - cv::merge(merge_channel_mats, content.merge_mat); - - content.fgr_mat.convertTo(content.fgr_mat, CV_8UC3); - content.merge_mat.convertTo(content.merge_mat, CV_8UC3); - } - - content.flag = true; -} diff --git a/lite/ncnn/cv/ncnn_modnet.h b/lite/ncnn/cv/ncnn_modnet.h deleted file mode 100644 index 3424d234..00000000 --- a/lite/ncnn/cv/ncnn_modnet.h +++ /dev/null @@ -1,43 +0,0 @@ -// -// Created by DefTruth on 2022/3/27. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_MODNET_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_MODNET_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNMODNet : public BasicNCNNHandler - { - public: - explicit NCNNMODNet(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1, - unsigned int _input_height = 512, - unsigned int _input_width = 512); - - ~NCNNMODNet() override = default; - - private: - const int input_height; - const int input_width; - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; - const float norm_vals[3] = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - void generate_matting(ncnn::Extractor &extractor, - const cv::Mat &mat, types::MattingContent &content, - bool remove_noise = false, bool minimum_post_process = false); - - public: - void detect(const cv::Mat &mat, types::MattingContent &content, bool remove_noise = false, - bool minimum_post_process = false); - - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_MODNET_H diff --git a/lite/ncnn/cv/ncnn_nanodet.cpp b/lite/ncnn/cv/ncnn_nanodet.cpp deleted file mode 100644 index c8379a84..00000000 --- a/lite/ncnn/cv/ncnn_nanodet.cpp +++ /dev/null @@ -1,243 +0,0 @@ -// -// Created by DefTruth on 2021/10/7. -// - -#include "ncnn_nanodet.h" -#include "lite/utils.h" - -using ncnncv::NCNNNanoDet; - -NCNNNanoDet::NCNNNanoDet(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads, - int _input_height, int _input_width) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ - input_height = _input_height; - input_width = _input_width; -} - -void NCNNNanoDet::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - NanoScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(0, 0, 0)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.ratio = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.flag = true; -} - -void NCNNNanoDet::transform(const cv::Mat &mat_rs, ncnn::Mat &in) -{ - // BGR NHWC -> BGR NCHW - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNNanoDet::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - auto img_height = static_cast(mat.rows); - auto img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - NanoScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat_rs, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input.1", input); - // 3.rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, extractor, score_threshold, img_height, img_width); - // 4. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void NCNNNanoDet::generate_points(unsigned int target_height, unsigned int target_width) -{ - if (center_points_is_update && (!is_dynamic_input)) return; - - for (auto stride : strides) - { - unsigned int num_grid_w = target_width / stride; - unsigned int num_grid_h = target_height / stride; - std::vector points; - - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - float grid0 = (float) g0 + 0.5f; - float grid1 = (float) g1 + 0.5f; -#ifdef LITE_WIN32 - NanoCenterPoint point; - point.grid0 = grid0; - point.grid1 = grid1; - point.stride = (float) stride; - points.push_back(point); -#else - points.push_back((NanoCenterPoint) {grid0, grid1, (float) stride}); -#endif - } - } - center_points[stride] = points; - } - - center_points_is_update = true; -} - -void NCNNNanoDet::generate_bboxes(const NanoScaleParams &scale_params, - std::vector &bbox_collection, - ncnn::Extractor &extractor, - float score_threshold, - float img_height, float img_width) -{ - ncnn::Mat cls_pred_stride_8; - ncnn::Mat cls_pred_stride_16; - ncnn::Mat cls_pred_stride_32; - ncnn::Mat dis_pred_stride_8; - ncnn::Mat dis_pred_stride_16; - ncnn::Mat dis_pred_stride_32; - extractor.extract("cls_pred_stride_8", cls_pred_stride_8); - extractor.extract("cls_pred_stride_16", cls_pred_stride_16); - extractor.extract("cls_pred_stride_32", cls_pred_stride_32); - extractor.extract("dis_pred_stride_8", dis_pred_stride_8); - extractor.extract("dis_pred_stride_16", dis_pred_stride_16); - extractor.extract("dis_pred_stride_32", dis_pred_stride_32); - this->generate_points(input_height, input_width); - - bbox_collection.clear(); - // level 8 & 16 & 32 - this->generate_bboxes_single_stride(scale_params, cls_pred_stride_8, dis_pred_stride_8, 8, - score_threshold, img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, cls_pred_stride_16, dis_pred_stride_16, 16, - score_threshold, img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, cls_pred_stride_32, dis_pred_stride_32, 32, - score_threshold, img_height, img_width, bbox_collection); -#if LITENCNN_DEBUG - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void NCNNNanoDet::generate_bboxes_single_stride(const NanoScaleParams &scale_params, - ncnn::Mat &cls_pred, ncnn::Mat &dis_pred, - unsigned int stride, float score_threshold, - float img_height, float img_width, - std::vector &bbox_collection) -{ - unsigned int nms_pre_ = (stride / 8) * nms_pre; // 1 * 1000,2*1000,... - nms_pre_ = nms_pre_ >= nms_pre ? nms_pre_ : nms_pre; - - const unsigned int f_h = (unsigned int) input_height / stride; - const unsigned int f_w = (unsigned int) input_width / stride; - const unsigned int num_points = f_h * f_w; - const unsigned int num_classes = 80; - - float ratio = scale_params.ratio; - int dw = scale_params.dw; - int dh = scale_params.dh; - - unsigned int count = 0; - auto &stride_points = center_points[stride]; - - for (unsigned int i = 0; i < num_points; ++i) - { - const float *scores = cls_pred.row(i); // row ptr - float cls_conf = scores[0]; - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = scores[j]; - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - if (cls_conf < score_threshold) continue; // filter - - auto &point = stride_points.at(i); - const float cx = point.grid0; // cx - const float cy = point.grid1; // cy - const float s = point.stride; // stride - - const float *offsets = dis_pred.row(i); - float l = offsets[0]; // left - float t = offsets[1]; // top - float r = offsets[2]; // right - float b = offsets[3]; // bottom - - types::Boxf box; - float x1 = ((cx - l) * s - (float) dw) / ratio; // cx - l x1 - float y1 = ((cy - t) * s - (float) dh) / ratio; // cy - t y1 - float x2 = ((cx + r) * s - (float) dw) / ratio; // cx + r x2 - float y2 = ((cy + b) * s - (float) dh) / ratio; // cy + b y2 - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(img_width - 1.f, x2); - box.y2 = std::min(img_height - 1.f, y2); - box.score = cls_conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - - if (bbox_collection.size() > nms_pre_) - { - std::sort(bbox_collection.begin(), bbox_collection.end(), - [](const types::Boxf &a, const types::Boxf &b) - { return a.score > b.score; }); // sort inplace - // trunc - bbox_collection.resize(nms_pre_); - } - -} - -void NCNNNanoDet::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} diff --git a/lite/ncnn/cv/ncnn_nanodet.h b/lite/ncnn/cv/ncnn_nanodet.h deleted file mode 100644 index 86287d8b..00000000 --- a/lite/ncnn/cv/ncnn_nanodet.h +++ /dev/null @@ -1,115 +0,0 @@ -// -// Created by DefTruth on 2021/10/7. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_NANODET_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_NANODET_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNNanoDet : public BasicNCNNHandler - { - public: - explicit NCNNNanoDet(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1, - int _input_height = 320, - int _input_width = 320); // - ~NCNNNanoDet() override = default; - - private: - // nested classes - typedef struct - { - float grid0; - float grid1; - float stride; - } NanoCenterPoint; - - typedef struct - { - float ratio; - int dw; - int dh; - bool flag; - } NanoScaleParams; - - private: - const float mean_vals[3] = {103.53f, 116.28f, 123.675f}; // BGR - const float norm_vals[3] = {0.017429f, 0.017507f, 0.017125f}; - - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int nms_pre = 1000; - static constexpr const unsigned int max_nms = 30000; - static constexpr const bool is_dynamic_input = false; - - // multi-levels center points - int input_height = 320; - int input_width = 320; - std::vector strides = {8, 16, 32}; - std::unordered_map> center_points; - bool center_points_is_update = false; - - private: - - void transform(const cv::Mat &mat_rs, ncnn::Mat &in) override; - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - NanoScaleParams &scale_params); - - // only generate once - void generate_points(unsigned int target_height, unsigned int target_width); - - void generate_bboxes_single_stride(const NanoScaleParams &scale_params, - ncnn::Mat &cls_pred, - ncnn::Mat &dis_pred, - unsigned int stride, - float score_threshold, - float img_height, - float img_width, - std::vector &bbox_collection); - - void generate_bboxes(const NanoScaleParams &scale_params, - std::vector &bbox_collection, - ncnn::Extractor &extractor, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - /** - * @param mat cv::Mat BGR format - * @param detected_boxes vector of Boxf to catch detected boxes. - * @param score_threshold default 0.45f, only keep the result which >= score_threshold. - * @param iou_threshold default 0.3f, iou threshold for NMS. - * @param topk default 100, maximum output boxes after NMS. - * @param nms_type the method. - */ - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.45f, float iou_threshold = 0.3f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_NANODET_H diff --git a/lite/ncnn/cv/ncnn_nanodet_depreciated.cpp b/lite/ncnn/cv/ncnn_nanodet_depreciated.cpp deleted file mode 100644 index 0e43414b..00000000 --- a/lite/ncnn/cv/ncnn_nanodet_depreciated.cpp +++ /dev/null @@ -1,259 +0,0 @@ -// -// Created by DefTruth on 2021/10/9. -// - -#include "ncnn_nanodet_depreciated.h" -#include "lite/utils.h" - -using ncnncv::NCNNNanoDetDepreciated; - -NCNNNanoDetDepreciated::NCNNNanoDetDepreciated(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads, - int _input_height, - int _input_width) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ - input_height = _input_height; - input_width = _input_width; -} - -void NCNNNanoDetDepreciated::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - NanoDepreciatedScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(0, 0, 0)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.ratio = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.flag = true; -} - -void NCNNNanoDetDepreciated::transform(const cv::Mat &mat_rs, ncnn::Mat &in) -{ - // BGR NHWC -> BGR NCHW - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNNanoDetDepreciated::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - auto img_height = static_cast(mat.rows); - auto img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - NanoDepreciatedScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat_rs, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input.1", input); - // 3.rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, extractor, score_threshold, img_height, img_width); - // 4. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void NCNNNanoDetDepreciated::generate_points(unsigned int target_height, unsigned int target_width) -{ - if (center_points_is_update && (!is_dynamic_input)) return; - - for (auto stride : strides) - { - unsigned int num_grid_w = target_width / stride; - unsigned int num_grid_h = target_height / stride; - std::vector points; - - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - float grid0 = (float) g0 + 0.5f; - float grid1 = (float) g1 + 0.5f; -#ifdef LITE_WIN32 - NanoDepreciatedCenterPoint point; - point.grid0 = grid0; - point.grid1 = grid1; - point.stride = (float) stride; - points.push_back(point); -#else - points.push_back((NanoDepreciatedCenterPoint) {grid0, grid1, (float) stride}); -#endif - } - } - center_points[stride] = points; - } - - center_points_is_update = true; -} - -void NCNNNanoDetDepreciated::generate_bboxes(const NanoDepreciatedScaleParams &scale_params, - std::vector &bbox_collection, - ncnn::Extractor &extractor, - float score_threshold, - float img_height, - float img_width) -{ - ncnn::Mat cls_pred_stride_8; - ncnn::Mat cls_pred_stride_16; - ncnn::Mat cls_pred_stride_32; - ncnn::Mat dis_pred_stride_8; - ncnn::Mat dis_pred_stride_16; - ncnn::Mat dis_pred_stride_32; - extractor.extract("cls_pred_stride_8", cls_pred_stride_8); - extractor.extract("cls_pred_stride_16", cls_pred_stride_16); - extractor.extract("cls_pred_stride_32", cls_pred_stride_32); - extractor.extract("dis_pred_stride_8", dis_pred_stride_8); - extractor.extract("dis_pred_stride_16", dis_pred_stride_16); - extractor.extract("dis_pred_stride_32", dis_pred_stride_32); - this->generate_points(input_height, input_width); - - bbox_collection.clear(); - // level 8 & 16 & 32 - this->generate_bboxes_single_stride(scale_params, cls_pred_stride_8, dis_pred_stride_8, 8, - score_threshold, img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, cls_pred_stride_16, dis_pred_stride_16, 16, - score_threshold, img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, cls_pred_stride_32, dis_pred_stride_32, 32, - score_threshold, img_height, img_width, bbox_collection); -#if LITENCNN_DEBUG - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void NCNNNanoDetDepreciated::generate_bboxes_single_stride(const NanoDepreciatedScaleParams &scale_params, - ncnn::Mat &cls_pred, ncnn::Mat &dis_pred, - unsigned int stride, float score_threshold, - float img_height, float img_width, - std::vector &bbox_collection) -{ - unsigned int nms_pre_ = (stride / 8) * nms_pre; // 1 * 1000,2*1000,... - nms_pre_ = nms_pre_ >= nms_pre ? nms_pre_ : nms_pre; - - const unsigned int f_h = (unsigned int) input_height / stride; - const unsigned int f_w = (unsigned int) input_width / stride; - const unsigned int num_points = f_h * f_w; - const unsigned int num_classes = 80; - - const unsigned int dis_pred_w = dis_pred.w; - const unsigned int reg_max = dis_pred_w / 4; // e.g 8=7+1 - - float ratio = scale_params.ratio; - int dw = scale_params.dw; - int dh = scale_params.dh; - - unsigned int count = 0; - auto &stride_points = center_points[stride]; - - for (unsigned int i = 0; i < num_points; ++i) - { - const float *scores = cls_pred.row(i); // row ptr - float cls_conf = scores[0]; - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = scores[j]; - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - if (cls_conf < score_threshold) continue; // filter - - auto &point = stride_points.at(i); - const float cx = point.grid0; // cx - const float cy = point.grid1; // cy - const float s = point.stride; // stride - - const float *logits = dis_pred.row(i); // 32|44... - std::vector offsets(4); - for (unsigned int k = 0; k < 4; ++k) - { - float offset = 0.f; - unsigned int max_id; - auto probs = lite::utils::math::softmax( - logits + (k * reg_max), reg_max, max_id); - for (unsigned int l = 0; l < reg_max; ++l) - offset += (float) l * probs[l]; - offsets[k] = offset; - } - float l = offsets[0]; // left - float t = offsets[1]; // top - float r = offsets[2]; // right - float b = offsets[3]; // bottom - - types::Boxf box; - float x1 = ((cx - l) * s - (float) dw) / ratio; // cx - l x1 - float y1 = ((cy - t) * s - (float) dh) / ratio; // cy - t y1 - float x2 = ((cx + r) * s - (float) dw) / ratio; // cx + r x2 - float y2 = ((cy + b) * s - (float) dh) / ratio; // cy + b y2 - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(img_width - 1.f, x2); - box.y2 = std::min(img_height - 1.f, y2); - box.score = cls_conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - - if (bbox_collection.size() > nms_pre_) - { - std::sort(bbox_collection.begin(), bbox_collection.end(), - [](const types::Boxf &a, const types::Boxf &b) - { return a.score > b.score; }); // sort inplace - // trunc - bbox_collection.resize(nms_pre_); - } - -} - -void NCNNNanoDetDepreciated::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_nanodet_depreciated.h b/lite/ncnn/cv/ncnn_nanodet_depreciated.h deleted file mode 100644 index 02be39d5..00000000 --- a/lite/ncnn/cv/ncnn_nanodet_depreciated.h +++ /dev/null @@ -1,115 +0,0 @@ -// -// Created by DefTruth on 2021/10/9. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_NANODET_DEPRECIATED_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_NANODET_DEPRECIATED_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNNanoDetDepreciated : public BasicNCNNHandler - { - public: - explicit NCNNNanoDetDepreciated(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1, - int _input_height = 320, - int _input_width = 320); // - ~NCNNNanoDetDepreciated() override = default; - - private: - // nested classes - typedef struct - { - float grid0; - float grid1; - float stride; - } NanoDepreciatedCenterPoint; - - typedef struct - { - float ratio; - int dw; - int dh; - bool flag; - } NanoDepreciatedScaleParams; - - private: - const float mean_vals[3] = {103.53f, 116.28f, 123.675f}; // BGR - const float norm_vals[3] = {0.017429f, 0.017507f, 0.017125f}; - - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int nms_pre = 1000; - static constexpr const unsigned int max_nms = 30000; - static constexpr const bool is_dynamic_input = false; - - // multi-levels center points - int input_height = 320; - int input_width = 320; - std::vector strides = {8, 16, 32}; - std::unordered_map> center_points; - bool center_points_is_update = false; - - private: - - void transform(const cv::Mat &mat_rs, ncnn::Mat &in) override; - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - NanoDepreciatedScaleParams &scale_params); - - // only generate once - void generate_points(unsigned int target_height, unsigned int target_width); - - void generate_bboxes_single_stride(const NanoDepreciatedScaleParams &scale_params, - ncnn::Mat &cls_pred, - ncnn::Mat &dis_pred, - unsigned int stride, - float score_threshold, - float img_height, - float img_width, - std::vector &bbox_collection); - - void generate_bboxes(const NanoDepreciatedScaleParams &scale_params, - std::vector &bbox_collection, - ncnn::Extractor &extractor, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - /** - * @param mat cv::Mat BGR format - * @param detected_boxes vector of Boxf to catch detected boxes. - * @param score_threshold default 0.45f, only keep the result which >= score_threshold. - * @param iou_threshold default 0.3f, iou threshold for NMS. - * @param topk default 100, maximum output boxes after NMS. - * @param nms_type the method. - */ - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.45f, float iou_threshold = 0.3f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_NANODET_DEPRECIATED_H diff --git a/lite/ncnn/cv/ncnn_nanodet_efficientdet_lite_depreciated.h b/lite/ncnn/cv/ncnn_nanodet_efficientdet_lite_depreciated.h deleted file mode 100644 index f3e9995b..00000000 --- a/lite/ncnn/cv/ncnn_nanodet_efficientdet_lite_depreciated.h +++ /dev/null @@ -1,115 +0,0 @@ -// -// Created by DefTruth on 2021/10/9. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_NANODET_EFFICIENTDET_LITE_DEPRECIATED_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_NANODET_EFFICIENTDET_LITE_DEPRECIATED_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNNanoDetEfficientNetLiteDepreciated : public BasicNCNNHandler - { - public: - explicit NCNNNanoDetEfficientNetLiteDepreciated(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1, - int _input_height = 320, - int _input_width = 320); // - ~NCNNNanoDetEfficientNetLiteDepreciated() override = default; - - private: - // nested classes - typedef struct - { - float grid0; - float grid1; - float stride; - } NanoLiteDepreciatedCenterPoint; - - typedef struct - { - float ratio; - int dw; - int dh; - bool flag; - } NanoLiteDepreciatedScaleParams; - - private: - const float mean_vals[3] = {103.53f, 116.28f, 123.675f}; // BGR - const float norm_vals[3] = {0.017429f, 0.017507f, 0.017125f}; - - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int nms_pre = 1000; - static constexpr const unsigned int max_nms = 30000; - static constexpr const bool is_dynamic_input = false; - - // multi-levels center points - int input_height = 320; - int input_width = 320; - std::vector strides = {8, 16, 32}; - std::unordered_map> center_points; - bool center_points_is_update = false; - - private: - - void transform(const cv::Mat &mat_rs, ncnn::Mat &in) override; - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - NanoLiteDepreciatedScaleParams &scale_params); - - // only generate once - void generate_points(unsigned int target_height, unsigned int target_width); - - void generate_bboxes_single_stride(const NanoLiteDepreciatedScaleParams &scale_params, - ncnn::Mat &cls_pred, - ncnn::Mat &dis_pred, - unsigned int stride, - float score_threshold, - float img_height, - float img_width, - std::vector &bbox_collection); - - void generate_bboxes(const NanoLiteDepreciatedScaleParams &scale_params, - std::vector &bbox_collection, - ncnn::Extractor &extractor, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - /** - * @param mat cv::Mat BGR format - * @param detected_boxes vector of Boxf to catch detected boxes. - * @param score_threshold default 0.45f, only keep the result which >= score_threshold. - * @param iou_threshold default 0.3f, iou threshold for NMS. - * @param topk default 100, maximum output boxes after NMS. - * @param nms_type the method. - */ - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.45f, float iou_threshold = 0.3f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_NANODET_EFFICIENTDET_LITE_DEPRECIATED_H diff --git a/lite/ncnn/cv/ncnn_nanodet_efficientnet_lite.cpp b/lite/ncnn/cv/ncnn_nanodet_efficientnet_lite.cpp deleted file mode 100644 index ea60a511..00000000 --- a/lite/ncnn/cv/ncnn_nanodet_efficientnet_lite.cpp +++ /dev/null @@ -1,244 +0,0 @@ -// -// Created by DefTruth on 2021/10/7. -// - -#include "ncnn_nanodet_efficientnet_lite.h" -#include "lite/utils.h" - -using ncnncv::NCNNNanoDetEfficientNetLite; - -NCNNNanoDetEfficientNetLite::NCNNNanoDetEfficientNetLite(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads, - int _input_height, int _input_width) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ - input_height = _input_height; - input_width = _input_width; -} - -void NCNNNanoDetEfficientNetLite::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - NanoLiteScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(0, 0, 0)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.ratio = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.flag = true; -} - -void NCNNNanoDetEfficientNetLite::transform(const cv::Mat &mat_rs, ncnn::Mat &in) -{ - // BGR NHWC -> BGR NCHW - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNNanoDetEfficientNetLite::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - auto img_height = static_cast(mat.rows); - auto img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - NanoLiteScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat_rs, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input.1", input); - // 3.rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, extractor, score_threshold, img_height, img_width); - // 4. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void NCNNNanoDetEfficientNetLite::generate_points(unsigned int target_height, unsigned int target_width) -{ - if (center_points_is_update && (!is_dynamic_input)) return; - - for (auto stride : strides) - { - unsigned int num_grid_w = target_width / stride; - unsigned int num_grid_h = target_height / stride; - std::vector points; - - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - float grid0 = (float) g0 + 0.5f; - float grid1 = (float) g1 + 0.5f; -#ifdef LITE_WIN32 - NanoLiteCenterPoint point; - point.grid0 = grid0; - point.grid1 = grid1; - point.stride = (float) stride; - points.push_back(point); -#else - points.push_back((NanoLiteCenterPoint) {grid0, grid1, (float) stride}); -#endif - } - } - center_points[stride] = points; - } - - center_points_is_update = true; -} - -void NCNNNanoDetEfficientNetLite::generate_bboxes(const NanoLiteScaleParams &scale_params, - std::vector &bbox_collection, - ncnn::Extractor &extractor, - float score_threshold, - float img_height, float img_width) -{ - ncnn::Mat cls_pred_stride_8; - ncnn::Mat cls_pred_stride_16; - ncnn::Mat cls_pred_stride_32; - ncnn::Mat dis_pred_stride_8; - ncnn::Mat dis_pred_stride_16; - ncnn::Mat dis_pred_stride_32; - extractor.extract("cls_pred_stride_8", cls_pred_stride_8); - extractor.extract("cls_pred_stride_16", cls_pred_stride_16); - extractor.extract("cls_pred_stride_32", cls_pred_stride_32); - extractor.extract("dis_pred_stride_8", dis_pred_stride_8); - extractor.extract("dis_pred_stride_16", dis_pred_stride_16); - extractor.extract("dis_pred_stride_32", dis_pred_stride_32); - this->generate_points(input_height, input_width); - - bbox_collection.clear(); - // level 8 & 16 & 32 - this->generate_bboxes_single_stride(scale_params, cls_pred_stride_8, dis_pred_stride_8, 8, - score_threshold, img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, cls_pred_stride_16, dis_pred_stride_16, 16, - score_threshold, img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, cls_pred_stride_32, dis_pred_stride_32, 32, - score_threshold, img_height, img_width, bbox_collection); -#if LITENCNN_DEBUG - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void NCNNNanoDetEfficientNetLite::generate_bboxes_single_stride(const NanoLiteScaleParams &scale_params, - ncnn::Mat &cls_pred, ncnn::Mat &dis_pred, - unsigned int stride, float score_threshold, - float img_height, float img_width, - std::vector &bbox_collection) -{ - unsigned int nms_pre_ = (stride / 8) * nms_pre; // 1 * 1000,2*1000,... - nms_pre_ = nms_pre_ >= nms_pre ? nms_pre_ : nms_pre; - - const unsigned int f_h = (unsigned int) input_height / stride; - const unsigned int f_w = (unsigned int) input_width / stride; - const unsigned int num_points = f_h * f_w; - const unsigned int num_classes = 80; - - float ratio = scale_params.ratio; - int dw = scale_params.dw; - int dh = scale_params.dh; - - unsigned int count = 0; - auto &stride_points = center_points[stride]; - - for (unsigned int i = 0; i < num_points; ++i) - { - const float *scores = cls_pred.row(i); // row ptr - float cls_conf = scores[0]; - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = scores[j]; - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - if (cls_conf < score_threshold) continue; // filter - - auto &point = stride_points.at(i); - const float cx = point.grid0; // cx - const float cy = point.grid1; // cy - const float s = point.stride; // stride - - const float *offsets = dis_pred.row(i); - float l = offsets[0]; // left - float t = offsets[1]; // top - float r = offsets[2]; // right - float b = offsets[3]; // bottom - - types::Boxf box; - float x1 = ((cx - l) * s - (float) dw) / ratio; // cx - l x1 - float y1 = ((cy - t) * s - (float) dh) / ratio; // cy - t y1 - float x2 = ((cx + r) * s - (float) dw) / ratio; // cx + r x2 - float y2 = ((cy + b) * s - (float) dh) / ratio; // cy + b y2 - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(img_width - 1.f, x2); - box.y2 = std::min(img_height - 1.f, y2); - box.score = cls_conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - - if (bbox_collection.size() > nms_pre_) - { - std::sort(bbox_collection.begin(), bbox_collection.end(), - [](const types::Boxf &a, const types::Boxf &b) - { return a.score > b.score; }); // sort inplace - // trunc - bbox_collection.resize(nms_pre_); - } - -} - -void NCNNNanoDetEfficientNetLite::nms( - std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} diff --git a/lite/ncnn/cv/ncnn_nanodet_efficientnet_lite.h b/lite/ncnn/cv/ncnn_nanodet_efficientnet_lite.h deleted file mode 100644 index e6b9474e..00000000 --- a/lite/ncnn/cv/ncnn_nanodet_efficientnet_lite.h +++ /dev/null @@ -1,115 +0,0 @@ -// -// Created by DefTruth on 2021/10/7. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_NANODET_EFFICIENTNET_LITE_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_NANODET_EFFICIENTNET_LITE_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNNanoDetEfficientNetLite : public BasicNCNNHandler - { - public: - explicit NCNNNanoDetEfficientNetLite(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1, - int _input_height = 320, - int _input_width = 320); // - ~NCNNNanoDetEfficientNetLite() override = default; - - private: - // nested classes - typedef struct - { - float grid0; - float grid1; - float stride; - } NanoLiteCenterPoint; - - typedef struct - { - float ratio; - int dw; - int dh; - bool flag; - } NanoLiteScaleParams; - - private: - const float mean_vals[3] = {127.f, 127.f, 127.f}; // BGR - const float norm_vals[3] = {1.f / 128.f, 1.f / 128.f, 1.f / 128.f}; - - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int nms_pre = 1000; - static constexpr const unsigned int max_nms = 30000; - static constexpr const bool is_dynamic_input = false; - - // multi-levels center points - int input_height = 320; - int input_width = 320; - std::vector strides = {8, 16, 32}; - std::unordered_map> center_points; - bool center_points_is_update = false; - - private: - - void transform(const cv::Mat &mat_rs, ncnn::Mat &in) override; - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - NanoLiteScaleParams &scale_params); - - // only generate once - void generate_points(unsigned int target_height, unsigned int target_width); - - void generate_bboxes_single_stride(const NanoLiteScaleParams &scale_params, - ncnn::Mat &cls_pred, - ncnn::Mat &dis_pred, - unsigned int stride, - float score_threshold, - float img_height, - float img_width, - std::vector &bbox_collection); - - void generate_bboxes(const NanoLiteScaleParams &scale_params, - std::vector &bbox_collection, - ncnn::Extractor &extractor, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - /** - * @param mat cv::Mat BGR format - * @param detected_boxes vector of Boxf to catch detected boxes. - * @param score_threshold default 0.45f, only keep the result which >= score_threshold. - * @param iou_threshold default 0.3f, iou threshold for NMS. - * @param topk default 100, maximum output boxes after NMS. - * @param nms_type the method. - */ - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.45f, float iou_threshold = 0.3f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_NANODET_EFFICIENTNET_LITE_H diff --git a/lite/ncnn/cv/ncnn_nanodet_efficientnet_lite_depreciated.cpp b/lite/ncnn/cv/ncnn_nanodet_efficientnet_lite_depreciated.cpp deleted file mode 100644 index ceaf86f9..00000000 --- a/lite/ncnn/cv/ncnn_nanodet_efficientnet_lite_depreciated.cpp +++ /dev/null @@ -1,264 +0,0 @@ -// -// Created by DefTruth on 2021/10/9. -// - -#include "ncnn_nanodet_efficientdet_lite_depreciated.h" -#include "lite/utils.h" - -using ncnncv::NCNNNanoDetEfficientNetLiteDepreciated; - -NCNNNanoDetEfficientNetLiteDepreciated::NCNNNanoDetEfficientNetLiteDepreciated( - const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads, - int _input_height, int _input_width) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ - input_height = _input_height; - input_width = _input_width; -} - -void NCNNNanoDetEfficientNetLiteDepreciated::resize_unscale( - const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - NanoLiteDepreciatedScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(0, 0, 0)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.ratio = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.flag = true; -} - -void NCNNNanoDetEfficientNetLiteDepreciated::transform(const cv::Mat &mat_rs, ncnn::Mat &in) -{ - // BGR NHWC -> BGR NCHW - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNNanoDetEfficientNetLiteDepreciated::detect( - const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - auto img_height = static_cast(mat.rows); - auto img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - NanoLiteDepreciatedScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat_rs, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input.1", input); - // 3.rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, extractor, score_threshold, img_height, img_width); - // 4. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void NCNNNanoDetEfficientNetLiteDepreciated::generate_points( - unsigned int target_height, unsigned int target_width) -{ - if (center_points_is_update && (!is_dynamic_input)) return; - - for (auto stride : strides) - { - unsigned int num_grid_w = target_width / stride; - unsigned int num_grid_h = target_height / stride; - std::vector points; - - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - float grid0 = (float) g0 + 0.5f; - float grid1 = (float) g1 + 0.5f; -#ifdef LITE_WIN32 - NanoLiteDepreciatedCenterPoint point; - point.grid0 = grid0; - point.grid1 = grid1; - point.stride = (float) stride; - points.push_back(point); -#else - points.push_back((NanoLiteDepreciatedCenterPoint) {grid0, grid1, (float) stride}); -#endif - } - } - center_points[stride] = points; - } - - center_points_is_update = true; -} - -void NCNNNanoDetEfficientNetLiteDepreciated::generate_bboxes( - const NanoLiteDepreciatedScaleParams &scale_params, - std::vector &bbox_collection, - ncnn::Extractor &extractor, - float score_threshold, - float img_height, float img_width) -{ - ncnn::Mat cls_pred_stride_8; - ncnn::Mat cls_pred_stride_16; - ncnn::Mat cls_pred_stride_32; - ncnn::Mat dis_pred_stride_8; - ncnn::Mat dis_pred_stride_16; - ncnn::Mat dis_pred_stride_32; - extractor.extract("cls_pred_stride_8", cls_pred_stride_8); - extractor.extract("cls_pred_stride_16", cls_pred_stride_16); - extractor.extract("cls_pred_stride_32", cls_pred_stride_32); - extractor.extract("dis_pred_stride_8", dis_pred_stride_8); - extractor.extract("dis_pred_stride_16", dis_pred_stride_16); - extractor.extract("dis_pred_stride_32", dis_pred_stride_32); - this->generate_points(input_height, input_width); - - bbox_collection.clear(); - // level 8 & 16 & 32 - this->generate_bboxes_single_stride(scale_params, cls_pred_stride_8, dis_pred_stride_8, 8, - score_threshold, img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, cls_pred_stride_16, dis_pred_stride_16, 16, - score_threshold, img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, cls_pred_stride_32, dis_pred_stride_32, 32, - score_threshold, img_height, img_width, bbox_collection); -#if LITENCNN_DEBUG - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void NCNNNanoDetEfficientNetLiteDepreciated::generate_bboxes_single_stride( - const NanoLiteDepreciatedScaleParams &scale_params, - ncnn::Mat &cls_pred, ncnn::Mat &dis_pred, - unsigned int stride, float score_threshold, - float img_height, float img_width, - std::vector &bbox_collection) -{ - unsigned int nms_pre_ = (stride / 8) * nms_pre; // 1 * 1000,2*1000,... - nms_pre_ = nms_pre_ >= nms_pre ? nms_pre_ : nms_pre; - - const unsigned int f_h = (unsigned int) input_height / stride; - const unsigned int f_w = (unsigned int) input_width / stride; - const unsigned int num_points = f_h * f_w; - const unsigned int num_classes = 80; - - const unsigned int dis_pred_w = dis_pred.w; - const unsigned int reg_max = dis_pred_w / 4; // e.g 8 - - float ratio = scale_params.ratio; - int dw = scale_params.dw; - int dh = scale_params.dh; - - unsigned int count = 0; - auto &stride_points = center_points[stride]; - - for (unsigned int i = 0; i < num_points; ++i) - { - const float *scores = cls_pred.row(i); // row ptr - float cls_conf = scores[0]; - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = scores[j]; - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - if (cls_conf < score_threshold) continue; // filter - - auto &point = stride_points.at(i); - const float cx = point.grid0; // cx - const float cy = point.grid1; // cy - const float s = point.stride; // stride - - const float *logits = dis_pred.row(i); // 32|44... - std::vector offsets(4); - for (unsigned int k = 0; k < 4; ++k) - { - float offset = 0.f; - unsigned int max_id; - auto probs = lite::utils::math::softmax( - logits + (k * reg_max), reg_max, max_id); - for (unsigned int l = 0; l < reg_max; ++l) - offset += (float) l * probs[l]; - offsets[k] = offset; - } - float l = offsets[0]; // left - float t = offsets[1]; // top - float r = offsets[2]; // right - float b = offsets[3]; // bottom - - types::Boxf box; - float x1 = ((cx - l) * s - (float) dw) / ratio; // cx - l x1 - float y1 = ((cy - t) * s - (float) dh) / ratio; // cy - t y1 - float x2 = ((cx + r) * s - (float) dw) / ratio; // cx + r x2 - float y2 = ((cy + b) * s - (float) dh) / ratio; // cy + b y2 - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(img_width - 1.f, x2); - box.y2 = std::min(img_height - 1.f, y2); - box.score = cls_conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - - if (bbox_collection.size() > nms_pre_) - { - std::sort(bbox_collection.begin(), bbox_collection.end(), - [](const types::Boxf &a, const types::Boxf &b) - { return a.score > b.score; }); // sort inplace - // trunc - bbox_collection.resize(nms_pre_); - } - -} - -void NCNNNanoDetEfficientNetLiteDepreciated::nms( - std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_nanodet_plus.cpp b/lite/ncnn/cv/ncnn_nanodet_plus.cpp deleted file mode 100644 index 130f45f3..00000000 --- a/lite/ncnn/cv/ncnn_nanodet_plus.cpp +++ /dev/null @@ -1,222 +0,0 @@ -// -// Created by DefTruth on 2021/12/27. -// - -#include "ncnn_nanodet_plus.h" -#include "lite/utils.h" - -using ncnncv::NCNNNanoDetPlus; - -NCNNNanoDetPlus::NCNNNanoDetPlus(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads, - int _input_height, - int _input_width) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ - input_height = _input_height; - input_width = _input_width; -} - -void NCNNNanoDetPlus::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - NanoPlusScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(0, 0, 0)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.ratio = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.flag = true; -} - -void NCNNNanoDetPlus::transform(const cv::Mat &mat_rs, ncnn::Mat &in) -{ - // BGR NHWC -> BGR NCHW - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNNanoDetPlus::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - auto img_height = static_cast(mat.rows); - auto img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - NanoPlusScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat_rs, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("data", input); - // 3.rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, extractor, score_threshold, img_height, img_width); - // 4. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void NCNNNanoDetPlus::generate_points(unsigned int target_height, unsigned int target_width) -{ - if (center_points_is_update) return; - // 8, 16, 32, 64 - for (auto stride : strides) - { - unsigned int num_grid_w = target_width / stride; - unsigned int num_grid_h = target_height / stride; - - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - float grid0 = (float) g0; - float grid1 = (float) g1; -#ifdef LITE_WIN32 - NanoPlusCenterPoint point; - point.grid0 = grid0; - point.grid1 = grid1; - point.stride = (float) stride; - center_points.push_back(point); -#else - center_points.push_back((NanoPlusCenterPoint) {grid0, grid1, (float) stride}); -#endif - } - } - } - - center_points_is_update = true; -} - - -void NCNNNanoDetPlus::generate_bboxes(const NanoPlusScaleParams &scale_params, - std::vector &bbox_collection, - ncnn::Extractor &extractor, - float score_threshold, - float img_height, - float img_width) -{ - ncnn::Mat output_pred; // [1,2125,112] - extractor.extract("output", output_pred); -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(output_pred, "output"); -#endif - this->generate_points(input_height, input_width); - - // level 8, 16, 32, 64 - const unsigned int num_classes = 80; - const unsigned int num_cls_reg = output_pred.w; // 112 - const unsigned int reg_max = (num_cls_reg - num_classes) / 4; // e.g 8=7+1 - const unsigned int num_points = center_points.size(); - - float ratio = scale_params.ratio; - int dw = scale_params.dw; - int dh = scale_params.dh; - - unsigned int count = 0; - - bbox_collection.clear(); - for (unsigned int i = 0; i < num_points; ++i) - { - const float *scores = output_pred.row(i); // row ptr - float cls_conf = scores[0]; - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = scores[j]; - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - if (cls_conf < score_threshold) continue; // filter - - auto &point = center_points.at(i); - const float cx = point.grid0; // cx - const float cy = point.grid1; // cy - const float s = point.stride; // stride - - const float *logits = output_pred.row(i) + num_classes; // 32|44... - std::vector offsets(4); - for (unsigned int k = 0; k < 4; ++k) - { - float offset = 0.f; - unsigned int max_id; - auto probs = lite::utils::math::softmax( - logits + (k * reg_max), reg_max, max_id); - for (unsigned int l = 0; l < reg_max; ++l) - offset += (float) l * probs[l]; - offsets[k] = offset; - } - - float l = offsets[0]; // left - float t = offsets[1]; // top - float r = offsets[2]; // right - float b = offsets[3]; // bottom - - types::Boxf box; - float x1 = ((cx - l) * s - (float) dw) / ratio; // cx - l x1 - float y1 = ((cy - t) * s - (float) dh) / ratio; // cy - t y1 - float x2 = ((cx + r) * s - (float) dw) / ratio; // cx + r x2 - float y2 = ((cy + b) * s - (float) dh) / ratio; // cy + b y2 - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(img_width - 1.f, x2); - box.y2 = std::min(img_height - 1.f, y2); - box.score = cls_conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - -#if LITENCNN_DEBUG - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void NCNNNanoDetPlus::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_nanodet_plus.h b/lite/ncnn/cv/ncnn_nanodet_plus.h deleted file mode 100644 index 5beaa10d..00000000 --- a/lite/ncnn/cv/ncnn_nanodet_plus.h +++ /dev/null @@ -1,104 +0,0 @@ -// -// Created by DefTruth on 2021/12/27. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_NANODET_PLUS_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_NANODET_PLUS_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNNanoDetPlus : public BasicNCNNHandler - { - public: - explicit NCNNNanoDetPlus(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1, - int _input_height = 320, - int _input_width = 320); // - ~NCNNNanoDetPlus() override = default; - - private: - // nested classes - typedef struct - { - float grid0; - float grid1; - float stride; - } NanoPlusCenterPoint; - - typedef struct - { - float ratio; - int dw; - int dh; - bool flag; - } NanoPlusScaleParams; - - private: - const float mean_vals[3] = {103.53f, 116.28f, 123.675f}; // BGR - const float norm_vals[3] = {0.017429f, 0.017507f, 0.017125f}; - - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - // multi-levels center points - int input_height = 320; - int input_width = 320; - std::vector strides = {8, 16, 32, 64}; - std::vector center_points; - bool center_points_is_update = false; - - private: - - void transform(const cv::Mat &mat_rs, ncnn::Mat &in) override; - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - NanoPlusScaleParams &scale_params); - - // only generate once - void generate_points(unsigned int target_height, unsigned int target_width); - - void generate_bboxes(const NanoPlusScaleParams &scale_params, - std::vector &bbox_collection, - ncnn::Extractor &extractor, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - /** - * @param mat cv::Mat BGR format - * @param detected_boxes vector of Boxf to catch detected boxes. - * @param score_threshold default 0.45f, only keep the result which >= score_threshold. - * @param iou_threshold default 0.3f, iou threshold for NMS. - * @param topk default 100, maximum output boxes after NMS. - * @param nms_type the method. - */ - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.45f, float iou_threshold = 0.3f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_NANODET_PLUS_H diff --git a/lite/ncnn/cv/ncnn_pfld.cpp b/lite/ncnn/cv/ncnn_pfld.cpp deleted file mode 100644 index e69dcdfa..00000000 --- a/lite/ncnn/cv/ncnn_pfld.cpp +++ /dev/null @@ -1,58 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#include "ncnn_pfld.h" - -using ncnncv::NCNNPFLD; - -NCNNPFLD::NCNNPFLD(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNPFLD::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNPFLD::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - // 3. fetch landmarks. - ncnn::Mat landmarks_norm; - extractor.extract("output", landmarks_norm); // c=1,w=106*2,h=1 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(landmarks_norm, "output"); -#endif - const unsigned int num_landmarks = landmarks_norm.w; - const float *landmarks_ptr = (float *) landmarks_norm.data; - - for (unsigned int i = 0; i < num_landmarks; i += 2) - { - float x = landmarks_ptr[i]; - float y = landmarks_ptr[i + 1]; - - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - landmarks.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_pfld.h b/lite/ncnn/cv/ncnn_pfld.h deleted file mode 100644 index 8735dce7..00000000 --- a/lite/ncnn/cv/ncnn_pfld.h +++ /dev/null @@ -1,35 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_PFLD_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_PFLD_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNPFLD : public BasicNCNNHandler - { - public: - explicit NCNNPFLD(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNPFLD() override = default; - - private: - const int input_height = 112; - const int input_width = 112; - const float mean_vals[3] = {0.f, 0.f, 0.f}; - const float norm_vals[3] = {1.0f / 255.f, 1.0f / 255.f, 1.0f / 255.f}; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_PFLD_H diff --git a/lite/ncnn/cv/ncnn_pfld68.cpp b/lite/ncnn/cv/ncnn_pfld68.cpp deleted file mode 100644 index 78a38fbc..00000000 --- a/lite/ncnn/cv/ncnn_pfld68.cpp +++ /dev/null @@ -1,58 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#include "ncnn_pfld68.h" - -using ncnncv::NCNNPFLD68; - -NCNNPFLD68::NCNNPFLD68(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNPFLD68::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNPFLD68::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - // 3. fetch landmarks. - ncnn::Mat landmarks_norm; - extractor.extract("output", landmarks_norm); // c=1,w=68*2,h=1 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(landmarks_norm, "output"); -#endif - const unsigned int num_landmarks = landmarks_norm.w; - const float *landmarks_ptr = (float *) landmarks_norm.data; - - for (unsigned int i = 0; i < num_landmarks; i += 2) - { - float x = landmarks_ptr[i]; - float y = landmarks_ptr[i + 1]; - - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - landmarks.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_pfld68.h b/lite/ncnn/cv/ncnn_pfld68.h deleted file mode 100644 index e5749ee4..00000000 --- a/lite/ncnn/cv/ncnn_pfld68.h +++ /dev/null @@ -1,35 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_PFLD68_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_PFLD68_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNPFLD68 : public BasicNCNNHandler - { - public: - explicit NCNNPFLD68(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNPFLD68() override = default; - - private: - const int input_height = 112; - const int input_width = 112; - const float mean_vals[3] = {0.f, 0.f, 0.f}; - const float norm_vals[3] = {1.0f / 255.f, 1.0f / 255.f, 1.0f / 255.f}; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_PFLD68_H diff --git a/lite/ncnn/cv/ncnn_pfld98.cpp b/lite/ncnn/cv/ncnn_pfld98.cpp deleted file mode 100644 index aa9e8f03..00000000 --- a/lite/ncnn/cv/ncnn_pfld98.cpp +++ /dev/null @@ -1,58 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#include "ncnn_pfld98.h" - -using ncnncv::NCNNPFLD98; - -NCNNPFLD98::NCNNPFLD98(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNPFLD98::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNPFLD98::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - // 3. fetch landmarks. - ncnn::Mat landmarks_norm; - extractor.extract("landmarks", landmarks_norm); // c=1,w=98*2,h=1 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(landmarks_norm, "landmarks"); -#endif - const unsigned int num_landmarks = landmarks_norm.w; - const float *landmarks_ptr = (float *) landmarks_norm.data; - - for (unsigned int i = 0; i < num_landmarks; i += 2) - { - float x = landmarks_ptr[i]; - float y = landmarks_ptr[i + 1]; - - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - landmarks.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_pfld98.h b/lite/ncnn/cv/ncnn_pfld98.h deleted file mode 100644 index e7ec3256..00000000 --- a/lite/ncnn/cv/ncnn_pfld98.h +++ /dev/null @@ -1,35 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_PFLD98_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_PFLD98_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNPFLD98 : public BasicNCNNHandler - { - public: - explicit NCNNPFLD98(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNPFLD98() override = default; - - private: - const int input_height = 112; - const int input_width = 112; - const float mean_vals[3] = {0.f, 0.f, 0.f}; - const float norm_vals[3] = {1.0f / 255.f, 1.0f / 255.f, 1.0f / 255.f}; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_PFLD98_H diff --git a/lite/ncnn/cv/ncnn_pipnet19.cpp b/lite/ncnn/cv/ncnn_pipnet19.cpp deleted file mode 100644 index f71f413e..00000000 --- a/lite/ncnn/cv/ncnn_pipnet19.cpp +++ /dev/null @@ -1,193 +0,0 @@ -// -// Created by DefTruth on 2022/3/20. -// - -#include "ncnn_pipnet19.h" - -using ncnncv::NCNNPIPNet19; - -NCNNPIPNet19::NCNNPIPNet19(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNPIPNet19::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - // will do deepcopy inside ncnn - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - - -void NCNNPIPNet19::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("img", input); - // 3. generate landmarks - this->generate_landmarks(landmarks, extractor, img_height, img_width); -} - -void NCNNPIPNet19::generate_landmarks(types::Landmarks &landmarks, - ncnn::Extractor &extractor, - float img_height, float img_width) -{ - ncnn::Mat outputs_cls, outputs_x, outputs_y, outputs_nb_x, outputs_nb_y; - extractor.extract("outputs_cls", outputs_cls); // (19,8,8) - extractor.extract("outputs_x", outputs_x); // (19,8,8) - extractor.extract("outputs_y", outputs_y); // (19,8,8) - extractor.extract("outputs_nb_x", outputs_nb_x); // (19*10,8,8) - extractor.extract("outputs_nb_y", outputs_nb_y); // (19*10,8,8) -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(outputs_cls, "outputs_cls"); - BasicNCNNHandler::print_shape(outputs_x, "outputs_x"); - BasicNCNNHandler::print_shape(outputs_y, "outputs_y"); - BasicNCNNHandler::print_shape(outputs_nb_x, "outputs_nb_x"); - BasicNCNNHandler::print_shape(outputs_nb_y, "outputs_nb_y"); -#endif - const unsigned int input_h = input_height; - const unsigned int input_w = input_width; - - // fetch data from pointers - const float *outputs_cls_ptr = (float *) outputs_cls.data; - const float *outputs_x_ptr = (float *) outputs_x.data; - const float *outputs_y_ptr = (float *) outputs_y.data; - const float *outputs_nb_x_ptr = (float *) outputs_nb_x.data; - const float *outputs_nb_y_ptr = (float *) outputs_nb_y.data; - - // find max_ids - std::vector max_ids(num_lms); - for (unsigned int i = 0; i < num_lms; ++i) - { - const float *score_ptr = outputs_cls_ptr + i * grid_length; - unsigned int max_id = 0; - float max_score = score_ptr[0]; - for (unsigned int j = 0; j < grid_length; ++j) - { - if (score_ptr[j] > max_score) - { - max_score = score_ptr[j]; - max_id = j; - } - } - max_ids[i] = max_id; // range 0~64 - } - - // find x & y offsets - std::vector output_x_select(num_lms); - std::vector output_y_select(num_lms); - for (unsigned int i = 0; i < num_lms; ++i) - { - const float *offset_x_ptr = outputs_x_ptr + i * grid_length; - const float *offset_y_ptr = outputs_y_ptr + i * grid_length; - const unsigned int max_id = max_ids.at(i); - output_x_select[i] = offset_x_ptr[max_id]; - output_y_select[i] = offset_y_ptr[max_id]; - } - - // find nb_x & nb_y offsets - std::unordered_map> output_nb_x_select; - std::unordered_map> output_nb_y_select; - // initialize offsets map - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector nb_x_offset(num_nb); - std::vector nb_y_offset(num_nb); - output_nb_x_select[i] = nb_x_offset; - output_nb_y_select[i] = nb_y_offset; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - for (unsigned int j = 0; j < num_nb; ++j) - { - const float *offset_nb_x_ptr = outputs_nb_x_ptr + (i * num_nb + j) * grid_length; - const float *offset_nb_y_ptr = outputs_nb_y_ptr + (i * num_nb + j) * grid_length; - const unsigned int max_id = max_ids.at(i); - output_nb_x_select[i][j] = offset_nb_x_ptr[max_id]; - output_nb_y_select[i][j] = offset_nb_y_ptr[max_id]; - } - } - - // calculate coords - std::vector lms_pred_x(num_lms); // 19 - std::vector lms_pred_y(num_lms); // 19 - std::unordered_map> lms_pred_nb_x; // 19,10 - std::unordered_map> lms_pred_nb_y; // 19,10 - // initialize pred maps - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector nb_x_offset(num_nb); - std::vector nb_y_offset(num_nb); - lms_pred_nb_x[i] = nb_x_offset; - lms_pred_nb_y[i] = nb_y_offset; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - float cx = static_cast(max_ids.at(i) % grid_w); - float cy = static_cast(max_ids.at(i) / grid_w); - // calculate coords & normalize - lms_pred_x[i] = ((cx + output_x_select[i]) * (float) net_stride) / (float) input_w; - lms_pred_y[i] = ((cy + output_y_select[i]) * (float) net_stride) / (float) input_h; - for (unsigned int j = 0; j < num_nb; ++j) - { - lms_pred_nb_x[i][j] = ((cx + output_nb_x_select[i][j]) * (float) net_stride) / (float) input_w; - lms_pred_nb_y[i][j] = ((cy + output_nb_y_select[i][j]) * (float) net_stride) / (float) input_h; - } - } - - // reverse indexes - std::unordered_map> tmp_nb_x; // 19,max_len - std::unordered_map> tmp_nb_y; // 19,max_len - // initialize reverse maps - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector tmp_x(max_len); - std::vector tmp_y(max_len); - tmp_nb_x[i] = tmp_x; - tmp_nb_y[i] = tmp_y; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - for (unsigned int j = 0; j < max_len; ++j) - { - unsigned int ri = reverse_index1[i * max_len + j]; - unsigned int rj = reverse_index2[i * max_len + j]; - tmp_nb_x[i][j] = lms_pred_nb_x[ri][rj]; - tmp_nb_y[i][j] = lms_pred_nb_y[ri][rj]; - } - } - - // merge predictions - landmarks.points.clear(); - for (unsigned int i = 0; i < num_lms; ++i) - { - float total_x = lms_pred_x[i]; - float total_y = lms_pred_y[i]; - for (unsigned int j = 0; j < max_len; ++j) - { - total_x += tmp_nb_x[i][j]; - total_y += tmp_nb_y[i][j]; - } - float x = total_x / ((float) max_len + 1.f); - float y = total_y / ((float) max_len + 1.f); - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - - landmarks.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_pipnet19.h b/lite/ncnn/cv/ncnn_pipnet19.h deleted file mode 100644 index 3dfd2ae5..00000000 --- a/lite/ncnn/cv/ncnn_pipnet19.h +++ /dev/null @@ -1,74 +0,0 @@ -// -// Created by DefTruth on 2022/3/20. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_PIPNET19_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_PIPNET19_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNPIPNet19 : public BasicNCNNHandler - { - public: - explicit NCNNPIPNet19(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNPIPNet19() override = default; - - private: - // hardcode input size - static constexpr const unsigned int input_height = 256; - static constexpr const unsigned int input_width = 256; - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - static constexpr const unsigned int num_nb = 10; - static constexpr const unsigned int num_lms = 19; - static constexpr const unsigned int max_len = 18; - static constexpr const unsigned int net_stride = 32; - // hardcode grid size - static constexpr const unsigned int grid_h = 8; - static constexpr const unsigned int grid_w = 8; - static constexpr const unsigned int grid_length = 8 * 8; // 64 - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - void generate_landmarks(types::Landmarks &landmarks, - ncnn::Extractor &extractor, - float img_height, float img_width); - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - - private: - const unsigned int reverse_index1[19 * 18] = { - 1, 2, 6, 7, 8, 1, 2, 6, 7, 8, 1, 2, 6, 7, 8, 1, 2, 6, 0, 2, 3, 4, 6, 7, 8, 0, 2, 3, 4, 6, 7, 8, 0, 2, 3, 4, 0, 1, 3, 4, 5, 6, 7, 8, - 9, 10, 11, 12, 0, 1, 3, 4, 5, 6, 0, 1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 14, 0, 1, 2, 4, 5, 6, 1, 2, 3, 5, 9, 10, 11, 1, 2, 3, 5, 9, 10, - 11, 1, 2, 3, 5, 3, 4, 9, 10, 11, 3, 4, 9, 10, 11, 3, 4, 9, 10, 11, 3, 4, 9, 0, 1, 2, 3, 7, 8, 12, 13, 15, 0, 1, 2, 3, 7, 8, 12, 13, - 15, 0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 15, 16, 18, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 0, - 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15, 16, 17, 18, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 13, 14, 16, 17, 18, 0, 1, 3, 4, 5, 9, - 10, 14, 17, 3, 4, 5, 9, 10, 14, 17, 3, 4, 5, 9, 0, 1, 6, 7, 8, 13, 14, 15, 16, 17, 18, 0, 1, 6, 7, 8, 13, 14, 0, 2, 5, 6, 7, 8, 9, - 10, 11, 12, 14, 15, 16, 17, 18, 0, 2, 5, 4, 5, 9, 10, 11, 12, 13, 15, 16, 17, 18, 4, 5, 9, 10, 11, 12, 13, 12, 13, 14, 16, 17, 18, - 12, 13, 14, 16, 17, 18, 12, 13, 14, 16, 17, 18, 12, 13, 14, 15, 17, 18, 12, 13, 14, 15, 17, 18, 12, 13, 14, 15, 17, 18, 12, 13, 14, - 15, 16, 18, 12, 13, 14, 15, 16, 18, 12, 13, 14, 15, 16, 18, 15, 16, 17, 15, 16, 17, 15, 16, 17, 15, 16, 17, 15, 16, 17, 15, 16, 17 - }; - const unsigned int reverse_index2[19 * 18] = { - 0, 6, 1, 4, 6, 0, 6, 1, 4, 6, 0, 6, 1, 4, 6, 0, 6, 1, 0, 1, 8, 7, 2, 2, 3, 0, 1, 8, 7, 2, 2, 3, 0, 1, 8, 7, 3, 1, 3, 5, 5, 4, 3, 1, - 5, 6, 6, 9, 3, 1, 3, 5, 5, 4, 5, 5, 3, 1, 3, 7, 5, 5, 1, 3, 4, 9, 5, 5, 3, 1, 3, 7, 7, 8, 1, 0, 3, 2, 2, 7, 8, 1, 0, 3, 2, 2, 7, 8, - 1, 0, 6, 0, 6, 4, 1, 6, 0, 6, 4, 1, 6, 0, 6, 4, 1, 6, 0, 6, 1, 3, 4, 9, 1, 2, 6, 9, 8, 1, 3, 4, 9, 1, 2, 6, 9, 8, 2, 2, 2, 7, 8, 9, - 0, 0, 9, 9, 9, 5, 7, 7, 8, 8, 2, 2, 4, 4, 0, 5, 6, 6, 3, 0, 4, 5, 7, 4, 3, 8, 6, 6, 9, 6, 7, 6, 5, 0, 4, 4, 8, 6, 4, 0, 3, 8, 4, 4, - 9, 7, 6, 7, 9, 8, 7, 2, 2, 2, 9, 9, 9, 0, 0, 8, 5, 9, 7, 9, 9, 8, 4, 3, 1, 2, 1, 6, 8, 4, 3, 1, 2, 1, 6, 8, 4, 3, 1, 2, 6, 9, 5, 7, - 8, 0, 2, 1, 3, 4, 4, 6, 9, 5, 7, 8, 0, 2, 8, 9, 8, 6, 8, 7, 7, 8, 8, 0, 0, 2, 2, 2, 5, 8, 9, 8, 9, 7, 8, 7, 5, 2, 1, 4, 4, 1, 3, 9, - 7, 8, 7, 5, 2, 1, 1, 5, 7, 0, 3, 1, 1, 5, 7, 0, 3, 1, 1, 5, 7, 0, 3, 1, 3, 2, 3, 0, 0, 0, 3, 2, 3, 0, 0, 0, 3, 2, 3, 0, 0, 0, 7, 6, - 1, 3, 1, 2, 7, 6, 1, 3, 1, 2, 7, 6, 1, 3, 1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 - }; - - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_PIPNET19_H diff --git a/lite/ncnn/cv/ncnn_pipnet29.cpp b/lite/ncnn/cv/ncnn_pipnet29.cpp deleted file mode 100644 index 0e4f7457..00000000 --- a/lite/ncnn/cv/ncnn_pipnet29.cpp +++ /dev/null @@ -1,193 +0,0 @@ -// -// Created by DefTruth on 2022/3/20. -// - -#include "ncnn_pipnet29.h" - -using ncnncv::NCNNPIPNet29; - -NCNNPIPNet29::NCNNPIPNet29(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNPIPNet29::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - // will do deepcopy inside ncnn - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - - -void NCNNPIPNet29::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("img", input); - // 3. generate landmarks - this->generate_landmarks(landmarks, extractor, img_height, img_width); -} - -void NCNNPIPNet29::generate_landmarks(types::Landmarks &landmarks, - ncnn::Extractor &extractor, - float img_height, float img_width) -{ - ncnn::Mat outputs_cls, outputs_x, outputs_y, outputs_nb_x, outputs_nb_y; - extractor.extract("outputs_cls", outputs_cls); // (29,8,8) - extractor.extract("outputs_x", outputs_x); // (29,8,8) - extractor.extract("outputs_y", outputs_y); // (29,8,8) - extractor.extract("outputs_nb_x", outputs_nb_x); // (29*10,8,8) - extractor.extract("outputs_nb_y", outputs_nb_y); // (29*10,8,8) -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(outputs_cls, "outputs_cls"); - BasicNCNNHandler::print_shape(outputs_x, "outputs_x"); - BasicNCNNHandler::print_shape(outputs_y, "outputs_y"); - BasicNCNNHandler::print_shape(outputs_nb_x, "outputs_nb_x"); - BasicNCNNHandler::print_shape(outputs_nb_y, "outputs_nb_y"); -#endif - const unsigned int input_h = input_height; - const unsigned int input_w = input_width; - - // fetch data from pointers - const float *outputs_cls_ptr = (float *) outputs_cls.data; - const float *outputs_x_ptr = (float *) outputs_x.data; - const float *outputs_y_ptr = (float *) outputs_y.data; - const float *outputs_nb_x_ptr = (float *) outputs_nb_x.data; - const float *outputs_nb_y_ptr = (float *) outputs_nb_y.data; - - // find max_ids - std::vector max_ids(num_lms); - for (unsigned int i = 0; i < num_lms; ++i) - { - const float *score_ptr = outputs_cls_ptr + i * grid_length; - unsigned int max_id = 0; - float max_score = score_ptr[0]; - for (unsigned int j = 0; j < grid_length; ++j) - { - if (score_ptr[j] > max_score) - { - max_score = score_ptr[j]; - max_id = j; - } - } - max_ids[i] = max_id; // range 0~64 - } - - // find x & y offsets - std::vector output_x_select(num_lms); - std::vector output_y_select(num_lms); - for (unsigned int i = 0; i < num_lms; ++i) - { - const float *offset_x_ptr = outputs_x_ptr + i * grid_length; - const float *offset_y_ptr = outputs_y_ptr + i * grid_length; - const unsigned int max_id = max_ids.at(i); - output_x_select[i] = offset_x_ptr[max_id]; - output_y_select[i] = offset_y_ptr[max_id]; - } - - // find nb_x & nb_y offsets - std::unordered_map> output_nb_x_select; - std::unordered_map> output_nb_y_select; - // initialize offsets map - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector nb_x_offset(num_nb); - std::vector nb_y_offset(num_nb); - output_nb_x_select[i] = nb_x_offset; - output_nb_y_select[i] = nb_y_offset; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - for (unsigned int j = 0; j < num_nb; ++j) - { - const float *offset_nb_x_ptr = outputs_nb_x_ptr + (i * num_nb + j) * grid_length; - const float *offset_nb_y_ptr = outputs_nb_y_ptr + (i * num_nb + j) * grid_length; - const unsigned int max_id = max_ids.at(i); - output_nb_x_select[i][j] = offset_nb_x_ptr[max_id]; - output_nb_y_select[i][j] = offset_nb_y_ptr[max_id]; - } - } - - // calculate coords - std::vector lms_pred_x(num_lms); // 29 - std::vector lms_pred_y(num_lms); // 29 - std::unordered_map> lms_pred_nb_x; // 29,10 - std::unordered_map> lms_pred_nb_y; // 29,10 - // initialize pred maps - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector nb_x_offset(num_nb); - std::vector nb_y_offset(num_nb); - lms_pred_nb_x[i] = nb_x_offset; - lms_pred_nb_y[i] = nb_y_offset; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - float cx = static_cast(max_ids.at(i) % grid_w); - float cy = static_cast(max_ids.at(i) / grid_w); - // calculate coords & normalize - lms_pred_x[i] = ((cx + output_x_select[i]) * (float) net_stride) / (float) input_w; - lms_pred_y[i] = ((cy + output_y_select[i]) * (float) net_stride) / (float) input_h; - for (unsigned int j = 0; j < num_nb; ++j) - { - lms_pred_nb_x[i][j] = ((cx + output_nb_x_select[i][j]) * (float) net_stride) / (float) input_w; - lms_pred_nb_y[i][j] = ((cy + output_nb_y_select[i][j]) * (float) net_stride) / (float) input_h; - } - } - - // reverse indexes - std::unordered_map> tmp_nb_x; // 29,max_len - std::unordered_map> tmp_nb_y; // 29,max_len - // initialize reverse maps - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector tmp_x(max_len); - std::vector tmp_y(max_len); - tmp_nb_x[i] = tmp_x; - tmp_nb_y[i] = tmp_y; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - for (unsigned int j = 0; j < max_len; ++j) - { - unsigned int ri = reverse_index1[i * max_len + j]; - unsigned int rj = reverse_index2[i * max_len + j]; - tmp_nb_x[i][j] = lms_pred_nb_x[ri][rj]; - tmp_nb_y[i][j] = lms_pred_nb_y[ri][rj]; - } - } - - // merge predictions - landmarks.points.clear(); - for (unsigned int i = 0; i < num_lms; ++i) - { - float total_x = lms_pred_x[i]; - float total_y = lms_pred_y[i]; - for (unsigned int j = 0; j < max_len; ++j) - { - total_x += tmp_nb_x[i][j]; - total_y += tmp_nb_y[i][j]; - } - float x = total_x / ((float) max_len + 1.f); - float y = total_y / ((float) max_len + 1.f); - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - - landmarks.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_pipnet29.h b/lite/ncnn/cv/ncnn_pipnet29.h deleted file mode 100644 index 465c5b07..00000000 --- a/lite/ncnn/cv/ncnn_pipnet29.h +++ /dev/null @@ -1,86 +0,0 @@ -// -// Created by DefTruth on 2022/3/20. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_PIPNET29_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_PIPNET29_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNPIPNet29 : public BasicNCNNHandler - { - public: - explicit NCNNPIPNet29(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNPIPNet29() override = default; - - private: - // hardcode input size - static constexpr const unsigned int input_height = 256; - static constexpr const unsigned int input_width = 256; - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - static constexpr const unsigned int num_nb = 10; - static constexpr const unsigned int num_lms = 29; - static constexpr const unsigned int max_len = 19; - static constexpr const unsigned int net_stride = 32; - // hardcode grid size - static constexpr const unsigned int grid_h = 8; - static constexpr const unsigned int grid_w = 8; - static constexpr const unsigned int grid_length = 8 * 8; // 64 - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - void generate_landmarks(types::Landmarks &landmarks, - ncnn::Extractor &extractor, - float img_height, float img_width); - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - - private: - const unsigned int reverse_index1[29 * 19] = { - 2, 4, 5, 8, 12, 13, 16, 2, 4, 5, 8, 12, 13, 16, 2, 4, 5, 8, 12, 3, 6, 7, 9, 14, 15, 17, 3, 6, 7, 9, 14, 15, 17, 3, 6, 7, 9, 14, 0, - 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 16, 0, 3, 4, 5, 6, 7, 0, 1, 2, 4, 5, 6, 7, 9, 10, 11, 12, 14, 15, 17, 0, 1, 2, 4, 5, 0, 2, 5, - 8, 10, 12, 13, 16, 0, 2, 5, 8, 10, 12, 13, 16, 0, 2, 5, 0, 2, 4, 8, 10, 12, 13, 16, 0, 2, 4, 8, 10, 12, 13, 16, 0, 2, 4, 1, 3, 7, 9, - 11, 14, 15, 17, 1, 3, 7, 9, 11, 14, 15, 17, 1, 3, 7, 1, 3, 6, 9, 11, 14, 15, 17, 1, 3, 6, 9, 11, 14, 15, 17, 1, 3, 6, 0, 2, 4, 5, - 10, 12, 13, 16, 0, 2, 4, 5, 10, 12, 13, 16, 0, 2, 4, 1, 3, 6, 7, 11, 14, 15, 17, 1, 3, 6, 7, 11, 14, 15, 17, 1, 3, 6, 0, 2, 3, 4, 5, - 8, 12, 13, 16, 18, 20, 0, 2, 3, 4, 5, 8, 12, 13, 1, 2, 3, 6, 7, 9, 14, 15, 17, 19, 20, 21, 1, 2, 3, 6, 7, 9, 14, 0, 2, 4, 5, 8, 10, - 13, 16, 0, 2, 4, 5, 8, 10, 13, 16, 0, 2, 4, 0, 2, 4, 5, 8, 10, 12, 16, 18, 22, 0, 2, 4, 5, 8, 10, 12, 16, 18, 1, 3, 6, 7, 9, 11, 15, - 17, 1, 3, 6, 7, 9, 11, 15, 17, 1, 3, 6, 1, 3, 6, 7, 9, 11, 14, 17, 19, 23, 1, 3, 6, 7, 9, 11, 14, 17, 19, 0, 2, 4, 5, 8, 10, 12, 13, - 18, 0, 2, 4, 5, 8, 10, 12, 13, 18, 0, 1, 3, 6, 7, 9, 11, 14, 15, 19, 1, 3, 6, 7, 9, 11, 14, 15, 19, 1, 0, 4, 5, 8, 10, 12, 13, 16, - 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 0, 1, 6, 7, 9, 11, 14, 15, 17, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 1, 1, 8, 9, 10, 11, - 13, 15, 16, 17, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 18, 19, 20, 22, 23, 24, 25, 26, 27, 28, 18, 19, 20, 22, 23, 24, 25, 26, 27, - 18, 20, 21, 24, 25, 26, 27, 28, 18, 20, 21, 24, 25, 26, 27, 28, 18, 20, 21, 19, 21, 24, 25, 26, 27, 28, 19, 21, 24, 25, 26, 27, 28, - 19, 21, 24, 25, 26, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 18, 19, 20, 21, 22, 23, 25, 26, 27, 18, 19, 20, 21, 22, 23, 24, 26, 27, - 28, 18, 19, 20, 21, 22, 23, 24, 26, 27, 18, 19, 20, 21, 22, 23, 24, 25, 27, 28, 18, 19, 20, 21, 22, 23, 24, 25, 27, 20, 21, 22, 23, - 24, 25, 26, 28, 20, 21, 22, 23, 24, 25, 26, 28, 20, 21, 22, 22, 23, 24, 25, 26, 27, 22, 23, 24, 25, 26, 27, 22, 23, 24, 25, 26, 27, - 22 - }; - const unsigned int reverse_index2[29 * 19] = { - 9, 3, 5, 3, 7, 7, 7, 9, 3, 5, 3, 7, 7, 7, 9, 3, 5, 3, 7, 9, 3, 5, 3, 7, 7, 7, 9, 3, 5, 3, 7, 7, 7, 9, 3, 5, 3, 7, 7, 6, 6, 6, 8, 9, - 7, 0, 9, 6, 5, 9, 6, 7, 6, 6, 6, 8, 9, 9, 7, 6, 8, 9, 6, 6, 7, 8, 0, 9, 6, 6, 6, 9, 7, 6, 8, 9, 2, 5, 0, 5, 5, 3, 6, 5, 2, 5, 0, 5, - 5, 3, 6, 5, 2, 5, 0, 1, 3, 0, 4, 4, 2, 4, 2, 1, 3, 0, 4, 4, 2, 4, 2, 1, 3, 0, 2, 4, 0, 5, 5, 3, 5, 5, 2, 4, 0, 5, 5, 3, 5, 5, 2, 4, - 0, 1, 3, 0, 4, 4, 2, 4, 2, 1, 3, 0, 4, 4, 2, 4, 2, 1, 3, 0, 0, 7, 4, 3, 6, 5, 3, 4, 0, 7, 4, 3, 6, 5, 3, 4, 0, 7, 4, 0, 7, 4, 3, 6, - 5, 2, 4, 0, 7, 4, 3, 6, 5, 2, 4, 0, 7, 4, 6, 0, 8, 7, 7, 6, 4, 2, 3, 5, 6, 6, 0, 8, 7, 7, 6, 4, 2, 6, 8, 0, 7, 7, 6, 4, 3, 3, 5, 7, - 9, 6, 8, 0, 7, 7, 6, 4, 3, 1, 1, 1, 2, 3, 1, 0, 3, 1, 1, 1, 2, 3, 1, 0, 3, 1, 1, 5, 4, 5, 4, 0, 2, 1, 1, 6, 9, 5, 4, 5, 4, 0, 2, 1, - 1, 6, 3, 1, 1, 1, 2, 3, 1, 0, 3, 1, 1, 1, 2, 3, 1, 0, 3, 1, 1, 5, 5, 5, 4, 0, 2, 1, 1, 7, 9, 5, 5, 5, 4, 0, 2, 1, 1, 7, 4, 2, 2, 2, - 1, 1, 0, 0, 9, 4, 2, 2, 2, 1, 1, 0, 0, 9, 4, 4, 2, 2, 2, 1, 1, 0, 0, 9, 4, 2, 2, 2, 1, 1, 0, 0, 9, 4, 8, 9, 8, 8, 7, 8, 8, 8, 8, 1, - 3, 0, 8, 5, 8, 9, 9, 9, 8, 8, 9, 8, 8, 7, 8, 8, 8, 8, 2, 4, 8, 0, 6, 7, 8, 8, 7, 8, 9, 9, 9, 9, 8, 9, 9, 9, 9, 0, 0, 0, 6, 6, 4, 4, - 6, 7, 8, 1, 1, 0, 5, 5, 2, 3, 3, 4, 6, 1, 1, 0, 5, 5, 2, 3, 3, 4, 2, 8, 7, 7, 5, 4, 6, 5, 2, 8, 7, 7, 5, 4, 6, 5, 2, 8, 7, 2, 8, 8, - 6, 5, 5, 4, 2, 8, 8, 6, 5, 5, 4, 2, 8, 8, 6, 5, 3, 3, 3, 1, 2, 3, 0, 2, 2, 3, 3, 3, 3, 1, 2, 3, 0, 2, 2, 4, 4, 4, 2, 1, 1, 0, 0, 1, - 2, 4, 4, 4, 2, 1, 1, 0, 0, 1, 7, 6, 5, 5, 3, 2, 1, 1, 0, 1, 7, 6, 5, 5, 3, 2, 1, 1, 0, 9, 6, 4, 4, 3, 2, 1, 0, 9, 6, 4, 4, 3, 2, 1, - 0, 9, 6, 4, 7, 7, 9, 9, 7, 3, 7, 7, 9, 9, 7, 3, 7, 7, 9, 9, 7, 3, 7 - }; - }; -} - - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_PIPNET29_H diff --git a/lite/ncnn/cv/ncnn_pipnet68.cpp b/lite/ncnn/cv/ncnn_pipnet68.cpp deleted file mode 100644 index 23868018..00000000 --- a/lite/ncnn/cv/ncnn_pipnet68.cpp +++ /dev/null @@ -1,193 +0,0 @@ -// -// Created by DefTruth on 2022/3/20. -// - -#include "ncnn_pipnet68.h" - -using ncnncv::NCNNPIPNet68; - -NCNNPIPNet68::NCNNPIPNet68(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNPIPNet68::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - // will do deepcopy inside ncnn - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - - -void NCNNPIPNet68::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("img", input); - // 3. generate landmarks - this->generate_landmarks(landmarks, extractor, img_height, img_width); -} - -void NCNNPIPNet68::generate_landmarks(types::Landmarks &landmarks, - ncnn::Extractor &extractor, - float img_height, float img_width) -{ - ncnn::Mat outputs_cls, outputs_x, outputs_y, outputs_nb_x, outputs_nb_y; - extractor.extract("outputs_cls", outputs_cls); // (68,8,8) - extractor.extract("outputs_x", outputs_x); // (68,8,8) - extractor.extract("outputs_y", outputs_y); // (68,8,8) - extractor.extract("outputs_nb_x", outputs_nb_x); // (68*10,8,8) - extractor.extract("outputs_nb_y", outputs_nb_y); // (68*10,8,8) -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(outputs_cls, "outputs_cls"); - BasicNCNNHandler::print_shape(outputs_x, "outputs_x"); - BasicNCNNHandler::print_shape(outputs_y, "outputs_y"); - BasicNCNNHandler::print_shape(outputs_nb_x, "outputs_nb_x"); - BasicNCNNHandler::print_shape(outputs_nb_y, "outputs_nb_y"); -#endif - const unsigned int input_h = input_height; - const unsigned int input_w = input_width; - - // fetch data from pointers - const float *outputs_cls_ptr = (float *) outputs_cls.data; - const float *outputs_x_ptr = (float *) outputs_x.data; - const float *outputs_y_ptr = (float *) outputs_y.data; - const float *outputs_nb_x_ptr = (float *) outputs_nb_x.data; - const float *outputs_nb_y_ptr = (float *) outputs_nb_y.data; - - // find max_ids - std::vector max_ids(num_lms); - for (unsigned int i = 0; i < num_lms; ++i) - { - const float *score_ptr = outputs_cls_ptr + i * grid_length; - unsigned int max_id = 0; - float max_score = score_ptr[0]; - for (unsigned int j = 0; j < grid_length; ++j) - { - if (score_ptr[j] > max_score) - { - max_score = score_ptr[j]; - max_id = j; - } - } - max_ids[i] = max_id; // range 0~64 - } - - // find x & y offsets - std::vector output_x_select(num_lms); - std::vector output_y_select(num_lms); - for (unsigned int i = 0; i < num_lms; ++i) - { - const float *offset_x_ptr = outputs_x_ptr + i * grid_length; - const float *offset_y_ptr = outputs_y_ptr + i * grid_length; - const unsigned int max_id = max_ids.at(i); - output_x_select[i] = offset_x_ptr[max_id]; - output_y_select[i] = offset_y_ptr[max_id]; - } - - // find nb_x & nb_y offsets - std::unordered_map> output_nb_x_select; - std::unordered_map> output_nb_y_select; - // initialize offsets map - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector nb_x_offset(num_nb); - std::vector nb_y_offset(num_nb); - output_nb_x_select[i] = nb_x_offset; - output_nb_y_select[i] = nb_y_offset; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - for (unsigned int j = 0; j < num_nb; ++j) - { - const float *offset_nb_x_ptr = outputs_nb_x_ptr + (i * num_nb + j) * grid_length; - const float *offset_nb_y_ptr = outputs_nb_y_ptr + (i * num_nb + j) * grid_length; - const unsigned int max_id = max_ids.at(i); - output_nb_x_select[i][j] = offset_nb_x_ptr[max_id]; - output_nb_y_select[i][j] = offset_nb_y_ptr[max_id]; - } - } - - // calculate coords - std::vector lms_pred_x(num_lms); // 68 - std::vector lms_pred_y(num_lms); // 68 - std::unordered_map> lms_pred_nb_x; // 68,10 - std::unordered_map> lms_pred_nb_y; // 68,10 - // initialize pred maps - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector nb_x_offset(num_nb); - std::vector nb_y_offset(num_nb); - lms_pred_nb_x[i] = nb_x_offset; - lms_pred_nb_y[i] = nb_y_offset; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - float cx = static_cast(max_ids.at(i) % grid_w); - float cy = static_cast(max_ids.at(i) / grid_w); - // calculate coords & normalize - lms_pred_x[i] = ((cx + output_x_select[i]) * (float) net_stride) / (float) input_w; - lms_pred_y[i] = ((cy + output_y_select[i]) * (float) net_stride) / (float) input_h; - for (unsigned int j = 0; j < num_nb; ++j) - { - lms_pred_nb_x[i][j] = ((cx + output_nb_x_select[i][j]) * (float) net_stride) / (float) input_w; - lms_pred_nb_y[i][j] = ((cy + output_nb_y_select[i][j]) * (float) net_stride) / (float) input_h; - } - } - - // reverse indexes - std::unordered_map> tmp_nb_x; // 68,max_len - std::unordered_map> tmp_nb_y; // 68,max_len - // initialize reverse maps - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector tmp_x(max_len); - std::vector tmp_y(max_len); - tmp_nb_x[i] = tmp_x; - tmp_nb_y[i] = tmp_y; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - for (unsigned int j = 0; j < max_len; ++j) - { - unsigned int ri = reverse_index1[i * max_len + j]; - unsigned int rj = reverse_index2[i * max_len + j]; - tmp_nb_x[i][j] = lms_pred_nb_x[ri][rj]; - tmp_nb_y[i][j] = lms_pred_nb_y[ri][rj]; - } - } - - // merge predictions - landmarks.points.clear(); - for (unsigned int i = 0; i < num_lms; ++i) - { - float total_x = lms_pred_x[i]; - float total_y = lms_pred_y[i]; - for (unsigned int j = 0; j < max_len; ++j) - { - total_x += tmp_nb_x[i][j]; - total_y += tmp_nb_y[i][j]; - } - float x = total_x / ((float) max_len + 1.f); - float y = total_y / ((float) max_len + 1.f); - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - - landmarks.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_pipnet68.h b/lite/ncnn/cv/ncnn_pipnet68.h deleted file mode 100644 index 20c81544..00000000 --- a/lite/ncnn/cv/ncnn_pipnet68.h +++ /dev/null @@ -1,134 +0,0 @@ -// -// Created by DefTruth on 2022/3/20. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_PIPNET68_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_PIPNET68_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNPIPNet68 : public BasicNCNNHandler - { - public: - explicit NCNNPIPNet68(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNPIPNet68() override = default; - - private: - // hardcode input size - static constexpr const unsigned int input_height = 256; - static constexpr const unsigned int input_width = 256; - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - static constexpr const unsigned int num_nb = 10; - static constexpr const unsigned int num_lms = 68; - static constexpr const unsigned int max_len = 22; - static constexpr const unsigned int net_stride = 32; - // hardcode grid size - static constexpr const unsigned int grid_h = 8; - static constexpr const unsigned int grid_w = 8; - static constexpr const unsigned int grid_length = 8 * 8; // 64 - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - void generate_landmarks(types::Landmarks &landmarks, - ncnn::Extractor &extractor, - float img_height, float img_width); - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - - private: - const unsigned int reverse_index1[68 * 22] = { - 1, 2, 17, 18, 36, 1, 2, 17, 18, 36, 1, 2, 17, 18, 36, 1, 2, 17, 18, 36, 1, 2, 0, 2, 3, 17, 0, 2, 3, 17, 0, 2, 3, 17, 0, 2, 3, 17, 0, - 2, 3, 17, 0, 2, 0, 1, 3, 4, 0, 1, 3, 4, 0, 1, 3, 4, 0, 1, 3, 4, 0, 1, 3, 4, 0, 1, 1, 2, 4, 5, 1, 2, 4, 5, 1, 2, 4, 5, 1, 2, 4, 5, 1, - 2, 4, 5, 1, 2, 2, 3, 5, 6, 2, 3, 5, 6, 2, 3, 5, 6, 2, 3, 5, 6, 2, 3, 5, 6, 2, 3, 3, 4, 6, 7, 3, 4, 6, 7, 3, 4, 6, 7, 3, 4, 6, 7, 3, - 4, 6, 7, 3, 4, 3, 4, 5, 7, 8, 3, 4, 5, 7, 8, 3, 4, 5, 7, 8, 3, 4, 5, 7, 8, 3, 4, 5, 6, 8, 9, 5, 6, 8, 9, 5, 6, 8, 9, 5, 6, 8, 9, 5, - 6, 8, 9, 5, 6, 6, 7, 9, 10, 6, 7, 9, 10, 6, 7, 9, 10, 6, 7, 9, 10, 6, 7, 9, 10, 6, 7, 7, 8, 10, 11, 7, 8, 10, 11, 7, 8, 10, 11, 7, - 8, 10, 11, 7, 8, 10, 11, 7, 8, 8, 9, 11, 12, 13, 8, 9, 11, 12, 13, 8, 9, 11, 12, 13, 8, 9, 11, 12, 13, 8, 9, 9, 10, 12, 13, 9, 10, - 12, 13, 9, 10, 12, 13, 9, 10, 12, 13, 9, 10, 12, 13, 9, 10, 10, 11, 13, 14, 10, 11, 13, 14, 10, 11, 13, 14, 10, 11, 13, 14, 10, 11, - 13, 14, 10, 11, 11, 12, 14, 15, 11, 12, 14, 15, 11, 12, 14, 15, 11, 12, 14, 15, 11, 12, 14, 15, 11, 12, 12, 13, 15, 16, 12, 13, 15, - 16, 12, 13, 15, 16, 12, 13, 15, 16, 12, 13, 15, 16, 12, 13, 13, 14, 16, 26, 13, 14, 16, 26, 13, 14, 16, 26, 13, 14, 16, 26, 13, 14, - 16, 26, 13, 14, 14, 15, 25, 26, 45, 14, 15, 25, 26, 45, 14, 15, 25, 26, 45, 14, 15, 25, 26, 45, 14, 15, 0, 1, 2, 18, 19, 36, 37, 41, - 0, 1, 2, 18, 19, 36, 37, 41, 0, 1, 2, 18, 19, 36, 0, 1, 17, 19, 20, 36, 37, 38, 41, 0, 1, 17, 19, 20, 36, 37, 38, 41, 0, 1, 17, 19, - 0, 17, 18, 20, 21, 36, 37, 38, 40, 41, 0, 17, 18, 20, 21, 36, 37, 38, 40, 41, 0, 17, 17, 18, 19, 21, 36, 37, 38, 39, 40, 41, 17, 18, - 19, 21, 36, 37, 38, 39, 40, 41, 17, 18, 18, 19, 20, 22, 27, 28, 37, 38, 39, 40, 41, 18, 19, 20, 22, 27, 28, 37, 38, 39, 40, 41, 21, - 23, 24, 25, 27, 28, 42, 43, 44, 46, 47, 21, 23, 24, 25, 27, 28, 42, 43, 44, 46, 47, 22, 24, 25, 26, 42, 43, 44, 45, 46, 47, 22, 24, - 25, 26, 42, 43, 44, 45, 46, 47, 22, 24, 16, 22, 23, 25, 26, 43, 44, 45, 46, 47, 16, 22, 23, 25, 26, 43, 44, 45, 46, 47, 16, 22, 15, - 16, 23, 24, 26, 43, 44, 45, 46, 15, 16, 23, 24, 26, 43, 44, 45, 46, 15, 16, 23, 24, 14, 15, 16, 24, 25, 44, 45, 46, 14, 15, 16, 24, - 25, 44, 45, 46, 14, 15, 16, 24, 25, 44, 20, 21, 22, 23, 28, 29, 38, 39, 40, 42, 43, 47, 20, 21, 22, 23, 28, 29, 38, 39, 40, 42, 21, - 22, 27, 29, 30, 39, 40, 42, 47, 21, 22, 27, 29, 30, 39, 40, 42, 47, 21, 22, 27, 29, 27, 28, 30, 31, 35, 39, 42, 27, 28, 30, 31, 35, - 39, 42, 27, 28, 30, 31, 35, 39, 42, 27, 28, 29, 31, 32, 33, 34, 35, 28, 29, 31, 32, 33, 34, 35, 28, 29, 31, 32, 33, 34, 35, 28, 2, - 3, 29, 30, 32, 33, 48, 49, 2, 3, 29, 30, 32, 33, 48, 49, 2, 3, 29, 30, 32, 33, 29, 30, 31, 33, 34, 35, 49, 50, 29, 30, 31, 33, 34, - 35, 49, 50, 29, 30, 31, 33, 34, 35, 29, 30, 31, 32, 34, 35, 50, 51, 52, 29, 30, 31, 32, 34, 35, 50, 51, 52, 29, 30, 31, 32, 29, 30, - 31, 32, 33, 35, 52, 53, 29, 30, 31, 32, 33, 35, 52, 53, 29, 30, 31, 32, 33, 35, 13, 14, 29, 30, 32, 33, 34, 53, 54, 13, 14, 29, 30, - 32, 33, 34, 53, 54, 13, 14, 29, 30, 0, 1, 2, 17, 18, 19, 20, 37, 38, 39, 40, 41, 0, 1, 2, 17, 18, 19, 20, 37, 38, 39, 0, 1, 17, 18, - 19, 20, 21, 36, 38, 39, 40, 41, 0, 1, 17, 18, 19, 20, 21, 36, 38, 39, 0, 1, 17, 18, 19, 20, 21, 27, 28, 36, 37, 39, 40, 41, 0, 1, - 17, 18, 19, 20, 21, 27, 19, 20, 21, 27, 28, 29, 36, 37, 38, 40, 41, 19, 20, 21, 27, 28, 29, 36, 37, 38, 40, 41, 0, 1, 17, 18, 19, - 20, 21, 27, 28, 36, 37, 38, 39, 41, 0, 1, 17, 18, 19, 20, 21, 27, 0, 1, 2, 17, 18, 19, 20, 21, 36, 37, 38, 39, 40, 0, 1, 2, 17, 18, - 19, 20, 21, 36, 22, 23, 24, 27, 28, 29, 43, 44, 45, 46, 47, 22, 23, 24, 27, 28, 29, 43, 44, 45, 46, 47, 15, 16, 22, 23, 24, 25, 26, - 27, 42, 44, 45, 46, 47, 15, 16, 22, 23, 24, 25, 26, 27, 42, 15, 16, 22, 23, 24, 25, 26, 42, 43, 45, 46, 47, 15, 16, 22, 23, 24, 25, - 26, 42, 43, 45, 14, 15, 16, 23, 24, 25, 26, 42, 43, 44, 46, 47, 14, 15, 16, 23, 24, 25, 26, 42, 43, 44, 14, 15, 16, 22, 23, 24, 25, - 26, 42, 43, 44, 45, 47, 14, 15, 16, 22, 23, 24, 25, 26, 42, 15, 16, 22, 23, 24, 25, 26, 27, 28, 42, 43, 44, 45, 46, 15, 16, 22, 23, - 24, 25, 26, 27, 2, 3, 4, 5, 6, 49, 59, 60, 2, 3, 4, 5, 6, 49, 59, 60, 2, 3, 4, 5, 6, 49, 3, 4, 5, 31, 32, 48, 50, 51, 59, 60, 61, - 67, 3, 4, 5, 31, 32, 48, 50, 51, 59, 60, 30, 31, 32, 33, 34, 48, 49, 51, 52, 58, 59, 60, 61, 62, 66, 67, 30, 31, 32, 33, 34, 48, 30, - 31, 32, 33, 34, 35, 48, 49, 50, 52, 53, 54, 56, 58, 60, 61, 62, 63, 64, 65, 66, 67, 30, 32, 33, 34, 35, 50, 51, 53, 54, 55, 56, 62, - 63, 64, 65, 30, 32, 33, 34, 35, 50, 51, 11, 12, 13, 34, 35, 52, 54, 55, 63, 64, 65, 11, 12, 13, 34, 35, 52, 54, 55, 63, 64, 65, 10, - 11, 12, 13, 14, 53, 55, 64, 10, 11, 12, 13, 14, 53, 55, 64, 10, 11, 12, 13, 14, 53, 8, 9, 10, 11, 12, 13, 53, 54, 56, 57, 63, 64, - 65, 8, 9, 10, 11, 12, 13, 53, 54, 56, 7, 8, 9, 10, 11, 12, 54, 55, 57, 58, 63, 64, 65, 66, 7, 8, 9, 10, 11, 12, 54, 55, 6, 7, 8, 9, - 10, 55, 56, 58, 59, 62, 65, 66, 67, 6, 7, 8, 9, 10, 55, 56, 58, 59, 4, 5, 6, 7, 8, 9, 48, 56, 57, 59, 60, 61, 62, 66, 67, 4, 5, 6, - 7, 8, 9, 48, 3, 4, 5, 6, 7, 8, 48, 49, 57, 58, 60, 61, 67, 3, 4, 5, 6, 7, 8, 48, 49, 57, 2, 3, 4, 5, 6, 31, 48, 49, 59, 2, 3, 4, 5, - 6, 31, 48, 49, 59, 2, 3, 4, 5, 31, 32, 33, 48, 49, 50, 51, 52, 57, 58, 59, 60, 62, 63, 66, 67, 31, 32, 33, 48, 49, 50, 33, 34, 48, - 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 63, 64, 65, 66, 67, 33, 34, 35, 50, 51, 52, 53, 54, 55, 56, 57, 61, 62, 64, 65, - 66, 34, 35, 50, 51, 52, 53, 54, 10, 11, 12, 13, 14, 35, 53, 54, 55, 10, 11, 12, 13, 14, 35, 53, 54, 55, 10, 11, 12, 13, 9, 10, 11, - 12, 51, 52, 53, 54, 55, 56, 57, 58, 61, 62, 63, 64, 66, 67, 9, 10, 11, 12, 7, 8, 9, 50, 51, 52, 55, 56, 57, 58, 59, 60, 61, 62, 63, - 64, 65, 67, 7, 8, 9, 50, 4, 5, 6, 7, 48, 49, 50, 51, 56, 57, 58, 59, 60, 61, 62, 63, 65, 66, 4, 5, 6, 7 - }; - const unsigned int reverse_index2[68 * 22] = { - 0, 3, 1, 7, 8, 0, 3, 1, 7, 8, 0, 3, 1, 7, 8, 0, 3, 1, 7, 8, 0, 3, 1, 1, 4, 9, 1, 1, 4, 9, 1, 1, 4, 9, 1, 1, 4, 9, 1, 1, 4, 9, 1, 1, - 6, 1, 1, 5, 6, 1, 1, 5, 6, 1, 1, 5, 6, 1, 1, 5, 6, 1, 1, 5, 6, 1, 5, 0, 0, 6, 5, 0, 0, 6, 5, 0, 0, 6, 5, 0, 0, 6, 5, 0, 0, 6, 5, 0, - 2, 0, 1, 7, 2, 0, 1, 7, 2, 0, 1, 7, 2, 0, 1, 7, 2, 0, 1, 7, 2, 0, 2, 1, 1, 6, 2, 1, 1, 6, 2, 1, 1, 6, 2, 1, 1, 6, 2, 1, 1, 6, 2, 1, - 9, 4, 0, 1, 4, 9, 4, 0, 1, 4, 9, 4, 0, 1, 4, 9, 4, 0, 1, 4, 9, 4, 5, 0, 1, 3, 5, 0, 1, 3, 5, 0, 1, 3, 5, 0, 1, 3, 5, 0, 1, 3, 5, 0, - 4, 0, 0, 4, 4, 0, 0, 4, 4, 0, 0, 4, 4, 0, 0, 4, 4, 0, 0, 4, 4, 0, 3, 0, 0, 5, 3, 0, 0, 5, 3, 0, 0, 5, 3, 0, 0, 5, 3, 0, 0, 5, 3, 0, - 3, 1, 0, 4, 9, 3, 1, 0, 4, 9, 3, 1, 0, 4, 9, 3, 1, 0, 4, 9, 3, 1, 6, 1, 0, 2, 6, 1, 0, 2, 6, 1, 0, 2, 6, 1, 0, 2, 6, 1, 0, 2, 6, 1, - 7, 1, 0, 2, 7, 1, 0, 2, 7, 1, 0, 2, 7, 1, 0, 2, 7, 1, 0, 2, 7, 1, 6, 1, 1, 4, 6, 1, 1, 4, 6, 1, 1, 4, 6, 1, 1, 4, 6, 1, 1, 4, 6, 1, - 5, 1, 0, 6, 5, 1, 0, 6, 5, 1, 0, 6, 5, 1, 0, 6, 5, 1, 0, 6, 5, 1, 3, 0, 0, 9, 3, 0, 0, 9, 3, 0, 0, 9, 3, 0, 0, 9, 3, 0, 0, 9, 3, 0, - 3, 1, 7, 2, 8, 3, 1, 7, 2, 8, 3, 1, 7, 2, 8, 3, 1, 7, 2, 8, 3, 1, 0, 3, 9, 0, 4, 4, 8, 6, 0, 3, 9, 0, 4, 4, 8, 6, 0, 3, 9, 0, 4, 4, - 3, 8, 0, 0, 6, 5, 7, 9, 7, 3, 8, 0, 0, 6, 5, 7, 9, 7, 3, 8, 0, 0, 7, 4, 1, 1, 6, 6, 5, 7, 9, 5, 7, 4, 1, 1, 6, 6, 5, 7, 9, 5, 7, 4, - 8, 4, 1, 0, 9, 6, 4, 7, 6, 8, 8, 4, 1, 0, 9, 6, 4, 7, 6, 8, 8, 4, 9, 6, 0, 4, 2, 7, 9, 6, 5, 5, 9, 9, 6, 0, 4, 2, 7, 9, 6, 5, 5, 9, - 4, 1, 6, 9, 3, 8, 5, 6, 9, 9, 6, 4, 1, 6, 9, 3, 8, 5, 6, 9, 9, 6, 0, 1, 4, 8, 7, 5, 7, 9, 8, 5, 0, 1, 4, 8, 7, 5, 7, 9, 8, 5, 0, 1, - 7, 6, 0, 1, 4, 7, 5, 6, 6, 9, 7, 6, 0, 1, 4, 7, 5, 6, 6, 9, 7, 6, 8, 3, 5, 0, 0, 9, 6, 5, 7, 8, 3, 5, 0, 0, 9, 6, 5, 7, 8, 3, 5, 0, - 8, 3, 1, 4, 0, 8, 4, 5, 8, 3, 1, 4, 0, 8, 4, 5, 8, 3, 1, 4, 0, 8, 9, 1, 1, 9, 1, 2, 8, 4, 7, 2, 8, 7, 9, 1, 1, 9, 1, 2, 8, 4, 7, 2, - 8, 8, 0, 0, 6, 6, 8, 6, 8, 8, 8, 0, 0, 6, 6, 8, 6, 8, 8, 8, 0, 0, 5, 0, 0, 9, 9, 9, 9, 5, 0, 0, 9, 9, 9, 9, 5, 0, 0, 9, 9, 9, 9, 5, - 4, 1, 2, 2, 2, 2, 2, 4, 1, 2, 2, 2, 2, 2, 4, 1, 2, 2, 2, 2, 2, 4, 8, 8, 6, 5, 0, 7, 7, 9, 8, 8, 6, 5, 0, 7, 7, 9, 8, 8, 6, 5, 0, 7, - 4, 3, 0, 0, 4, 5, 8, 7, 4, 3, 0, 0, 4, 5, 8, 7, 4, 3, 0, 0, 4, 5, 7, 2, 1, 1, 1, 1, 5, 8, 5, 7, 2, 1, 1, 1, 1, 5, 8, 5, 7, 2, 1, 1, - 3, 1, 5, 4, 1, 0, 6, 9, 3, 1, 5, 4, 1, 0, 6, 9, 3, 1, 5, 4, 1, 0, 8, 9, 5, 4, 9, 6, 0, 8, 7, 8, 9, 5, 4, 9, 6, 0, 8, 7, 8, 9, 5, 4, - 2, 2, 4, 2, 3, 5, 8, 1, 5, 8, 4, 1, 2, 2, 4, 2, 3, 5, 8, 1, 5, 8, 5, 6, 3, 2, 2, 3, 7, 1, 1, 3, 3, 0, 5, 6, 3, 2, 2, 3, 7, 1, 1, 3, - 9, 9, 6, 6, 3, 2, 2, 7, 9, 3, 2, 1, 0, 3, 9, 9, 6, 6, 3, 2, 2, 7, 9, 4, 3, 4, 3, 9, 7, 4, 2, 1, 4, 9, 4, 3, 4, 3, 9, 7, 4, 2, 1, 4, - 8, 7, 7, 8, 8, 5, 5, 8, 5, 2, 3, 0, 0, 2, 8, 7, 7, 8, 8, 5, 5, 8, 4, 4, 5, 5, 5, 7, 7, 9, 0, 0, 3, 2, 2, 4, 4, 5, 5, 5, 7, 7, 9, 0, - 3, 4, 9, 1, 2, 8, 2, 4, 7, 4, 2, 3, 4, 9, 1, 2, 8, 2, 4, 7, 4, 2, 9, 9, 2, 2, 3, 6, 6, 6, 1, 2, 3, 3, 0, 9, 9, 2, 2, 3, 6, 6, 6, 1, - 6, 5, 7, 3, 2, 2, 3, 4, 1, 1, 1, 3, 6, 5, 7, 3, 2, 2, 3, 4, 1, 1, 4, 2, 2, 8, 5, 3, 1, 8, 4, 1, 0, 4, 4, 2, 2, 8, 5, 3, 1, 8, 4, 1, - 5, 5, 4, 9, 7, 7, 5, 5, 3, 3, 0, 0, 1, 5, 5, 4, 9, 7, 7, 5, 5, 3, 7, 8, 5, 6, 8, 8, 7, 9, 6, 0, 0, 3, 2, 2, 7, 8, 5, 6, 8, 8, 7, 9, - 6, 3, 2, 2, 5, 3, 3, 0, 6, 3, 2, 2, 5, 3, 3, 0, 6, 3, 2, 2, 5, 3, 6, 7, 8, 4, 6, 1, 3, 9, 4, 1, 5, 8, 6, 7, 8, 4, 6, 1, 3, 9, 4, 1, - 7, 3, 3, 4, 8, 5, 1, 1, 7, 9, 8, 5, 1, 6, 9, 5, 7, 3, 3, 4, 8, 5, 9, 6, 5, 3, 5, 6, 9, 6, 1, 1, 6, 9, 8, 8, 8, 3, 0, 3, 8, 6, 6, 6, - 8, 8, 5, 3, 3, 8, 2, 1, 5, 8, 9, 7, 1, 5, 4, 8, 8, 5, 3, 3, 8, 2, 8, 7, 6, 6, 4, 3, 1, 3, 5, 1, 8, 8, 7, 6, 6, 4, 3, 1, 3, 5, 1, 8, - 5, 2, 2, 4, 6, 2, 4, 0, 5, 2, 2, 4, 6, 2, 4, 0, 5, 2, 2, 4, 6, 2, 7, 5, 2, 3, 6, 7, 5, 2, 2, 9, 8, 2, 5, 7, 5, 2, 3, 6, 7, 5, 2, 2, - 7, 5, 2, 3, 7, 8, 6, 0, 1, 5, 7, 6, 3, 8, 7, 5, 2, 3, 7, 8, 6, 0, 8, 4, 2, 4, 8, 7, 0, 0, 7, 8, 7, 4, 7, 8, 4, 2, 4, 8, 7, 0, 0, 7, - 9, 7, 3, 2, 6, 7, 6, 5, 0, 0, 6, 7, 9, 7, 3, 9, 7, 3, 2, 6, 7, 6, 7, 6, 3, 2, 5, 8, 2, 5, 8, 2, 2, 8, 4, 7, 6, 3, 2, 5, 8, 2, 5, 8, - 7, 5, 3, 4, 6, 8, 0, 0, 1, 7, 5, 3, 4, 6, 8, 0, 0, 1, 7, 5, 3, 4, 7, 7, 9, 3, 2, 0, 3, 9, 6, 4, 5, 3, 2, 6, 3, 0, 7, 7, 9, 3, 2, 0, - 8, 9, 8, 7, 2, 0, 2, 7, 8, 9, 6, 5, 6, 9, 7, 2, 2, 7, 2, 0, 2, 8, 7, 7, 9, 4, 0, 3, 3, 5, 4, 7, 6, 3, 3, 0, 5, 7, 7, 9, 4, 0, 3, 3, - 6, 4, 3, 5, 7, 8, 0, 0, 1, 6, 4, 3, 5, 7, 8, 0, 0, 1, 6, 4, 3, 5, 8, 9, 9, 9, 7, 4, 4, 4, 2, 1, 4, 7, 9, 5, 0, 4, 2, 9, 8, 9, 9, 9, - 9, 9, 9, 6, 5, 8, 6, 3, 2, 3, 6, 9, 4, 1, 4, 9, 1, 1, 9, 9, 9, 6, 8, 9, 9, 8, 4, 4, 4, 6, 7, 3, 1, 2, 4, 0, 4, 9, 9, 1, 8, 9, 9, 8 - }; - - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_PIPNET68_H diff --git a/lite/ncnn/cv/ncnn_pipnet98.cpp b/lite/ncnn/cv/ncnn_pipnet98.cpp deleted file mode 100644 index e8ec044f..00000000 --- a/lite/ncnn/cv/ncnn_pipnet98.cpp +++ /dev/null @@ -1,193 +0,0 @@ -// -// Created by DefTruth on 2022/3/20. -// - -#include "ncnn_pipnet98.h" - -using ncnncv::NCNNPIPNet98; - -NCNNPIPNet98::NCNNPIPNet98(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNPIPNet98::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - // will do deepcopy inside ncnn - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - - -void NCNNPIPNet98::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("img", input); - // 3. generate landmarks - this->generate_landmarks(landmarks, extractor, img_height, img_width); -} - -void NCNNPIPNet98::generate_landmarks(types::Landmarks &landmarks, - ncnn::Extractor &extractor, - float img_height, float img_width) -{ - ncnn::Mat outputs_cls, outputs_x, outputs_y, outputs_nb_x, outputs_nb_y; - extractor.extract("outputs_cls", outputs_cls); // (98,8,8) - extractor.extract("outputs_x", outputs_x); // (98,8,8) - extractor.extract("outputs_y", outputs_y); // (98,8,8) - extractor.extract("outputs_nb_x", outputs_nb_x); // (98*10,8,8) - extractor.extract("outputs_nb_y", outputs_nb_y); // (98*10,8,8) -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(outputs_cls, "outputs_cls"); - BasicNCNNHandler::print_shape(outputs_x, "outputs_x"); - BasicNCNNHandler::print_shape(outputs_y, "outputs_y"); - BasicNCNNHandler::print_shape(outputs_nb_x, "outputs_nb_x"); - BasicNCNNHandler::print_shape(outputs_nb_y, "outputs_nb_y"); -#endif - const unsigned int input_h = input_height; - const unsigned int input_w = input_width; - - // fetch data from pointers - const float *outputs_cls_ptr = (float *) outputs_cls.data; - const float *outputs_x_ptr = (float *) outputs_x.data; - const float *outputs_y_ptr = (float *) outputs_y.data; - const float *outputs_nb_x_ptr = (float *) outputs_nb_x.data; - const float *outputs_nb_y_ptr = (float *) outputs_nb_y.data; - - // find max_ids - std::vector max_ids(num_lms); - for (unsigned int i = 0; i < num_lms; ++i) - { - const float *score_ptr = outputs_cls_ptr + i * grid_length; - unsigned int max_id = 0; - float max_score = score_ptr[0]; - for (unsigned int j = 0; j < grid_length; ++j) - { - if (score_ptr[j] > max_score) - { - max_score = score_ptr[j]; - max_id = j; - } - } - max_ids[i] = max_id; // range 0~64 - } - - // find x & y offsets - std::vector output_x_select(num_lms); - std::vector output_y_select(num_lms); - for (unsigned int i = 0; i < num_lms; ++i) - { - const float *offset_x_ptr = outputs_x_ptr + i * grid_length; - const float *offset_y_ptr = outputs_y_ptr + i * grid_length; - const unsigned int max_id = max_ids.at(i); - output_x_select[i] = offset_x_ptr[max_id]; - output_y_select[i] = offset_y_ptr[max_id]; - } - - // find nb_x & nb_y offsets - std::unordered_map> output_nb_x_select; - std::unordered_map> output_nb_y_select; - // initialize offsets map - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector nb_x_offset(num_nb); - std::vector nb_y_offset(num_nb); - output_nb_x_select[i] = nb_x_offset; - output_nb_y_select[i] = nb_y_offset; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - for (unsigned int j = 0; j < num_nb; ++j) - { - const float *offset_nb_x_ptr = outputs_nb_x_ptr + (i * num_nb + j) * grid_length; - const float *offset_nb_y_ptr = outputs_nb_y_ptr + (i * num_nb + j) * grid_length; - const unsigned int max_id = max_ids.at(i); - output_nb_x_select[i][j] = offset_nb_x_ptr[max_id]; - output_nb_y_select[i][j] = offset_nb_y_ptr[max_id]; - } - } - - // calculate coords - std::vector lms_pred_x(num_lms); // 98 - std::vector lms_pred_y(num_lms); // 98 - std::unordered_map> lms_pred_nb_x; // 98,10 - std::unordered_map> lms_pred_nb_y; // 98,10 - // initialize pred maps - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector nb_x_offset(num_nb); - std::vector nb_y_offset(num_nb); - lms_pred_nb_x[i] = nb_x_offset; - lms_pred_nb_y[i] = nb_y_offset; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - float cx = static_cast(max_ids.at(i) % grid_w); - float cy = static_cast(max_ids.at(i) / grid_w); - // calculate coords & normalize - lms_pred_x[i] = ((cx + output_x_select[i]) * (float) net_stride) / (float) input_w; - lms_pred_y[i] = ((cy + output_y_select[i]) * (float) net_stride) / (float) input_h; - for (unsigned int j = 0; j < num_nb; ++j) - { - lms_pred_nb_x[i][j] = ((cx + output_nb_x_select[i][j]) * (float) net_stride) / (float) input_w; - lms_pred_nb_y[i][j] = ((cy + output_nb_y_select[i][j]) * (float) net_stride) / (float) input_h; - } - } - - // reverse indexes - std::unordered_map> tmp_nb_x; // 98,max_len - std::unordered_map> tmp_nb_y; // 98,max_len - // initialize reverse maps - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector tmp_x(max_len); - std::vector tmp_y(max_len); - tmp_nb_x[i] = tmp_x; - tmp_nb_y[i] = tmp_y; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - for (unsigned int j = 0; j < max_len; ++j) - { - unsigned int ri = reverse_index1[i * max_len + j]; - unsigned int rj = reverse_index2[i * max_len + j]; - tmp_nb_x[i][j] = lms_pred_nb_x[ri][rj]; - tmp_nb_y[i][j] = lms_pred_nb_y[ri][rj]; - } - } - - // merge predictions - landmarks.points.clear(); - for (unsigned int i = 0; i < num_lms; ++i) - { - float total_x = lms_pred_x[i]; - float total_y = lms_pred_y[i]; - for (unsigned int j = 0; j < max_len; ++j) - { - total_x += tmp_nb_x[i][j]; - total_y += tmp_nb_y[i][j]; - } - float x = total_x / ((float) max_len + 1.f); - float y = total_y / ((float) max_len + 1.f); - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - - landmarks.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_pipnet98.h b/lite/ncnn/cv/ncnn_pipnet98.h deleted file mode 100644 index af34a855..00000000 --- a/lite/ncnn/cv/ncnn_pipnet98.h +++ /dev/null @@ -1,143 +0,0 @@ -// -// Created by DefTruth on 2022/3/20. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_PIPNET98_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_PIPNET98_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNPIPNet98 : public BasicNCNNHandler - { - public: - explicit NCNNPIPNet98(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNPIPNet98() override = default; - - private: - // hardcode input size - static constexpr const unsigned int input_height = 256; - static constexpr const unsigned int input_width = 256; - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - static constexpr const unsigned int num_nb = 10; - static constexpr const unsigned int num_lms = 98; - static constexpr const unsigned int max_len = 17; - static constexpr const unsigned int net_stride = 32; - // hardcode grid size - static constexpr const unsigned int grid_h = 8; - static constexpr const unsigned int grid_w = 8; - static constexpr const unsigned int grid_length = 8 * 8; // 64 - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - void generate_landmarks(types::Landmarks &landmarks, - ncnn::Extractor &extractor, - float img_height, float img_width); - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - - private: - const unsigned int reverse_index1[98 * 17] = { - 1, 2, 3, 4, 5, 33, 1, 2, 3, 4, 5, 33, 1, 2, 3, 4, 5, 0, 2, 3, 4, 5, 6, 33, 0, 2, 3, 4, 5, 6, 33, 0, 2, 3, 0, 1, 3, 4, 5, 6, 0, 1, 3, - 4, 5, 6, 0, 1, 3, 4, 5, 0, 1, 2, 4, 5, 6, 7, 0, 1, 2, 4, 5, 6, 7, 0, 1, 2, 0, 1, 2, 3, 5, 6, 7, 8, 0, 1, 2, 3, 5, 6, 7, 8, 0, 1, 2, - 3, 4, 6, 7, 8, 9, 1, 2, 3, 4, 6, 7, 8, 9, 1, 2, 3, 4, 5, 7, 8, 9, 10, 2, 3, 4, 5, 7, 8, 9, 10, 2, 3, 4, 5, 6, 8, 9, 10, 3, 4, 5, 6, - 8, 9, 10, 3, 4, 5, 4, 5, 6, 7, 9, 10, 11, 4, 5, 6, 7, 9, 10, 11, 4, 5, 6, 4, 5, 6, 7, 8, 10, 11, 12, 4, 5, 6, 7, 8, 10, 11, 12, 4, - 5, 6, 7, 8, 9, 11, 12, 13, 76, 5, 6, 7, 8, 9, 11, 12, 13, 7, 8, 9, 10, 12, 13, 14, 76, 88, 7, 8, 9, 10, 12, 13, 14, 76, 8, 9, 10, - 11, 13, 14, 15, 8, 9, 10, 11, 13, 14, 15, 8, 9, 10, 10, 11, 12, 14, 15, 16, 10, 11, 12, 14, 15, 16, 10, 11, 12, 14, 15, 11, 12, 13, - 15, 16, 17, 11, 12, 13, 15, 16, 17, 11, 12, 13, 15, 16, 12, 13, 14, 16, 17, 18, 12, 13, 14, 16, 17, 18, 12, 13, 14, 16, 17, 13, 14, - 15, 17, 18, 19, 13, 14, 15, 17, 18, 19, 13, 14, 15, 17, 18, 14, 15, 16, 18, 19, 20, 14, 15, 16, 18, 19, 20, 14, 15, 16, 18, 19, 15, - 16, 17, 19, 20, 21, 15, 16, 17, 19, 20, 21, 15, 16, 17, 19, 20, 16, 17, 18, 20, 21, 22, 16, 17, 18, 20, 21, 22, 16, 17, 18, 20, 21, - 17, 18, 19, 21, 22, 23, 24, 17, 18, 19, 21, 22, 23, 24, 17, 18, 19, 18, 19, 20, 22, 23, 24, 25, 82, 18, 19, 20, 22, 23, 24, 25, 82, - 18, 19, 20, 21, 23, 24, 25, 26, 27, 19, 20, 21, 23, 24, 25, 26, 27, 19, 20, 21, 22, 24, 25, 26, 27, 28, 20, 21, 22, 24, 25, 26, 27, - 28, 20, 21, 22, 23, 25, 26, 27, 28, 21, 22, 23, 25, 26, 27, 28, 21, 22, 23, 21, 22, 23, 24, 26, 27, 28, 29, 21, 22, 23, 24, 26, 27, - 28, 29, 21, 22, 23, 24, 25, 27, 28, 29, 30, 22, 23, 24, 25, 27, 28, 29, 30, 22, 23, 24, 25, 26, 28, 29, 30, 31, 23, 24, 25, 26, 28, - 29, 30, 31, 23, 24, 25, 26, 27, 29, 30, 31, 32, 24, 25, 26, 27, 29, 30, 31, 32, 24, 25, 26, 27, 28, 30, 31, 32, 25, 26, 27, 28, 30, - 31, 32, 25, 26, 27, 26, 27, 28, 29, 31, 32, 26, 27, 28, 29, 31, 32, 26, 27, 28, 29, 31, 26, 27, 28, 29, 30, 32, 46, 26, 27, 28, 29, - 30, 32, 46, 26, 27, 28, 27, 28, 29, 30, 31, 46, 27, 28, 29, 30, 31, 46, 27, 28, 29, 30, 31, 0, 1, 2, 3, 34, 41, 60, 0, 1, 2, 3, 34, - 41, 60, 0, 1, 2, 0, 33, 35, 40, 41, 60, 0, 33, 35, 40, 41, 60, 0, 33, 35, 40, 41, 33, 34, 36, 37, 39, 40, 41, 60, 61, 62, 33, 34, - 36, 37, 39, 40, 41, 34, 35, 37, 38, 39, 40, 63, 64, 34, 35, 37, 38, 39, 40, 63, 64, 34, 36, 38, 39, 51, 64, 36, 38, 39, 51, 64, 36, - 38, 39, 51, 64, 36, 38, 36, 37, 39, 51, 52, 63, 64, 65, 36, 37, 39, 51, 52, 63, 64, 65, 36, 35, 36, 37, 38, 40, 62, 63, 64, 65, 66, - 67, 96, 35, 36, 37, 38, 40, 33, 34, 35, 36, 37, 38, 39, 41, 60, 61, 62, 63, 65, 66, 67, 96, 33, 0, 1, 2, 33, 34, 35, 40, 60, 61, 67, - 0, 1, 2, 33, 34, 35, 40, 43, 49, 50, 51, 68, 43, 49, 50, 51, 68, 43, 49, 50, 51, 68, 43, 49, 42, 44, 45, 48, 49, 50, 68, 69, 42, 44, - 45, 48, 49, 50, 68, 69, 42, 42, 43, 45, 46, 47, 48, 49, 70, 42, 43, 45, 46, 47, 48, 49, 70, 42, 32, 44, 46, 47, 48, 71, 72, 73, 32, - 44, 46, 47, 48, 71, 72, 73, 32, 29, 30, 31, 32, 45, 47, 72, 29, 30, 31, 32, 45, 47, 72, 29, 30, 31, 30, 31, 32, 44, 45, 46, 48, 71, - 72, 73, 30, 31, 32, 44, 45, 46, 48, 42, 43, 44, 45, 46, 47, 49, 50, 69, 70, 71, 72, 73, 74, 75, 97, 42, 42, 43, 44, 48, 50, 68, 69, - 70, 74, 75, 97, 42, 43, 44, 48, 50, 68, 42, 43, 49, 51, 52, 68, 69, 75, 42, 43, 49, 51, 52, 68, 69, 75, 42, 37, 38, 42, 50, 52, 53, - 64, 68, 37, 38, 42, 50, 52, 53, 64, 68, 37, 51, 53, 54, 51, 53, 54, 51, 53, 54, 51, 53, 54, 51, 53, 54, 51, 53, 51, 52, 54, 55, 56, - 57, 59, 51, 52, 54, 55, 56, 57, 59, 51, 52, 54, 52, 53, 55, 56, 57, 58, 59, 52, 53, 55, 56, 57, 58, 59, 52, 53, 55, 53, 54, 56, 57, - 76, 77, 78, 88, 53, 54, 56, 57, 76, 77, 78, 88, 53, 53, 54, 55, 57, 58, 77, 78, 79, 88, 53, 54, 55, 57, 58, 77, 78, 79, 53, 54, 55, - 56, 58, 59, 78, 79, 80, 90, 53, 54, 55, 56, 58, 59, 78, 53, 54, 56, 57, 59, 79, 80, 81, 82, 92, 53, 54, 56, 57, 59, 79, 80, 53, 54, - 57, 58, 80, 81, 82, 92, 53, 54, 57, 58, 80, 81, 82, 92, 53, 0, 1, 2, 3, 4, 33, 34, 41, 61, 62, 66, 67, 96, 0, 1, 2, 3, 0, 1, 33, 34, - 35, 40, 41, 60, 62, 63, 65, 66, 67, 96, 0, 1, 33, 33, 34, 35, 36, 37, 38, 39, 40, 41, 60, 61, 63, 64, 65, 66, 67, 96, 35, 36, 37, - 38, 39, 40, 51, 52, 61, 62, 64, 65, 66, 67, 96, 35, 36, 36, 37, 38, 39, 51, 52, 53, 63, 65, 66, 96, 36, 37, 38, 39, 51, 52, 36, 37, - 38, 39, 52, 61, 62, 63, 64, 66, 67, 96, 36, 37, 38, 39, 52, 41, 60, 61, 62, 63, 64, 65, 67, 96, 41, 60, 61, 62, 63, 64, 65, 67, 0, - 1, 2, 3, 33, 34, 35, 40, 41, 60, 61, 62, 65, 66, 96, 0, 1, 42, 43, 49, 50, 51, 52, 53, 69, 74, 75, 97, 42, 43, 49, 50, 51, 52, 42, - 43, 44, 48, 49, 50, 51, 68, 70, 71, 73, 74, 75, 97, 42, 43, 44, 42, 43, 44, 45, 46, 47, 48, 49, 50, 68, 69, 71, 72, 73, 74, 75, 97, - 31, 32, 44, 45, 46, 47, 48, 69, 70, 72, 73, 74, 75, 97, 31, 32, 44, 28, 29, 30, 31, 32, 45, 46, 47, 70, 71, 73, 74, 97, 28, 29, 30, - 31, 29, 30, 31, 32, 44, 45, 46, 47, 48, 70, 71, 72, 74, 75, 97, 29, 30, 47, 68, 69, 70, 71, 72, 73, 75, 97, 47, 68, 69, 70, 71, 72, - 73, 75, 42, 43, 49, 50, 52, 68, 69, 70, 71, 72, 73, 74, 97, 42, 43, 49, 50, 6, 7, 8, 9, 10, 11, 12, 55, 77, 87, 88, 89, 95, 6, 7, 8, - 9, 55, 56, 76, 78, 86, 87, 88, 89, 95, 55, 56, 76, 78, 86, 87, 88, 89, 54, 55, 56, 57, 58, 76, 77, 79, 80, 85, 86, 87, 88, 89, 90, - 94, 95, 54, 55, 56, 57, 58, 59, 77, 78, 80, 81, 84, 85, 86, 89, 90, 91, 94, 54, 57, 58, 59, 78, 79, 81, 82, 83, 84, 85, 90, 91, 92, - 93, 94, 54, 58, 59, 80, 82, 83, 84, 91, 92, 93, 58, 59, 80, 82, 83, 84, 91, 92, 20, 21, 22, 23, 24, 25, 26, 59, 81, 83, 91, 92, 93, - 20, 21, 22, 23, 17, 18, 19, 20, 21, 22, 23, 81, 82, 84, 91, 92, 93, 17, 18, 19, 20, 16, 17, 18, 19, 20, 81, 82, 83, 85, 91, 92, 93, - 94, 16, 17, 18, 19, 14, 15, 16, 17, 18, 83, 84, 86, 87, 90, 93, 94, 95, 14, 15, 16, 17, 11, 12, 13, 14, 15, 16, 76, 77, 85, 87, 88, - 89, 94, 95, 11, 12, 13, 9, 10, 11, 12, 13, 14, 76, 77, 86, 88, 89, 95, 9, 10, 11, 12, 13, 7, 8, 9, 10, 11, 12, 13, 55, 76, 77, 86, - 87, 89, 95, 7, 8, 9, 55, 56, 76, 77, 78, 79, 86, 87, 88, 90, 95, 55, 56, 76, 77, 78, 79, 56, 57, 58, 78, 79, 80, 83, 84, 85, 86, 87, - 89, 91, 92, 93, 94, 95, 58, 59, 79, 80, 81, 82, 83, 84, 85, 90, 92, 93, 94, 58, 59, 79, 80, 19, 20, 21, 22, 23, 24, 25, 59, 81, 82, - 83, 84, 91, 93, 19, 20, 21, 18, 19, 79, 80, 81, 82, 83, 84, 85, 90, 91, 92, 94, 18, 19, 79, 80, 15, 16, 17, 78, 79, 80, 83, 84, 85, - 86, 87, 89, 90, 91, 93, 95, 15, 13, 14, 15, 76, 77, 78, 85, 86, 87, 88, 89, 90, 94, 13, 14, 15, 76, 34, 35, 36, 38, 39, 40, 41, 60, - 61, 62, 63, 64, 65, 66, 67, 34, 35, 43, 44, 45, 47, 48, 49, 50, 68, 69, 70, 71, 72, 73, 74, 75, 43, 44 - }; - const unsigned int reverse_index2[98 * 17] = { - 0, 2, 4, 6, 8, 4, 0, 2, 4, 6, 8, 4, 0, 2, 4, 6, 8, 0, 0, 2, 4, 6, 8, 8, 0, 0, 2, 4, 6, 8, 8, 0, 0, 2, 1, 1, 0, 2, 4, 6, 1, 1, 0, 2, - 4, 6, 1, 1, 0, 2, 4, 3, 2, 1, 0, 2, 4, 6, 3, 2, 1, 0, 2, 4, 6, 3, 2, 1, 6, 3, 3, 1, 0, 2, 4, 7, 6, 3, 3, 1, 0, 2, 4, 7, 6, 6, 4, 3, - 1, 0, 2, 4, 8, 6, 4, 3, 1, 0, 2, 4, 8, 6, 7, 5, 3, 1, 0, 2, 4, 9, 7, 5, 3, 1, 0, 2, 4, 9, 7, 6, 5, 3, 1, 0, 2, 4, 6, 5, 3, 1, 0, 2, - 4, 6, 5, 3, 7, 5, 3, 1, 0, 2, 4, 7, 5, 3, 1, 0, 2, 4, 7, 5, 3, 9, 7, 5, 3, 1, 0, 2, 5, 9, 7, 5, 3, 1, 0, 2, 5, 9, 9, 7, 5, 3, 1, 0, - 2, 5, 8, 9, 7, 5, 3, 1, 0, 2, 5, 7, 5, 3, 1, 0, 2, 5, 9, 9, 7, 5, 3, 1, 0, 2, 5, 9, 9, 5, 3, 1, 0, 2, 4, 9, 5, 3, 1, 0, 2, 4, 9, 5, - 3, 6, 3, 1, 0, 2, 6, 6, 3, 1, 0, 2, 6, 6, 3, 1, 0, 2, 7, 3, 1, 0, 3, 7, 7, 3, 1, 0, 3, 7, 7, 3, 1, 0, 3, 6, 3, 1, 1, 3, 6, 6, 3, 1, - 1, 3, 6, 6, 3, 1, 1, 3, 7, 3, 1, 1, 3, 7, 7, 3, 1, 1, 3, 7, 7, 3, 1, 1, 3, 6, 3, 0, 1, 3, 6, 6, 3, 0, 1, 3, 6, 6, 3, 0, 1, 3, 7, 2, - 0, 1, 3, 5, 7, 2, 0, 1, 3, 5, 7, 2, 0, 1, 3, 5, 2, 0, 1, 3, 5, 5, 2, 0, 1, 3, 5, 5, 2, 0, 1, 3, 4, 2, 0, 1, 3, 5, 8, 4, 2, 0, 1, 3, - 5, 8, 4, 2, 0, 5, 2, 0, 1, 3, 5, 7, 9, 5, 2, 0, 1, 3, 5, 7, 9, 5, 4, 2, 0, 1, 3, 5, 7, 9, 4, 2, 0, 1, 3, 5, 7, 9, 4, 4, 2, 0, 1, 3, - 5, 7, 9, 4, 2, 0, 1, 3, 5, 7, 9, 4, 4, 2, 0, 1, 3, 5, 7, 4, 2, 0, 1, 3, 5, 7, 4, 2, 0, 9, 4, 2, 0, 1, 3, 5, 6, 9, 4, 2, 0, 1, 3, 5, - 6, 9, 9, 4, 2, 0, 1, 3, 5, 6, 9, 4, 2, 0, 1, 3, 5, 6, 9, 8, 4, 2, 0, 1, 3, 4, 6, 8, 4, 2, 0, 1, 3, 4, 6, 8, 6, 4, 2, 0, 1, 3, 3, 5, - 6, 4, 2, 0, 1, 3, 3, 5, 6, 6, 4, 2, 0, 1, 2, 3, 6, 4, 2, 0, 1, 2, 3, 6, 4, 2, 6, 4, 2, 0, 1, 1, 6, 4, 2, 0, 1, 1, 6, 4, 2, 0, 1, 8, - 6, 4, 2, 0, 0, 9, 8, 6, 4, 2, 0, 0, 9, 8, 6, 4, 8, 6, 4, 2, 0, 6, 8, 6, 4, 2, 0, 6, 8, 6, 4, 2, 0, 2, 4, 5, 8, 3, 1, 6, 2, 4, 5, 8, - 3, 1, 6, 2, 4, 5, 7, 1, 1, 5, 0, 8, 7, 1, 1, 5, 0, 8, 7, 1, 1, 5, 0, 7, 1, 2, 8, 6, 0, 5, 9, 8, 8, 7, 1, 2, 8, 6, 0, 5, 8, 2, 1, 4, - 0, 6, 7, 9, 8, 2, 1, 4, 0, 6, 7, 9, 8, 1, 0, 5, 5, 7, 1, 0, 5, 5, 7, 1, 0, 5, 5, 7, 1, 0, 4, 0, 2, 2, 6, 6, 2, 8, 4, 0, 2, 2, 6, 6, - 2, 8, 4, 4, 0, 2, 1, 4, 7, 4, 4, 5, 9, 9, 7, 4, 0, 2, 1, 4, 5, 2, 0, 3, 9, 9, 4, 2, 7, 5, 4, 8, 9, 8, 6, 6, 5, 5, 7, 9, 0, 0, 3, 3, - 2, 6, 7, 5, 7, 9, 0, 0, 3, 3, 2, 5, 0, 6, 7, 2, 5, 0, 6, 7, 2, 5, 0, 6, 7, 2, 5, 1, 1, 8, 5, 0, 4, 9, 7, 1, 1, 8, 5, 0, 4, 9, 7, 1, - 8, 1, 1, 7, 4, 0, 6, 9, 8, 1, 1, 7, 4, 0, 6, 9, 8, 7, 2, 1, 0, 6, 9, 8, 9, 7, 2, 1, 0, 6, 9, 8, 9, 7, 8, 5, 4, 2, 2, 1, 6, 8, 5, 4, - 2, 2, 1, 6, 8, 5, 4, 9, 7, 6, 3, 0, 0, 3, 6, 2, 7, 9, 7, 6, 3, 0, 0, 3, 7, 3, 0, 3, 5, 2, 2, 9, 8, 4, 5, 7, 6, 7, 9, 6, 7, 2, 0, 4, - 2, 1, 3, 2, 7, 9, 5, 8, 2, 0, 4, 2, 1, 3, 0, 4, 3, 1, 5, 2, 6, 8, 0, 4, 3, 1, 5, 2, 6, 8, 0, 5, 6, 5, 5, 1, 5, 8, 8, 5, 6, 5, 5, 1, - 5, 8, 8, 5, 0, 1, 9, 0, 1, 9, 0, 1, 9, 0, 1, 9, 0, 1, 9, 0, 1, 7, 0, 1, 9, 9, 9, 9, 7, 0, 1, 9, 9, 9, 9, 7, 0, 1, 4, 0, 5, 2, 0, 2, - 4, 4, 0, 5, 2, 0, 2, 4, 4, 0, 5, 6, 5, 0, 8, 6, 6, 9, 6, 6, 5, 0, 8, 6, 6, 9, 6, 6, 3, 2, 0, 2, 7, 7, 5, 7, 8, 3, 2, 0, 2, 7, 7, 5, - 7, 2, 0, 2, 1, 1, 2, 4, 3, 5, 7, 2, 0, 2, 1, 1, 2, 4, 4, 3, 7, 1, 0, 5, 4, 8, 8, 8, 4, 3, 7, 1, 0, 5, 4, 7, 4, 7, 0, 9, 6, 6, 6, 7, - 4, 7, 0, 9, 6, 6, 6, 7, 4, 5, 6, 7, 8, 2, 5, 4, 1, 9, 6, 1, 9, 4, 5, 6, 7, 8, 9, 3, 4, 6, 2, 3, 1, 2, 9, 7, 4, 0, 5, 8, 9, 3, 9, 6, - 5, 6, 7, 7, 3, 1, 7, 4, 2, 3, 6, 4, 1, 4, 0, 8, 5, 3, 3, 1, 8, 8, 9, 7, 3, 1, 0, 5, 8, 3, 8, 5, 8, 4, 2, 8, 4, 3, 9, 1, 1, 7, 8, 8, - 4, 2, 8, 4, 3, 9, 6, 5, 9, 7, 9, 6, 0, 0, 3, 5, 2, 9, 6, 5, 9, 7, 9, 3, 4, 1, 5, 5, 3, 2, 1, 9, 3, 4, 1, 5, 5, 3, 2, 9, 8, 8, 9, 6, - 7, 9, 9, 6, 0, 0, 5, 6, 2, 4, 9, 8, 4, 8, 8, 2, 3, 2, 8, 1, 8, 1, 9, 4, 8, 8, 2, 3, 2, 3, 5, 8, 8, 1, 3, 9, 0, 3, 7, 8, 5, 0, 5, 3, - 5, 8, 9, 6, 5, 6, 8, 6, 1, 4, 7, 6, 4, 2, 5, 4, 2, 4, 0, 9, 8, 6, 4, 3, 3, 4, 9, 1, 1, 0, 4, 7, 2, 9, 8, 6, 8, 7, 7, 5, 4, 5, 2, 5, - 8, 1, 1, 6, 7, 8, 7, 7, 5, 9, 8, 8, 9, 9, 7, 4, 7, 9, 5, 0, 0, 1, 6, 3, 9, 8, 9, 5, 5, 2, 4, 3, 2, 3, 1, 9, 5, 5, 2, 4, 3, 2, 3, 6, - 9, 9, 6, 8, 1, 0, 6, 8, 9, 5, 3, 4, 6, 9, 9, 6, 9, 8, 6, 6, 5, 6, 7, 8, 4, 2, 0, 8, 7, 9, 8, 6, 6, 1, 5, 2, 7, 5, 3, 2, 0, 3, 1, 5, - 2, 7, 5, 3, 2, 0, 7, 4, 3, 4, 9, 7, 5, 1, 3, 7, 7, 6, 7, 2, 2, 3, 4, 6, 7, 4, 3, 4, 6, 9, 0, 0, 9, 9, 6, 9, 7, 0, 7, 2, 8, 5, 3, 3, - 3, 2, 5, 7, 6, 7, 8, 3, 2, 7, 4, 4, 8, 5, 1, 6, 2, 3, 5, 0, 2, 3, 5, 1, 6, 2, 3, 5, 0, 2, 7, 6, 6, 6, 7, 8, 9, 8, 4, 2, 8, 0, 8, 7, - 6, 6, 6, 8, 7, 6, 5, 7, 8, 9, 3, 1, 1, 3, 1, 2, 8, 7, 6, 5, 7, 5, 4, 5, 9, 7, 5, 5, 1, 4, 5, 1, 5, 7, 5, 4, 5, 8, 5, 4, 6, 8, 8, 2, - 2, 8, 4, 9, 0, 9, 8, 5, 4, 6, 9, 8, 4, 4, 6, 8, 5, 8, 2, 5, 5, 4, 6, 1, 9, 8, 4, 9, 8, 5, 4, 6, 7, 1, 3, 1, 1, 3, 2, 9, 8, 5, 4, 6, - 9, 8, 7, 7, 8, 9, 9, 6, 0, 2, 8, 1, 5, 5, 9, 8, 7, 3, 6, 3, 0, 2, 8, 3, 4, 3, 6, 0, 3, 6, 3, 0, 2, 8, 8, 6, 8, 1, 0, 1, 9, 6, 3, 6, - 9, 6, 6, 9, 7, 1, 8, 6, 5, 6, 2, 0, 3, 4, 3, 9, 5, 3, 0, 9, 6, 5, 6, 2, 9, 8, 8, 7, 7, 9, 9, 7, 2, 0, 1, 8, 5, 5, 9, 8, 8, 9, 8, 9, - 8, 1, 4, 0, 0, 4, 8, 1, 4, 7, 9, 8, 9, 8, 8, 9, 9, 6, 4, 7, 7, 4, 0, 4, 7, 9, 1, 9, 6, 6, 8, 8, 9, 9, 4, 1, 8, 5, 0, 0, 4, 1, 9, 8, - 8, 9, 9, 4, 9, 7, 7, 8, 7, 7, 8, 5, 3, 0, 2, 3, 2, 0, 3, 9, 7, 7, 7, 9, 8, 7, 7, 8, 4, 3, 0, 3, 4, 3, 0, 2, 7, 7 - }; - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_PIPNET98_H diff --git a/lite/ncnn/cv/ncnn_plantid.cpp b/lite/ncnn/cv/ncnn_plantid.cpp deleted file mode 100644 index 56783e5b..00000000 --- a/lite/ncnn/cv/ncnn_plantid.cpp +++ /dev/null @@ -1,63 +0,0 @@ -// -// Created by DefTruth on 2022/3/27. -// - -#include "ncnn_plantid.h" -#include "lite/utils.h" - -using ncnncv::NCNNPlantID; - -NCNNPlantID::NCNNPlantID(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNPlantID::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - // will do deepcopy inside ncnn - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNPlantID::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input.1", input); - // 3. fetch. - ncnn::Mat logits_mat; - extractor.extract("477", logits_mat); // c=1,h=1,w=4066 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(logits_mat, "477"); -#endif - - const unsigned int num_classes = logits_mat.w; - const float *logits = (float *) logits_mat.data; - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} diff --git a/lite/ncnn/cv/ncnn_plantid.h b/lite/ncnn/cv/ncnn_plantid.h deleted file mode 100644 index d9d0de8b..00000000 --- a/lite/ncnn/cv/ncnn_plantid.h +++ /dev/null @@ -1,820 +0,0 @@ -// -// Created by DefTruth on 2022/3/27. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_PLANTID_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_PLANTID_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNPlantID : public BasicNCNNHandler - { - public: - explicit NCNNPlantID(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNPlantID() override = default; - - private: - const int input_height = 224; - const int input_width = 224; - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[4066] = { - "Saururus chinensis", "Houttuynia cordata", "Aucuba chinensis", "Aucuba japonica var. variegata", "Aucuba obcordata", - "Blechnum novae-zelandiae", "Woodwardia fimbriata", "Woodwardia prolifera", "Pentaphylax euryoides", "Ternstroemia gymnanthera", - "Eurya chinensis", "Eurya distichophylla", "Eurya emarginata", "Eurya japonica", "Eurya macartneyi", "Eurya muricata", - "Eurya rubiginosa var. attenuata", "Eurya saxicola", "Cleyera japonica", "Anneslea fragrans", "Eleutherococcus nodiflorus", - "Eleutherococcus senticosus", "Eleutherococcus trifoliatus", "Panax ginseng", "Fatsia japonica", "Kalopanax septemlobus", - "Trevesia palmata", "Schefflera arboricola", "Schefflera elegantissima", "Schefflera heptaphylla", "Schefflera macrostachya", - "Hydrocotyle sibthorpioides", "Hydrocotyle sibthorpioides var. batrachium", "Hydrocotyle verticillata", "Hydrocotyle wilfordii", - "Hedera helix", "Hedera nepalensis var. sinensis", "Metapanax davidii", "Aralia cordata", "Aralia elata", "Aralia nudicaulis", - "Schisandra chinensis", "Schisandra sphenanthera", "Illicium angustisepalum", "Illicium dunnianum", "Illicium lanceolatum", - "Illicium verum", "Kadsura coccinea", "Kadsura heteroclita", "Kadsura longipedunculata", "Dillenia turbinata", - "Tetracera sarmentosa", "Adoxa moschatellina", "Sambucus adnata", "Sambucus javanica", "Sambucus nigra", "Sambucus nigra caerulea", - "Sambucus racemosa", "Sambucus williamsii", "Viburnum acerifolium", "Viburnum betulifolium", "Viburnum chinshanense", - "Viburnum dilatatum", "Viburnum foetidum var. rectangulatum", "Viburnum fordiae", "Viburnum lantanoides", "Viburnum macrocephalum", - "Viburnum macrocephalum f. keteleeri", "Viburnum melanocarpum", "Viburnum odoratissimum", "Viburnum odoratissimum var. awabuki", - "Viburnum opulus", "Viburnum opulus subsp. calvescens", "Viburnum plicatum", "Viburnum plicatum f. tomentosum", - "Viburnum prunifolium", "Viburnum setigerum", "Viburnum tinus", "Linum usitatissimum&perenne", "Reinwardtia indica", - "Lophophora williamsii", "Schlumbergera truncata", "Opuntia basilaris", "Opuntia ficus-indica", "Opuntia humifusa", - "Opuntia littoralis", "Opuntia microdasys", "Echinopsis chamaecereus", "Nopalxochia ackermannii", "Cylindropuntia imbricata", - "Cylindropuntia leptocaulis", "Ferocactus peninsulae", "Epiphyllum oxypetalum", "Astrophytum myriostigma", "Pereskia bleo", - "Cleistocactus colademononis", "Hylocereus undatus", "Echinocactus grusonii", "Aporocactus flagelliformis", "Curculigo capitulata", - "Hypoxis hirsuta", "Hypoxis juncea", "Pauridia capensis", "Eryngium leavenworthii", "Eryngium planum", "Eryngium yuccifolium", - "Sanicula lamelligera", "Sanicula orthacantha", "Angelica dahurica", "Angelica decursiva", "Angelica polymorpha", - "Changium smyrnioides", "Astrantia major", "Bupleurum smithii", "Pastinaca sativa", "Conium maculatum", "Oenanthe javanica", - "Heracleum maximum", "Glehnia littoralis", "Centella asiatica", "Torilis arvensis", "Torilis scabra", "Daucus carota", - "Daucus carota var. sativa", "Coriandrum sativum", "Apium graveolens", "Foeniculum vulgare", "Cnidium monnieri", "Zizia aurea", - "Quisqualis indica", "Terminalia arjuna", "Terminalia catappa", "Terminalia nigrovenulosa", "Combretum alfredii", - "Combretum constrictum", "", "", "Alstroemeria hybrida", "Isotria verticillata", "Sacoila lanceolata", "Limodorum abortivum", - "Anacamptis coriophora", "Anacamptis laxiflora", "Anacamptis morio", "Anacamptis palustris", "Anacamptis papilionacea", - "Anacamptis pyramidalis", "Eriochilus cucullatus", "Paphiopedilum", "Paphiopedilum emersonii", "Paphiopedilum hirsutissimum", - "Paphiopedilum purpuratum", "Neottianthe cucullata", "Cymbidium ensifolium", "Cymbidium faberi", "Cymbidium floribundum", - "Cymbidium goeringii", "Cymbidium kanran", "Cymbidium lancifolium", "Cymbidium serratum", "Cymbidium sinense", "Cattleya hybrida", - "Epigeneium fargesii", "Malaxis monophyllos", "Malaxis unifolia", "Cheirostylis yunnanensis", "Dipodium roseum", - "Chiloglottis valida", "Encyclia tampensis", "Polystachya concreta", "Cephalanthera damasonium", "Cephalanthera falcata", - "Cephalanthera longifolia", "Cephalanthera rubra", "Cryptochilus roseus", "Robiquetia succisa", "Oberonioides microtatantha", - "Ponerorchis brevicalcarata", "Dracula simia", "Oreorchis nana", "Galeola lindleyana", "Calypso bulbosa var. speciosa", - "Tainia dunnii", "Tainia hongkongensis", "Anoectochilus roxburghii", "Gymnadenia nigra", "Gymnadenia odoratissima", - "Gymnadenia rhellicani", "Bletia purpurea", "Aerides rosea", "Dactylorhiza fuchsii", "Dactylorhiza majalis", - "Dactylorhiza traunsteineri", "Dactylorhiza viridis", "Oncidium", "Goodyera foliosa", "Goodyera oblongifolia", "Goodyera procera", - "Goodyera pubescens", "Goodyera repens", "Goodyera schlechtendaliana", "Goodyera tesselata", "Goodyera viridiflora", - "Neotinea maculata", "Neotinea tridentata", "Amitostigma monanthum", "Amitostigma pinguicula", "Dienia ophrydis", - "Cypripedium acaule", "Cypripedium bardolphianum", "Cypripedium calceolus", "Cypripedium calcicola", "Cypripedium candidum", - "Cypripedium flavum", "Cypripedium franchetii", "Cypripedium guttatum", "Cypripedium henryi", "Cypripedium japonicum", - "Cypripedium lichiangense", "Cypripedium macranthos", "Cypripedium montanum", "Cypripedium parviflorum", - "Cypripedium plectrochilum", "Cypripedium reginae", "Cypripedium shanxiense", "Cypripedium tibeticum", "Cypripedium wardii", - "Cypripedium yunnanense", "Cypripedium × ventricosum", "Cremastra appendiculata", "Thelymitra antennifera", "Thelymitra longifolia", - "Epidendrum radicans", "Eria corneri", "Calopogon tuberosus", "Epipactis atrorubens", "Epipactis gigantea", "Epipactis helleborine", - "Epipactis mairei", "Epipactis microphylla", "Epipactis palustris", "Renanthera coccinea", "Appendicula cornuta", - "Pleione bulbocodioides", "Pleione formosana", "Habenaria ciliolaris", "Habenaria dentata", "Habenaria floribunda", - "Habenaria glaucifolia", "Habenaria leptoloba", "Habenaria limprichtii", "Habenaria monorrhiza", "Habenaria petelotii", - "Habenaria repens", "Habenaria rhodocheila", "Habenaria schindleri", "Corallorhiza maculata", "Corallorhiza mertensiana", - "Corallorhiza striata", "Corallorhiza trifida", "Corallorhiza wisteriana", "Bletilla ochracea", "Bletilla striata", - "Pseudorchis albida", "Pseudorchis straminea", "Thrixspermum centipeda", "Pecteilis susannae", "Gastrochilus calceolaris", - "Galearis rotundifolia", "Chamorchis alpina", "Pholidota articulata", "Pholidota cantonensis", "Pholidota chinensis", - "Dendrobium chrysotoxum", "Dendrobium crepidatum", "Dendrobium cucullatum", "Dendrobium densiflorum", "Dendrobium hancockii", - "Dendrobium henryi", "Dendrobium hercoglossum", "Dendrobium loddigesii", "Dendrobium moniliforme", "Dendrobium moschatum", - "Dendrobium officinale", "Dendrobium sinominutiflorum", "Dendrobium thyrsiflorum", "Bulbophyllum ambrosia", - "Bulbophyllum kwangtungense", "Bulbophyllum levinei", "Bulbophyllum odoratissimum", "Bulbophyllum orientale", - "Bulbophyllum pecten-veneris", "Bulbophyllum retusiusculum", "Prosthechea cochleata", "Arundina graminifolia", - "Orchis anthropophora", "Orchis italica", "Orchis mascula", "Orchis militaris", "Orchis pallens", "Orchis provincialis", - "Orchis simia", "Zeuxine parvifolia", "Zeuxine strateumatica", "Dendrolirium lasiopetalum", "Spiranthes cernua", - "Spiranthes lacera", "Spiranthes lucida", "Spiranthes magnicamporum", "Spiranthes praecox", "Spiranthes sinensis", - "Spiranthes spiralis", "Spiranthes tuberosa", "Spiranthes vernalis", "Liparis bootanensis", "Liparis nervosa", - "Liparis stricklandiana", "Liparis viridiflora", "Eulophia alta", "Eulophia cucullata", "Eulophia graminea", "Eulophia zollingeri", - "Arethusa bulbosa", "Pterostylis banksii", "Pterostylis nana", "Pterostylis nutans", "Acampe rigida", "Platanthera aquilonis", - "Platanthera blephariglottis", "Platanthera clavellata", "Platanthera dilatata", "Platanthera elegans", "Platanthera flava", - "Platanthera grandiflora", "Platanthera huronensis", "Platanthera hyperborea", "Platanthera lacera", "Platanthera minor", - "Platanthera obtusata", "Platanthera orbiculata", "Platanthera psycodes", "Platanthera sparsiflora", "Platanthera stricta", - "Platanthera ussuriensis", "Hemipilia flabellata", "Spathoglottis plicata", "Spathoglottis pubescens", "Disa bracteata", - "Microtis unifolia", "Traunsteinera globosa", "Ponthieva racemosa", "Epipogium aphyllum", "Epipogium roseum", "Calanthe brevicornu", - "Calanthe clavata", "Calanthe graciliflora", "Calanthe sylvatica", "Calanthe tricarinata", "Calanthe triplicata", - "Diploprora championii", "Conchidium pusillum", "Ophrys apifera", "Ophrys bertolonii", "Ophrys bombyliflora", "Ophrys fuciflora", - "Ophrys fusca", "Ophrys insectifera", "Ophrys lutea", "Ophrys scolopax", "Ophrys speculum", "Ophrys sphegodes", - "Ophrys tenthredinifera", "Arachnis labrosa", "Phalaenopsis aphrodite", "Ludisia discolor", "Caladenia caerulea", - "Caladenia carnea", "Caladenia flava", "Caladenia fuscata", "Caladenia major", "Caladenia tentaculata", "Herminium monorchis", - "Ansellia africana", "Coelogyne corymbosa", "Coelogyne fimbriata", "Acianthus exsertus", "Erythrodes blumei", "Corybas taliensis", - "Serapias cordigera", "Serapias lingua", "Serapias vomeracea", "Cleisostoma paniculatum", "Cleisostoma rostratum", - "Cleisostoma simondii var. guangdongense", "Neofinetia falcata", "Caleana major", "Neottia banksiana", "Neottia convallarioides", - "Neottia nidus-avis", "Neottia ovata", "Satyrium yunnanense", "Phaius", "Phaius flavus", "Phaius tancarvilleae", - "Cephalantheropsis obcordata", "Ilex aculeolata", "Ilex asprella", "Ilex centrochinensis", "Ilex cornuta", - "Ilex cornuta 'National'", "Ilex decidua", "Ilex latifolia", "Ilex macrocarpa", "Ilex opaca", "Ilex pubescens", "Ilex rotunda", - "Ilex verticillata", "Ilex vomitoria", "Impatiens arguta", "Impatiens balsamina", "Impatiens blepharosepala", "Impatiens capensis", - "Impatiens chekiangensis", "Impatiens chinensis", "Impatiens commelinoides", "Impatiens hawkeri", "Impatiens hongkongensis", - "Impatiens macrovexilla", "Impatiens niamniamensis", "Impatiens noli-tangere", "Impatiens pallida", "Impatiens platychlaena", - "Impatiens platysepala", "Impatiens tubulosa", "Impatiens walleriana", "Pellaea andromedifolia", "Adiantum aleuticum", - "Adiantum capillus-veneris", "Adiantum nelumboides", "Adiantum pedatum", "Aechmea fulgens", "Ananas comosus", "Cryptanthus acaulis", - "Billbergia pyramidalis", "Tillandsia cyanea", "Tillandsia recurvata", "Tillandsia usneoides", "Rehmannia chingii", - "Rehmannia glutinosa", "Cymbaria mongolica", "Euphrasia pectinata", "Euphrasia regelii", "Melampyrum laxum", "Melampyrum roseum", - "Brandisia hancei", "Phtheirospermum japonicum", "Phtheirospermum tenuisectum", "Castilleja exserta", "Castilleja indivisa", - "Striga asiatica", "Cistanche deserticola", "Conopholis americana", "Boschniakia himalaica", "Aeginetia indica", - "Siphonostegia chinensis", "Siphonostegia laeta", "Pedicularis cheilanthifolia", "Pedicularis chinensis", "Pedicularis cranolopha", - "Pedicularis davidii", "Pedicularis densiflora", "Pedicularis densispica", "Pedicularis kansuensis", "Pedicularis muscicola", - "Pedicularis rhinanthoides subsp. labellata", "Monochasma sheareri", "Portulacaria afra", "Portulacaria afra 'Variegata'", - "Solms-laubachia pulcherrima", "Pegaeophyton scapiflorum", "Iberis amara", "Barbarea orthoceras", "Barbarea vulgaris", - "Descurainia sophia", "Cakile maritima", "Lepidium apetalum", "Lepidium latifolium", "Lepidium virginicum", "Cardamine californica", - "Cardamine concatenata", "Cardamine diphylla", "Cardamine hirsuta", "Cardamine impatiens", "Cardamine leucantha", - "Cardamine lyrata", "Cardamine purpurascens", "Erysimum amurense", "Erysimum capitatum", "Erysimum × cheiri", "Matthiola incana", - "Eruca vesicaria subsp. sativa", "Dontostemon dentatus", "Dontostemon glandulosus", "Dontostemon tibeticus", "Brassica juncea", - "Brassica juncea var. gemmifera", "Brassica juncea var. multicep", "Brassica oleracea", "Brassica oleracea var. acephala", - "Brassica oleracea var. botrytis", "Brassica oleracea var. capitata", "Brassica oleracea var. gemmifera", - "Brassica oleracea var. gongylodes", "Brassica oleracea var. italica", "Brassica rapa var. chinensis", "Brassica rapa var. glabra", - "Brassica rapa var. oleifera", "Capsella bursa-pastoris", "Thlaspi arvense", "Raphanus raphanistrum", "Raphanus sativus", - "Alliaria petiolata", "Rorippa globosa", "Rorippa indica", "Orychophragmus violaceus", "Nasturtium officinale", - "Yinshania fumarioides", "Hesperis matronalis", "Lobularia maritima", "Megacarpaea delavayi", "Duabanga grandiflora", - "Lythrum salicaria", "Lawsonia inermis", "Sonneratia apetala", "Sonneratia caseolaris", "Punica granatum", - "Punica granatum 'Albescens'", "Lagerstroemia fordii", "Lagerstroemia indica", "Lagerstroemia indica f. alba", - "Lagerstroemia limii", "Lagerstroemia speciosa", "Lagerstroemia subcostata", "Rotala rotundifolia", "Trapa natans", - "Cuphea hookeriana", "Cuphea hyssopifolia", "Woodfordia fruticosa", "Heimia myrtifolia", "Celastrus monospermus", - "Celastrus orbiculatus", "Euonymus alatus", "Euonymus carnosus", "Euonymus centidens", "Euonymus cornutus", "Euonymus fortunei", - "Euonymus japonicus", "Euonymus japonicus 'Aurea-marginatus'", "Euonymus laxiflorus", "Euonymus maackii", "Euonymus myrianthus", - "Euonymus nitidus", "Euonymus phellomanus", "Euonymus schensianus", "Euonymus semenovii", "Parnassia wightiana", - "Brexia madagascariensis", "Tripterygium wilfordii", "Selaginella uncinata", "Bretschneidera sinensis", "", "", - "Erythroxylum sinense", "Antidesma bunius", "Antidesma japonicum", "Phyllanthus acidus", "Phyllanthus chekiangensis", - "Phyllanthus emblica", "Phyllanthus flexuosus", "Phyllanthus glaucus", "Phyllanthus hainanensis", "Phyllanthus pulcher", - "Phyllanthus sootepensis", "Phyllanthus urinaria", "Phyllanthus ussuriensis", "Actephila collinsiae", "Baccaurea ramiflora", - "Flueggea suffruticosa", "Bischofia polycarpa", "Glochidion eriocarpum", "Glochidion puberum", "Glochidion wrightii", - "Glochidion zeylanicum", "Aporosa dioica", "Cleistanthus sumatranus", "Breynia disticha", "Breynia fruticosa", "Rotheca myricoides", - "Petraeovitex wolfei", "Paraphlomis javanica", "Paraphlomis javanica var. angustifolia", "Paraphlomis javanica var. coronata", - "Physostegia virginiana", "Holmskioldia sanguinea", "Mesona chinensis", "Perovskia abrotanoides", "Pogostemon auricularius", - "Hanceola exserta", "Lycopus lucidus", "Lycopus lucidus var. hirtus", "Prunella hispida", "Prunella vulgaris", "Lagopsis supina", - "Clerodendrum bungei", "Clerodendrum canescens", "Clerodendrum chinense", "Clerodendrum chinense var. simplex", - "Clerodendrum cyrtophyllum", "Clerodendrum fortunatum", "Clerodendrum inerme", "Clerodendrum japonicum", "Clerodendrum lindleyi", - "Clerodendrum paniculatum", "Clerodendrum quadriloculare", "Clerodendrum serratum", "Clerodendrum speciosum", - "Clerodendrum splendens", "Clerodendrum thomsoniae", "Clerodendrum trichotomum", "Clerodendrum wallichii", "Galeobdolon chinense", - "Anisomeles indica", "Tectona grandis", "Phlomis fruticosa", "Phlomis mongolica", "Marrubium vulgare", "Stachys byzantina", - "Stachys geobombycis", "Stachys japonica", "Stachys oblongifolia", "Glechoma hederacea", "Glechoma longituba", - "Colquhounia seguinii", "Origanum vulgare", "Vitex agnus-castus", "Vitex negundo", "Vitex negundo var. cannabifolia", - "Vitex negundo var. heterophylla", "Vitex rotundifolia", "Vitex trifolia", "Lamiophlomis rotata", "Leonotis leonurus", - "Leonotis nepetifolia", "Leonurus japonicus", "Leonurus sibiricus", "Gmelina asiatica", "Gmelina hainanensis", - "Gmelina philippensis", "Mosla dianthera", "Mosla scabra", "Mosla soochowensis", "Karomia speciosa", "Ajuga ciliata", - "Ajuga decumbens", "Ajuga lupulina", "Ajuga reptans", "Callicarpa americana", "Callicarpa bodinieri&dichotoma", - "Callicarpa cathayana", "Callicarpa formosana", "Callicarpa giraldii", "Callicarpa rubella", "Perilla frutescens", - "Eriophyton wallichii", "Ocimum basilicum", "Monarda citriodora", "Monarda didyma", "Monarda fistulosa", "Monarda punctata", - "Clerodendranthus spicatus", "Nepeta cataria", "Nepeta × faassenii 'Six Hills Giant'", "Caryopteris incana", - "Caryopteris nepetifolia", "Caryopteris × clandonensis", "Mentha canadensis", "Lavandula dentata", "Lavandula stoechas", - "Agastache rugosa", "Premna microphylla", "Moluccella laevis", "Rosmarinus officinalis", "Lamium amplexicaule", "Lamium barbatum", - "Lamium purpureum", "Gomphostemma chinense", "Gomphostemma lucidum", "Dracocephalum heterophyllum", - "Coleus hybridu&scutellarioides", "Clinopodium chinense", "Clinopodium confine", "Clinopodium megalanthum", "Teucrium canadense", - "Teucrium fruticans", "Teucrium viscidum", "Keiskea elsholtzioides", "Isodon adenanthus", "Isodon amethystoides", - "Isodon lophanthoides", "Isodon sculponeatus", "Isodon serra", "Elsholtzia argyi", "Elsholtzia ciliata", "Elsholtzia fruticosa", - "Elsholtzia stauntonii", "Plectranthus ecklonii", "Plectranthus glabratus", "Plectranthus hadiensis var. tomentosus", - "Plectranthus prostratus", "Scutellaria baicalensis", "Scutellaria barbata", "Scutellaria indica", "Scutellaria viscidula", - "Scutellaria wongkei", "Salvia", "Salvia apiana", "Salvia bowleyana", "Salvia chinensis", "Salvia coccinea", "Salvia columbariae", - "Salvia farinacea", "Salvia greggii", "Salvia guaranitica 'Black and Blue'", "Salvia leucantha", "Salvia liguliloba", - "Salvia lyrata", "Salvia mellifera", "Salvia miltiorrhiza", "Salvia nemorosa", "Salvia plebeia", "Salvia pratensis", - "Salvia splendens", "Salvia uliginosa", "Meehania fargesii", "Meehania montis-koyae", "Phytolacca acinosa", "Phytolacca americana", - "Talinum paniculatum", "Marchantia polymorpha", "Rinorea bengalensis", "Viola acuminata", "Viola arcuata", "Viola betonicifolia", - "Viola cornuta", "Viola delavayi", "Viola diffusa", "Viola fargesii", "Viola grypoceras", "Viola inconspicua", "Viola japonica", - "Viola mongolica", "Viola philippica", "Viola sororia", "Viola stewardiana", "Viola tricolor", "Melicytus ramiflorus", - "Notholithocarpus densiflorus", "Lithocarpus corneus", "Lithocarpus glaber", "Lithocarpus hancei", "Quercus acutissima", - "Quercus agrifolia", "Quercus alba", "Quercus aliena", "Quercus kelloggii", "Quercus lobata", "Quercus macrocarpa", - "Quercus palustris", "Quercus phellos", "Quercus robur", "Quercus rubra", "Quercus stellata", "Quercus variabilis", - "Castanea dentata", "Castanea mollissima", "Castanea seguinii", "Fagus grandifolia", "Castanopsis fargesii", "Castanopsis fissa", - "Castanopsis lamontii", "Cyclobalanopsis fleuryi", "Trigonostemon chinensis", "Trigonostemon flavidus", "Triadica cochinchinensis", - "Triadica sebifera", "Codiaeum variegatum", "Codiaeum variegatum 'Excellent'", "Hura crepitans", "Euphorbia antiquorum", - "Euphorbia bicolor", "Euphorbia characias", "Euphorbia cotinifolia", "Euphorbia cyathophora", "Euphorbia dentata", - "Euphorbia helioscopia&esula", "Euphorbia humifusa", "Euphorbia hypericifolia", "Euphorbia kansuensis", "Euphorbia lathyris", - "Euphorbia leucocephala", "Euphorbia maculata", "Euphorbia marginata", "Euphorbia milii", "Euphorbia milii var. alba", - "Euphorbia neorubella", "Euphorbia obesa", "Euphorbia prostrata", "Euphorbia pulcherrima", "Euphorbia resinifera", - "Euphorbia tirucalli", "Euphorbia viguieri", "Sauropus androgynus", "Strophioblachia fimbricalyx", "Alchornea davidii", - "Alchornea trewioides", "Croton capitatus", "Croton setiger", "Croton tiglium", "Plukenetia volubilis", "Manihot esculenta", - "Garcia nutans", "Vernicia fordii&montana", "Excoecaria acerifolia", "Excoecaria agallocha", "Excoecaria cochinchinensis", - "Aleurites moluccana", "Pedilanthus tithymaloides", "Cnidoscolus texanus", "Ricinus communis", "Macaranga tanarius var. tomentosa", - "Mallotus apelta", "Mallotus barbatus", "Mallotus japonicus", "Mallotus paniculatus", "Mallotus philippensis", "Mallotus repandus", - "Mallotus repandus var. chrysocarpus", "Mallotus tenuifolius", "Acalypha australis", "Acalypha hispida", "Acalypha reptans", - "Jatropha curcas", "Jatropha integerrima", "Jatropha podagrica", "Cannabis sativa", "Trema cannabina var. dielsiana", - "Celtis biondii", "Celtis sinensis", "Humulus lupulus", "Humulus scandens", "Pteroceltis tatarinowii", "Caladium bicolor", - "Pinellia cordata", "Pinellia pedatisecta", "Pinellia ternata", "Syngonium podophyllum", "Philodendron erubescens", - "Philodendron selloum", "Pistia stratiotes", "Arisaema bockii", "Arisaema erubescens", "Arisaema heterophyllum", - "Arisaema hunanense", "Arisaema silvestrii", "Arisaema triphyllum", "Aglaonema modestum", "Lysichiton americanus", "Lemna minor", - "Alocasia 'Amazonica'", "Alocasia odora", "Typhonium blumei", "Spathiphyllum kochii", "Symplocarpus foetidus", - "Colocasia antiquorum", "Colocasia esculenta", "Anthurium andraeanum", "Zamioculcas zamiifolia", "Zantedeschia", - "Amorphophallus dunnii", "Amorphophallus kiusianus", "Amorphophallus konjac", "Amorphophallus paeoniifolius", "Epipremnum aureum", - "Dieffenbachia seguine", "Monstera deliciosa", "Yucca gloriosa", "Paradisea liliastrum", "Ruscus aculeatus", "Eucomis comosa", - "Chlorophytum comosum", "Albuca namaquensis", "Hesperocallis undulata", "Asparagus cochinchinensis", "Asparagus densiflorus", - "Asparagus officinalis", "Asparagus setaceus", "Liriope muscari", "Liriope spicata", "Campylandra delavayi", "Thysanotus chinensis", - "Triteleia laxa", "Ornithogalum caudatum", "Ornithogalum divergens", "Ornithogalum dubium", "Ornithogalum narbonense", - "Ornithogalum umbellatum", "Cordyline australis", "Cordyline fruticosa", "Ledebouria socialis", "Ophiopogon bodinieri", - "Ophiopogon chingii", "Ophiopogon japonicus", "Hosta albomarginata", "Hosta plantaginea", "Hosta ventricosa", "Speirantha gardenii", - "Chlorogalum pomeridianum", "Disporopsis aspersa", "Disporopsis fuscopicta", "Disporopsis longifolia", "Disporopsis pernyi", - "Dichopogon strictus", "Camassia leichtlinii", "Camassia quamash", "Camassia scilloides", "Lachenalia viridiflora", - "Barnardia japonica", "Maianthemum bifolium", "Maianthemum canadense", "Maianthemum henryi", "Maianthemum japonicum", - "Maianthemum racemosum", "Maianthemum stellatum", "Muscari botryoides", "Dichelostemma capitatum", "Scilla bifolia", - "Scilla luciliae", "Scilla siberica", "Scilla verna", "Hyacinthoides hispanica", "Hyacinthoides non-scripta", - "Sansevieria gracilis", "Sansevieria trifasciata", "Sansevieria trifasciata var. laurentii", "Puschkinia scilloides", - "Aspidistra fimbriata", "Aspidistra grandiflora", "Aspidistra minutiflora", "Hesperoyucca whipplei", "Beaucarnea recurvata", - "Convallaria majalis", "Hyacinthus orientalis", "Polygonatum cyrtonema", "Polygonatum filipes", "Polygonatum hookeri", - "Polygonatum odoratum", "Polygonatum sibiricum", "Polygonatum verticillatum", "Agave americana", "Dracaena cambodiana", - "Dracaena draco", "Dracaena fragrans", "Dracaena reflexa", "Dracaena sanderiana", "Dracaena surculosa var. maculata", - "Wrightia laevis", "Wrightia pubescens", "Wrightia religiosa", "Carissa macrocarpa", "Pseudolithos migiurtinus", - "Gymnema sylvestre", "Dregea sinensis", "Dregea volubilis", "Dregea yunnanensis", "Ceropegia trichantha", "Ceropegia woodii", - "Parsonsia alboflavescens", "Telosma cordata", "Graphistemma pictum", "Nerium oleander", "Nerium oleander 'Paihua'", - "Tylophora ovata", "Tylophora silvestris", "Melodinus suaveolens", "Tabernaemontana divaricata", "Periploca sepium", - "Cryptostegia grandiflora", "Pachypodium lamerei", "Urceola rosea", "Amsonia tabernaemontana", "Adenium obesum", "Cerbera manghas", - "Beaumontia brevituba", "Beaumontia grandiflora", "Calotropis gigantea", "Stapelia", "Hoya carnosa", "Hoya multiflora", - "Cryptolepis buchananii", "Dischidia chinensis", "Dischidia ruscifolia&nummularia", "Pentasachme caudatum", "Vallaris indecora", - "Trachelospermum axillare", "Trachelospermum jasminoides", "Trachelospermum jasminoides 'Flame'", "Apocynum androsaemifolium", - "Apocynum venetum", "Strophanthus divaricatus", "Strophanthus gratus", "Stephanotis floribunda", "Metaplexis japonica", - "Vinca major", "Vinca major 'Variegata'", "Vinca minor", "Kopsia arborea", "Kopsia fruticosa", "Heterostemma brownii", - "Gomphocarpus fruticosus", "Gomphocarpus physocarpus", "Catharanthus roseus", "Catharanthus roseus 'Albus'", "Mandevilla sanderi", - "Asclepias asperula", "Asclepias curassavica", "Asclepias curassavica 'Flaviflora'", "Asclepias fascicularis", - "Asclepias incarnata", "Asclepias oenotheroides", "Asclepias speciosa", "Asclepias syriaca", "Asclepias tuberosa", - "Asclepias verticillata", "Asclepias viridiflora", "Asclepias viridis", "Merrillanthus hainanensis", "Anodendron affine", - "Plumeria obtusa", "Plumeria pudica", "Plumeria rubra", "Plumeria rubra 'Acutifolia'", "Alstonia scholaris", - "Cynanchum acuminatifolium", "Cynanchum atratum", "Cynanchum auriculatum", "Cynanchum chekiangense", "Cynanchum chinense", - "Cynanchum corymbosum", "Cynanchum stauntonii", "Cynanchum thesioides", "Chonemorpha eriostylis", "Thevetia peruviana", - "Thevetia peruviana 'Aurantiaca'", "Allamanda blanchetii", "Allamanda schottii&cathartica", "Jasminanthes mucronata", - "Zingiber cochleariforme", "Zingiber mioga", "Zingiber officinale", "Zingiber striolatum", "Zingiber zerumbet", - "Hedychium coccineum", "Hedychium coronarium", "Hedychium flavescens", "Hedychium flavum", "Hedychium yunnanense", - "Curcuma alismatifolia", "Curcuma longa", "Curcuma phaeocaulis", "Curcuma wenyujin", "Alpinia hainanensis", "Alpinia japonica", - "Alpinia oblongifolia", "Alpinia officinarum", "Alpinia zerumbet", "Alpinia zerumbet 'Variegata'", "Kaempferia elegans", - "Kaempferia galanga", "Kaempferia rotunda", "Globba schomburgkii", "Etlingera elatior", "Amomum tsaoko", "Amomum villosum", - "Roscoea schneideriana", "Cheilocostus speciosus", "Styrax chinensis", "Styrax confusus", "Styrax faberi", "Styrax japonicus", - "Styrax odoratissimus", "Styrax suberifolius", "Huodendron biaristatum var. parviflorum", "Rehderodendron kwangtungense", - "Pterostyrax corymbosus", "Sinojackia xylocarpa", "Alniphyllum fortunei", "Halesia macgregorii", "Melliodendron xylocarpum", - "Myriophyllum aquaticum", "Myriophyllum verticillatum", "Podophyllum peltatum", "Mahonia bealei", "Mahonia fortunei", - "Mahonia oiwakensis", "Mahonia shenii", "Nandina domestica", "Berberis diaphana", "Berberis jamesiana", "Berberis julianae", - "Berberis lempergiana", "Berberis pruinosa", "Berberis thunbergii", "Berberis thunbergii 'Atropurpurea'", "Berberis trifoliolata", - "Berberis vulgaris", "Berberis wilsoniae", "Diphylleia grayi", "Sinopodophyllum hexandrum", "Epimedium brevicornu", - "Epimedium davidii", "Epimedium sagittatum", "Epimedium wushanense", "Gymnospermium kiangnanense", "Dysosma pleiantha", - "Dysosma versipellis", "Microdesmis caseariifolia", "Capparis acutifolia", "Capparis bodinieri", "Crateva formosensis", - "Crateva religiosa", "Crateva unilocularis", "Pouteria caimito", "Pouteria campechiana", "Synsepalum dulcificum", - "Madhuca pasquieri", "Chrysophyllum cainito", "Sinosideroxylon wightianum", "Manilkara zapota", "Mimusops elengi", - "Symplocos cochinchinensis", "Symplocos congesta", "Symplocos lancifolia", "Symplocos lucida", "Symplocos paniculata", - "Symplocos stellaris", "Symplocos sumuntia", "Alangium chinense", "Alangium kurzii", "Alangium platanifolium", - "Alangium salviifolium", "Cornus alba", "Cornus canadensis", "Cornus capitata", "Cornus controversa", "Cornus drummondii", - "Cornus florida", "Cornus hongkongensis", "Cornus hongkongensis subsp. elegans", "Cornus kousa subsp. chinensis", "Cornus mas", - "Cornus officinalis", "Cornus quinquenervis", "Cornus sanguinea", "Cornus sericea", "Polyspora axillaris", "Camellia amplexicaulis", - "Camellia azalea", "Camellia chekiangoleosa", "Camellia crapnelliana", "Camellia cuspidata", "Camellia grijsii", - "Camellia japonica", "Camellia oleifera", "Camellia petelotii", "Camellia pitardii", "Camellia salicifolia", "Camellia saluenensis", - "Camellia sasanqua", "Camellia sinensis", "Camellia sinensis var. assamica", "Camellia uraku", "Camellia yunnanensis", - "Schima superba", "Pyrenaria microcarpa", "Pyrenaria spectabilis", "Stewartia sinensis", "Helicia reticulata", "Protea cynaroides", - "Buckinghamia celsissima", "Macadamia integrifolia", "Leucospermum nutans", "Grevillea banksii", "Diapensia purpurea", - "Heptacodium miconioides", "Zabelia biflora", "Zabelia dielsii", "Acanthocalyx alba", "Linnaea borealis", "Dipsacus asper", - "Dipsacus fullonum", "Lonicera caerulea", "Lonicera chrysantha", "Lonicera elisae", "Lonicera ferdinandi", - "Lonicera fragrantissima", "Lonicera fragrantissima var. lancifolia", "Lonicera hispida", "Lonicera hispidula", - "Lonicera involucrata", "Lonicera japonica", "Lonicera japonica var. chinensis", "Lonicera korolkowi", "Lonicera maackii", - "Lonicera modesta", "Lonicera praeflorens", "Lonicera sempervirens", "Lonicera tangutica", "Lonicera tatarica", - "Lonicera tatarica 'Lutea'", "Lonicera trichosantha", "Symphoricarpos albus", "Symphoricarpos orbiculatus", "Kolkwitzia amabilis", - "Abelia chinensis", "Abelia macrotera", "Abelia uniflora", "Abelia × grandiflora", "Triosteum himalayanum", "Scabiosa atropurpurea", - "Scabiosa comosa", "Patrinia monandra", "Patrinia villosa", "Centranthus ruber", "Weigela coraeensis", "Weigela florida", - "Weigela florida 'Red Prince'", "Weigela florida 'Variegata'", "Weigela japonica var. sinica", "Leycesteria formosa", "Platanus", - "Platanus occidentalis", "Platanus racemosa", "Penthorum chinense", "Trientalis borealis", "Trientalis europaea", - "Trientalis latifolia", "Cyclamen persicum", "Stimpsonia chamaedryoides", "Primula acaulis", "Primula agleniana", - "Primula beesiana", "Primula bella", "Primula blinii", "Primula chionantha", "Primula cicutariifolia", "Primula denticulata", - "Primula denticulata subsp. sinodenticulata", "Primula dryadifolia subsp. jonardunii", "Primula hendersonii", - "Primula maximowiczii", "Primula nutans", "Primula obconica", "Primula palmata", "Primula pelargoniifolia", "Primula pinnatifida", - "Primula poissonii", "Primula polyneura", "Primula pseudodenticulata", "Primula pulverulenta", "Primula saxatilis", - "Primula secundiflora", "Primula sikkimensis", "Primula sinolisteri", "Primula sonchifolia", "Primula stenocalyx", - "Primula tangutica", "Primula valentiniana", "Primula vialii", "Maesa montana", "Maesa perlarius", "Androsace henryi", - "Androsace mariae", "Androsace rigida", "Androsace spinulifera", "Androsace umbellata", "Androsace wardii", - "Androsace yargongensis", "Omphalogramma vinciflorum", "Lysimachia alfredii", "Lysimachia barystachys", "Lysimachia candida", - "Lysimachia christiniae", "Lysimachia ciliata", "Lysimachia clethroides", "Lysimachia congestiflora", "Lysimachia fortunei", - "Lysimachia grammica", "Lysimachia hemsleyana", "Lysimachia heterogenea", "Lysimachia klattiana", "Lysimachia longipes", - "Lysimachia nanpingensis", "Lysimachia nummularia", "Lysimachia nummularia 'Aurea'", "Lysimachia paridiformis var. stenophylla", - "Lysimachia patungensis", "Lysimachia pseudohenryi", "Lysimachia pumila", "Lysimachia punctata", "Anagallis arvensis", - "Anagallis arvensis f. coerulea", "Ardisia crenata", "Ardisia crispa", "Ardisia elliptica", "Ardisia humilis", "Ardisia japonica", - "Ardisia mamillata", "Ardisia obtusa", "Ardisia villosa", "Aegiceras corniculatum", "Embelia parviflora", "Embelia ribes", - "Myrsine africana", "Myrsine seguinii", "Muntingia calabura", "Erycibe expansa", "Evolvulus alsinoides", "Evolvulus nuttallianus", - "Xenostegia tridentata", "Calystegia hederacea", "Calystegia sepium", "Calystegia soldanella", "Convolvulus arvensis", - "Convolvulus tragacanthoides", "Pharbitis limbata", "Operculina turpethum", "Quamoclit coccinea", "Quamoclit pennata", - "Quamoclit × sloteri", "Cuscuta japonica", "Ipomoea alba", "Ipomoea aquatica", "Ipomoea batatas", "Ipomoea biflora", - "Ipomoea cairica", "Ipomoea carnea subsp. fistulosa", "Ipomoea cordatotriloba", "Ipomoea indica", "Ipomoea lacunosa", - "Ipomoea nil&purpurea", "Ipomoea obscura", "Ipomoea pes-caprae", "Ipomoea triloba", "Dinetus racemosus", "Dichondra micrantha", - "Merremia dissecta", "Merremia hederacea", "Merremia sibirica", "Merremia tuberosa", "Merremia vitifolia", "Stachyurus chinensis", - "Stachyurus himalaicus", "Aesculus californica", "Aesculus chinensis", "Aesculus glabra", "Aesculus hippocastanum", - "Aesculus pavia", "Cardiospermum grandiflorum", "Cardiospermum halicacabum", "Blighia sapida", "Xanthoceras sorbifolium", - "Sapindus saponaria", "Koelreuteria bipinnata", "Koelreuteria paniculata", "Acer buergerianum", "Acer cordatum", "Acer davidii", - "Acer fabri", "Acer henryi", "Acer macrophyllum", "Acer negundo", "Acer palmatum", "Acer pensylvanicum", "Acer platanoides", - "Acer pubinerve", "Acer rubrum", "Acer saccharinum", "Acer saccharum", "Acer spicatum", "Acer tataricum subsp. ginnala", - "Acer tataricum subsp. theiferum", "Acer truncatum", "Arytera littoralis", "Delavaya toxocarpa", "Litchi chinensis", - "Dodonaea viscosa", "Nephelium lappaceum", "Dimocarpus longan", "Tropaeolum majus", "Kingdonia uniflora", "Kalanchoe beauverdii", - "Kalanchoe blossfeldiana", "Kalanchoe delagoensis", "Kalanchoe fedtschenkoi", "Kalanchoe marmorata", "Kalanchoe porphyrocalyx", - "Kalanchoe tomentosa", "Hylotelephium spectabile", "Hylotelephium tatarinowii", "× Pachyveria 'Powder Puff'", - "Adromischus cristatus var. clavifolius", "Greenovia", "Sedum acre 'Aurea'", "Sedum alfredii", "Sedum bulbiferum", - "Sedum drymarioides", "Sedum emarginatum", "Sedum lineare", "Sedum sarmentosum", "Sedum sediforme", "Sedum spathulifolium", - "Orostachys fimbriata", "Orostachys malacophylla", "Echeveria 'Neon Breakers'", "Echeveria 'Perle von Nurnberg'", - "Echeveria lilacina", "Echeveria pulidonis", "Echeveria pulvinata", "Echeveria runyonii 'Topsy Turvy'", "Rhodiola rosea", - "Rhodiola yunnanensis", "Aeonium 'Zwartkop'", "Bryophyllum pinnatum", "Phedimus aizoon", "Cotyledon tomentosa", - "Sempervivum arachnoideum subsp. tomentosum", "Crassula arborescens", "Crassula corymbulosa", "Crassula marnieriana", - "Crassula obliqua 'Gollum'", "Graptopetalum amethystinum", "Magnolia grandiflora", "Magnolia tripetala", "Houpoea officinalis", - "Michelia champaca", "Michelia chapensis", "Michelia crassipes", "Michelia figo", "Michelia foveolata", "Michelia guangdongensis", - "Michelia macclurei", "Michelia maudiae", "Michelia skinneriana", "Michelia × alba", "Oyama sieboldii", "Manglietia fordiana", - "Manglietia insignis", "Woonyoungia septentrionalis", "Yulania biondii", "Yulania denudata", "Yulania denudata 'Fei Huang'", - "Yulania liliiflora", "Yulania stellata", "Yulania zenii", "Yulania × soulangeana", "Talauma hodgsonii", "Lirianthe championii", - "Lirianthe coco", "Liriodendron chinense", "Liriodendron tulipifera", "Syringa", "Syringa meyeri", "Syringa oblata", - "Syringa pubescens", "Syringa reticulata subsp. amurensis", "Syringa reticulata subsp. pekinensis", - "Syringa reticulata subsp. pekinensis 'Jinyuan'", "Syringa vulgaris", "Syringa yunnanensis", "Syringa × persica", - "Ligustrum lucidum", "Ligustrum obtusifolium", "Ligustrum quihoui", "Ligustrum sinense", "Ligustrum × vicaryi", "Osmanthus armatus", - "Osmanthus fragrans", "Olea europaea", "Fraxinus chinensis", "Fraxinus pennsylvanica", "Fraxinus sieboldiana", - "Chionanthus retusus", "Jasminum elongatum", "Jasminum floridum", "Jasminum grandiflorum", "Jasminum humile", - "Jasminum lanceolaria", "Jasminum mesnyi", "Jasminum multiflorum", "Jasminum nervosum", "Jasminum nudiflorum", - "Jasminum odoratissimum", "Jasminum officinale", "Jasminum pentaneurum", "Jasminum polyanthum", "Jasminum sambac", - "Jasminum sinense", "Jasminum subhumile", "Forsythia suspensa", "Forsythia viridissima", "Fontanesia phillyreoides subsp. fortunei", - "", "Equisetum arvense", "Equisetum hyemale", "Equisetum ramosissimum", "Equisetum ramosissimum subsp. debile", "Akebia quinata", - "Akebia trifoliata", "Stauntonia chinensis", "Stauntonia obovatifoliola subsp. urophylla", "Eucommia ulmoides", - "Elaeocarpus apiculatus", "Elaeocarpus decipiens", "Elaeocarpus glabripetalus", "Elaeocarpus hainanensis", "Elaeocarpus serratus", - "Sloanea sinensis", "Monotropastrum humile", "Enkianthus campanulatus", "Enkianthus chinensis", "Enkianthus deflexus", - "Enkianthus quinqueflorus", "Enkianthus serrulatus", "Chimaphila maculata", "Kalmia latifolia", "Cassiope selaginoides", - "Diplarche multiflora", "Rhododendron argyrophyllum", "Rhododendron bachii", "Rhododendron campylogynum", "Rhododendron capitatum", - "Rhododendron championiae", "Rhododendron charitopes subsp. tsangpoense", "Rhododendron florulentum", "Rhododendron hongkongense", - "Rhododendron kwangtungense", "Rhododendron latoucheae", "Rhododendron lepidotum", "Rhododendron maculiferum subsp. anwheiense", - "Rhododendron micranthum", "Rhododendron molle", "Rhododendron mucronatum", "Rhododendron oreodoxa", "Rhododendron ovatum", - "Rhododendron rhuyuenense", "Rhododendron rivulare", "Rhododendron seniavinii", "Rhododendron simiarum", "Rhododendron simsii", - "Rhododendron stamineum", "Rhododendron yunnanense", "Rhododendron × pulchrum", "Pterospora andromedea", "Agapetes burmanica", - "Agapetes lacei", "Monotropa hypopitys", "Monotropa uniflora", "Moneses uniflora", "Lyonia ovalifolia var. hebecarpa", - "Gaultheria procumbens", "Gaultheria shallon", "Gaultheria trichophylla", "Arbutus menziesii", "Sarcodes sanguinea", - "Vaccinium bracteatum", "Vaccinium corymbosum", "Vaccinium macrocarpon", "Vaccinium mandarinorum", "Vaccinium ovatum", - "Vaccinium uliginosum", "Pieris formosa", "Pieris japonica", "Pyrola calliantha", "Homalium ceylanicum", "Homalium cochinchinense", - "Idesia polycarpa", "Populus alba", "Populus deltoides", "Populus simonii var. przewalskii", "Salix", "Salix integra", - "Salix integra 'Hakuro Nishiki'", "Salix wallichiana", "Itoa orientalis", "Casearia velutina", "Myrica rubra", "Picea abies", - "Picea likiangensis var. linzhiensis", "Abies balsamea", "Pinus densiflora", "Pinus massoniana", "Pinus palustris", - "Pinus parviflora", "Pinus ponderosa", "Pinus strobus", "Pinus sylvestris", "Pinus taeda", "Larix gmelinii", "Larix kaempferi", - "Pseudolarix amabilis", "Tsuga canadensis", "Pseudotsuga menziesii", "Platycladus orientalis", "Juniperus chinensis", - "Juniperus communis", "Juniperus virginiana", "Sequoia sempervirens", "Thuja occidentalis", "Taxodium distichum", - "Taxodium mucronatum", "Ludwigia adscendens", "Ludwigia octovalvis", "Ludwigia peploides subsp. stipulacea", "Ludwigia sedioides", - "Clarkia amoena", "Clarkia unguiculata", "Fuchsia hybrida", "Gaura lindheimeri", "Gaura parviflora", - "Oenothera biennis&glazioviana", "Oenothera drummondii", "Oenothera laciniata", "Oenothera macrocarpa", "Oenothera rosea", - "Oenothera speciosa", "Oenothera tetraptera", "Chamerion angustifolium", "Epilobium canum", "Epilobium hirsutum", - "Epilobium pyrricholophum", "Circaea cordata", "Tamarix chinensis", "Tamarix ramosissima", "Myricaria squamosa", "Diospyros armata", - "Diospyros cathayensis", "Diospyros japonica", "Diospyros kaki", "Diospyros lotus", "Diospyros nitida", "Diospyros rhombifolia", - "Diospyros vaccinioides", "Diospyros virginiana", "Corymbia ptychocarpa", "Plinia cauliflora", "Rhodomyrtus tomentosa", - "Eucalyptus cinerea", "Eugenia brasiliensis", "Eugenia uniflora", "Psidium guajava", "Melaleuca cajuputi subsp. cumingiana", - "Callistemon citrinus", "Callistemon rigidus", "Syzygium acuminatissimum", "Syzygium australe", "Syzygium cumini", - "Syzygium fluviatile", "Syzygium grijsii", "Syzygium jambos", "Syzygium malaccense", "Syzygium rehderianum", - "Syzygium samarangense", "Acca sellowiana", "Xanthostemon chrysanthus", "Chamelaucium uncinatum", "Myrtus communis", - "Leptospermum scoparium", "Dendrophthoe pentandra", "Scurrula parasitica", "Taxillus chinensis", "Broussonetia kaempferi", - "Broussonetia kaempferi var. australis", "Broussonetia kazinoki", "Broussonetia papyrifera", "Morus alba", "Morus australis", - "Ficus altissima", "Ficus auriculata", "Ficus carica", "Ficus deltoidea", "Ficus elastica", "Ficus erecta", - "Ficus gasparriniana var. laceratifolia", "Ficus hispida", "Ficus pandurata", "Ficus pumila", "Ficus racemosa", "Ficus religiosa", - "Ficus subpisocarpa", "Ficus vaccinioides", "Ficus virens", "Maclura cochinchinensis", "Maclura pomifera", "Maclura tricuspidata", - "Artocarpus communis", "Artocarpus heterophyllus", "Artocarpus hypargyreus", "Dorstenia elata", "Codonopsis lanceolata", - "Codonopsis subglobosa", "Lobelia cardinalis", "Lobelia chinensis", "Lobelia davidii", "Lobelia erinus", "Lobelia melliana", - "Lobelia nummularia", "Lobelia sessilifolia", "Lobelia siphilitica", "Lobelia zeylanica", "Triodanis perfoliata", - "Triodanis perfoliata subsp. biflora", "Platycodon grandiflorus", "Adenophora himalayana", "Adenophora petiolata subsp. hunanensis", - "Adenophora polyantha", "Adenophora potaninii", "Adenophora stricta", "Adenophora trachelioides", "Wahlenbergia marginata", - "Cyananthus formosus", "Cyananthus incanus", "Cyananthus macrocalyx", "Cyclocodon lancifolius", "Campanumoea javanica", - "Lithotoma axillaris", "Campanula", "Campanula glomerata subsp. speciosa", "Campanula punctata", "Campanula rotundifolia", - "Hippobroma longiflora", "Clethra barbinervis", "Clethra delavayi", "Alnus trabeculosa", "Corylus avellana", "Ostrya rehderiana", - "Washingtonia filifera", "Washingtonia robusta", "Chrysalidocarpus lutescens", "Trachycarpus fortunei", "Cocos nucifera", - "Areca catechu", "Phoenix sylvestris", "Wodyetia bifurcata", "Calamus thysanolepis", "Sabal minor", "Livistona chinensis", - "Salacca edulis", "Caryota maxima", "Aphanamixis polystachya", "Swietenia macrophylla", "Melia azedarach", "Aglaia odorata", - "Toona sinensis", "Heynea trijuga", "Chukrasia tabularis", "Ulmus americana", "Ulmus pumila", "Salvinia molesta", - "Azolla pinnata subsp. asiatica", "Umbellularia californica", "Lindera aggregata", "Lindera benzoin", "Lindera communis", - "Lindera megaphylla", "Laurus nobilis", "Litsea cubeba", "Litsea glutinosa", "Phoebe bournei", "Phoebe chekiangensis", - "Phoebe sheareri", "Cinnamomum burmannii", "Cinnamomum camphora", "Cinnamomum cassia", "Cinnamomum japonicum", - "Cinnamomum kotoense", "Sassafras albidum", "Sassafras tzumu", "Machilus grijsii", "Machilus leptophylla", "Machilus thunbergii", - "Machilus velutina", "Persea americana", "Canarium album", "Torenia concolor", "Torenia fournieri", "Torenia violacea", - "Lindernia anagallis", "Lindernia crustacea", "Lindernia ruellioides", "Aconitum barbatum var. puberulum", "Aconitum coreanum", - "Aconitum gymnandrum", "Aconitum hemsleyanum", "Aconitum kusnezoffii", "Aconitum tanguticum", "Dichocarpum dalzielii", "Adonis", - "Thalictrum acutifolium", "Thalictrum aquilegiifolium var. sibiricum", "Thalictrum delavayi", "Thalictrum fargesii", - "Thalictrum fortunei", "Thalictrum ichangense", "Thalictrum petaloideum", "Thalictrum thalictroides", "Semiaquilegia adoxoides", - "Paraquilegia microphylla", "Ficaria verna", "Ranunculus asiaticus", "Ranunculus cantoniensis", "Ranunculus muricatus", - "Ranunculus repens", "Ranunculus sceleratus", "Ranunculus ternatus", "Batrachium bungei", "Batrachium pekinense", - "Pulsatilla chinensis", "Actaea erythrocarpa", "Actaea pachypoda", "Actaea rubra", "Anemoclema glaucifolium", - "Delphinium anthriscifolium", "Delphinium anthriscifolium var. majus", "Delphinium anthriscifolium var. savatieri", - "Delphinium elatum", "Delphinium grandiflorum", "Aquilegia canadensis", "Aquilegia chrysantha", "Aquilegia ecalcarata", - "Aquilegia formosa", "Aquilegia oxysepala", "Aquilegia oxysepala var. oxysepala f. pallidiflora", "Aquilegia viridiflora", - "Aquilegia viridiflora var. atropurpurea", "Aquilegia vulgaris", "Aquilegia yabeana", "Anemonopsis macrophylla", - "Trollius chinensis", "Trollius yunnanensis", "Helleborus thibetanus", "Clematis 'Rooguchi'", "Clematis acerifolia", - "Clematis apiifolia", "Clematis apiifolia var. argentilucida", "Clematis armandii", "Clematis brevicaudata", "Clematis chinensis", - "Clematis chrysocoma", "Clematis courtoisii", "Clematis crassifolia", "Clematis finetiana", "Clematis florida", - "Clematis fruticosa", "Clematis fusca var. violacea", "Clematis henryi", "Clematis heracleifolia", "Clematis hexapetala", - "Clematis integrifolia", "Clematis lasiandra", "Clematis macropetala", "Clematis meyeniana", "Clematis montana", - "Clematis montana var. sterilis", "Clematis nannophylla", "Clematis peterae", "Clematis potaninii", "Clematis pseudootophora", - "Clematis pseudopogonandra", "Clematis ranunculoides", "Clematis rehderiana", "Clematis repens", "Clematis sibirica", - "Clematis sibirica var. ochotensis", "Clematis tangutica", "Clematis terniflora", "Clematis terniflora var. mandshurica", - "Clematis uncinata", "Clematis virginiana", "Anemone acutiloba", "Anemone americana", "Anemone coronaria", "Anemone demissa", - "Anemone flaccida", "Anemone geum subsp. ovalifolia", "Anemone hupehensis", "Anemone obtusiloba", "Anemone rivularis", - "Anemone rivularis var. flore-minore", "Anemone rupicola", "Anemone tomentosa&vitifolia", "Consolida ajacis", "Caltha palustris", - "Caltha sinogracilis", "Oxygraphis glacialis", "Souliea vaginata", "Nigella damascena", "Claytonia caroliniana", - "Claytonia perfoliata", "Claytonia virginica", "Lewisia cotyledon", "Burmannia disticha", "Burmannia itoana", - "Burmannia nepalensis", "Egeria densa", "Ottelia acuminata", "Ottelia acuminata var. crispa", "Ottelia alismoides", - "Hydrocharis dubia", "Polypodium virginianum", "Microsorum pustulatum", "Platycerium bifurcatum", "Platycerium wallichii", - "Aletris scopulorum", "Aletris spicata", "Paulownia", "Paulownia tomentosa", "Sagittaria latifolia", "Sagittaria montevidensis", - "Sagittaria pygmaea", "Sagittaria sagittifolia", "Sagittaria trifolia", "Hydrocleys nymphoides", "Alisma canaliculatum", - "Alisma plantago-aquatica", "Echinodorus grisebachii", "Limnocharis flava", "Pittosporum illicioides", "Pittosporum tobira", - "Lygodium japonicum", "Meliosma flexuosa", "Meliosma rigida", "Meliosma rigida var. pannosa", "Meliosma squamulata", - "Sabia campanulata subsp. ritchieae", "Sabia discolor", "Sabia japonica", "Sabia limoniacea", "Sabia swinhoei", "Malosma laurina", - "Choerospondias axillaris", "Mangifera indica", "Toxicodendron diversilobum", "Toxicodendron radicans", "Toxicodendron succedaneum", - "Rhus aromatica", "Rhus chinensis", "Rhus glabra", "Rhus integrifolia", "Rhus ovata", "Rhus typhina", "Anacardium occidentale", - "Cotinus coggygria", "Pistacia vera", "Juncus allioides", "Juncus effusus", "Juncus prismatocarpus", "Barleria cristata", - "Barleria lupulina", "Asystasia gangetica", "Asystasia gangetica subsp. micrantha", "Asystasia neesiana", - "Crossandra infundibuliformis", "Aphelandra sinclairiana", "Aphelandra squarrosa", "Eranthemum pulchellum", "Rungia densiflora", - "Pseuderanthemum carruthersii", "Pseuderanthemum crenulatum", "Pseuderanthemum laxiflorum", - "Pseuderanthemum reticulatum var. ovarifolium", "Thunbergia alata", "Thunbergia coccinea", "Thunbergia erecta", - "Thunbergia fragrans", "Thunbergia grandiflora", "Thunbergia laurifolia", "Thunbergia mysorensis", "Hygrophila ringens", - "Rhinacanthus nasutus", "Justicia adhatoda", "Justicia austrosinensis", "Justicia betonica", "Justicia brandegeeana", - "Justicia brasiliana", "Justicia procumbens", "Justicia quadrifaria", "Dicliptera chinensis", "Cyrtanthera carnea", - "Andrographis paniculata", "Fittonia albivenis", "Acanthus ilicifolius", "Acanthus mollis", "Perilepta dyeriana", "Ruellia elegans", - "Ruellia simplex", "Ruellia venusta", "Peristrophe hyssopifolia 'Aureo-variegata'", "Peristrophe japonica", - "Megaskepasma erythrochlamys", "Brillantaisia owariensis", "Pachystachys lutea", "Codonacanthus pauciflorus", - "Strobilanthes aprica", "Strobilanthes cusia", "Strobilanthes dimorphotricha", "Strobilanthes hamiltoniana", - "Strobilanthes sarcorrhiza", "Strobilanthes schomburgkii", "Strobilanthes tetrasperma", "Clinacanthus nutans", - "Cystacanthus pyramidalis", "Odontonema strictum", "Sanchezia speciosa", "Rourea microphylla", "Pelargonium graveolens", - "Pelargonium hortorum", "Pelargonium peltatum", "Pelargonium zonale", "Erodium cicutarium", "Erodium stephanianum", - "Geranium carolinianum", "Geranium maculatum", "Geranium nepalense", "Geranium pratense", "Geranium pylzowianum", - "Geranium refractum", "Geranium robertianum", "Geranium sibiricum", "Geranium sinense", "Geranium wilfordii", - "Geranium wlassovianum", "Pinguicula alpina", "Utricularia aurea", "Utricularia australis", "Utricularia bifida", - "Utricularia caerulea", "Utricularia striatula", "Utricularia warburgii", "Saurauia tristyla", "Actinidia arguta", - "Actinidia callosa var. discolor", "Actinidia chinensis", "Actinidia eriantha", "Actinidia lanceolata", "Actinidia latifolia", - "Actinidia macrosperma", "Actinidia rubricaulis var. coriacea", "Nepenthes mirabilis", "Diascia barberae", "Verbascum blattaria", - "Verbascum thapsus", "Scrophularia californica", "Scrophularia ningpoensis", "Leucophyllum frutescens", "Buddleja asiatica", - "Buddleja davidii", "Buddleja fallowiana", "Buddleja lindleyana", "Buddleja officinalis", "Nemesia strumosa", - "Couroupita guianensis", "Barringtonia acutangula", "Barringtonia asiatica", "Barringtonia racemosa", "Onoclea sensibilis", - "Matteuccia struthiopteris", "Aquilaria sinensis", "Stellera chamaejasme", "Daphne aurantiaca", "Daphne championii", - "Daphne genkwa", "Daphne giraldii", "Daphne kiusiana var. atrocaulis", "Daphne longilobata", "Daphne odora", "Daphne papyracea", - "Daphne tangutica", "Edgeworthia chrysantha", "Wikstroemia indica", "Wikstroemia monnula", "Wikstroemia nutans", - "Wikstroemia pilosa", "Sarracenia purpurea", "Eriodictyon californicum", "Hydrophyllum virginianum", "Philydrum lanuginosum", - "Carica papaya", "Mesembryanthemum cordifolium", "Mesembryanthemum crystallinum", "Lampranthus spectabilis", "Carpobrotus edulis", - "Lithops pseudotruncatella subsp. archerae", "Fenestraria aurantiaca", "Glottiphyllum longum", "Rhombophyllum nelii", - "Astridia velutina", "Cananga odorata", "Cananga odorata var. fruticosa", "Desmos chinensis", "Asimina triloba", "Polyalthia laui", - "Polyalthia longifolia", "Polyalthia suberosa", "Fissistigma oldhamii", "Fissistigma polyanthum", "Annona glabra", "Annona montana", - "Annona muricata", "Annona squamosa", "Uvaria boniana", "Uvaria grandiflora", "Uvaria macrophylla", "Uvaria tonkinensis", - "Chieniodendron hainanense", "Mitrephora tomentosa", "Artabotrys hainanensis", "Artabotrys hexapetalus", "Artabotrys hongkongensis", - "Peganum harmala", "Armeria maritima", "Plumbago auriculata", "Plumbago indica", "Plumbago zeylanica", "Limonium bicolor", - "Limonium sinense", "Limonium tenellum", "Peritoma arborea", "Tarenaya hassleriana", "Clintonia borealis", "Calochortus albus", - "Calochortus amabilis", "Calochortus leichtlinii", "Calochortus luteus", "Calochortus plummerae", "Calochortus pulchellus", - "Calochortus splendens", "Calochortus tolmiei", "Calochortus venustus", "Notholirion bulbuliferum", "Cardiocrinum cathayanum", - "Cardiocrinum giganteum", "Cardiocrinum giganteum var. yunnanense", "Medeola virginiana", "Streptopus simplex", - "Tricyrtis formosana", "Tricyrtis macropoda", "Tricyrtis pilosa", "Erythronium albidum", "Erythronium americanum", - "Erythronium grandiflorum", "Erythronium japonicum", "Erythronium oregonum", "Erythronium sibiricum", - "Lilium bakerianum var. rubrum", "Lilium brownii", "Lilium canadense", "Lilium columbianum", "Lilium concolor", - "Lilium concolor var. pulchellum", "Lilium dauricum", "Lilium davidii", "Lilium davidii var. unicolor", "Lilium distichum", - "Lilium duchartrei", "Lilium lankongense", "Lilium longiflorum", "Lilium lophophorum", "Lilium martagon", - "Lilium nanum var. flavidum", "Lilium pardalinum", "Lilium parvum", "Lilium philadelphicum", "Lilium primulinum var. ochraceum", - "Lilium pumilum", "Lilium regale", "Lilium rosthornii", "Lilium souliei", "Lilium speciosum var. gloriosoides", "Lilium taliense", - "Lilium tigrinum", "Amana edulis", "Nomocharis aperta", "Nomocharis pardanthina", "Fritillaria affinis", - "Fritillaria camschatcensis", "Fritillaria imperialis", "Fritillaria maximowiczii", "Fritillaria meleagris", "Fritillaria persica", - "Fritillaria thunbergii", "Fritillaria ussuriensis", "Tulipa gesneriana", "Tulipa iliensis", "Welwitschia mirabilis", - "Stemona japonica", "Stemona mairei", "Stemona tuberosa", "Turpinia arguta", "Euscaphis japonica", "Potamogeton crispus", - "Potamogeton distinctus", "Victoria amazonica", "Victoria cruziana", "Nymphaea", "Nymphaea alba", "Nymphaea nouchali", - "Nymphaea odorata", "Euryale ferox", "Nuphar pumila", "Menyanthes trifoliata", "Nymphoides coreana", "Nymphoides cristata", - "Nymphoides indica", "Nymphoides peltata", "Palhinhaea cernua", "Diphasiastrum digitatum", "Dendrolycopodium obscurum", - "Lycopodiastrum casuarinoides", "Lychnis chalcedonica", "Lychnis fulgens", "Lychnis senno", "Cerastium glomeratum", - "Arenaria smithiana", "Sagina japonica", "Gypsophila oldhamiana", "Gypsophila paniculata", "Dianthus armeria", "Dianthus barbatus", - "Dianthus caryophyllus", "Dianthus chinensis", "Dianthus superbus", "Stellaria alsine", "Stellaria chinensis", "Stellaria media", - "Saponaria officinalis", "Silene armeria", "Silene baccifera", "Silene conoidea", "Silene davidii", "Silene gallica", - "Silene latifolia", "Silene vulgaris", "Myosoton aquaticum", "Agrostemma githago", "Vaccaria hispanica", "Nothoscordum bivalve", - "Boophone disticha", "Eucharis amazonica", "Clivia miniata", "Clivia nobilis", "Clivia × hybrida", "Amaryllis belladonna", - "Crinum amabile", "Crinum asiaticum var. sinicum", "Crinum moorei", "Ipheion uniflorum", "Polianthes tuberosa", - "Cyrtanthus mackenii", "Hippeastrum reticulatum", "Hippeastrum rutilum", "Narcissus bulbocodium", "Narcissus poeticus", - "Narcissus pseudonarcissus", "Narcissus tazetta var. chinensis", "Narcissus triandrus", "Hymenocallis speciosa&littoralis", - "Agapanthus africanus", "Agapanthus praecox", "Lycoris aurea", "Lycoris chinensis", "Lycoris haywardii", "Lycoris incarnata", - "Lycoris longituba", "Lycoris radiata", "Lycoris sprengeri", "Lycoris squamigera", "Lycoris straminea", "Lycoris × rosea", - "Tulbaghia violacea", "Allium carolinianum", "Allium cepa", "Allium chinense", "Allium fistulosum", "Allium giganteum", - "Allium prattii", "Allium sativum", "Allium senescens", "Allium sikkimense", "Allium triquetrum", "Allium tuberosum", - "Allium wallichii", "Zephyranthes candida", "Zephyranthes carinata", "Zephyranthes citrina", "Haemanthus albiflos", - "Haemanthus multiflorus", "Galanthus elwesii", "Leucojum aestivum", "Leucojum vernum", "Eucrosia bicolor", "Histiopteris incisa", - "Pteridium aquilinum", "Lagurus ovatus", "Phyllostachys nigra", "Hordeum jubatum", "Bothriochloa ischaemum", - "Chasmanthium latifolium", "Triticum aestivum", "Poa annua", "Phaenosperma globosa", "Isachne globosa", "Polypogon monspeliensis", - "Oplismenus undulatifolius", "Avena fatua", "Setaria italica var. germanica", "Setaria palmifolia", "Setaria pumila", - "Setaria viridis", "Cynodon dactylon", "Pennisetum alopecuroides", "Pennisetum glaucum", "Pennisetum orientale", - "Pennisetum setaceum 'Rubrum'", "Zea mays", "Saccharum officinarum", "Imperata cylindrica", "Alopecurus aequalis", - "Echinochloa caudata", "Echinochloa crus-galli", "Oryza sativa", "Eleusine indica", "Bambusoideae", "Indocalamus latifolius", - "Bambusa ventricosa", "Miscanthus sinensis 'Gracillimus'", "Miscanthus sinensis 'Zebrinus'", "Arundo donax", "Phragmites australis", - "Microstegium vimineum", "Zizania latifolia", "Cortaderia selloana", "Coix lacryma-jobi", "Phalaris arundinacea", - "Paspalum dilatatum", "Sorghum bicolor", "Sorghum halepense", "Dactylis glomerata", "Panicum virgatum", "Lolium perenne", - "Disporum cantoniense", "Disporum longistylum", "Disporum megalanthum", "Disporum uniflorum", "Disporum viridescens", - "Gloriosa superba", "Sandersonia aurantiaca", "Colchicum autumnale", "Begonia boliviensis", "Begonia circumlobata", - "Begonia cucullata", "Begonia fimbristipula", "Begonia grandis subsp. sinensis", "Begonia leprosa", "Begonia maculata", - "Begonia masoniana", "Begonia palmata", "Begonia soli-mutata", "Begonia × hiemalis", "Ctenanthe setosa", "Thalia dealbata", - "Thalia geniculata", "Maranta leuconeura", "Maranta&Calathea", "Stromanthe sanguinea", "Calathea warscewiczii", "Calathea zebrina", - "Bougainvillea spectabilis&glabra", "Mirabilis jalapa", "Boerhavia diffusa", "Myosotis alpestris", "Ehretia acuminata", - "Ehretia longiflora", "Carmona microphylla", "Heliotropium arborescens", "Heliotropium curassavicum", "Heliotropium indicum", - "Microula sikkimensis", "Bothriospermum chinense", "Bothriospermum zeylanicum", "Onosma hookeri var. longiflorum", - "Mertensia virginica", "Borago officinalis", "Cynoglossum amabile", "Cynoglossum grande", "Cynoglossum lanceolatum", - "Thyrocarpus sampsonii", "Cordia dichotoma", "Cordia subcordata", "Nemophila maculata", "Nemophila menziesii", - "Tournefortia montana", "Tournefortia sibirica", "Stenosolenium saxatile", "Lithospermum incisum", "Lithospermum zollingeri", - "Symphytum officinale", "Echium vulgare", "Echium wildpretii", "Trigonotis peduncularis", "Osmundastrum cinnamomeum", - "Osmunda claytoniana", "Campsis grandiflora", "Campsis radicans", "Kigelia africana", "Catalpa bungei", "Catalpa fargesii", - "Catalpa ovata", "Catalpa speciosa", "Mayodendron igneum", "Spathodea campanulata", "Pyrostegia venusta", - "Markhamia stipulata var. kerrii", "Macfadyena unguis-cati", "Pandorea jasminoides", "Tabebuia impetiginosa", "Tabebuia rosea", - "Radermachera sinica&hainanensis", "Crescentia alata", "Mansoa alliacea", "Jacaranda mimosifolia", "Incarvillea arguta", - "Incarvillea mairei var. multifoliolata", "Incarvillea sinensis", "Clytostoma callistegioides", "Podranea ricasoliana", - "Handroanthus chrysanthus", "Tecoma capensis", "Tecoma stans", "Calophyllum inophyllum", "Calophyllum membranaceum", "Mesua ferrea", - "Bixa orellana", "Bruguiera gymnorhiza", "Kandelia obovata", "Cephalotaxus sinensis", "Torreya grandis 'Merrillii'", - "Taxus baccata", "Taxus wallichiana var. chinensis", "Philadelphus laxiflorus", "Philadelphus pekinensis", - "Philadelphus zhejiangensis", "Dichroa febrifuga", "Deutzia baroniana", "Deutzia crenata", "Deutzia glauca", - "Deutzia glomeruliflora", "Deutzia gracilis", "Deutzia longifolia", "Deutzia ningpoensis", "Deutzia scabra", - "Deutzia scabra var. plena", "Hydrangea", "Hydrangea chinensis", "Hydrangea lingii", "Hydrangea paniculata", - "Hydrangea quercifolia", "Hydrangea strigosa", "Platycrater arguta", "Macleaya cordata", "Chelidonium majus", - "Dicranostigma leptopodum", "Corydalis bungeana", "Corydalis caudata", "Corydalis curviflora", "Corydalis decumbens", - "Corydalis edulis", "Corydalis fangshanensis", "Corydalis flexuosa", "Corydalis hamata", "Corydalis hemidicentra", - "Corydalis incisa", "Corydalis linarioides", "Corydalis melanochlora", "Corydalis mucronata", "Corydalis pachycentra", - "Corydalis pallida", "Corydalis pseudobarbisepala", "Corydalis racemosa", "Corydalis repens", "Corydalis sheareri", - "Corydalis speciosa", "Corydalis turtschaninovii", "Corydalis yanhusuo", "Meconopsis", "Meconopsis balangensis", - "Meconopsis betonicifolia", "Meconopsis chelidoniifolia", "Meconopsis delavayi", "Meconopsis henrici", "Meconopsis horridula", - "Meconopsis impedita", "Meconopsis integrifolia", "Meconopsis lancifolia", "Meconopsis paniculata", "Meconopsis pseudointegrifolia", - "Meconopsis punicea", "Meconopsis quintuplinervia", "Meconopsis racemosa", "Meconopsis simplicifolia", "Meconopsis speciosa", - "Meconopsis sulphurea", "Meconopsis venusta", "Meconopsis wilsonii", "Papaver orientale", "Papaver radicatum var. pseudoradicatum", - "Papaver rhoeas", "Papaver somniferum", "Eschscholzia californica", "Lamprocapnos spectabilis", "Lamprocapnos spectabilis f. alba", - "Hylomecon japonica", "Argemone mexicana", "Sanguinaria canadensis", "Eomecon chionantha", "Dicentra cucullaria", - "Dicentra formosa", "Nageia nagi", "Podocarpus macrophyllus", "Canna", "Canna generalis", "Canna glauca", "Canna indica", - "Canna indica var. flava", "Canna orchioides", "Canna warscewiezii", "Astelia fragrans", "Nephrolepis cordifolia", - "Platycarya strobilacea", "Carya illinoinensis", "Pterocarya stenoptera", "Engelhardia roxburghiana", "Juglans mandshurica", - "Juglans nigra", "Juglans regia", "Cyclocarya paliurus", "Piper aduncum", "Piper hancei", "Piper kadsura", "Piper nigrum", - "Piper sarmentosum", "Peperomia argyreia", "Peperomia caperata", "Peperomia pellucida", "Peperomia polybotrya", - "Peperomia tetraphylla", "Hippophae rhamnoides", "Elaeagnus angustifolia", "Elaeagnus argyi", "Elaeagnus conferta", - "Elaeagnus glabra", "Elaeagnus lanceolata", "Elaeagnus mollis", "Elaeagnus multiflora", "Elaeagnus pungens", - "Elaeagnus Pungens 'Aurea'", "Elaeagnus umbellata", "Paeonia delavayi", "Paeonia lactiflora", "Paeonia obovata", - "Paeonia suffruticosa", "Sesamum indicum", "Uncarina grandidieri", "Musella lasiocarpa", "Musa nana", "Ensete glaucum", - "Stylidium uliginosum", "Cobaea scandens", "Phlox", "Phlox drummondii", "Phlox paniculata", "Phlox subulata", "Ipomopsis aggregata", - "Polemonium caeruleum", "Polemonium chinense", "Butomus umbellatus", "Murraya exotica", "Tetradium austrosinense", - "Tetradium glabrifolium", "Tetradium ruticarpum", "Glycosmis pentaphylla", "Acronychia pedunculata", "Citrus australasica", - "Citrus japonica", "Citrus maxima", "Citrus medica 'Fingered'", "Citrus reticulata", "Citrus reticulata", "Citrus sinensis", - "Citrus trifoliata", "Citrus × limon", "Ptelea trifoliata", "Dictamnus dasycarpus", "Boenninghausenia albiflora", - "Zanthoxylum ailanthoides", "Zanthoxylum bungeanum", "Zanthoxylum nitidum", "Zanthoxylum piperitum", "Zanthoxylum scandens", - "Zanthoxylum simulans", "Skimmia reevesiana", "Melicope pteleifolia", "Toddalia asiatica", "Clausena excavata", "Clausena lansium", - "Gomphrena globosa", "Kochia scoparia", "Cyathula prostrata", "Achyranthes bidentata", "Beta vulgaris", "Salsola tragus", - "Amaranthus caudatus", "Amaranthus hypochondriacus", "Amaranthus spinosus", "Amaranthus tricolor", "Alternanthera bettzickiana", - "Alternanthera philoxeroides", "Spinacia oleracea", "Chenopodium album", "Celosia argentea", "Celosia cristata", "Cycas revoluta", - "Ailanthus altissima", "Brucea javanica", "Hemiboea cavaleriei", "Hemiboea subcapitata", "Didymostigma obtusum", - "Titanotrichum oldhamii", "Lysionotus pauciflorus", "Lysionotus serratus", "Chirita eburnea", "Chirita fimbrisepala", - "Chirita lutea", "Chirita pinnatifida", "Chirita pumila", "Episcia cupreata", "Gyrocheilos chorisepalus", "Sinningia leucotricha", - "Sinningia speciosa", "Gloxinia sylvatica", "Primulina xiziae", "Streptocarpus hybrids", "Streptocarpus saxorum", - "Briggsia chienii", "Rhynchotechum ellipticum", "Didissandra sesquifolia", "Aeschynanthus acuminatus", "Aeschynanthus buxifolius", - "Aeschynanthus sp", "Aeschynanthus speciosus", "Aeschynanthus superbus", "Paraboea sinensis", "Nematanthus wettsteinii", - "Saintpaulia ionantha", "Oreocharis auricula", "Oreocharis benthamii var. reticulata", "Oreocharis maximowiczii", - "Nicandra physalodes", "Cestrum aurantiacum", "Cestrum nocturnum", "Hyoscyamus niger", "Anisodus tanguticus", "Datura inoxia", - "Datura stramonium", "Datura wrightii", "Brugmansia arborea", "Brugmansia aurea", "Brugmansia suaveolens", "Lycium chinense", - "Cyphomandra betacea", "Juanulloa aurantiaca", "Nicotiana alata", "Nicotiana glauca", "Nicotiana tabacum", - "Lycopersicon esculentum", "Petunia × hybrida", "Lycianthes biflora", "Calibrachoa hybrids", "Mandragora caulescens", - "Solanum aculeatissimum", "Solanum capsicoides", "Solanum carolinense", "Solanum dulcamara", "Solanum elaeagnifolium", - "Solanum erianthum", "Solanum jasminoides", "Solanum laciniatum", "Solanum lyratum", "Solanum mammosum", "Solanum melongena", - "Solanum muricatum", "Solanum nigrum&americanum", "Solanum pseudocapsicum", "Solanum pseudocapsicum var. diflorum", - "Solanum rantonnetii", "Solanum rostratum", "Solanum septemlobum", "Solanum texanum", "Solanum torvum", "Solanum tuberosum", - "Solanum virginianum", "Solanum wrightii", "Schizanthus pinnatus", "Capsicum annuum", "Capsicum annuum subsp. cerasiforme", - "Capsicum annuum var. conoides", "Physalis", "Physalis minima", "Physalis philadelphica", "Solandra longiflora", "Solandra maxima", - "Brunfelsia brasiliensis", "Brunfelsia calycina", "Dionaea muscipula", "Drosera burmanni", "Drosera peltata", - "Drosera rotundifolia", "Drosera spatulata", "Psychotria serpens", "Pentas lanceolata", "Coffea", "Pavetta hongkongensis", - "Bouvardia ternifolia", "Morinda citrifolia", "Morinda parvifolia", "Galium aparine", "Galium spurium", "Galium verum", - "Gardenia jasminoides", "Gardenia scabrella", "Adina pilulifera", "Adina rubella", "Coptosapelta diffusa", "Luculia pinceana", - "Diplospora dubia", "Canthium horridum", "Mussaenda 'Alicia'", "Mussaenda erosa", "Mussaenda erythrophylla", "Mussaenda parviflora", - "Mussaenda pubescens", "Mussaenda shikokiana", "Sherardia arvensis", "Serissa japonica", "Serissa japonica 'Variegata'", - "Serissa serissoides", "Neohymenopogon parasiticus", "Lasianthus chinensis", "Houstonia caerulea", "Hedyotis caudatifolia", - "Hedyotis chrysotricha", "Hedyotis diffusa", "Hedyotis hedyotidea", "Hedyotis tenuipes", "Mycetia sinensis", "Coprosma robusta", - "Mitchella repens", "Damnacanthus giganteus", "Ophiorrhiza japonica", "Ophiorrhiza pumila", "Rondeletia leucophylla", - "Rondeletia odorata", "Leptodermis oblonga", "Uncaria hirsuta", "Spermacoce alata", "Hamelia patens", "Cephalanthus occidentalis", - "Cephalanthus tetrandrus", "Paederia foetida", "Ixora chinensis", "Ixora coccinea f. lutea", "Ixora finlaysoniana", - "Ixora paraopaca", "Mappianthus iodoides", "Ribes burejense", "Ribes himalense var. verruculosum", "Ribes nigrum", "Ribes odoratum", - "Ribes reclinatum", "Ribes rubrum", "Ribes rubrum", "Scaevola aemula", "Scaevola taccada", "Goodenia pilosa subsp. chinensis", - "Pilea aquarum", "Pilea cadierei", "Pilea microphylla", "Pilea notata", "Pilea pumila", "Cecropia peltata", "Elatostema cuspidatum", - "Debregeasia orientalis", "Gonostegia hirta", "Oreocnide frutescens", "Nanocnide lobata", "Boehmeria japonica", "Boehmeria nivea", - "Boehmeria tricuspis", "Urtica dioica", "Girardinia diversifolia subsp. suborbiculata", "Pellionia repens", "Pouzolzia zeylanica", - "Calceolaria crenatiflora", "Rhynchospora colorata", "Schoenoplectus tabernaemontani", "Kyllinga brevifolia", "Kyllinga polyphylla", - "Eleocharis dulcis", "Cyperus difformis", "Cyperus glomeratus", "Cyperus involucratus", "Cyperus prolifer", "Cyperus rotundus", - "Trichophorum subcapitatum", "Carex baccans", "Carex scaposa", "Fimbristylis dichotoma", "Illigera celebica", "Illigera rhodantha", - "Nelumbo nucifera", "Brasenia schreberi", "Mycelis muralis", "Solidago canadensis", "Emilia prenanthoidea", "Emilia sonchifolia", - "Tagetes erecta", "Calyptocarpus vialis", "Parasyncalathium souliei", "Mikania micrantha", "Paraprenanthes sororia", - "Praxelis clematidea", "Crepidiastrum lanceolatum", "Crepidiastrum sonchifolium", "Heterotheca subaxillaris", - "Syneilesis aconitifolia", "Ainsliaea fragrans", "Ainsliaea kawakamii", "Gazania rigens", "Smallanthus sonchifolius", - "Senecio analogus", "Senecio cineraria", "Senecio faberi", "Senecio haworthii", "Senecio rowleyanus", "Senecio scandens", - "Senecio serpens", "Senecio vulgaris", "Helianthus annuus", "Helianthus decapetalus", "Helianthus maxillianii", - "Helianthus tuberosus", "Cremanthodium campanulatum", "Helenium amarum", "Helenium autumnale", "Dahlia pinnata", - "Farfugium japonicum", "Gaillardia pulchella&aristata", "Carpesium abrotanoides", "Tragopogon dubius", "Tragopogon porrifolius", - "Tragopogon pratensis", "Wollastonia biflora", "Ixeridium dentatum", "Hieracium aurantiacum", "Dolomiaea souliei", - "Pseudognaphalium hypoleucum", "Inula helenium", "Inula helianthusaquatilis", "Inula japonica", "Argyranthemum frutescens", - "Echinacea purpurea", "Silphium laciniatum", "Silphium perfoliatum", "Nouelia insignis", "Engelmannia peristenia", - "Ligularia sibirica", "Tussilago farfara", "Matricaria chamomilla", "Matricaria discoidea", "Melanoseris atropurpurea", - "Silybum marianum", "Hemisteptia lyrata", "Eupatorium fortunei", "Eupatorium perfoliatum", "Eupatorium serotinum", - "Leucanthemum maximum", "Leucanthemum vulgare", "Rhaponticum chinense", "Rhaponticum uniflorum", "Gerbera jamesonii", - "Leontopodium japonicum", "Leontopodium leontopodioides", "Galinsoga parviflora", "Galinsoga quadriradiata", - "Helminthotheca echioides", "Arctium lappa", "Hypochaeris radicata", "Pericallis hybrida", "Stevia rebaudiana", - "Centaurea solstitialis", "Zinnia elegans", "Cyanus segetum", "Cosmos bipinnatus", "Cosmos sulphureus", "Lapsanastrum apogonoides", - "Ageratina adenophora", "Ageratina altissima", "Aster altaicus", "Aster baccharoides", "Aster hispidus", "Aster indicus", - "Aster likiangensis", "Aster novi-belgii", "Aster pekinensis", "Aster scaber", "Aster trinervius subsp. ageratoides", - "Aster turbinatus", "Carthamus tinctorius", "Eriophyllum confertiflorum", "Eriophyllum staechadifolium", "Thelesperma filifolium", - "Callistephus chinensis", "Symphyotrichum novae-angliae", "Symphyotrichum subulatum", "Tithonia diversifolia", - "Encelia californica", "Blumea megacephala", "Crossostephium chinensis", "Xanthium strumarium", "Sonchus asper", - "Sonchus oleraceus", "Ixeris chinensis", "Glebionis coronaria", "Glebionis segetum", "Ratibida columnifera", "Lactuca indica", - "Lactuca sativa", "Lactuca sativa var. ramosa", "Lactuca serriola", "Lactuca sibirica", "Gynura aurantiaca", "Gynura bicolor", - "Gynura divaricata", "Chrysanthemum multicaule", "Chrysanthemum × morifolium", "Cichorium endivia", "Cichorium intybus", - "Tanacetum vulgare", "Cynara cardunculus", "Cynara scolymus", "Sinosenecio oldhamianus", "Taraxacum mongolicum", - "Taraxacum officinale", "Artemisia argyi", "Artemisia californica", "Artemisia caruifolia", "Artemisia douglasiana", - "Artemisia lactiflora", "Artemisia selengensis", "Achillea millefolium", "Centratherum punctatum", "Echinops gmelinii", - "Cirsium arvense", "Cirsium arvense var. integrifolium", "Cirsium japonicum", "Cirsium leo", "Cirsium souliei", "Cirsium vulgare", - "Ageratum conyzoides", "Ageratum houstonianum", "Myripnois dioica", "Liatris spicata", "Petasites japonicus", - "Xerochrysum bracteatum", "Sphagneticola calendulacea", "Sphagneticola trilobata", "Ambrosia artemisiifolia", "Ambrosia trifida", - "Sigesbeckia orientalis", "Heliopsis helianthoides", "Heliopsis helianthoides var. scabra", "Baccharis halimifolia", - "Baccharis pilularis", "Baccharis salicifolia", "Crassocephalum crepidioides", "Crassocephalum rubens", "Rudbeckia bicolor", - "Rudbeckia fulgida", "Rudbeckia fulgida 'Goldsturm'", "Rudbeckia hirta", "Rudbeckia laciniata", - "Rudbeckia laciniata var. hortensia", "Calendula officinalis", "Synedrella nodiflora", "Acmella paniculata", "Coreopsis basalis", - "Coreopsis lanceolata", "Coreopsis tinctoria", "Coreopsis verticillata", "Vernonia baldwinii", "Vernonia gratiosa", - "Vernonia volkameriifolia", "Parthenium hysterophorus", "Conoclinium coelestinum", "Bellis perennis", "Saussurea involucrata", - "Saussurea medusa", "Saussurea przewalskii", "Saussurea stella", "Saussurea tibetica", "Saussurea velutina", "Carduus crispus", - "Carduus nutans", "Carduus pycnocephalus", "Erigeron annuus", "Erigeron canadensis", "Erigeron glaucus", "Erigeron philadelphicus", - "Erigeron sumatrensis", "Anaphalis margaritacea", "Anaphalis nepalensis", "Anaphalis nepalensis var. monocephala", - "Verbesina virginica", "Osteospermum ecklonis", "Bidens biternata", "Bidens cernua", "Bidens frondosa", "Bidens pilosa", - "Eclipta prostrata", "Brachyscome angustifolia", "Brachyscome iberidifolia", "Euryops pectinatus", "Flaveria bidentis", - "Youngia heterophylla", "Youngia japonica", "Gnaphalium", "Gnaphalium japonicum", "Acorus calamus", "Smilax bona-nox", - "Smilax china", "Smilax davidiana", "Smilax riparia", "Biondia microcentra", "Basella alba", "Anredera cordifolia", - "Cayratia albifolia", "Cayratia japonica", "Yua austro-orientalis", "Parthenocissus laetevirens", "Parthenocissus quinquefolia", - "Parthenocissus tricuspidata", "Tetrastigma hemsleyanum", "Tetrastigma planicaule", "Cissus hexangularis", "Vitis bryoniifolia", - "Vitis flexuosa", "Vitis vinifera", "Ampelopsis aconitifolia", "Ampelopsis delavayana", "Ampelopsis glandulosa", - "Ampelopsis glandulosa var. heterophylla", "Marah fabacea", "Marah macrocarpa", "Luffa aegyptiaca", "Sechium edule", - "Benincasa hispida", "Cucurbita foetidissima", "Cucurbita moschata", "Cucurbita pepo", "Trichosanthes anguina", - "Trichosanthes cucumeroides", "Trichosanthes kirilowii", "Trichosanthes rubriflos", "Diplocyclos palmatus", "Melothria pendula", - "Melothria scabra", "Actinostemma tenerum", "Coccinia grandis", "Gynostemma pentaphyllum", "Momordica charantia", - "Momordica cochinchinensis", "Lagenaria siceraria", "Lagenaria siceraria ‘Hispida’", "Citrullus lanatus", "Thladiantha dubia", - "Thladiantha longifolia", "Thladiantha nudiflora", "Gymnopetalum chinense", "Zehneria japonica", "Cucumis melo", "Cucumis melo", - "Cucumis melo", "Cucumis melo subsp. agrestis", "Cucumis metuliferus", "Cucumis sativus", "Rivina humilis", "Larrea tridentata", - "Tribulus terrestris", "Zygophyllum mucronatum", "Camptotheca acuminata", "Davidia involucrata", "Nyssa sinensis", - "Fallopia multiflora", "Muehlenbeckia complexa", "Rheum alexandrae", "Rheum nobile", "Rheum rhabarbarum", "Oxyria sinensis", - "Coccoloba uvifera", "Antigonon leptopus", "Eriogonum fasciculatum", "Eriogonum latifolium", "Fagopyrum dibotrys", - "Fagopyrum esculentum", "Polygonum aviculare", "Polygonum capitatum", "Polygonum chinense", "Polygonum coriaceum", - "Polygonum japonicum", "Polygonum longisetum", "Polygonum macrophyllum", "Polygonum muricatum", "Polygonum orientale", - "Polygonum perfoliatum", "Polygonum plebeium", "Polygonum pubescens", "Polygonum runcinatum", "Polygonum senticosum", - "Polygonum thunbergii", "Polygonum viscosum", "Persicaria virginiana", "Reynoutria japonica", "Rumex acetosa", "Rumex acetosella", - "Rumex crispus", "Rumex hastatus", "Rumex japonicus", "Rumex obtusifolius", "Antenoron filiforme", - "Antenoron filiforme var. neofiliforme", "Dryas octopetala", "Aruncus sylvester", "Amelanchier canadensis", - "Sanguisorba officinalis", "Potentilla anserina", "Potentilla discolor", "Potentilla fragarioides", "Potentilla freyniana", - "Potentilla fruticosa", "Potentilla glabra", "Potentilla kleiniana", "Potentilla recta", "Potentilla supina", - "Stephanandra chinensis", "Crataegus cuneata", "Crataegus maximowiczii", "Crataegus monogyna", "Crataegus pinnatifida", - "Rubus alceifolius", "Rubus armeniacus", "Rubus buergeri", "Rubus chingii", "Rubus corchorifolius", "Rubus coreanus", - "Rubus crataegifolius", "Rubus fockeanus", "Rubus fruticosus", "Rubus idaeus&hirsutus", "Rubus lambertianus", "Rubus odoratus", - "Rubus pacificus", "Rubus parviflorus", "Rubus parvifolius", "Rubus phoenicolasius", "Rubus pirifolius", "Rubus rosifolius", - "Rubus setchuenensis", "Rubus spectabilis", "Rubus sumatranus", "Rubus swinhoei", "Rubus trianthus", "Rubus ursinus", - "Prinsepia utilis", "Chaenomeles cathayensis", "Chaenomeles sinensis", "Chaenomeles speciosa", "Prunus cerasifera f. atropurpurea", - "Prunus laurocerasus", "Prunus salicina", "Prunus serotina", "Prunus spinosa", "Prunus virginiana", "Armeniaca mume", - "Armeniaca mume var. mume f. alphandii", "Armeniaca mume var. mume f. purpurea", "Armeniaca mume var. mume f. viridicalyx", - "Armeniaca vulgaris", "Eriobotrya japonica", "Adenostoma fasciculatum", "Heteromeles arbutifolia", "Cotoneaster adpressus", - "Cotoneaster horizontalis", "Cotoneaster microphyllus", "Cotoneaster multiflorus", "Amygdalus communis", "Amygdalus persica", - "Amygdalus persica 'Compressa'", "Amygdalus persica 'Juhuatao'", "Amygdalus triloba", "Pyrus", "Pyrus betulifolia", - "Pyrus calleryana", "Pyrus phaeocarpa", "Pyrus sinkiangensis", "Kerria japonica", "Kerria japonica f. pleniflora", - "Cydonia oblonga", "Cerasus campanulata", "Cerasus cerasoides", "Cerasus dielsiana", "Cerasus glandulosa", "Cerasus japonica", - "Cerasus pseudocerasus", "Cerasus serrulata var. lannesiana", "Cerasus tomentosa", "Pyracantha angustifolia", - "Pyracantha fortuneana", "Pyracantha fortuneana 'Harlequin'", "Sorbaria sorbifolia", "Exochorda racemosa", "Rhaphiolepis indica", - "Rhaphiolepis umbellata", "Photinia beauverdiana", "Photinia bodinieri", "Photinia glomerata", "Photinia komarovii", - "Photinia serratifolia", "Photinia × fraseri", "Padus avium", "Padus buergeriana", "Holodiscus discolor", "Neillia sinensis", - "Spiraea alpina", "Spiraea blumei", "Spiraea cantoniensis", "Spiraea fritschiana", "Spiraea japonica", "Spiraea mongolica", - "Spiraea myrtilloides", "Spiraea prunifolia", "Spiraea prunifolia var. simpliciflora", "Spiraea pubescens", "Spiraea thunbergii", - "Spiraea trilobata", "Spiraea × bumalda 'coldfiame'", "Spiraea × bumalda 'Goalden Mound'", "Spiraea × vanhouttei", - "Potaninia mongolica", "Sorbus alnifolia", "Sorbus folgneri", "Sorbus pohuashanensis", "Malus 'American'", "Malus baccata", - "Malus halliana", "Malus hupehensis", "Malus pumila", "Malus × micromalus", "Malus × robusta", "Fragaria orientalis", - "Fragaria vesca", "Fragaria virginiana", "Fragaria × ananassa", "Rosa banksiae", "Rosa banksiae f. lutea", "Rosa bracteata", - "Rosa californica", "Rosa chinensis", "Rosa cymosa", "Rosa davurica", "Rosa henryi", "Rosa laevigata", "Rosa multiflora", - "Rosa multiflora var. carnea", "Rosa multiflora var. cathayensis", "Rosa omeiensis", "Rosa roxburghii", - "Rosa roxburghii f. normalis", "Rosa rugosa", "Rosa rugosa f. albo-plena", "Rosa xanthina", "Rosa xanthina var. normalis", - "Filipendula palmata", "Duchesnea indica", "Geum aleppicum", "Geum canadense", "Geum japonicum var. chinense", - "Physocarpus amurensis", "Spenceria ramalana", "Agrimonia pilosa", "Liquidambar formosana", "Liquidambar styraciflua", - "Altingia chinensis", "Tacca chantrieri", "Tacca plantaginea", "Dioscorea bulbifera", "Dioscorea cirrhosa", - "Dioscorea elephantipes", "Dioscorea japonica", "Dioscorea polystachya", "Ypsilandra thibetica", "Trillium cernuum", - "Trillium chloropetalum", "Trillium cuneatum", "Trillium erectum", "Trillium grandiflorum", "Trillium luteum", "Trillium ovatum", - "Trillium recurvatum", "Trillium undulatum", "Toxicoscordion fremontii", "Chionographis chinensis", "Veratrum californicum", - "Veratrum nigrum", "Veratrum schindleri", "Veratrum viride", "Paris", "Paris luquanensis", "Paris polyphylla", - "Paris polyphylla var. chinensis", "Paris verticillata", "Garcinia cowa", "Garcinia mangostana", "Garcinia multiflora", - "Garcinia oblongifolia", "Garcinia subelliptica", "Garcinia xanthochymus", "Daphniphyllum calycinum", "Daphniphyllum macropodum", - "Mukdenia rossii", "Oresitrophe rupifraga", "Heuchera", "Astilbe chinensis", "Saxifraga egregia", "Saxifraga przewalskii", - "Saxifraga stolonifera", "Tiarella cordifolia", "Tiarella polyphylla", "Balanophora harlandii", "Balanophora laxiflora", - "Calycanthus chinensis", "Calycanthus floridus", "Chimonanthus nitens", "Chimonanthus praecox", "Heliconia latispatha", - "Heliconia metallica", "Heliconia rostrata", "Turnera subulata", "Turnera ulmifolia", "Passiflora alata", "Passiflora amethystina", - "Passiflora caerulea", "Passiflora coccinea", "Passiflora edulis", "Passiflora foetida", "Passiflora incarnata", "Passiflora lutea", - "Passiflora suberosa", "Passiflora yucatanensis", "Eriocaulon buergerianum", "Eriocaulon sexangulare", "Acmispon glaber", - "Amphicarpaea edgeworthii", "Caesalpinia bonduc", "Caesalpinia decapetala", "Caesalpinia minax", "Caesalpinia pulcherrima", - "Caesalpinia pulcherrima 'Flava'", "Caesalpinia sappan", "Lysidice brevicalyx", "Lysidice rhodostegia", "Dendrolobium triangulare", - "Senna alata", "Senna bicapsularis", "Senna occidentalis", "Senna sophera", "Senna spectabilis", "Senna surattensis", - "Delonix regia", "Canavalia gladiata", "Canavalia rosea", "Erythrina corallodendron", "Erythrina crista-galli", - "Erythrina variegata", "Robinia pseudoacacia", "Robinia pseudoacacia f. decaisneana", "Albizia julibrissin", "Albizia kalkora", - "Albizia lebbeck", "Aeschynomene indica", "Mimosa bimucronata", "Mimosa pudica", "Apios carnea", "Apios fortunei", "Glycine max", - "Glycine soja", "Coronilla varia", "Chamaecrista fasciculata", "Chamaecrista mimosoides", "Desmodium heterocarpon", - "Desmodium microphyllum", "Desmodium triflorum", "Lathyrus latifolius", "Lathyrus odoratus", "Fordia cauliflora", - "Lablab purpureus", "Phyllodium pulchellum", "Saraca dives", "Indigofera bungeana", "Indigofera decora", "Indigofera hendecaphylla", - "Indigofera kirilowii", "Cajanus cajan", "Calliandra haematocephala", "Calliandra tergemina var. emarginata", - "Campylotropis macrocarpa", "Campylotropis polyantha", "Castanospermum australe", "Erythrophleum fordii", "Oxytropis aciphylla", - "Oxytropis caerulea", "Oxytropis myriophylla", "Styphnolobium japonicum", "Ammopiptanthus mongolicus", "Sindora glabra", - "Mucuna bennettii", "Mucuna birdwoodiana", "Mucuna lamellata", "Mucuna macrocarpa", "Mucuna sempervirens", - "Adenanthera microsperma", "Prosopis glandulosa", "Uraria crinita", "Uraria picta", "Crotalaria assamica", "Crotalaria pallida", - "Crotalaria sessiliflora", "Crotalaria spectabilis", "Crotalaria trichotoma", "Archidendron clypearia", "Glycyrrhiza uralensis", - "Sesbania cannabina", "Sesbania grandiflora", "Lotus corniculatus", "Gleditsia japonica", "Gleditsia triacanthos", - "Abrus precatorius", "Acacia auriculiformis", "Acacia catechu", "Acacia confusa", "Acacia farnesiana", "Acacia podalyriifolia", - "Peltophorum pterocarpum", "Butea monosperma", "Amorpha fruticosa", "Cercis canadensis", "Cercis chinensis", "Cercis chingii", - "Cercis chuniana", "Cercis glabra", "Wisteria sinensis&villosa", "Ormosia henryi", "Corethrodendron scoparium", - "Bauhinia acuminata", "Bauhinia brachycarpa", "Bauhinia championii", "Bauhinia corymbosa", "Bauhinia didyma", "Bauhinia galpinii", - "Bauhinia glauca", "Bauhinia glauca subsp. tenuiflora", "Bauhinia kockiana", "Bauhinia tomentosa", "Bauhinia touranensis", - "Bauhinia variegata", "Bauhinia variegata var. candida", "Bauhinia × blakeana", "Lupinus arboreus", - "Lupinus micranthus&polyphyllus", "Lupinus texensis", "Strongylodon macrobotrys", "Lespedeza bicolor", "Lespedeza buergeri", - "Lespedeza chinensis", "Lespedeza cuneata", "Lespedeza davidii", "Lespedeza dunnii", "Lespedeza floribunda", "Lespedeza pilosa", - "Lespedeza thunbergii subsp. formosa", "Lespedeza tomentosa", "Lespedeza virgata", "Cassia fistula", "Codoriocalyx motorius", - "Medicago lupulina", "Medicago polymorpha", "Medicago sativa", "Sophora davidii", "Sophora flavescens", "Sphaerophysa salsula", - "Ulex europaeus", "Melilotus albus", "Melilotus indicus", "Melilotus officinalis", "Phaseolus coccineus", "Phaseolus vulgaris", - "Arachis duranensis", "Arachis hypogaea", "Pueraria montana", "Pueraria wallichii", "Bowringia callicarpa", "Clitoria ternatea", - "Cullen corylifolium", "Pachyrhizus erosus", "Vigna radiata", "Vigna umbellata", "Vigna unguiculata", "Vigna vexillata", - "Pisum sativum", "Baptisia australis", "Centrosema pubescens", "Trifolium pratense", "Trifolium repens", "Tamarindus indica", - "Thermopsis barbata", "Thermopsis lanceolata", "Vicia amoena", "Vicia cracca", "Vicia faba", "Vicia sativa", "Vicia sepium", - "Vicia tetrasperma", "Vicia villosa", "Cytisus scoparius", "Leucaena leucocephala", "Caragana jubata", "Caragana rosea", - "Caragana sinica", "Caragana tibetica", "Hylodesmum podocarpum", "Hylodesmum podocarpum subsp. fallax", - "Hylodesmum podocarpum subsp. oxyphyllum", "Chesneya polystichoides", "Tibetia yunnanensis", "Derris alborubra", "Derris fordii", - "Colutea arborescens", "Kummerowia striata", "Callerya dielsiana", "Callerya nitida", "Callerya reticulata", "Callerya speciosa", - "Spartium junceum", "Rhynchosia volubilis", "Dalbergia assamica", "Dalbergia hupeana", "Astragalus sinicus", - "Athyrium filix-femina", "Bacopa diffusa", "Pseudolysimachion longifolium", "Pseudolysimachion spicatum", "Lagotis brevituba", - "Veronica anagallis-aquatica", "Veronica arvensis", "Veronica henryi", "Veronica persica", "Veronica undulata", "Linaria maroccana", - "Linaria vulgaris", "Linaria vulgaris subsp. chinensis", "Digitalis purpurea", "Adenosma glutinosum", "Russelia equisetiformis", - "Veronicastrum axillare", "Otacanthus azureus", "Cymbalaria muralis", "Plantago asiatica", "Plantago depressa", - "Plantago lanceolata", "Plantago major", "Plantago virginica", "Antirrhinum majus", "Penstemon", "Penstemon barbatus", - "Penstemon digitalis", "Collinsia heterophylla", "Hemiphragma heterophyllum", "Angelonia angustifolia", "Chelone glabra", - "Moringa drouhardii", "Moringa oleifera", "Polygala arillata", "Polygala fallax", "Polygala hongkongensis", - "Polygala hongkongensis var. stenophylla", "Polygala japonica", "Polygala latouchei", "Polygala myrtifolia", "Polygala sibirica", - "Polygala tenuifolia", "Salomonia cantoniensis", "Cercidiphyllum japonicum", "Mimulus aurantiacus", "Mimulus guttatus", - "Mimulus szechuanensis", "Lancea tibetica", "Mazus caducifer", "Mazus pumilus", "Oxalis", "Oxalis articulata", "Oxalis barrelieri", - "Oxalis corniculata", "Oxalis corymbosa", "Oxalis griffithii", "Oxalis oregana", "Oxalis palmifrons", "Oxalis pes-caprae", - "Oxalis purpurea", "Oxalis stricta", "Oxalis triangularis 'Urpurea'", "Oxalis violacea", "Averrhoa carambola", - "Oxyspora paniculata", "Blastus cochinchinensis", "Blastus pauciflorus", "Fordiophyton faberi", "Tibouchina semidecandra", - "Tigridiopalma exalata", "Tigridiopalma magnifica", "Sonerila cantonensis", "Memecylon ligustrifolium", "Memecylon octocostatum", - "Medinilla formosana", "Medinilla magnifica", "Bredia fordii", "Bredia quadrangularis", "Melastoma dodecandrum", - "Melastoma malabathricum", "Melastoma malabathricum var. alba", "Melastoma sanguineum", "Osbeckia chinensis", "Osbeckia stellata", - "Phyllagathis cavaleriei", "Hypericum 'Excellent Flair'", "Hypericum androsaemum", "Hypericum faberi", "Hypericum japonicum", - "Hypericum monogynum", "Hypericum patulum", "Hypericum perforatum", "Hypericum sampsonii", "Cratoxylum cochinchinense", - "Phegopteris connectilis", "Sarcandra glabra", "Chloranthus fortunei", "Chloranthus henryi", "Chloranthus japonicus", - "Chloranthus serratus", "Chloranthus spicatus", "Mytilaria laosensis", "Loropetalum chinense", "Loropetalum chinense var. rubrum", - "Loropetalum subcordatum", "Sycopsis sinensis", "Fortunearia sinensis", "Eustigma oblongifolium", "Rhodoleia championii", - "Distylium buxifolium", "Distylium racemosum", "Corylopsis multiflora var. nivea", "Corylopsis sinensis", "Hamamelis mollis", - "Hamamelis virginiana", "Hamamelis × intermedia", "Ochna integerrima", "Ochna serrulata", "Ochna thomasiana", - "Tristellateia australasiae", "Heteropterys glabra", "Thryallis gracilis", "Malpighia glabra", "Hiptage benghalensis", - "Ceratophyllum demersum", "Gelsemium elegans", "Gelsemium sempervirens", "Ancistrocladus tectorius", "Asplenium bulbiferum", - "Asplenium nidus", "Asplenium oblongifolium", "Asplenium platyneuron", "Asplenium trichomanes", "Erythropalum scandens", - "Ginkgo biloba", "", "Byttneria grandifolia", "Triumfetta annua", "Triumfetta cana", "Triumfetta rhomboidea", - "Pentapetes phoenicea", "Anisodontea capensis", "Theobroma cacao", "Ceiba pentandra", "Ceiba speciosa", "Helicteres angustifolia", - "Helicteres hirsuta", "Malvaviscus arboreus", "Malvaviscus arboreus var. mexicanus", "Malvaviscus penduliflorus", "Grewia biloba", - "Grewia biloba var. parviflora", "Grewia occidentalis", "Ambroma augustum", "Bombax ceiba", "Hibiscus acetosella", - "Hibiscus aridicola", "Hibiscus coccineus", "Hibiscus grandiflorus", "Hibiscus grewiifolius", "Hibiscus hamabo", - "Hibiscus moscheutos", "Hibiscus mutabilis", "Hibiscus rosa-sinensis", "Hibiscus sabdariffa", "Hibiscus schizopetalus", - "Hibiscus syriacus", "Hibiscus syriacus var. syriacus f. totus-albus", "Hibiscus tiliaceus", "Hibiscus trionum", - "Firmiana kwangsiensis", "Firmiana simplex", "Reevesia pubescens", "Reevesia thyrsoidea", "Urena lobata", "Urena procumbens", - "Urena procumbens var. microphylla", "Gossypium", "Sidalcea malviflora", "Tilia americana", "Durio zibethinus", - "Diplodiscus trichospermus", "Adansonia digitata", "Pachira glabra", "Corchoropsis crenata", "Microcos paniculata", - "Abelmoschus esculentus", "Abelmoschus manihot", "Abelmoschus sagittifolius", "Pavonia hastata", "Callirhoe involucrata", - "Pterygota alata", "Scaphium wallichii", "Abutilon indicum", "Abutilon megapotamicum", "Abutilon pictum", "Abutilon theophrasti", - "Sterculia lanceolata", "Sterculia monosperma", "Althaea officinalis", "Waltheria indica", "Alcea rosea", - "Malvastrum coromandelianum", "Brachychiton acerifolius", "Brachychiton rupestris", "Heritiera littoralis", "Heritiera parvifolia", - "Malva cathayensis", "Malva pusilla", "Malva verticillata var. crispa", "Dombeya wallichii", "Melochia corchorifolia", - "Kleinhovia hospita", "Sida subcordata", "Corchorus aestuans", "Costus barbatus", "Costus lucanusianus", "Costus woodsonii", - "Stephania cephalantha", "Stephania epigaea&cephalantha", "Stephania longa", "Stephania tetrandra", "Cocculus orbiculatus", - "Diploclisia affinis", "Diploclisia glaucescens", "Menispermum dauricum", "Cyclea racemosa", "Sinomenium acutum", - "Haworthia cooperi var. pilifera", "Haworthia fasciata", "Haworthia truncata", "Dianella ensifolia", "Stypandra glauca", - "Asphodeline lutea", "Kniphofia uvaria", "Geitonoplesium cymosum", "Aloe arborescens", "Aloe ferox", "Aloe mitriformis", - "Aloe vera", "Hemerocallis citrina", "Hemerocallis fulva", "Hemerocallis fulva 'Golden Doll'", "Hemerocallis hybridus", - "Asphodelus fistulosus", "Asphodelus ramosus", "Bulbine bulbosa", "Tricoryne elatior", "Gasteria gracilis var. minima", - "Phormium tenax", "Eichhornia crassipes", "Pontederia cordata", "Pontederia cordata var. alba", "Monochoria korsakowii", - "Monochoria vaginalis", "Sciaphila secundiflora", "Pandanus tectorius", "Schoepfia chinensis", "Helwingia chinensis", - "Helwingia japonica", "Helwingia omeiensis", "Hydnocarpus anthelminthicus", "Hydnocarpus hainanensis", "Typha", - "Typha angustifolia", "Typha latifolia", "Typha orientalis", "Sparganium stoloniferum", "Asarum canadense", "Asarum caudigerum", - "Asarum forbesii", "Asarum heterotropoides", "Aristolochia arborea", "Aristolochia contorta", "Aristolochia debilis", - "Aristolochia elegans", "Aristolochia gentilis", "Aristolochia gibertii", "Aristolochia grandiflora", "Aristolochia griffithii", - "Aristolochia hainanensis", "Aristolochia kwangsiensis", "Aristolochia manshuriensis", "Aristolochia mollissima", - "Aristolochia ringens", "Aristolochia tagala", "Aristolochia tubiflora", "Aristolochia westlandii", "Coriaria nepalensis", - "Mitrasacme pygmaea", "Gardneria multiflora", "Strychnos angustiflora", "Duranta erecta", "Duranta erecta 'Alba'", - "Glandularia bipinnatifida", "Glandularia tenera", "Glandularia × hybrida", "Petrea volubilis", "Phyla canescens", - "Phyla nodiflora", "Lantana camara", "Lantana fucata", "Lantana montevidensis", "Verbena bonariensis", "Verbena brasiliensis", - "Verbena halei", "Verbena hastata", "Verbena officinalis", "Verbena stricta", "Portulaca gilliesii", "Portulaca grandiflora", - "Portulaca molokiniensis", "Portulaca oleracea", "Portulaca pilosa", "Portulaca umbraticola", "", "", "Polystichum acrostichoides", - "Polystichum munitum", "Polystichum vestitum", "Gladiolus communis", "Gladiolus dalenii", "Gladiolus gandavensis", - "Gladiolus imbricatus", "Belamcanda chinensis", "Neomarica gracilis", "Sisyrinchium albidum", "Sisyrinchium angustifolium", - "Sisyrinchium bellum", "Sisyrinchium campestre", "Sisyrinchium micranthum", "Sisyrinchium montanum", "sisyrinchium rosulatum", - "Alophia drummondii", "Olsynium douglasii", "Romulea columnae", "Romulea rosea", "Herbertia lahue", "Crocus biflorus", - "Crocus nudiflorus", "Crocus sativus", "Crocus tommasinianus", "Crocus vernus", "Dietes bicolor", "Nemastylis geminiflora", - "Tigridia pavonia", "Ixia viridiflora", "Trimezia martinicensis", "Crocosmia × crocosmiiflora", "Freesia refracta", - "Sparaxis tricolor", "Iris bulleyana", "Iris chrysographes", "Iris confusa", "Iris cristata", "Iris douglasiana", "Iris ensata", - "Iris foetidissima", "Iris fulva 'Louisiana Hybrids'", "Iris germanica", "Iris hartwegii", "Iris japonica", "Iris lactea", - "Iris lutescens", "Iris macrosiphon", "Iris missouriensis", "Iris pseudacorus", "Iris pumila", "Iris ruthenica", "Iris sanguinea", - "Iris setosa", "Iris sibirica", "Iris speculatrix", "Iris tectorum", "Iris tenax", "Iris verna", "Iris versicolor", - "Iris virginica", "Tinantia anomala", "Tinantia erecta", "Pollia japonica", "Murdannia loriformis", "Murdannia nudiflora", - "Murdannia triquetra", "Amischotolype hispida", "Tradescantia cerinthoides 'Nanouk'", "Tradescantia fluminensis", - "Tradescantia ohiensis", "Tradescantia pallida", "Tradescantia sillamontana", "Tradescantia spathacea", "Tradescantia virginiana", - "Tradescantia zanonia", "Tradescantia zebrina", "Floscopa scandens", "Cyanotis arachnoidea", "Commelina benghalensis", - "Commelina communis", "Commelina diffusa", "Commelina erecta", "Strelitzia nicolai", "Strelitzia reginae", "Ephedra aspera", - "Ephedra californica", "Ephedra distachya", "Ephedra trifurca", "Ephedra viridis", "Pachysandra terminalis", - "Sarcococca hookeriana", "Sarcococca ruscifolia", "Buxus harlandii", "Buxus sinica", "Itea omeiensis", "Berchemia floribunda", - "Berchemia lineata", "Berchemia sinica", "Ziziphus jujuba", "Ziziphus mauritiana", "Hovenia acerba", "Ceanothus", - "Ventilago leiocarpa", "Frangula californica", "Sageretia thea", "Paliurus hemsleyanus", "Paliurus ramosissimus", - "Rhamnus cathartica", "Rhamnus crenata", "Rhamnus davurica", "Rhamnus utilis", "Gentianella azurea", "Latouchea fokienensis", - "Tripterospermum chinense", "Tripterospermum nienkui", "Comastoma pulmonarium", "Megacodon stylophorus", "Gentianopsis barbata", - "Cotylanthera paucisquama", "Eustoma grandiflorum", "Fagraea ceilanica", "Fagraea ceilanica 'Variegata'", "Swertia bimaculata", - "Swertia decora", "Swertia hickinii", "Swertia pseudochinensis", "Centaurium pulchellum var. altaicum", "Canscora lucidissima", - "Sabatia campestris", "Halenia elliptica", "Exacum affine", "Gentiana arethusae var. delicatula", "Gentiana aristata", - "Gentiana dahurica", "Gentiana davidii", "Gentiana lawrencei var. farreri", "Gentiana loureiroi", "Gentiana panthaica", - "Gentiana pseudoaquatica", "Gentiana pudica", "Gentiana rubicunda", "Gentiana squarrosa", "Gentiana straminea", "Gentiana striata", - "Gentiana tatsienensis", "Gentiana urnula", "Gentiana veitchiorum", "Gentiana zollingeri", "Hopea chinensis", "Hopea hainanensis", - "Vatica mangachapoi", "Marsilea quadrifolia" - }; - }; -} - - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_PLANTID_H diff --git a/lite/ncnn/cv/ncnn_resnet.cpp b/lite/ncnn/cv/ncnn_resnet.cpp deleted file mode 100644 index 225263d0..00000000 --- a/lite/ncnn/cv/ncnn_resnet.cpp +++ /dev/null @@ -1,63 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "ncnn_resnet.h" -#include "lite/utils.h" - -using ncnncv::NCNNResNet; - -NCNNResNet::NCNNResNet(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNResNet::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - // will do deepcopy inside ncnn - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNResNet::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - // 3. fetch. - ncnn::Mat logits_mat; - extractor.extract("logits", logits_mat); // c=1,h=1,w=1000 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(logits_mat, "logits"); -#endif - - const unsigned int num_classes = logits_mat.w; - const float *logits = (float *) logits_mat.data; - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} diff --git a/lite/ncnn/cv/ncnn_resnet.h b/lite/ncnn/cv/ncnn_resnet.h deleted file mode 100644 index 6105375f..00000000 --- a/lite/ncnn/cv/ncnn_resnet.h +++ /dev/null @@ -1,413 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_RESNET_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_RESNET_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNResNet : public BasicNCNNHandler - { - public: - explicit NCNNResNet(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNResNet() override = default; - - private: - const int input_height = 224; - const int input_width = 224; - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_RESNET_H diff --git a/lite/ncnn/cv/ncnn_resnext.cpp b/lite/ncnn/cv/ncnn_resnext.cpp deleted file mode 100644 index 4b900143..00000000 --- a/lite/ncnn/cv/ncnn_resnext.cpp +++ /dev/null @@ -1,62 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "ncnn_resnext.h" -#include "lite/utils.h" - -using ncnncv::NCNNResNeXt; - -NCNNResNeXt::NCNNResNeXt(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNResNeXt::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNResNeXt::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - // 3. fetch. - ncnn::Mat logits_mat; - extractor.extract("logits", logits_mat); // c=1,h=1,w=1000 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(logits_mat, "logits"); -#endif - - const unsigned int num_classes = logits_mat.w; - const float *logits = (float *) logits_mat.data; - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_resnext.h b/lite/ncnn/cv/ncnn_resnext.h deleted file mode 100644 index bed5340c..00000000 --- a/lite/ncnn/cv/ncnn_resnext.h +++ /dev/null @@ -1,413 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_RESNEXT_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_RESNEXT_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNResNeXt : public BasicNCNNHandler - { - public: - explicit NCNNResNeXt(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNResNeXt() override = default; - - private: - const int input_height = 224; - const int input_width = 224; - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_RESNEXT_H diff --git a/lite/ncnn/cv/ncnn_retinaface.cpp b/lite/ncnn/cv/ncnn_retinaface.cpp deleted file mode 100644 index 18a4531e..00000000 --- a/lite/ncnn/cv/ncnn_retinaface.cpp +++ /dev/null @@ -1,216 +0,0 @@ -// -// Created by DefTruth on 2021/11/20. -// - -#include "ncnn_retinaface.h" -#include "lite/utils.h" - -using ncnncv::NCNNRetinaFace; - -NCNNRetinaFace::NCNNRetinaFace(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads, - int _input_height, - int _input_width) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads), - input_height(_input_height), input_width(_input_width) -{ -} - -void NCNNRetinaFace::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNRetinaFace::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - // 3.rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(bbox_collection, extractor, score_threshold, img_height, img_width); - // 4. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void NCNNRetinaFace::generate_anchors(const int target_height, const int target_width, - std::vector &anchors) -{ - std::vector> feature_maps; - for (auto step: steps) - { - feature_maps.push_back( - { - (int) std::ceil((float) target_height / (float) step), - (int) std::ceil((float) target_width / (float) step) - } // ceil - ); - } - - anchors.clear(); - const int num_feature_map = feature_maps.size(); - - for (int k = 0; k < num_feature_map; ++k) - { - auto f_map = feature_maps.at(k); // e.g [640//8,640//8] - auto tmp_min_sizes = min_sizes.at(k); // e.g [8,16] - int f_h = f_map.at(0); - int f_w = f_map.at(1); - - for (int i = 0; i < f_h; ++i) - { - for (int j = 0; j < f_w; ++j) - { - for (auto min_size: tmp_min_sizes) - { - float s_kx = (float) min_size / (float) target_width; // e.g 16/w - float s_ky = (float) min_size / (float) target_height; // e.g 16/h - // (x + 0.5) * step / w normalized loc mapping to input width - // (y + 0.5) * step / h normalized loc mapping to input height - float cx = ((float) j + 0.5f) * (float) steps.at(k) / (float) target_width; - float cy = ((float) i + 0.5f) * (float) steps.at(k) / (float) target_height; - - anchors.push_back(RetinaAnchor{cx, cy, s_kx, s_ky}); // without clip - } - } - } - } -} - -void NCNNRetinaFace::generate_bboxes(std::vector &bbox_collection, - ncnn::Extractor &extractor, float score_threshold, - float img_height, float img_width) -{ - ncnn::Mat bboxes, probs; - extractor.extract("bbox", bboxes); // c=1 h=? w=4 - extractor.extract("conf", probs); // c=1 h=? w=2 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(bboxes, "bbox"); - BasicNCNNHandler::print_shape(probs, "conf"); -#endif - const unsigned int bbox_num = bboxes.h; // n = ? - - std::vector anchors; - this->generate_anchors(input_height, input_width, anchors); - - const unsigned int num_anchors = anchors.size(); - if (num_anchors != bbox_num) - throw std::runtime_error("mismatch num_anchors != bbox_num"); - - const float *bboxes_ptr = (float *) bboxes.data; - const float *probs_ptr = (float *) probs.data; - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - float conf = probs_ptr[2 * i + 1]; - if (conf < score_threshold) continue; // filter first. - - float prior_cx = anchors.at(i).cx; - float prior_cy = anchors.at(i).cy; - float prior_s_kx = anchors.at(i).s_kx; - float prior_s_ky = anchors.at(i).s_ky; - - float dx = bboxes_ptr[4 * i + 0]; - float dy = bboxes_ptr[4 * i + 1]; - float dw = bboxes_ptr[4 * i + 2]; - float dh = bboxes_ptr[4 * i + 3]; - // ref: https://github.com/biubug6/Pytorch_Retinaface/blob/master/utils/box_utils.py - float cx = prior_cx + dx * variance[0] * prior_s_kx; - float cy = prior_cy + dy * variance[0] * prior_s_ky; - float w = prior_s_kx * std::exp(dw * variance[1]); - float h = prior_s_ky * std::exp(dh * variance[1]); // norm coor (0.,1.) - - types::Boxf box; - box.x1 = (cx - w / 2.f) * img_width; - box.y1 = (cy - h / 2.f) * img_height; - box.x2 = (cx + w / 2.f) * img_width; - box.y2 = (cy + h / 2.f) * img_height; - box.score = conf; - box.label = 1; - box.label_text = "face"; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - -#if LITENCNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void NCNNRetinaFace::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lite/ncnn/cv/ncnn_retinaface.h b/lite/ncnn/cv/ncnn_retinaface.h deleted file mode 100644 index 1cff882d..00000000 --- a/lite/ncnn/cv/ncnn_retinaface.h +++ /dev/null @@ -1,78 +0,0 @@ -// -// Created by DefTruth on 2021/11/20. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_RETINAFACE_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_RETINAFACE_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNRetinaFace : public BasicNCNNHandler - { - public: - explicit NCNNRetinaFace(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1, - int _input_height = 640, - int _input_width = 640); - - ~NCNNRetinaFace() override = default; - - private: - // nested classes - struct RetinaAnchor - { - float cx; - float cy; - float s_kx; - float s_ky; - }; - - private: - const int input_height; // 640/320 - const int input_width; // 640/320 - - const float mean_vals[3] = {104.f, 117.f, 123.f}; // bgr order - const float norm_vals[3] = {1.f, 1.f, 1.f}; - const float variance[2] = {0.1f, 0.2f}; - std::vector steps = {8, 16, 32}; - std::vector> min_sizes = { - {16, 32}, - {64, 128}, - {256, 512} - }; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - void generate_anchors(const int target_height, - const int target_width, - std::vector &anchors); - - - void generate_bboxes(std::vector &bbox_collection, - ncnn::Extractor &extractor, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.7f, float iou_threshold = 0.3f, - unsigned int topk = 300, unsigned int nms_type = 0); - - - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_RETINAFACE_H diff --git a/lite/ncnn/cv/ncnn_rvm.cpp b/lite/ncnn/cv/ncnn_rvm.cpp deleted file mode 100644 index 07e9063d..00000000 --- a/lite/ncnn/cv/ncnn_rvm.cpp +++ /dev/null @@ -1,224 +0,0 @@ -// -// Created by DefTruth on 2021/10/10. -// - -#include "ncnn_rvm.h" - -using ncnncv::NCNNRobustVideoMatting; - - -NCNNRobustVideoMatting::NCNNRobustVideoMatting( - const std::string &_param_path, const std::string &_bin_path, - unsigned int _num_threads, int _input_height, - int _input_width, unsigned int _variant_type -) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads), - input_height(_input_height), input_width(_input_width), - variant_type(_variant_type) -{ - initialize_context(); -} - -void NCNNRobustVideoMatting::initialize_context() -{ - if (variant_type == VARIANT::MOBILENETV3) - { - if (input_width == 1920 && input_height == 1080) - { - r1i = ncnn::Mat(240, 135, 16); // w,h,c in NCNN - r2i = ncnn::Mat(120, 68, 20); - r3i = ncnn::Mat(60, 34, 40); - r4i = ncnn::Mat(30, 17, 64); - } // hxw 480x640 480x480 640x480 - else - { - r1i = ncnn::Mat(input_width / 2, input_height / 2, 16); - r2i = ncnn::Mat(input_width / 4, input_height / 4, 20); - r3i = ncnn::Mat(input_width / 8, input_height / 8, 40); - r4i = ncnn::Mat(input_width / 16, input_height / 16, 64); - } - } // RESNET50 - else - { - if (input_width == 1920 && input_height == 1080) - { - r1i = ncnn::Mat(240, 135, 16); - r2i = ncnn::Mat(120, 68, 32); - r3i = ncnn::Mat(60, 34, 64); - r4i = ncnn::Mat(30, 17, 128); - } // hxw 480x640 480x480 640x480 - else - { - r1i = ncnn::Mat(input_width / 2, input_height / 2, 16); - r2i = ncnn::Mat(input_width / 4, input_height / 4, 20); - r3i = ncnn::Mat(input_width / 8, input_height / 8, 40); - r4i = ncnn::Mat(input_width / 16, input_height / 16, 64); - } - } - // init 0. - r1i.fill(0.f); - r2i.fill(0.f); - r3i.fill(0.f); - r4i.fill(0.f); - - context_is_initialized = true; -} - -void NCNNRobustVideoMatting::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW & resize - int h = mat.rows; - int w = mat.cols; - in = ncnn::Mat::from_pixels_resize( - mat.data, ncnn::Mat::PIXEL_BGR2RGB, - w, h, input_width, input_height - ); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNRobustVideoMatting::detect(const cv::Mat &mat, types::MattingContent &content, bool video_mode) -{ - if (mat.empty()) return; - int img_h = mat.rows; - int img_w = mat.cols; - if (!context_is_initialized) return; - - // 1. make input tensor - ncnn::Mat src; - this->transform(mat, src); - - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("src", src); - extractor.input("r1i", r1i); - extractor.input("r2i", r2i); - extractor.input("r3i", r3i); - extractor.input("r4i", r4i); - - // 3. generate matting - this->generate_matting(extractor, content, img_h, img_w); - - // 4. update context (needed for video detection.) - if (video_mode) - { - context_is_update = false; // init state. - this->update_context(extractor); - } -} - -void NCNNRobustVideoMatting::detect_video(const std::string &video_path, - const std::string &output_path, - std::vector &contents, - bool save_contents, unsigned int writer_fps) -{ - // 0. init video capture - cv::VideoCapture video_capture(video_path); - const unsigned int width = video_capture.get(cv::CAP_PROP_FRAME_WIDTH); - const unsigned int height = video_capture.get(cv::CAP_PROP_FRAME_HEIGHT); - const unsigned int frame_count = video_capture.get(cv::CAP_PROP_FRAME_COUNT); - if (!video_capture.isOpened()) - { - std::cout << "Can not open video: " << video_path << "\n"; - return; - } - // 1. init video writer - cv::VideoWriter video_writer(output_path, cv::VideoWriter::fourcc('m', 'p', '4', 'v'), - writer_fps, cv::Size(width, height)); - if (!video_writer.isOpened()) - { - std::cout << "Can not open writer: " << output_path << "\n"; - return; - } - - // 2. matting loop - cv::Mat mat; - unsigned int i = 0; - while (video_capture.read(mat)) - { - i += 1; - types::MattingContent content; - this->detect(mat, content); - // 3. save contents and writing out. - if (content.flag) - { - if (save_contents) contents.push_back(content); - if (!content.merge_mat.empty()) video_writer.write(content.merge_mat); - } - // 4. check context states. - if (!context_is_update) break; -#ifdef LITENCNN_DEBUG - std::cout << i << "/" << frame_count << " done!" << "\n"; -#endif - } - - // 5. release - video_capture.release(); - video_writer.release(); -} - -void NCNNRobustVideoMatting::generate_matting(ncnn::Extractor &extractor, - types::MattingContent &content, - int img_h, int img_w) -{ - ncnn::Mat fgr, pha; - extractor.extract("fgr", fgr); - extractor.extract("pha", pha); - float *fgr_ptr = (float *) fgr.data; - float *pha_ptr = (float *) pha.data; - - const unsigned int channel_step = input_height * input_width; - - // fast assign & channel transpose(CHW->HWC). - cv::Mat rmat(input_height, input_width, CV_32FC1, fgr_ptr); - cv::Mat gmat(input_height, input_width, CV_32FC1, fgr_ptr + channel_step); - cv::Mat bmat(input_height, input_width, CV_32FC1, fgr_ptr + 2 * channel_step); - cv::Mat pmat(input_height, input_width, CV_32FC1, pha_ptr); // ref only, zero-copy. - rmat *= 255.f; - bmat *= 255.f; - gmat *= 255.f; - cv::Mat rest = 1.f - pmat; - cv::Mat mbmat = bmat.mul(pmat) + rest * 153.f; - cv::Mat mgmat = gmat.mul(pmat) + rest * 255.f; - cv::Mat mrmat = rmat.mul(pmat) + rest * 120.f; - std::vector fgr_channel_mats, merge_channel_mats; - fgr_channel_mats.push_back(bmat); - fgr_channel_mats.push_back(gmat); - fgr_channel_mats.push_back(rmat); - merge_channel_mats.push_back(mbmat); - merge_channel_mats.push_back(mgmat); - merge_channel_mats.push_back(mrmat); - - // need clone to allocate a new continuous memory. - content.pha_mat = pmat.clone(); // allocated - cv::merge(fgr_channel_mats, content.fgr_mat); - cv::merge(merge_channel_mats, content.merge_mat); - content.fgr_mat.convertTo(content.fgr_mat, CV_8UC3); - content.merge_mat.convertTo(content.merge_mat, CV_8UC3); - - if (img_w != input_width || img_h != input_height) - { - cv::resize(content.pha_mat, content.pha_mat, cv::Size(img_w, img_h)); - cv::resize(content.fgr_mat, content.fgr_mat, cv::Size(img_w, img_h)); - cv::resize(content.merge_mat, content.merge_mat, cv::Size(img_w, img_h)); - } - - content.flag = true; -} - -void NCNNRobustVideoMatting::update_context(ncnn::Extractor &extractor) -{ - ncnn::Mat r1o, r2o, r3o, r4o; - extractor.extract("r1o", r1o); - extractor.extract("r2o", r2o); - extractor.extract("r3o", r3o); - extractor.extract("r4o", r4o); - - r1i.clone_from(r1o); // deepcopy - r2i.clone_from(r2o); // deepcopy - r3i.clone_from(r3o); // deepcopy - r4i.clone_from(r4o); // deepcopy - - context_is_update = true; -} diff --git a/lite/ncnn/cv/ncnn_rvm.h b/lite/ncnn/cv/ncnn_rvm.h deleted file mode 100644 index 08097c00..00000000 --- a/lite/ncnn/cv/ncnn_rvm.h +++ /dev/null @@ -1,158 +0,0 @@ -// -// Created by DefTruth on 2021/10/10. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_RVM_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_RVM_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNRobustVideoMatting : public BasicNCNNHandler - { - public: - explicit NCNNRobustVideoMatting(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1, - int _input_height = 480, - int _input_width = 640, - unsigned int _variant_type = 0); // - ~NCNNRobustVideoMatting() override = default; - - private: - const float mean_vals[3] = {0.f, 0.f, 0.f}; // RGB - const float norm_vals[3] = {1.f / 255.f, 1.f / 255.f, 1.f / 255.f}; - // hardcode input node names, hint only. - // downsample_ratio has been freeze while onnx exported - // and, the input size of each input has been freeze, also. - std::vector input_node_names = { - "src", - "r1i", - "r2i", - "r3i", - "r4i" - }; - // hardcode output node names, hint only. - std::vector output_node_names = { - "fgr", - "pha", - "r1o", - "r2o", - "r3o", - "r4o" - }; - bool context_is_update = false; - bool context_is_initialized = false; - - private: - enum VARIANT - { - MOBILENETV3 = 0, - RESNET50 = 1 - }; - // will be update inner video matting process. - ncnn::Mat r1i, r2i, r3i, r4i; - // input size & variant_type, initialize at runtime. - const int input_height; - const int input_width; - const unsigned int variant_type; - - private: - - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - void initialize_context(); - - void generate_matting(ncnn::Extractor &extractor, - types::MattingContent &content, - int img_h, int img_w); - - void update_context(ncnn::Extractor &extractor); - - public: - /** - * Image Matting Using RVM(https://github.com/PeterL1n/RobustVideoMatting) - * @param mat: cv::Mat BGR HWC - * @param content: types::MattingContent to catch the detected results. - * @param video_mode: false by default. - * See https://github.com/PeterL1n/RobustVideoMatting/blob/master/documentation/inference_zh_Hans.md - */ - void detect(const cv::Mat &mat, types::MattingContent &content, bool video_mode = false); - /** - * Video Matting Using RVM(https://github.com/PeterL1n/RobustVideoMatting) - * @param video_path: eg. xxx/xxx/input.mp4 - * @param output_path: eg. xxx/xxx/output.mp4 - * @param contents: vector of MattingContent to catch the detected results. - * @param save_contents: false by default, whether to save MattingContent. - * See https://github.com/PeterL1n/RobustVideoMatting/blob/master/documentation/inference_zh_Hans.md - * @param writer_fps: FPS for VideoWriter, 20 by default. - */ - void detect_video(const std::string &video_path, - const std::string &output_path, - std::vector &contents, - bool save_contents = false, - unsigned int writer_fps = 20); - - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_RVM_H - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lite/ncnn/cv/ncnn_scrfd.cpp b/lite/ncnn/cv/ncnn_scrfd.cpp deleted file mode 100644 index d9f35bdd..00000000 --- a/lite/ncnn/cv/ncnn_scrfd.cpp +++ /dev/null @@ -1,433 +0,0 @@ -// -// Created by DefTruth on 2021/12/30. -// - -#include "ncnn_scrfd.h" - -using ncnncv::NCNNSCRFD; - -NCNNSCRFD::NCNNSCRFD(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads, - int _input_height, - int _input_width) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ - input_height = _input_height; - input_width = _input_width; - initial_context(); -} - -void NCNNSCRFD::initial_context() -{ - if (num_outputs == 6) - { - fmc = 3; - feat_stride_fpn = {8, 16, 32}; - num_anchors = 2; - use_kps = false; - } // kps - else if (num_outputs == 9) - { - fmc = 3; - feat_stride_fpn = {8, 16, 32}; - num_anchors = 2; - use_kps = true; - } -} - -void NCNNSCRFD::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - SCRFDScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(0, 0, 0)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.ratio = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.flag = true; -} - -void NCNNSCRFD::transform(const cv::Mat &mat_rs, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNSCRFD::detect(const cv::Mat &mat, std::vector &detected_boxes_kps, - float score_threshold, float iou_threshold, unsigned int topk) -{ - if (mat.empty()) return; - auto img_height = static_cast(mat.rows); - auto img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - SCRFDScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - // 1. make input tensor - ncnn::Mat input; - this->transform(mat_rs, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input.1", input); - - // 3. rescale & exclude. - std::vector bbox_kps_collection; - this->generate_bboxes_kps(scale_params, bbox_kps_collection, extractor, - score_threshold, img_height, img_width); - // 4. hard nms with topk. - this->nms_bboxes_kps(bbox_kps_collection, detected_boxes_kps, iou_threshold, topk); -} - -void NCNNSCRFD::generate_points(const int target_height, const int target_width) -{ - if (center_points_is_update) return; - // 8, 16, 32 - for (auto stride : feat_stride_fpn) - { - unsigned int num_grid_w = target_width / stride; - unsigned int num_grid_h = target_height / stride; - // y - for (unsigned int i = 0; i < num_grid_h; ++i) - { - // x - for (unsigned int j = 0; j < num_grid_w; ++j) - { - // num_anchors, col major - for (unsigned int k = 0; k < num_anchors; ++k) - { - SCRFDPoint point; - point.cx = (float) j; - point.cy = (float) i; - point.stride = (float) stride; - center_points[stride].push_back(point); - } - - } - } - } - - center_points_is_update = true; -} - -void NCNNSCRFD::generate_bboxes_kps(const SCRFDScaleParams &scale_params, - std::vector &bbox_kps_collection, - ncnn::Extractor &extractor, float score_threshold, - float img_height, float img_width) -{ - // score_8,score_16,score_32,bbox_8,bbox_16,bbox_32 - ncnn::Mat score_8, score_16, score_32, bbox_8, bbox_16, bbox_32; - extractor.extract("score_8", score_8); - extractor.extract("score_16", score_16); - extractor.extract("score_32", score_32); - extractor.extract("bbox_8", bbox_8); - extractor.extract("bbox_16", bbox_16); - extractor.extract("bbox_32", bbox_32); -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(score_8, "score_8"); - BasicNCNNHandler::print_shape(score_16, "score_16"); - BasicNCNNHandler::print_shape(score_32, "score_32"); - BasicNCNNHandler::print_shape(bbox_8, "bbox_8"); - BasicNCNNHandler::print_shape(bbox_16, "bbox_16"); - BasicNCNNHandler::print_shape(bbox_32, "bbox_32"); -#endif - this->generate_points(input_height, input_width); - - bbox_kps_collection.clear(); - - if (use_kps) - { - ncnn::Mat kps_8, kps_16, kps_32; - extractor.extract("kps_8", kps_8); - extractor.extract("kps_16", kps_16); - extractor.extract("kps_32", kps_32); -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(kps_8, "kps_8"); - BasicNCNNHandler::print_shape(kps_16, "kps_16"); - BasicNCNNHandler::print_shape(kps_32, "kps_32"); -#endif - // level 8 & 16 & 32 with kps - this->generate_bboxes_kps_single_stride(scale_params, score_8, bbox_8, kps_8, 8, score_threshold, - img_height, img_width, bbox_kps_collection); - this->generate_bboxes_kps_single_stride(scale_params, score_16, bbox_16, kps_16, 16, score_threshold, - img_height, img_width, bbox_kps_collection); - this->generate_bboxes_kps_single_stride(scale_params, score_32, bbox_32, kps_32, 32, score_threshold, - img_height, img_width, bbox_kps_collection); - } // no kps - else - { - // level 8 & 16 & 32 - this->generate_bboxes_single_stride(scale_params, score_8, bbox_8, 8, score_threshold, - img_height, img_width, bbox_kps_collection); - this->generate_bboxes_single_stride(scale_params, score_16, bbox_16, 16, score_threshold, - img_height, img_width, bbox_kps_collection); - this->generate_bboxes_single_stride(scale_params, score_32, bbox_32, 32, score_threshold, - img_height, img_width, bbox_kps_collection); - } -#if LITENCNN_DEBUG - std::cout << "generate_bboxes_kps num: " << bbox_kps_collection.size() << "\n"; -#endif -} - -void NCNNSCRFD::generate_bboxes_single_stride( - const SCRFDScaleParams &scale_params, ncnn::Mat &score_pred, ncnn::Mat &bbox_pred, - unsigned int stride, float score_threshold, float img_height, float img_width, - std::vector &bbox_kps_collection) -{ - unsigned int nms_pre_ = (stride / 8) * nms_pre; // 1 * 1000,2*1000,... - nms_pre_ = nms_pre_ >= nms_pre ? nms_pre_ : nms_pre; - - const unsigned int num_points = score_pred.h; // 12800 - const float *score_ptr = (float *) score_pred.data; // [1,12800,1] - const float *bbox_ptr = (float *) bbox_pred.data; // [1,12800,4] - - float ratio = scale_params.ratio; - int dw = scale_params.dw; - int dh = scale_params.dh; - - unsigned int count = 0; - auto &stride_points = center_points[stride]; - - for (unsigned int i = 0; i < num_points; ++i) - { - const float cls_conf = score_ptr[i]; - if (cls_conf < score_threshold) continue; // filter - auto &point = stride_points.at(i); - const float cx = point.cx; // cx - const float cy = point.cy; // cy - const float s = point.stride; // stride - - // bbox - const float *offsets = bbox_ptr + i * 4; - float l = offsets[0]; // left - float t = offsets[1]; // top - float r = offsets[2]; // right - float b = offsets[3]; // bottom - - types::BoxfWithLandmarks box_kps; - float x1 = ((cx - l) * s - (float) dw) / ratio; // cx - l x1 - float y1 = ((cy - t) * s - (float) dh) / ratio; // cy - t y1 - float x2 = ((cx + r) * s - (float) dw) / ratio; // cx + r x2 - float y2 = ((cy + b) * s - (float) dh) / ratio; // cy + b y2 - box_kps.box.x1 = std::max(0.f, x1); - box_kps.box.y1 = std::max(0.f, y1); - box_kps.box.x2 = std::min(img_width - 1.f, x2); - box_kps.box.y2 = std::min(img_height - 1.f, y2); - box_kps.box.score = cls_conf; - box_kps.box.label = 1; - box_kps.box.label_text = "face"; - box_kps.box.flag = true; - box_kps.flag = true; - - bbox_kps_collection.push_back(box_kps); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - - if (bbox_kps_collection.size() > nms_pre_) - { - std::sort( - bbox_kps_collection.begin(), bbox_kps_collection.end(), - [](const types::BoxfWithLandmarks &a, const types::BoxfWithLandmarks &b) - { return a.box.score > b.box.score; } - ); // sort inplace - // trunc - bbox_kps_collection.resize(nms_pre_); - } -} - -void NCNNSCRFD::generate_bboxes_kps_single_stride( - const SCRFDScaleParams &scale_params, ncnn::Mat &score_pred, ncnn::Mat &bbox_pred, - ncnn::Mat &kps_pred, unsigned int stride, float score_threshold, float img_height, - float img_width, std::vector &bbox_kps_collection) -{ - unsigned int nms_pre_ = (stride / 8) * nms_pre; // 1 * 1000,2*1000,... - nms_pre_ = nms_pre_ >= nms_pre ? nms_pre_ : nms_pre; - - const unsigned int num_points = score_pred.h; // 12800 - const float *score_ptr = (float *) score_pred.data; // [1,12800,1] - const float *bbox_ptr = (float *) bbox_pred.data; // [1,12800,4] - const float *kps_ptr = (float *) kps_pred.data; // [1,12800,10] - - float ratio = scale_params.ratio; - int dw = scale_params.dw; - int dh = scale_params.dh; - - unsigned int count = 0; - auto &stride_points = center_points[stride]; - - for (unsigned int i = 0; i < num_points; ++i) - { - const float cls_conf = score_ptr[i]; - if (cls_conf < score_threshold) continue; // filter - auto &point = stride_points.at(i); - const float cx = point.cx; // cx - const float cy = point.cy; // cy - const float s = point.stride; // stride - - // bbox - const float *offsets = bbox_ptr + i * 4; - float l = offsets[0]; // left - float t = offsets[1]; // top - float r = offsets[2]; // right - float b = offsets[3]; // bottom - - types::BoxfWithLandmarks box_kps; - float x1 = ((cx - l) * s - (float) dw) / ratio; // cx - l x1 - float y1 = ((cy - t) * s - (float) dh) / ratio; // cy - t y1 - float x2 = ((cx + r) * s - (float) dw) / ratio; // cx + r x2 - float y2 = ((cy + b) * s - (float) dh) / ratio; // cy + b y2 - box_kps.box.x1 = std::max(0.f, x1); - box_kps.box.y1 = std::max(0.f, y1); - box_kps.box.x2 = std::min(img_width - 1.f, x2); - box_kps.box.y2 = std::min(img_height - 1.f, y2); - box_kps.box.score = cls_conf; - box_kps.box.label = 1; - box_kps.box.label_text = "face"; - box_kps.box.flag = true; - - // landmarks - const float *kps_offsets = kps_ptr + i * 10; - for (unsigned int j = 0; j < 10; j += 2) - { - cv::Point2f kps; - float kps_l = kps_offsets[j]; - float kps_t = kps_offsets[j + 1]; - float kps_x = ((cx + kps_l) * s - (float) dw) / ratio; // cx - l x - float kps_y = ((cy + kps_t) * s - (float) dh) / ratio; // cy - t y - kps.x = std::min(std::max(0.f, kps_x), img_width - 1.f); - kps.y = std::min(std::max(0.f, kps_y), img_height - 1.f); - box_kps.landmarks.points.push_back(kps); - } - box_kps.landmarks.flag = true; - box_kps.flag = true; - - bbox_kps_collection.push_back(box_kps); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - - if (bbox_kps_collection.size() > nms_pre_) - { - std::sort( - bbox_kps_collection.begin(), bbox_kps_collection.end(), - [](const types::BoxfWithLandmarks &a, const types::BoxfWithLandmarks &b) - { return a.box.score > b.box.score; } - ); // sort inplace - // trunc - bbox_kps_collection.resize(nms_pre_); - } - -} - -void NCNNSCRFD::nms_bboxes_kps(std::vector &input, - std::vector &output, - float iou_threshold, unsigned int topk) -{ - if (input.empty()) return; - std::sort( - input.begin(), input.end(), - [](const types::BoxfWithLandmarks &a, const types::BoxfWithLandmarks &b) - { return a.box.score > b.box.score; } - ); - const unsigned int box_num = input.size(); - std::vector merged(box_num, 0); - - unsigned int count = 0; - for (unsigned int i = 0; i < box_num; ++i) - { - if (merged[i]) continue; - std::vector buf; - - buf.push_back(input[i]); - merged[i] = 1; - - for (unsigned int j = i + 1; j < box_num; ++j) - { - if (merged[j]) continue; - - float iou = static_cast(input[i].box.iou_of(input[j].box)); - - if (iou > iou_threshold) - { - merged[j] = 1; - buf.push_back(input[j]); - } - - } - output.push_back(buf[0]); - - // keep top k - count += 1; - if (count >= topk) - break; - } -} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lite/ncnn/cv/ncnn_scrfd.h b/lite/ncnn/cv/ncnn_scrfd.h deleted file mode 100644 index 90719623..00000000 --- a/lite/ncnn/cv/ncnn_scrfd.h +++ /dev/null @@ -1,112 +0,0 @@ -// -// Created by DefTruth on 2021/12/30. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_SCRFD_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_SCRFD_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNSCRFD : public BasicNCNNHandler - { - public: - explicit NCNNSCRFD(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1, - int _input_height = 320, - int _input_width = 320); - - ~NCNNSCRFD() override = default; - - private: - // nested classes - typedef struct - { - float cx; - float cy; - float stride; - } SCRFDPoint; - typedef struct - { - float ratio; - int dw; - int dh; - bool flag; - } SCRFDScaleParams; - - private: - // blob = cv2.dnn.blobFromImage(img, 1.0/128, input_size, (127.5, 127.5, 127.5), swapRB=True) - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 128.f, 1.f / 128.f, 1.f / 128.f}; - // multi-levels center points - int input_height = 320; - int input_width = 320; - unsigned int fmc = 3; // feature map count - bool use_kps = false; - unsigned int num_anchors = 2; - std::vector feat_stride_fpn = {8, 16, 32}; // steps, may [8, 16, 32, 64, 128] - // if num_anchors>1, then stack points in col major -> (height*num_anchor*width,2) - // anchor_centers = np.stack([anchor_centers]*self._num_anchors, axis=1).reshape( (-1,2) ) - std::unordered_map> center_points; - bool center_points_is_update = false; - static constexpr const unsigned int nms_pre = 1000; - static constexpr const unsigned int max_nms = 30000; - - private: - - void transform(const cv::Mat &mat_rs, ncnn::Mat &in) override; - - // initial steps and num_anchors - // https://github.com/deepinsight/insightface/blob/master/detection/scrfd/tools/scrfd.py - void initial_context(); - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - SCRFDScaleParams &scale_params); - - // generate once. - void generate_points(const int target_height, const int target_width); - - void generate_bboxes_single_stride(const SCRFDScaleParams &scale_params, - ncnn::Mat &score_pred, - ncnn::Mat &bbox_pred, - unsigned int stride, - float score_threshold, - float img_height, - float img_width, - std::vector &bbox_kps_collection); - - void generate_bboxes_kps_single_stride(const SCRFDScaleParams &scale_params, - ncnn::Mat &score_pred, - ncnn::Mat &bbox_pred, - ncnn::Mat &kps_pred, - unsigned int stride, - float score_threshold, - float img_height, - float img_width, - std::vector &bbox_kps_collection); - - void generate_bboxes_kps(const SCRFDScaleParams &scale_params, - std::vector &bbox_kps_collection, - ncnn::Extractor &extractor, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms_bboxes_kps(std::vector &input, - std::vector &output, - float iou_threshold, unsigned int topk); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes_kps, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 400); - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_SCRFD_H - - diff --git a/lite/ncnn/cv/ncnn_shufflenetv2.cpp b/lite/ncnn/cv/ncnn_shufflenetv2.cpp deleted file mode 100644 index 8a9740a7..00000000 --- a/lite/ncnn/cv/ncnn_shufflenetv2.cpp +++ /dev/null @@ -1,62 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "ncnn_shufflenetv2.h" -#include "lite/utils.h" - -using ncnncv::NCNNShuffleNetV2; - -NCNNShuffleNetV2::NCNNShuffleNetV2(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNShuffleNetV2::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNShuffleNetV2::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - // 3. fetch. - ncnn::Mat logits_mat; - extractor.extract("output", logits_mat); // c=1,h=1,w=1000 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(logits_mat, "output"); -#endif - - const unsigned int num_classes = logits_mat.w; - const float *logits = (float *) logits_mat.data; - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_shufflenetv2.h b/lite/ncnn/cv/ncnn_shufflenetv2.h deleted file mode 100644 index e28ca9bd..00000000 --- a/lite/ncnn/cv/ncnn_shufflenetv2.h +++ /dev/null @@ -1,414 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_SHUFFLENETV2_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_SHUFFLENETV2_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNShuffleNetV2 : public BasicNCNNHandler - { - public: - explicit NCNNShuffleNetV2(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); - - ~NCNNShuffleNetV2() override = default; - - private: - const int input_height = 224; - const int input_width = 224; - const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f}; - const float norm_vals[3] = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - - }; -} - - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_SHUFFLENETV2_H diff --git a/lite/ncnn/cv/ncnn_sphere_face.cpp b/lite/ncnn/cv/ncnn_sphere_face.cpp deleted file mode 100644 index f95de198..00000000 --- a/lite/ncnn/cv/ncnn_sphere_face.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "ncnn_sphere_face.h" - -using ncnncv::NCNNSphereFace; - -void NCNNSphereFace::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW - int h = mat.rows; - int w = mat.cols; - in = ncnn::Mat::from_pixels_resize( - mat.data, ncnn::Mat::PIXEL_BGR, - w, h, input_width, input_height - ); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNSphereFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - ncnn::Mat embedding; - extractor.extract("embedding", embedding); - - const unsigned int hidden_dim = embedding.w; // 512 - const float *embedding_values = (float *) embedding.data; - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_sphere_face.h b/lite/ncnn/cv/ncnn_sphere_face.h deleted file mode 100644 index 966ee8a0..00000000 --- a/lite/ncnn/cv/ncnn_sphere_face.h +++ /dev/null @@ -1,37 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_SPHERE_FACE_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_SPHERE_FACE_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNSphereFace : public BasicNCNNHandler - { - public: - explicit NCNNSphereFace(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) - {}; - - ~NCNNSphereFace() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 128.0f, 1.f / 128.0f, 1.f / 128.0f}; - static constexpr const int input_width = 96; - static constexpr const int input_height = 112; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_SPHERE_FACE_H diff --git a/lite/ncnn/cv/ncnn_subpixel_cnn.cpp b/lite/ncnn/cv/ncnn_subpixel_cnn.cpp deleted file mode 100644 index e8243d2f..00000000 --- a/lite/ncnn/cv/ncnn_subpixel_cnn.cpp +++ /dev/null @@ -1,83 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "ncnn_subpixel_cnn.h" - -using ncnncv::NCNNSubPixelCNN; - -NCNNSubPixelCNN::NCNNSubPixelCNN( - const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) -{ -} - -void NCNNSubPixelCNN::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_y; // assume that input mat is Y of YCrCb - mat.convertTo(mat_y, CV_32FC1, 1.0f / 255.0f, 0.f); // (224,224,1) range (0.,1.0) - - in = ncnn::Mat(input_width, input_height, mat_y.data); -} - -void NCNNSubPixelCNN::detect(const cv::Mat &mat, types::SuperResolutionContent &super_resolution_content) -{ - if (mat.empty()) return; - cv::Mat mat_copy = mat.clone(); - cv::resize(mat_copy, mat_copy, cv::Size(input_width, input_height)); // (224,224,3) - cv::Mat mat_ycrcb, mat_y, mat_cr, mat_cb; - cv::cvtColor(mat_copy, mat_ycrcb, cv::COLOR_BGR2YCrCb); - - // 0. split - std::vector split_mats; - cv::split(mat_ycrcb, split_mats); - mat_y = split_mats.at(0); // (224,224,1) uchar CV_8UC1 - mat_cr = split_mats.at(1); - mat_cb = split_mats.at(2); - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat_y, input); // (1,1,224,224) - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - // 3. fetch. - ncnn::Mat pred; - extractor.extract("output", pred); // (1,1,672,672) -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(pred, "output"); -#endif - - const unsigned int rows = pred.h; // H - const unsigned int cols = pred.w; // W - - float *pred_ptr = (float *) pred.data; - - mat_y = cv::Mat(rows, cols, CV_32FC1, pred_ptr); // release & create - - mat_y *= 255.0f; - - mat_y.convertTo(mat_y, CV_8UC1); - - cv::resize(mat_cr, mat_cr, cv::Size(cols, rows)); - cv::resize(mat_cb, mat_cb, cv::Size(cols, rows)); - - std::vector out_mats; - out_mats.push_back(mat_y); - out_mats.push_back(mat_cr); - out_mats.push_back(mat_cb); - - // 3. merge - cv::merge(out_mats, super_resolution_content.mat); - if (super_resolution_content.mat.empty()) - { - super_resolution_content.flag = false; - return; - } - cv::cvtColor(super_resolution_content.mat, super_resolution_content.mat, cv::COLOR_YCrCb2BGR); - super_resolution_content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_subpixel_cnn.h b/lite/ncnn/cv/ncnn_subpixel_cnn.h deleted file mode 100644 index b0df980b..00000000 --- a/lite/ncnn/cv/ncnn_subpixel_cnn.h +++ /dev/null @@ -1,35 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_SUBPIXEL_CNN_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_SUBPIXEL_CNN_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNSubPixelCNN : public BasicNCNNHandler - { - public: - explicit NCNNSubPixelCNN(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1); // - ~NCNNSubPixelCNN() override = default; - - private: - int input_height = 224; - int input_width = 224; - - private: - - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::SuperResolutionContent &super_resolution_content); - - }; -} - - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_SUBPIXEL_CNN_H diff --git a/lite/ncnn/cv/ncnn_tencent_cifp_face.cpp b/lite/ncnn/cv/ncnn_tencent_cifp_face.cpp deleted file mode 100644 index 76fa561a..00000000 --- a/lite/ncnn/cv/ncnn_tencent_cifp_face.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "ncnn_tencent_cifp_face.h" - -using ncnncv::NCNNTencentCifpFace; - -void NCNNTencentCifpFace::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW - int h = mat.rows; - int w = mat.cols; - in = ncnn::Mat::from_pixels_resize( - mat.data, ncnn::Mat::PIXEL_BGR2RGB, - w, h, input_width, input_height - ); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNTencentCifpFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - ncnn::Mat embedding; - extractor.extract("embedding", embedding); - - const unsigned int hidden_dim = embedding.w; // 512 - const float *embedding_values = (float *) embedding.data; - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_tencent_cifp_face.h b/lite/ncnn/cv/ncnn_tencent_cifp_face.h deleted file mode 100644 index 5d0a0f00..00000000 --- a/lite/ncnn/cv/ncnn_tencent_cifp_face.h +++ /dev/null @@ -1,37 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_TENCENT_CIFP_FACE_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_TENCENT_CIFP_FACE_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNTencentCifpFace : public BasicNCNNHandler - { - public: - explicit NCNNTencentCifpFace(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) - {}; - - ~NCNNTencentCifpFace() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - static constexpr const int input_width = 112; - static constexpr const int input_height = 112; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_TENCENT_CIFP_FACE_H diff --git a/lite/ncnn/cv/ncnn_tencent_curricular_face.cpp b/lite/ncnn/cv/ncnn_tencent_curricular_face.cpp deleted file mode 100644 index 15018545..00000000 --- a/lite/ncnn/cv/ncnn_tencent_curricular_face.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "ncnn_tencent_curricular_face.h" - -using ncnncv::NCNNTencentCurricularFace; - -void NCNNTencentCurricularFace::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW - int h = mat.rows; - int w = mat.cols; - in = ncnn::Mat::from_pixels_resize( - mat.data, ncnn::Mat::PIXEL_BGR2RGB, - w, h, input_width, input_height - ); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNTencentCurricularFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - ncnn::Mat embedding; - extractor.extract("embedding", embedding); - - const unsigned int hidden_dim = embedding.w; // 512 - const float *embedding_values = (float *) embedding.data; - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_tencent_curricular_face.h b/lite/ncnn/cv/ncnn_tencent_curricular_face.h deleted file mode 100644 index dde5ddb1..00000000 --- a/lite/ncnn/cv/ncnn_tencent_curricular_face.h +++ /dev/null @@ -1,37 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_TENCENT_CURRICULAR_FACE_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_TENCENT_CURRICULAR_FACE_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNTencentCurricularFace : public BasicNCNNHandler - { - public: - explicit NCNNTencentCurricularFace(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads) - {}; - - ~NCNNTencentCurricularFace() override = default; - - private: - const float mean_vals[3] = {127.5f, 127.5f, 127.5f}; // RGB - const float norm_vals[3] = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - static constexpr const int input_width = 112; - static constexpr const int input_height = 112; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_TENCENT_CURRICULAR_FACE_H diff --git a/lite/ncnn/cv/ncnn_ultraface.cpp b/lite/ncnn/cv/ncnn_ultraface.cpp deleted file mode 100644 index e182082e..00000000 --- a/lite/ncnn/cv/ncnn_ultraface.cpp +++ /dev/null @@ -1,217 +0,0 @@ -// -// Created by DefTruth on 2021/11/20. -// - -#include "ncnn_ultraface.h" -#include "lite/utils.h" - -using ncnncv::NCNNUltraFace; - -NCNNUltraFace::NCNNUltraFace(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads, - int _input_height, - int _input_width) : - BasicNCNNHandler(_param_path, _bin_path, _num_threads), - input_height(_input_height), input_width(_input_width) -{ -} - -void NCNNUltraFace::transform(const cv::Mat &mat, ncnn::Mat &in) -{ - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNUltraFace::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - // 3.rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(bbox_collection, extractor, score_threshold, img_height, img_width); - // 4. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void NCNNUltraFace::generate_anchors(const int target_height, const int target_width, - std::vector &anchors) -{ - std::vector> feature_maps; - for (auto step: steps) - { - feature_maps.push_back( - { - (int) std::ceil((float) target_height / (float) step), - (int) std::ceil((float) target_width / (float) step) - } // ceil - ); - } - - anchors.clear(); - const int num_feature_map = feature_maps.size(); - - for (int k = 0; k < num_feature_map; ++k) - { - auto f_map = feature_maps.at(k); // e.g [640//8,640//8] - auto tmp_min_sizes = min_sizes.at(k); // e.g [8,16] - int f_h = f_map.at(0); - int f_w = f_map.at(1); - - for (int i = 0; i < f_h; ++i) - { - for (int j = 0; j < f_w; ++j) - { - for (auto min_size: tmp_min_sizes) - { - float s_kx = (float) min_size / (float) target_width; // e.g 16/w - float s_ky = (float) min_size / (float) target_height; // e.g 16/h - // (x + 0.5) * step / w normalized loc mapping to input width - // (y + 0.5) * step / h normalized loc mapping to input height - float cx = ((float) j + 0.5f) * (float) steps.at(k) / (float) target_width; - float cy = ((float) i + 0.5f) * (float) steps.at(k) / (float) target_height; - - anchors.push_back(UltraAnchor{cx, cy, s_kx, s_ky}); // without clip - } - } - } - } -} - -void NCNNUltraFace::generate_bboxes(std::vector &bbox_collection, - ncnn::Extractor &extractor, float score_threshold, - float img_height, float img_width) -{ - ncnn::Mat boxes, scores; - extractor.extract("boxes", boxes); // c=1 h=? w=4 - extractor.extract("scores", scores); // c=1 h=? w=2 -#ifdef LITENCNN_DEBUG - BasicNCNNHandler::print_shape(boxes, "boxes"); - BasicNCNNHandler::print_shape(scores, "scores"); -#endif - const unsigned int bbox_num = boxes.h; // n = ? - - std::vector anchors; - this->generate_anchors(input_height, input_width, anchors); - - const unsigned int num_anchors = anchors.size(); - if (num_anchors != bbox_num) - throw std::runtime_error("mismatch num_anchors != bbox_num"); - - const float *bboxes_ptr = (float *) boxes.data; - const float *probs_ptr = (float *) scores.data; - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - float conf = probs_ptr[2 * i + 1]; - if (conf < score_threshold) continue; // filter first. - - float prior_cx = anchors.at(i).cx; - float prior_cy = anchors.at(i).cy; - float prior_s_kx = anchors.at(i).s_kx; - float prior_s_ky = anchors.at(i).s_ky; - - float dx = bboxes_ptr[4 * i + 0]; - float dy = bboxes_ptr[4 * i + 1]; - float dw = bboxes_ptr[4 * i + 2]; - float dh = bboxes_ptr[4 * i + 3]; - // ref: https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB/ - // blob/master/ncnn/src/UltraFace.cpp - float cx = prior_cx + dx * variance[0] * prior_s_kx; - float cy = prior_cy + dy * variance[0] * prior_s_ky; - float w = prior_s_kx * std::exp(dw * variance[1]); - float h = prior_s_ky * std::exp(dh * variance[1]); // norm coor (0.,1.) - - types::Boxf box; - box.x1 = (cx - w / 2.f) * img_width; - box.y1 = (cy - h / 2.f) * img_height; - box.x2 = (cx + w / 2.f) * img_width; - box.y2 = (cy + h / 2.f) * img_height; - box.score = conf; - box.label = 1; - box.label_text = "face"; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - -#if LITENCNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void NCNNUltraFace::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lite/ncnn/cv/ncnn_ultraface.h b/lite/ncnn/cv/ncnn_ultraface.h deleted file mode 100644 index 702daba7..00000000 --- a/lite/ncnn/cv/ncnn_ultraface.h +++ /dev/null @@ -1,82 +0,0 @@ -// -// Created by DefTruth on 2021/11/20. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_ULTRAFACE_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_ULTRAFACE_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - // reference: - // https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB/ - // blob/master/ncnn/src/UltraFace.hpp - class LITE_EXPORTS NCNNUltraFace : public BasicNCNNHandler - { - public: - explicit NCNNUltraFace(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1, - int _input_height = 320, - int _input_width = 240); - - ~NCNNUltraFace() override = default; - - private: - // nested classes - struct UltraAnchor - { - float cx; - float cy; - float s_kx; - float s_ky; - }; - - private: - const int input_height; // 640/320 - const int input_width; // 640/320 - const float mean_vals[3] = {127.f, 127.f, 127.f}; - const float norm_vals[3] = {1.0f / 128.f, 1.0f / 128.f, 1.0f / 128.f}; - - const float variance[2] = {0.1f, 0.2f}; - std::vector steps = {8, 16, 32, 64}; - std::vector> min_sizes = { - {10, 16, 24}, - {32, 48}, - {64, 96}, - {128, 192, 256} - }; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void transform(const cv::Mat &mat, ncnn::Mat &in) override; - - void generate_anchors(const int target_height, - const int target_width, - std::vector &anchors); - - - void generate_bboxes(std::vector &bbox_collection, - ncnn::Extractor &extractor, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.7f, float iou_threshold = 0.3f, - unsigned int topk = 300, unsigned int nms_type = 0); - - }; -} - - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_ULTRAFACE_H diff --git a/lite/ncnn/cv/ncnn_yolo5face.cpp b/lite/ncnn/cv/ncnn_yolo5face.cpp deleted file mode 100644 index a30846ce..00000000 --- a/lite/ncnn/cv/ncnn_yolo5face.cpp +++ /dev/null @@ -1,467 +0,0 @@ -// -// Created by DefTruth on 2022/1/16. -// - -#include "ncnn_yolo5face.h" - -using ncnncv::NCNNYOLO5Face; - -NCNNYOLO5Face::NCNNYOLO5Face(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads, - int _input_height, - int _input_width) : - log_id(_param_path.data()), param_path(_param_path.data()), - bin_path(_bin_path.data()), num_threads(_num_threads), - input_height(_input_height), input_width(_input_width) -{ - net = new ncnn::Net(); - // init net, change this setting for better performance. - net->opt.use_fp16_arithmetic = false; - net->opt.use_vulkan_compute = false; // default - // yolo5face --> no Focus layer in yolo5face - net->load_param(param_path); - net->load_model(bin_path); -#ifdef LITENCNN_DEBUG - this->print_debug_string(); -#endif -} - -NCNNYOLO5Face::~NCNNYOLO5Face() -{ - if (net) delete net; - net = nullptr; -} - -void NCNNYOLO5Face::transform(const cv::Mat &mat_rs, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNYOLO5Face::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YOLO5FaceScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(0, 0, 0)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.ratio = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.flag = true; -} - -void NCNNYOLO5Face::detect(const cv::Mat &mat, std::vector &detected_boxes_kps, - float score_threshold, float iou_threshold, unsigned int topk) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - // resize & unscale - cv::Mat mat_rs; - YOLO5FaceScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - // 1. make input tensor - ncnn::Mat input; - this->transform(mat_rs, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("input", input); - - // 3. rescale & exclude. - std::vector bbox_kps_collection; - this->generate_bboxes_kps(scale_params, bbox_kps_collection, extractor, - score_threshold, img_height, img_width); - // 4. hard nms with topk. - this->nms_bboxes_kps(bbox_kps_collection, detected_boxes_kps, iou_threshold, topk); -} - -void NCNNYOLO5Face::generate_anchors(unsigned int target_height, unsigned int target_width) -{ - if (center_anchors_is_update) return; - - for (auto stride : strides) - { - unsigned int num_grid_w = target_width / stride; - unsigned int num_grid_h = target_height / stride; - std::vector anchors; - - if (stride == 8) - { - // 0 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YOLO5FaceAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 4.f; - anchor.height = 5.f; - anchors.push_back(anchor); - } - } - // 1 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YOLO5FaceAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 8.f; - anchor.height = 10.f; - anchors.push_back(anchor); - } - } - // 2 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YOLO5FaceAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 13.f; - anchor.height = 16.f; - anchors.push_back(anchor); - } - } - } // 16 - else if (stride == 16) - { - // 0 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YOLO5FaceAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 23.f; - anchor.height = 29.f; - anchors.push_back(anchor); - } - } - // 1 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YOLO5FaceAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 43.f; - anchor.height = 55.f; - anchors.push_back(anchor); - } - } - // 2 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YOLO5FaceAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 73.f; - anchor.height = 105.f; - anchors.push_back(anchor); - } - } - } // 32 - else - { - // 0 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YOLO5FaceAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 146.f; - anchor.height = 217.f; - anchors.push_back(anchor); - } - } - // 1 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YOLO5FaceAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 231.f; - anchor.height = 300.f; - anchors.push_back(anchor); - } - } - // 2 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YOLO5FaceAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 335.f; - anchor.height = 433.f; - anchors.push_back(anchor); - } - } - } - center_anchors[stride] = anchors; - } - - center_anchors_is_update = true; -} - -void NCNNYOLO5Face::generate_bboxes_kps(const YOLO5FaceScaleParams &scale_params, - std::vector &bbox_kps_collection, - ncnn::Extractor &extractor, float score_threshold, - float img_height, float img_width) -{ -// (1,n,16=4+1+10+1=cxcy+cwch+obj_conf+5kps+cls_conf) - ncnn::Mat det_stride_8, det_stride_16, det_stride_32; - extractor.extract("det_stride_8", det_stride_8); - extractor.extract("det_stride_16", det_stride_16); - extractor.extract("det_stride_32", det_stride_32); - - this->generate_anchors(input_height, input_width); - - // generate bounding boxes. - bbox_kps_collection.clear(); - - this->generate_bboxes_kps_single_stride(scale_params, det_stride_8, 8, score_threshold, - img_height, img_width, bbox_kps_collection); - this->generate_bboxes_kps_single_stride(scale_params, det_stride_16, 16, score_threshold, - img_height, img_width, bbox_kps_collection); - this->generate_bboxes_kps_single_stride(scale_params, det_stride_32, 32, score_threshold, - img_height, img_width, bbox_kps_collection); -#if LITENCNN_DEBUG - std::cout << "generate_bboxes_kps num: " << bbox_kps_collection.size() << "\n"; -#endif -} - -// inner function -static inline float sigmoid(float x) -{ - return static_cast(1.f / (1.f + std::exp(-x))); -} - -void NCNNYOLO5Face::generate_bboxes_kps_single_stride( - const YOLO5FaceScaleParams &scale_params, - ncnn::Mat &det_pred, unsigned int stride, - float score_threshold, float img_height, float img_width, - std::vector &bbox_kps_collection) -{ - unsigned int nms_pre_ = (stride / 8) * nms_pre; // 1 * 1000,2*1000,... - nms_pre_ = nms_pre_ >= nms_pre ? nms_pre_ : nms_pre; - - const unsigned int f_h = (unsigned int) input_height / stride; - const unsigned int f_w = (unsigned int) input_width / stride; - // e.g, 3*80*80 + 3*40*40 + 3*20*20 = 25200 - const unsigned int num_anchors = 3 * f_h * f_w; - const float *output_ptr = (float *) det_pred.data; - - float r_ = scale_params.ratio; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - // have c=3 indicate 3 anchors at one grid - unsigned int count = 0; - auto &stride_anchors = center_anchors[stride]; - - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *row_ptr = output_ptr + i * 16; - float obj_conf = sigmoid(row_ptr[4]); - if (obj_conf < score_threshold) continue; // filter first. - float cls_conf = sigmoid(row_ptr[15]); - if (cls_conf < score_threshold) continue; // face score. - - int grid0 = stride_anchors.at(i).grid0; // w - int grid1 = stride_anchors.at(i).grid1; // h - float anchor_w = stride_anchors.at(i).width; - float anchor_h = stride_anchors.at(i).height; - - // bounding box - const float *offsets = row_ptr; - float dx = sigmoid(offsets[0]); - float dy = sigmoid(offsets[1]); - float dw = sigmoid(offsets[2]); - float dh = sigmoid(offsets[3]); - - float cx = (dx * 2.f - 0.5f + (float) grid0) * (float) stride; - float cy = (dy * 2.f - 0.5f + (float) grid1) * (float) stride; - float w = std::pow(dw * 2.f, 2) * anchor_w; - float h = std::pow(dh * 2.f, 2) * anchor_h; - - types::BoxfWithLandmarks box_kps; - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - box_kps.box.x1 = std::max(0.f, x1); - box_kps.box.y1 = std::max(0.f, y1); - box_kps.box.x2 = std::min(img_width - 1.f, x2); - box_kps.box.y2 = std::min(img_height - 1.f, y2); - box_kps.box.score = cls_conf; - box_kps.box.label = 1; - box_kps.box.label_text = "face"; - box_kps.box.flag = true; - - // landmarks - const float *kps_offsets = row_ptr + 5; - for (unsigned int j = 0; j < 10; j += 2) - { - float kps_dx = kps_offsets[j]; - float kps_dy = kps_offsets[j + 1]; - float kps_x = (kps_dx * anchor_w + grid0 * (float) stride); - float kps_y = (kps_dy * anchor_h + grid1 * (float) stride); - - cv::Point2f kps; - kps_x = (kps_x - (float) dw_) / r_; - kps_y = (kps_y - (float) dh_) / r_; - kps.x = std::min(std::max(0.f, kps_x), img_width - 1.f); - kps.y = std::min(std::max(0.f, kps_y), img_height - 1.f); - box_kps.landmarks.points.push_back(kps); - } - box_kps.landmarks.flag = true; - box_kps.flag = true; - - bbox_kps_collection.push_back(box_kps); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - - if (bbox_kps_collection.size() > nms_pre_) - { - std::sort( - bbox_kps_collection.begin(), bbox_kps_collection.end(), - [](const types::BoxfWithLandmarks &a, const types::BoxfWithLandmarks &b) - { return a.box.score > b.box.score; } - ); // sort inplace - // trunc - bbox_kps_collection.resize(nms_pre_); - } -} - -void NCNNYOLO5Face::nms_bboxes_kps(std::vector &input, - std::vector &output, - float iou_threshold, unsigned int topk) -{ - if (input.empty()) return; - std::sort( - input.begin(), input.end(), - [](const types::BoxfWithLandmarks &a, const types::BoxfWithLandmarks &b) - { return a.box.score > b.box.score; } - ); - const unsigned int box_num = input.size(); - std::vector merged(box_num, 0); - - unsigned int count = 0; - for (unsigned int i = 0; i < box_num; ++i) - { - if (merged[i]) continue; - std::vector buf; - - buf.push_back(input[i]); - merged[i] = 1; - - for (unsigned int j = i + 1; j < box_num; ++j) - { - if (merged[j]) continue; - - float iou = static_cast(input[i].box.iou_of(input[j].box)); - - if (iou > iou_threshold) - { - merged[j] = 1; - buf.push_back(input[j]); - } - - } - output.push_back(buf[0]); - - // keep top k - count += 1; - if (count >= topk) - break; - } -} - -void NCNNYOLO5Face::print_debug_string() -{ - std::cout << "LITENCNN_DEBUG LogId: " << log_id << "\n"; - input_indexes = net->input_indexes(); - output_indexes = net->output_indexes(); -#ifdef NCNN_STRING - input_names = net->input_names(); - output_names = net->output_names(); -#endif - std::cout << "=============== Input-Dims ==============\n"; - for (int i = 0; i < input_indexes.size(); ++i) - { - std::cout << "Input: "; - auto tmp_in_blob = net->blobs().at(input_indexes.at(i)); -#ifdef NCNN_STRING - std::cout << input_names.at(i) << ": "; -#endif - std::cout << "shape: c=" << tmp_in_blob.shape.c - << " h=" << tmp_in_blob.shape.h << " w=" << tmp_in_blob.shape.w << "\n"; - } - - std::cout << "=============== Output-Dims ==============\n"; - for (int i = 0; i < output_indexes.size(); ++i) - { - auto tmp_out_blob = net->blobs().at(output_indexes.at(i)); - std::cout << "Output: "; -#ifdef NCNN_STRING - std::cout << output_names.at(i) << ": "; -#endif - std::cout << "shape: c=" << tmp_out_blob.shape.c - << " h=" << tmp_out_blob.shape.h << " w=" << tmp_out_blob.shape.w << "\n"; - } - std::cout << "========================================\n"; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_yolo5face.h b/lite/ncnn/cv/ncnn_yolo5face.h deleted file mode 100644 index dd7ccd8b..00000000 --- a/lite/ncnn/cv/ncnn_yolo5face.h +++ /dev/null @@ -1,114 +0,0 @@ -// -// Created by DefTruth on 2022/1/16. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLO5FACE_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLO5FACE_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNYOLO5Face - { - private: - ncnn::Net *net = nullptr; - const char *log_id = nullptr; - const char *param_path = nullptr; - const char *bin_path = nullptr; - std::vector input_names; - std::vector output_names; - std::vector input_indexes; - std::vector output_indexes; - - private: - // nested classes - typedef struct - { - int grid0; - int grid1; - int stride; - float width; - float height; - } YOLO5FaceAnchor; - - typedef struct - { - float ratio; - int dw; - int dh; - bool flag; - } YOLO5FaceScaleParams; - - public: - explicit NCNNYOLO5Face(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1, - int _input_height = 640, - int _input_width = 640); // - ~NCNNYOLO5Face(); - - private: - const unsigned int num_threads; // initialize at runtime. - // target image size after resize - const int input_height; // 640 - const int input_width; // 640 - - const float mean_vals[3] = {0.f, 0.f, 0.f}; // RGB - const float norm_vals[3] = {1.0 / 255.f, 1.0 / 255.f, 1.0 / 255.f}; - static constexpr const unsigned int nms_pre = 1000; - static constexpr const unsigned int max_nms = 30000; - - std::vector strides = {8, 16, 32}; - std::unordered_map> center_anchors; - bool center_anchors_is_update = false; - - protected: - NCNNYOLO5Face(const NCNNYOLO5Face &) = delete; // - NCNNYOLO5Face(NCNNYOLO5Face &&) = delete; // - NCNNYOLO5Face &operator=(const NCNNYOLO5Face &) = delete; // - NCNNYOLO5Face &operator=(NCNNYOLO5Face &&) = delete; // - - private: - void print_debug_string(); - - void transform(const cv::Mat &mat_rs, ncnn::Mat &in); - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YOLO5FaceScaleParams &scale_params); - - // only generate once - void generate_anchors(unsigned int target_height, unsigned int target_width); - - void generate_bboxes_kps_single_stride(const YOLO5FaceScaleParams &scale_params, - ncnn::Mat &det_pred, - unsigned int stride, - float score_threshold, - float img_height, - float img_width, - std::vector &bbox_kps_collection); - - void generate_bboxes_kps(const YOLO5FaceScaleParams &scale_params, - std::vector &bbox_kps_collection, - ncnn::Extractor &extractor, - float score_threshold, float img_height, - float img_width); - - void nms_bboxes_kps(std::vector &input, - std::vector &output, - float iou_threshold, unsigned int topk); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes_kps, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 400); - - }; - -} - - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLO5FACE_H diff --git a/lite/ncnn/cv/ncnn_yolop.cpp b/lite/ncnn/cv/ncnn_yolop.cpp deleted file mode 100644 index 3a2ed3ac..00000000 --- a/lite/ncnn/cv/ncnn_yolop.cpp +++ /dev/null @@ -1,507 +0,0 @@ -// -// Created by DefTruth on 2021/10/18. -// - -#include "ncnn_yolop.h" -#include "lite/utils.h" - -using ncnncv::NCNNYOLOP; - -NCNNYOLOP::NCNNYOLOP(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads, - int _input_height, - int _input_width) : - log_id(_param_path.data()), param_path(_param_path.data()), - bin_path(_bin_path.data()), num_threads(_num_threads), - input_height(_input_height), input_width(_input_width) -{ - net = new ncnn::Net(); - // init net, change this setting for better performance. - net->opt.use_fp16_arithmetic = false; - net->opt.use_vulkan_compute = false; // default - // setup Focus in yolov5 - net->register_custom_layer("YoloV5Focus", YoloV5Focus_layer_creator); - net->load_param(param_path); - net->load_model(bin_path); -#ifdef LITENCNN_DEBUG - this->print_debug_string(); -#endif -} - -NCNNYOLOP::~NCNNYOLOP() -{ - if (net) delete net; - net = nullptr; -} - -void NCNNYOLOP::transform(const cv::Mat &mat_rs, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNYOLOP::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YOLOPScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat = mat.clone(); - cv::resize(new_unpad_mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void NCNNYOLOP::detect(const cv::Mat &mat, - std::vector &detected_boxes, - types::SegmentContent &da_seg_content, - types::SegmentContent &ll_seg_content, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - // resize & unscale - cv::Mat mat_rs; - YOLOPScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - // 1. make input tensor - ncnn::Mat input; - this->transform(mat_rs, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("images", input); - // 4. rescale & fetch da|ll seg. - std::vector bbox_collection; - this->generate_bboxes_da_ll(scale_params, extractor, bbox_collection, - da_seg_content, ll_seg_content, score_threshold, - img_height, img_width); - // 5. hard|blend nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void NCNNYOLOP::generate_anchors(unsigned int target_height, unsigned int target_width) -{ - if (center_anchors_is_update) return; - - for (auto stride : strides) - { - unsigned int num_grid_w = target_width / stride; - unsigned int num_grid_h = target_height / stride; - std::vector anchors; - - if (stride == 8) - { - // 0 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YOLOPAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 3.f; - anchor.height = 9.f; - anchors.push_back(anchor); - } - } - // 1 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YOLOPAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 5.f; - anchor.height = 11.f; - anchors.push_back(anchor); - } - } - // 2 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YOLOPAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 4.f; - anchor.height = 20.f; - anchors.push_back(anchor); - } - } - } // 16 - else if (stride == 16) - { - // 0 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YOLOPAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 7.f; - anchor.height = 18.f; - anchors.push_back(anchor); - } - } - // 1 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YOLOPAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 6.f; - anchor.height = 39.f; - anchors.push_back(anchor); - } - } - // 2 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YOLOPAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 12.f; - anchor.height = 31.f; - anchors.push_back(anchor); - } - } - } // 32 - else - { - // 0 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YOLOPAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 19.f; - anchor.height = 50.f; - anchors.push_back(anchor); - } - } - // 1 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YOLOPAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 38.f; - anchor.height = 81.f; - anchors.push_back(anchor); - } - } - // 2 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YOLOPAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 68.f; - anchor.height = 157.f; - anchors.push_back(anchor); - } - } - } - center_anchors[stride] = anchors; - } - - center_anchors_is_update = true; -} - -void NCNNYOLOP::generate_bboxes_da_ll(const YOLOPScaleParams &scale_params, - ncnn::Extractor &extractor, - std::vector &bbox_collection, - types::SegmentContent &da_seg_content, - types::SegmentContent &ll_seg_content, - float score_threshold, float img_height, - float img_width) -{ - // (1,n,6=5+1=cxcy+cwch+obj_conf+cls_conf) (1,2,640,640) (1,2,640,640) - ncnn::Mat det_stride_8, det_stride_16, det_stride_32, da_seg_out, ll_seg_out; - extractor.extract("det_stride_8", det_stride_8); - extractor.extract("det_stride_16", det_stride_16); - extractor.extract("det_stride_32", det_stride_32); - extractor.extract("drive_area_seg", da_seg_out); - extractor.extract("lane_line_seg", ll_seg_out); - - this->generate_anchors(input_height, input_width); - - // generate bounding boxes. - bbox_collection.clear(); - this->generate_bboxes_single_stride(scale_params, det_stride_8, 8, score_threshold, - img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, det_stride_16, 16, score_threshold, - img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, det_stride_32, 32, score_threshold, - img_height, img_width, bbox_collection); -#if LITENCNN_DEBUG - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif - - int dw = scale_params.dw; - int dh = scale_params.dh; - int new_unpad_w = scale_params.new_unpad_w; - int new_unpad_h = scale_params.new_unpad_h; - // generate da && ll seg. - da_seg_content.names_map.clear(); - da_seg_content.class_mat = cv::Mat(new_unpad_h, new_unpad_w, CV_8UC1, cv::Scalar(0)); - da_seg_content.color_mat = cv::Mat(new_unpad_h, new_unpad_w, CV_8UC3, cv::Scalar(0, 0, 0)); - ll_seg_content.names_map.clear(); - ll_seg_content.class_mat = cv::Mat(new_unpad_h, new_unpad_w, CV_8UC1, cv::Scalar(0)); - ll_seg_content.color_mat = cv::Mat(new_unpad_h, new_unpad_w, CV_8UC3, cv::Scalar(0, 0, 0)); - - const unsigned int channel_step = input_height * input_width; - const float *da_seg_bg_ptr = (float *) da_seg_out.data; // background - const float *da_seg_fg_ptr = (float *) da_seg_out.data + channel_step; // foreground - const float *ll_seg_bg_ptr = (float *) ll_seg_out.data; // background - const float *ll_seg_fg_ptr = (float *) ll_seg_out.data + channel_step; // foreground - - for (int i = dh; i < dh + new_unpad_h; ++i) - { - // row ptr. - uchar *da_p_class = da_seg_content.class_mat.ptr(i - dh); - uchar *ll_p_class = ll_seg_content.class_mat.ptr(i - dh); - cv::Vec3b *da_p_color = da_seg_content.color_mat.ptr(i - dh); - cv::Vec3b *ll_p_color = ll_seg_content.color_mat.ptr(i - dh); - - for (int j = dw; j < dw + new_unpad_w; ++j) - { - // argmax - float da_bg_prob = da_seg_bg_ptr[i * input_height + j]; - float da_fg_prob = da_seg_fg_ptr[i * input_height + j]; - float ll_bg_prob = ll_seg_bg_ptr[i * input_height + j]; - float ll_fg_prob = ll_seg_fg_ptr[i * input_height + j]; - unsigned int da_label = da_bg_prob < da_fg_prob ? 1 : 0; - unsigned int ll_label = ll_bg_prob < ll_fg_prob ? 1 : 0; - - if (da_label == 1) - { - // assign label for pixel(i,j) - da_p_class[j - dw] = 1 * 255; // 255 indicate drivable area, for post resize - // assign color for detected class at pixel(i,j). - da_p_color[j - dw][0] = 0; - da_p_color[j - dw][1] = 255; // green - da_p_color[j - dw][2] = 0; - // assign names map - da_seg_content.names_map[255] = "drivable area"; - } - - if (ll_label == 1) - { - // assign label for pixel(i,j) - ll_p_class[j - dw] = 1 * 255; // 255 indicate lane line, for post resize - // assign color for detected class at pixel(i,j). - ll_p_color[j - dw][0] = 0; - ll_p_color[j - dw][1] = 0; - ll_p_color[j - dw][2] = 255; // red - // assign names map - ll_seg_content.names_map[255] = "lane line"; - } - - } - } - // resize to original size. - const unsigned int img_h = static_cast(img_height); - const unsigned int img_w = static_cast(img_width); - // da_seg_mask 255 or 0 - cv::resize(da_seg_content.class_mat, da_seg_content.class_mat, - cv::Size(img_w, img_h), cv::INTER_LINEAR); - cv::resize(da_seg_content.color_mat, da_seg_content.color_mat, - cv::Size(img_w, img_h), cv::INTER_LINEAR); - // ll_seg_mask 255 or 0 - cv::resize(ll_seg_content.class_mat, ll_seg_content.class_mat, - cv::Size(img_w, img_h), cv::INTER_LINEAR); - cv::resize(ll_seg_content.color_mat, ll_seg_content.color_mat, - cv::Size(img_w, img_h), cv::INTER_LINEAR); - - da_seg_content.flag = true; - ll_seg_content.flag = true; - -} - -// inner function -static inline float sigmoid(float x) -{ - return static_cast(1.f / (1.f + std::exp(-x))); -} - -// reference: https://github.com/Tencent/ncnn/blob/master/examples/yolov5.cpp -void NCNNYOLOP::generate_bboxes_single_stride(const YOLOPScaleParams &scale_params, - ncnn::Mat &det_pred, - unsigned int stride, - float score_threshold, - float img_height, - float img_width, - std::vector &bbox_collection) -{ - unsigned int nms_pre_ = (stride / 8) * nms_pre; // 1 * 1000,2*1000,... - nms_pre_ = nms_pre_ >= nms_pre ? nms_pre_ : nms_pre; - - const unsigned int f_h = (unsigned int) input_height / stride; - const unsigned int f_w = (unsigned int) input_width / stride; - // e.g, 3*80*80 + 3*40*40 + 3*20*20 = 25200 - const unsigned int num_anchors = 3 * f_h * f_w; - - float r_ = scale_params.r; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - // have c=3 indicate 3 anchors at one grid - unsigned int count = 0; - auto &stride_anchors = center_anchors[stride]; - - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *offset_obj_cls_ptr = (float *) det_pred.data + (i * 6); - float obj_conf = sigmoid(offset_obj_cls_ptr[4]); - if (obj_conf < score_threshold) continue; // filter first. - - unsigned int label = 1; // 1 class only - float cls_conf = sigmoid(offset_obj_cls_ptr[5]); - float conf = obj_conf * cls_conf; // cls_conf (0.,1.) - if (conf < score_threshold) continue; // filter - - int grid0 = stride_anchors.at(i).grid0; // w - int grid1 = stride_anchors.at(i).grid1; // h - float anchor_w = stride_anchors.at(i).width; - float anchor_h = stride_anchors.at(i).height; - - float dx = sigmoid(offset_obj_cls_ptr[0]); - float dy = sigmoid(offset_obj_cls_ptr[1]); - float dw = sigmoid(offset_obj_cls_ptr[2]); - float dh = sigmoid(offset_obj_cls_ptr[3]); - - float cx = (dx * 2.f - 0.5f + (float)grid0) * (float)stride; - float cy = (dy * 2.f - 0.5f + (float)grid1) * (float)stride; - float w = std::pow(dw * 2.f, 2) * anchor_w; - float h = std::pow(dh * 2.f, 2) * anchor_h; - - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - - types::Boxf box; - // de-padding & rescaling - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(x2, (float) img_width); - box.y2 = std::min(y2, (float) img_height); - box.score = conf; - box.label = label; - box.label_text = "traffic car"; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - - if (bbox_collection.size() > nms_pre_) - { - std::sort(bbox_collection.begin(), bbox_collection.end(), - [](const types::Boxf &a, const types::Boxf &b) - { return a.score > b.score; }); // sort inplace - // trunc - bbox_collection.resize(nms_pre_); - } - -} - -void NCNNYOLOP::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - -void NCNNYOLOP::print_debug_string() -{ - std::cout << "LITENCNN_DEBUG LogId: " << log_id << "\n"; - input_indexes = net->input_indexes(); - output_indexes = net->output_indexes(); -#ifdef NCNN_STRING - input_names = net->input_names(); - output_names = net->output_names(); -#endif - std::cout << "=============== Input-Dims ==============\n"; - for (int i = 0; i < input_indexes.size(); ++i) - { - std::cout << "Input: "; - auto tmp_in_blob = net->blobs().at(input_indexes.at(i)); -#ifdef NCNN_STRING - std::cout << input_names.at(i) << ": "; -#endif - std::cout << "shape: c=" << tmp_in_blob.shape.c - << " h=" << tmp_in_blob.shape.h << " w=" << tmp_in_blob.shape.w << "\n"; - } - - std::cout << "=============== Output-Dims ==============\n"; - for (int i = 0; i < output_indexes.size(); ++i) - { - auto tmp_out_blob = net->blobs().at(output_indexes.at(i)); - std::cout << "Output: "; -#ifdef NCNN_STRING - std::cout << output_names.at(i) << ": "; -#endif - std::cout << "shape: c=" << tmp_out_blob.shape.c - << " h=" << tmp_out_blob.shape.h << " w=" << tmp_out_blob.shape.w << "\n"; - } - std::cout << "========================================\n"; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_yolop.h b/lite/ncnn/cv/ncnn_yolop.h deleted file mode 100644 index b652c332..00000000 --- a/lite/ncnn/cv/ncnn_yolop.h +++ /dev/null @@ -1,124 +0,0 @@ -// -// Created by DefTruth on 2021/10/18. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOP_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOP_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNYOLOP - { - private: - ncnn::Net *net = nullptr; - const char *log_id = nullptr; - const char *param_path = nullptr; - const char *bin_path = nullptr; - std::vector input_names; - std::vector output_names; - std::vector input_indexes; - std::vector output_indexes; - - public: - explicit NCNNYOLOP(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1, - int _input_height = 640, - int _input_width = 640); // - ~NCNNYOLOP(); - - private: - // nested classes - typedef struct - { - int grid0; - int grid1; - int stride; - float width; - float height; - } YOLOPAnchor; - - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } YOLOPScaleParams; - - private: - const unsigned int num_threads; // initialize at runtime. - // target image size after resize - const int input_height; // 640/320/1280 - const int input_width; // 640/320/1280 - - const float mean_vals[3] = {255.f * 0.485f, 255.f * 0.456, 255.f * 0.406f}; // RGB - const float norm_vals[3] = {1.f / (255.f * 0.229f), 1.f / (255.f * 0.224f), 1.f / (255.f * 0.225f)}; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int nms_pre = 1000; - static constexpr const unsigned int max_nms = 30000; - - std::vector strides = {8, 16, 32}; - std::unordered_map> center_anchors; - bool center_anchors_is_update = false; - - protected: - NCNNYOLOP(const NCNNYOLOP &) = delete; // - NCNNYOLOP(NCNNYOLOP &&) = delete; // - NCNNYOLOP &operator=(const NCNNYOLOP &) = delete; // - NCNNYOLOP &operator=(NCNNYOLOP &&) = delete; // - - private: - void print_debug_string(); - - void transform(const cv::Mat &mat_rs, ncnn::Mat &in); - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YOLOPScaleParams &scale_params); - - // only generate once - void generate_anchors(unsigned int target_height, unsigned int target_width); - - void generate_bboxes_single_stride(const YOLOPScaleParams &scale_params, - ncnn::Mat &det_pred, - unsigned int stride, - float score_threshold, - float img_height, - float img_width, - std::vector &bbox_collection); - - void generate_bboxes_da_ll(const YOLOPScaleParams &scale_params, - ncnn::Extractor &extractor, - std::vector &bbox_collection, - types::SegmentContent &da_seg_content, - types::SegmentContent &ll_seg_content, - float score_threshold, float img_height, - float img_width); // det,da_seg,ll_seg - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, - std::vector &detected_boxes, - types::SegmentContent &da_seg_content, - types::SegmentContent &ll_seg_content, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - - }; - -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOP_H diff --git a/lite/ncnn/cv/ncnn_yolor.cpp b/lite/ncnn/cv/ncnn_yolor.cpp deleted file mode 100644 index f5897066..00000000 --- a/lite/ncnn/cv/ncnn_yolor.cpp +++ /dev/null @@ -1,477 +0,0 @@ -// -// Created by DefTruth on 2021/11/7. -// - -#include "ncnn_yolor.h" -#include "lite/utils.h" - -using ncnncv::NCNNYoloR; - -NCNNYoloR::NCNNYoloR(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads, - int _input_height, - int _input_width) : - log_id(_param_path.data()), param_path(_param_path.data()), - bin_path(_bin_path.data()), num_threads(_num_threads), - input_height(_input_height), input_width(_input_width) -{ - net = new ncnn::Net(); - // init net, change this setting for better performance. - net->opt.use_fp16_arithmetic = false; - net->opt.use_vulkan_compute = false; // default - // setup Focus in yolov5 - net->register_custom_layer("YoloV5Focus", YoloV5Focus_layer_creator); - net->load_param(param_path); - net->load_model(bin_path); -#ifdef LITENCNN_DEBUG - this->print_debug_string(); -#endif -} - -NCNNYoloR::~NCNNYoloR() -{ - if (net) delete net; - net = nullptr; -} - -void NCNNYoloR::transform(const cv::Mat &mat_rs, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNYoloR::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YoloRScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void NCNNYoloR::detect(const cv::Mat &mat, - std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - // resize & unscale - cv::Mat mat_rs; - YoloRScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - // 1. make input tensor - ncnn::Mat input; - this->transform(mat_rs, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("images", input); - // 4. rescale & fetch da|ll seg. - std::vector bbox_collection; - this->generate_bboxes(scale_params, extractor, bbox_collection, - score_threshold, img_height, img_width); - // 5. hard|blend nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void NCNNYoloR::generate_anchors(unsigned int target_height, unsigned int target_width) -{ - if (center_anchors_is_update) return; - - for (auto stride : strides) - { - unsigned int num_grid_w = target_width / stride; - unsigned int num_grid_h = target_height / stride; - std::vector anchors; - - if (stride == 8) - { - // 0 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloRAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 19.f; - anchor.height = 27.f; - anchors.push_back(anchor); - } - } - // 1 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloRAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 44.f; - anchor.height = 40.f; - anchors.push_back(anchor); - } - } - // 2 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloRAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 38.f; - anchor.height = 94.f; - anchors.push_back(anchor); - } - } - } // 16 - else if (stride == 16) - { - // 0 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloRAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 96.f; - anchor.height = 68.f; - anchors.push_back(anchor); - } - } - // 1 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloRAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 86.f; - anchor.height = 152.f; - anchors.push_back(anchor); - } - } - // 2 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloRAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 180.f; - anchor.height = 137.f; - anchors.push_back(anchor); - } - } - } // 32 - else if (stride == 32) - { - // 0 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloRAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 140.f; - anchor.height = 301.f; - anchors.push_back(anchor); - } - } - // 1 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloRAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 303.f; - anchor.height = 264.f; - anchors.push_back(anchor); - } - } - // 2 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloRAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 238.f; - anchor.height = 542.f; - anchors.push_back(anchor); - } - } - } // 64 - else - { - // 0 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloRAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 436.f; - anchor.height = 615.f; - anchors.push_back(anchor); - } - } - // 1 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloRAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 739.f; - anchor.height = 380.f; - anchors.push_back(anchor); - } - } - // 2 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloRAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 925.f; - anchor.height = 792.f; - anchors.push_back(anchor); - } - } - } - center_anchors[stride] = anchors; - } - - center_anchors_is_update = true; -} - -void NCNNYoloR::generate_bboxes(const YoloRScaleParams &scale_params, - ncnn::Extractor &extractor, - std::vector &bbox_collection, - float score_threshold, float img_height, - float img_width) -{ - // (1,n,85=5+80=cxcy+cwch+obj_conf+cls_conf) - ncnn::Mat det_stride_8, det_stride_16, det_stride_32, det_stride_64; - extractor.extract("det_stride_8", det_stride_8); - extractor.extract("det_stride_16", det_stride_16); - extractor.extract("det_stride_32", det_stride_32); - extractor.extract("det_stride_64", det_stride_64); - - this->generate_anchors(input_height, input_width); - - // generate bounding boxes. - bbox_collection.clear(); - this->generate_bboxes_single_stride(scale_params, det_stride_8, 8, score_threshold, - img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, det_stride_16, 16, score_threshold, - img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, det_stride_32, 32, score_threshold, - img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, det_stride_64, 64, score_threshold, - img_height, img_width, bbox_collection); -#if LITENCNN_DEBUG - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -// inner function -static inline float sigmoid(float x) -{ - return static_cast(1.f / (1.f + std::exp(-x))); -} - -void NCNNYoloR::generate_bboxes_single_stride(const YoloRScaleParams &scale_params, - ncnn::Mat &det_pred, - unsigned int stride, - float score_threshold, - float img_height, float img_width, - std::vector &bbox_collection) -{ - unsigned int nms_pre_ = (stride / 8) * nms_pre; // 1 * 1000,2*1000,... - nms_pre_ = nms_pre_ >= nms_pre ? nms_pre_ : nms_pre; - - const unsigned int f_h = (unsigned int) input_height / stride; - const unsigned int f_w = (unsigned int) input_width / stride; - // e.g, 3*80*80 + 3*40*40 + 3*20*20 = 25200 - const unsigned int num_anchors = 3 * f_h * f_w; - const unsigned int num_classes = 80; - - float r_ = scale_params.r; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - // have c=3 indicate 3 anchors at one grid - unsigned int count = 0; - auto &stride_anchors = center_anchors[stride]; - - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *offset_obj_cls_ptr = (float *) det_pred.data + (i * (num_classes + 5)); - float obj_conf = sigmoid(offset_obj_cls_ptr[4]); - if (obj_conf < score_threshold) continue; // filter first. - - float cls_conf = sigmoid(offset_obj_cls_ptr[5]); - unsigned int label = 0; // 80 class - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = sigmoid(offset_obj_cls_ptr[j + 5]); - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - - float conf = obj_conf * cls_conf; // cls_conf (0.,1.) - if (conf < score_threshold) continue; // filter - - int grid0 = stride_anchors.at(i).grid0; // w - int grid1 = stride_anchors.at(i).grid1; // h - float anchor_w = stride_anchors.at(i).width; - float anchor_h = stride_anchors.at(i).height; - - float dx = sigmoid(offset_obj_cls_ptr[0]); - float dy = sigmoid(offset_obj_cls_ptr[1]); - float dw = sigmoid(offset_obj_cls_ptr[2]); - float dh = sigmoid(offset_obj_cls_ptr[3]); - - float cx = (dx * 2.f - 0.5f + (float) grid0) * (float) stride; - float cy = (dy * 2.f - 0.5f + (float) grid1) * (float) stride; - float w = std::pow(dw * 2.f, 2) * anchor_w; - float h = std::pow(dh * 2.f, 2) * anchor_h; - - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - - types::Boxf box; - // de-padding & rescaling - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(x2, (float) img_width - 1.f); - box.y2 = std::min(y2, (float) img_height - 1.f); - box.score = conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - - if (bbox_collection.size() > nms_pre_) - { - std::sort(bbox_collection.begin(), bbox_collection.end(), - [](const types::Boxf &a, const types::Boxf &b) - { return a.score > b.score; }); // sort inplace - // trunc - bbox_collection.resize(nms_pre_); - } -} - -void NCNNYoloR::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - -void NCNNYoloR::print_debug_string() -{ - std::cout << "LITENCNN_DEBUG LogId: " << log_id << "\n"; - input_indexes = net->input_indexes(); - output_indexes = net->output_indexes(); -#ifdef NCNN_STRING - input_names = net->input_names(); - output_names = net->output_names(); -#endif - std::cout << "=============== Input-Dims ==============\n"; - for (int i = 0; i < input_indexes.size(); ++i) - { - std::cout << "Input: "; - auto tmp_in_blob = net->blobs().at(input_indexes.at(i)); -#ifdef NCNN_STRING - std::cout << input_names.at(i) << ": "; -#endif - std::cout << "shape: c=" << tmp_in_blob.shape.c - << " h=" << tmp_in_blob.shape.h << " w=" << tmp_in_blob.shape.w << "\n"; - } - - std::cout << "=============== Output-Dims ==============\n"; - for (int i = 0; i < output_indexes.size(); ++i) - { - auto tmp_out_blob = net->blobs().at(output_indexes.at(i)); - std::cout << "Output: "; -#ifdef NCNN_STRING - std::cout << output_names.at(i) << ": "; -#endif - std::cout << "shape: c=" << tmp_out_blob.shape.c - << " h=" << tmp_out_blob.shape.h << " w=" << tmp_out_blob.shape.w << "\n"; - } - std::cout << "========================================\n"; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_yolor.h b/lite/ncnn/cv/ncnn_yolor.h deleted file mode 100644 index fe406cbe..00000000 --- a/lite/ncnn/cv/ncnn_yolor.h +++ /dev/null @@ -1,131 +0,0 @@ -// -// Created by DefTruth on 2021/11/7. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOR_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOR_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNYoloR - { - private: - ncnn::Net *net = nullptr; - const char *log_id = nullptr; - const char *param_path = nullptr; - const char *bin_path = nullptr; - std::vector input_names; - std::vector output_names; - std::vector input_indexes; - std::vector output_indexes; - - private: - // nested classes - typedef struct - { - int grid0; - int grid1; - int stride; - float width; - float height; - } YoloRAnchor; - - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } YoloRScaleParams; - - public: - explicit NCNNYoloR(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1, - int _input_height = 640, - int _input_width = 640); // - ~NCNNYoloR(); - - private: - const unsigned int num_threads; // initialize at runtime. - // target image size after resize - const int input_height; // 640/320 - const int input_width; // 640/320 - - const float mean_vals[3] = {0.f, 0.f, 0.f}; // RGB - const float norm_vals[3] = {1.0 / 255.f, 1.0 / 255.f, 1.0 / 255.f}; - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int nms_pre = 1000; - static constexpr const unsigned int max_nms = 30000; - - std::vector strides = {8, 16, 32, 64}; - std::unordered_map> center_anchors; - bool center_anchors_is_update = false; - - protected: - NCNNYoloR(const NCNNYoloR &) = delete; // - NCNNYoloR(NCNNYoloR &&) = delete; // - NCNNYoloR &operator=(const NCNNYoloR &) = delete; // - NCNNYoloR &operator=(NCNNYoloR &&) = delete; // - - private: - void print_debug_string(); - - void transform(const cv::Mat &mat_rs, ncnn::Mat &in); - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YoloRScaleParams &scale_params); - - // only generate once - void generate_anchors(unsigned int target_height, unsigned int target_width); - - void generate_bboxes_single_stride(const YoloRScaleParams &scale_params, - ncnn::Mat &det_pred, - unsigned int stride, - float score_threshold, - float img_height, - float img_width, - std::vector &bbox_collection); - - void generate_bboxes(const YoloRScaleParams &scale_params, - ncnn::Extractor &extractor, - std::vector &bbox_collection, - float score_threshold, float img_height, - float img_width); - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, - std::vector &detected_boxes, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - - }; - -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOR_H diff --git a/lite/ncnn/cv/ncnn_yolor_ssss.cpp b/lite/ncnn/cv/ncnn_yolor_ssss.cpp deleted file mode 100644 index 37f218dd..00000000 --- a/lite/ncnn/cv/ncnn_yolor_ssss.cpp +++ /dev/null @@ -1,428 +0,0 @@ -// -// Created by DefTruth on 2021/11/7. -// - -#include "ncnn_yolor_ssss.h" -#include "lite/utils.h" - -using ncnncv::NCNNYoloRssss; - -NCNNYoloRssss::NCNNYoloRssss(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads, - int _input_height, - int _input_width) : - log_id(_param_path.data()), param_path(_param_path.data()), - bin_path(_bin_path.data()), num_threads(_num_threads), - input_height(_input_height), input_width(_input_width) -{ - net = new ncnn::Net(); - // init net, change this setting for better performance. - net->opt.use_fp16_arithmetic = false; - net->opt.use_vulkan_compute = false; // default - // setup Focus in yolov5 - net->register_custom_layer("YoloV5Focus", YoloV5Focus_layer_creator); - net->load_param(param_path); - net->load_model(bin_path); -#ifdef LITENCNN_DEBUG - this->print_debug_string(); -#endif -} - -NCNNYoloRssss::~NCNNYoloRssss() -{ - if (net) delete net; - net = nullptr; -} - -void NCNNYoloRssss::transform(const cv::Mat &mat_rs, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNYoloRssss::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YoloRssssScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat = mat.clone(); - cv::resize(new_unpad_mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void NCNNYoloRssss::detect(const cv::Mat &mat, - std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - // resize & unscale - cv::Mat mat_rs; - YoloRssssScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - // 1. make input tensor - ncnn::Mat input; - this->transform(mat_rs, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("images", input); - // 4. rescale & fetch da|ll seg. - std::vector bbox_collection; - this->generate_bboxes(scale_params, extractor, bbox_collection, - score_threshold, img_height, img_width); - // 5. hard|blend nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void NCNNYoloRssss::generate_anchors(unsigned int target_height, unsigned int target_width) -{ - if (center_anchors_is_update) return; - - for (auto stride : strides) - { - unsigned int num_grid_w = target_width / stride; - unsigned int num_grid_h = target_height / stride; - std::vector anchors; - - if (stride == 8) - { - // 0 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloRssssAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 12.f; - anchor.height = 16.f; - anchors.push_back(anchor); - } - } - // 1 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloRssssAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 19.f; - anchor.height = 36.f; - anchors.push_back(anchor); - } - } - // 2 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloRssssAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 40.f; - anchor.height = 28.f; - anchors.push_back(anchor); - } - } - } // 16 - else if (stride == 16) - { - // 0 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloRssssAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 36.f; - anchor.height = 75.f; - anchors.push_back(anchor); - } - } - // 1 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloRssssAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 76.f; - anchor.height = 55.f; - anchors.push_back(anchor); - } - } - // 2 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloRssssAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 72.f; - anchor.height = 146.f; - anchors.push_back(anchor); - } - } - } // 32 - else if (stride == 32) - { - // 0 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloRssssAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 142.f; - anchor.height = 110.f; - anchors.push_back(anchor); - } - } - // 1 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloRssssAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 192.f; - anchor.height = 243.f; - anchors.push_back(anchor); - } - } - // 2 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloRssssAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 459.f; - anchor.height = 401.f; - anchors.push_back(anchor); - } - } - } - center_anchors[stride] = anchors; - } - - center_anchors_is_update = true; -} - -void NCNNYoloRssss::generate_bboxes(const YoloRssssScaleParams &scale_params, - ncnn::Extractor &extractor, - std::vector &bbox_collection, - float score_threshold, float img_height, - float img_width) -{ - // (1,n,85=5+80=cxcy+cwch+obj_conf+cls_conf) - ncnn::Mat det_stride_8, det_stride_16, det_stride_32; - extractor.extract("det_stride_8", det_stride_8); - extractor.extract("det_stride_16", det_stride_16); - extractor.extract("det_stride_32", det_stride_32); - - this->generate_anchors(input_height, input_width); - - // generate bounding boxes. - bbox_collection.clear(); - this->generate_bboxes_single_stride(scale_params, det_stride_8, 8, score_threshold, - img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, det_stride_16, 16, score_threshold, - img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, det_stride_32, 32, score_threshold, - img_height, img_width, bbox_collection); -#if LITENCNN_DEBUG - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -// inner function -static inline float sigmoid(float x) -{ - return static_cast(1.f / (1.f + std::exp(-x))); -} - -void NCNNYoloRssss::generate_bboxes_single_stride(const YoloRssssScaleParams &scale_params, - ncnn::Mat &det_pred, - unsigned int stride, - float score_threshold, - float img_height, float img_width, - std::vector &bbox_collection) -{ - unsigned int nms_pre_ = (stride / 8) * nms_pre; // 1 * 1000,2*1000,... - nms_pre_ = nms_pre_ >= nms_pre ? nms_pre_ : nms_pre; - - const unsigned int f_h = (unsigned int) input_height / stride; - const unsigned int f_w = (unsigned int) input_width / stride; - // e.g, 3*80*80 + 3*40*40 + 3*20*20 = 25200 - const unsigned int num_anchors = 3 * f_h * f_w; - const unsigned int num_classes = 80; - - float r_ = scale_params.r; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - // have c=3 indicate 3 anchors at one grid - unsigned int count = 0; - auto &stride_anchors = center_anchors[stride]; - - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *offset_obj_cls_ptr = (float *) det_pred.data + (i * (num_classes + 5)); - float obj_conf = sigmoid(offset_obj_cls_ptr[4]); - if (obj_conf < score_threshold) continue; // filter first. - - float cls_conf = sigmoid(offset_obj_cls_ptr[5]); - unsigned int label = 0; // 80 class - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = sigmoid(offset_obj_cls_ptr[j + 5]); - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - - float conf = obj_conf * cls_conf; // cls_conf (0.,1.) - if (conf < score_threshold) continue; // filter - - int grid0 = stride_anchors.at(i).grid0; // w - int grid1 = stride_anchors.at(i).grid1; // h - float anchor_w = stride_anchors.at(i).width; - float anchor_h = stride_anchors.at(i).height; - - float dx = sigmoid(offset_obj_cls_ptr[0]); - float dy = sigmoid(offset_obj_cls_ptr[1]); - float dw = sigmoid(offset_obj_cls_ptr[2]); - float dh = sigmoid(offset_obj_cls_ptr[3]); - - float cx = (dx * 2.f - 0.5f + (float) grid0) * (float) stride; - float cy = (dy * 2.f - 0.5f + (float) grid1) * (float) stride; - float w = std::pow(dw * 2.f, 2) * anchor_w; - float h = std::pow(dh * 2.f, 2) * anchor_h; - - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - - types::Boxf box; - // de-padding & rescaling - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(x2, (float) img_width); - box.y2 = std::min(y2, (float) img_height); - box.score = conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - - if (bbox_collection.size() > nms_pre_) - { - std::sort(bbox_collection.begin(), bbox_collection.end(), - [](const types::Boxf &a, const types::Boxf &b) - { return a.score > b.score; }); // sort inplace - // trunc - bbox_collection.resize(nms_pre_); - } -} - -void NCNNYoloRssss::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - -void NCNNYoloRssss::print_debug_string() -{ - std::cout << "LITENCNN_DEBUG LogId: " << log_id << "\n"; - input_indexes = net->input_indexes(); - output_indexes = net->output_indexes(); -#ifdef NCNN_STRING - input_names = net->input_names(); - output_names = net->output_names(); -#endif - std::cout << "=============== Input-Dims ==============\n"; - for (int i = 0; i < input_indexes.size(); ++i) - { - std::cout << "Input: "; - auto tmp_in_blob = net->blobs().at(input_indexes.at(i)); -#ifdef NCNN_STRING - std::cout << input_names.at(i) << ": "; -#endif - std::cout << "shape: c=" << tmp_in_blob.shape.c - << " h=" << tmp_in_blob.shape.h << " w=" << tmp_in_blob.shape.w << "\n"; - } - - std::cout << "=============== Output-Dims ==============\n"; - for (int i = 0; i < output_indexes.size(); ++i) - { - auto tmp_out_blob = net->blobs().at(output_indexes.at(i)); - std::cout << "Output: "; -#ifdef NCNN_STRING - std::cout << output_names.at(i) << ": "; -#endif - std::cout << "shape: c=" << tmp_out_blob.shape.c - << " h=" << tmp_out_blob.shape.h << " w=" << tmp_out_blob.shape.w << "\n"; - } - std::cout << "========================================\n"; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_yolor_ssss.h b/lite/ncnn/cv/ncnn_yolor_ssss.h deleted file mode 100644 index 027cde9f..00000000 --- a/lite/ncnn/cv/ncnn_yolor_ssss.h +++ /dev/null @@ -1,131 +0,0 @@ -// -// Created by DefTruth on 2021/11/7. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOR_SSSS_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOR_SSSS_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNYoloRssss - { - private: - ncnn::Net *net = nullptr; - const char *log_id = nullptr; - const char *param_path = nullptr; - const char *bin_path = nullptr; - std::vector input_names; - std::vector output_names; - std::vector input_indexes; - std::vector output_indexes; - - private: - // nested classes - typedef struct - { - int grid0; - int grid1; - int stride; - float width; - float height; - } YoloRssssAnchor; - - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } YoloRssssScaleParams; - - public: - explicit NCNNYoloRssss(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1, - int _input_height = 640, - int _input_width = 640); // - ~NCNNYoloRssss(); - - private: - const unsigned int num_threads; // initialize at runtime. - // target image size after resize - const int input_height; // 640/320 - const int input_width; // 640/320 - - const float mean_vals[3] = {0.f, 0.f, 0.f}; // RGB - const float norm_vals[3] = {1.0 / 255.f, 1.0 / 255.f, 1.0 / 255.f}; - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int nms_pre = 1000; - static constexpr const unsigned int max_nms = 30000; - - std::vector strides = {8, 16, 32}; - std::unordered_map> center_anchors; - bool center_anchors_is_update = false; - - protected: - NCNNYoloRssss(const NCNNYoloRssss &) = delete; // - NCNNYoloRssss(NCNNYoloRssss &&) = delete; // - NCNNYoloRssss &operator=(const NCNNYoloRssss &) = delete; // - NCNNYoloRssss &operator=(NCNNYoloRssss &&) = delete; // - - private: - void print_debug_string(); - - void transform(const cv::Mat &mat_rs, ncnn::Mat &in); - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YoloRssssScaleParams &scale_params); - - // only generate once - void generate_anchors(unsigned int target_height, unsigned int target_width); - - void generate_bboxes_single_stride(const YoloRssssScaleParams &scale_params, - ncnn::Mat &det_pred, - unsigned int stride, - float score_threshold, - float img_height, - float img_width, - std::vector &bbox_collection); - - void generate_bboxes(const YoloRssssScaleParams &scale_params, - ncnn::Extractor &extractor, - std::vector &bbox_collection, - float score_threshold, float img_height, - float img_width); - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, - std::vector &detected_boxes, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - - }; - -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOR_SSSS_H diff --git a/lite/ncnn/cv/ncnn_yolov5.cpp b/lite/ncnn/cv/ncnn_yolov5.cpp deleted file mode 100644 index e8ab972a..00000000 --- a/lite/ncnn/cv/ncnn_yolov5.cpp +++ /dev/null @@ -1,429 +0,0 @@ -// -// Created by DefTruth on 2021/11/6. -// - -#include "ncnn_yolov5.h" -#include "lite/utils.h" - -using ncnncv::NCNNYoloV5; - -NCNNYoloV5::NCNNYoloV5(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads, - int _input_height, - int _input_width) : - log_id(_param_path.data()), param_path(_param_path.data()), - bin_path(_bin_path.data()), num_threads(_num_threads), - input_height(_input_height), input_width(_input_width) -{ - net = new ncnn::Net(); - // init net, change this setting for better performance. - net->opt.use_fp16_arithmetic = false; - net->opt.use_vulkan_compute = false; // default - // setup Focus in yolov5 - net->register_custom_layer("YoloV5Focus", YoloV5Focus_layer_creator); - net->load_param(param_path); - net->load_model(bin_path); -#ifdef LITENCNN_DEBUG - this->print_debug_string(); -#endif -} - -NCNNYoloV5::~NCNNYoloV5() -{ - if (net) delete net; - net = nullptr; -} - -void NCNNYoloV5::transform(const cv::Mat &mat_rs, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNYoloV5::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YoloV5ScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void NCNNYoloV5::detect(const cv::Mat &mat, - std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - // resize & unscale - cv::Mat mat_rs; - YoloV5ScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - // 1. make input tensor - ncnn::Mat input; - this->transform(mat_rs, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("images", input); - // 4. rescale & fetch da|ll seg. - std::vector bbox_collection; - this->generate_bboxes(scale_params, extractor, bbox_collection, - score_threshold, img_height, img_width); - // 5. hard|blend nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void NCNNYoloV5::generate_anchors(unsigned int target_height, unsigned int target_width) -{ - if (center_anchors_is_update) return; - - for (auto stride : strides) - { - unsigned int num_grid_w = target_width / stride; - unsigned int num_grid_h = target_height / stride; - std::vector anchors; - - if (stride == 8) - { - // 0 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 10.f; - anchor.height = 13.f; - anchors.push_back(anchor); - } - } - // 1 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 16.f; - anchor.height = 30.f; - anchors.push_back(anchor); - } - } - // 2 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 33.f; - anchor.height = 23.f; - anchors.push_back(anchor); - } - } - } // 16 - else if (stride == 16) - { - // 0 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 30.f; - anchor.height = 61.f; - anchors.push_back(anchor); - } - } - // 1 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 62.f; - anchor.height = 45.f; - anchors.push_back(anchor); - } - } - // 2 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 59.f; - anchor.height = 119.f; - anchors.push_back(anchor); - } - } - } // 32 - else - { - // 0 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 116.f; - anchor.height = 90.f; - anchors.push_back(anchor); - } - } - // 1 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 156.f; - anchor.height = 198.f; - anchors.push_back(anchor); - } - } - // 2 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 373.f; - anchor.height = 326.f; - anchors.push_back(anchor); - } - } - } - center_anchors[stride] = anchors; - } - - center_anchors_is_update = true; -} - -void NCNNYoloV5::generate_bboxes(const YoloV5ScaleParams &scale_params, - ncnn::Extractor &extractor, - std::vector &bbox_collection, - float score_threshold, float img_height, - float img_width) -{ - // (1,n,85=5+80=cxcy+cwch+obj_conf+cls_conf) - ncnn::Mat det_stride_8, det_stride_16, det_stride_32; - extractor.extract("det_stride_8", det_stride_8); - extractor.extract("det_stride_16", det_stride_16); - extractor.extract("det_stride_32", det_stride_32); - - this->generate_anchors(input_height, input_width); - - // generate bounding boxes. - bbox_collection.clear(); - this->generate_bboxes_single_stride(scale_params, det_stride_8, 8, score_threshold, - img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, det_stride_16, 16, score_threshold, - img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, det_stride_32, 32, score_threshold, - img_height, img_width, bbox_collection); -#if LITENCNN_DEBUG - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -// inner function -static inline float sigmoid(float x) -{ - return static_cast(1.f / (1.f + std::exp(-x))); -} - -void NCNNYoloV5::generate_bboxes_single_stride(const YoloV5ScaleParams &scale_params, - ncnn::Mat &det_pred, - unsigned int stride, - float score_threshold, - float img_height, float img_width, - std::vector &bbox_collection) -{ - unsigned int nms_pre_ = (stride / 8) * nms_pre; // 1 * 1000,2*1000,... - nms_pre_ = nms_pre_ >= nms_pre ? nms_pre_ : nms_pre; - - const unsigned int f_h = (unsigned int) input_height / stride; - const unsigned int f_w = (unsigned int) input_width / stride; - // e.g, 3*80*80 + 3*40*40 + 3*20*20 = 25200 - const unsigned int num_anchors = 3 * f_h * f_w; - const unsigned int num_classes = 80; - - float r_ = scale_params.r; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - // have c=3 indicate 3 anchors at one grid - unsigned int count = 0; - auto &stride_anchors = center_anchors[stride]; - - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *offset_obj_cls_ptr = (float *) det_pred.data + (i * (num_classes + 5)); - float obj_conf = sigmoid(offset_obj_cls_ptr[4]); - if (obj_conf < score_threshold) continue; // filter first. - - float cls_conf = sigmoid(offset_obj_cls_ptr[5]); - unsigned int label = 0; // 80 class - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = sigmoid(offset_obj_cls_ptr[j + 5]); - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - - float conf = obj_conf * cls_conf; // cls_conf (0.,1.) - if (conf < score_threshold) continue; // filter - - int grid0 = stride_anchors.at(i).grid0; // w - int grid1 = stride_anchors.at(i).grid1; // h - float anchor_w = stride_anchors.at(i).width; - float anchor_h = stride_anchors.at(i).height; - - float dx = sigmoid(offset_obj_cls_ptr[0]); - float dy = sigmoid(offset_obj_cls_ptr[1]); - float dw = sigmoid(offset_obj_cls_ptr[2]); - float dh = sigmoid(offset_obj_cls_ptr[3]); - - float cx = (dx * 2.f - 0.5f + (float) grid0) * (float) stride; - float cy = (dy * 2.f - 0.5f + (float) grid1) * (float) stride; - float w = std::pow(dw * 2.f, 2) * anchor_w; - float h = std::pow(dh * 2.f, 2) * anchor_h; - - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - - types::Boxf box; - // de-padding & rescaling - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(x2, (float) img_width - 1.f); - box.y2 = std::min(y2, (float) img_height - 1.f); - box.score = conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - - if (bbox_collection.size() > nms_pre_) - { - std::sort(bbox_collection.begin(), bbox_collection.end(), - [](const types::Boxf &a, const types::Boxf &b) - { return a.score > b.score; }); // sort inplace - // trunc - bbox_collection.resize(nms_pre_); - } -} - -void NCNNYoloV5::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - -void NCNNYoloV5::print_debug_string() -{ - std::cout << "LITENCNN_DEBUG LogId: " << log_id << "\n"; - input_indexes = net->input_indexes(); - output_indexes = net->output_indexes(); -#ifdef NCNN_STRING - input_names = net->input_names(); - output_names = net->output_names(); -#endif - std::cout << "=============== Input-Dims ==============\n"; - for (int i = 0; i < input_indexes.size(); ++i) - { - std::cout << "Input: "; - auto tmp_in_blob = net->blobs().at(input_indexes.at(i)); -#ifdef NCNN_STRING - std::cout << input_names.at(i) << ": "; -#endif - std::cout << "shape: c=" << tmp_in_blob.shape.c - << " h=" << tmp_in_blob.shape.h << " w=" << tmp_in_blob.shape.w << "\n"; - } - - std::cout << "=============== Output-Dims ==============\n"; - for (int i = 0; i < output_indexes.size(); ++i) - { - auto tmp_out_blob = net->blobs().at(output_indexes.at(i)); - std::cout << "Output: "; -#ifdef NCNN_STRING - std::cout << output_names.at(i) << ": "; -#endif - std::cout << "shape: c=" << tmp_out_blob.shape.c - << " h=" << tmp_out_blob.shape.h << " w=" << tmp_out_blob.shape.w << "\n"; - } - std::cout << "========================================\n"; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_yolov5.h b/lite/ncnn/cv/ncnn_yolov5.h deleted file mode 100644 index e3fddc6a..00000000 --- a/lite/ncnn/cv/ncnn_yolov5.h +++ /dev/null @@ -1,131 +0,0 @@ -// -// Created by DefTruth on 2021/11/6. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOV5_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOV5_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNYoloV5 - { - private: - ncnn::Net *net = nullptr; - const char *log_id = nullptr; - const char *param_path = nullptr; - const char *bin_path = nullptr; - std::vector input_names; - std::vector output_names; - std::vector input_indexes; - std::vector output_indexes; - - private: - // nested classes - typedef struct - { - int grid0; - int grid1; - int stride; - float width; - float height; - } YoloV5Anchor; - - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } YoloV5ScaleParams; - - public: - explicit NCNNYoloV5(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1, - int _input_height = 640, - int _input_width = 640); // - ~NCNNYoloV5(); - - private: - const unsigned int num_threads; // initialize at runtime. - // target image size after resize - const int input_height; // 640/320/1280 - const int input_width; // 640/320/1280 - - const float mean_vals[3] = {0.f, 0.f, 0.f}; // RGB - const float norm_vals[3] = {1.0 / 255.f, 1.0 / 255.f, 1.0 / 255.f}; - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int nms_pre = 1000; - static constexpr const unsigned int max_nms = 30000; - - std::vector strides = {8, 16, 32}; - std::unordered_map> center_anchors; - bool center_anchors_is_update = false; - - protected: - NCNNYoloV5(const NCNNYoloV5 &) = delete; // - NCNNYoloV5(NCNNYoloV5 &&) = delete; // - NCNNYoloV5 &operator=(const NCNNYoloV5 &) = delete; // - NCNNYoloV5 &operator=(NCNNYoloV5 &&) = delete; // - - private: - void print_debug_string(); - - void transform(const cv::Mat &mat_rs, ncnn::Mat &in); - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YoloV5ScaleParams &scale_params); - - // only generate once - void generate_anchors(unsigned int target_height, unsigned int target_width); - - void generate_bboxes_single_stride(const YoloV5ScaleParams &scale_params, - ncnn::Mat &det_pred, - unsigned int stride, - float score_threshold, - float img_height, - float img_width, - std::vector &bbox_collection); - - void generate_bboxes(const YoloV5ScaleParams &scale_params, - ncnn::Extractor &extractor, - std::vector &bbox_collection, - float score_threshold, float img_height, - float img_width); - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, - std::vector &detected_boxes, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - - }; - -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOV5_H diff --git a/lite/ncnn/cv/ncnn_yolov5_v6.0.cpp b/lite/ncnn/cv/ncnn_yolov5_v6.0.cpp deleted file mode 100644 index 51c340ee..00000000 --- a/lite/ncnn/cv/ncnn_yolov5_v6.0.cpp +++ /dev/null @@ -1,428 +0,0 @@ -// -// Created by DefTruth on 2021/11/10. -// -#include "ncnn_yolov5_v6.0.h" -#include "lite/utils.h" - -using ncnncv::NCNNYoloV5_V_6_0; - -NCNNYoloV5_V_6_0::NCNNYoloV5_V_6_0(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads, - int _input_height, - int _input_width) : - log_id(_param_path.data()), param_path(_param_path.data()), - bin_path(_bin_path.data()), num_threads(_num_threads), - input_height(_input_height), input_width(_input_width) -{ - net = new ncnn::Net(); - // init net, change this setting for better performance. - net->opt.use_fp16_arithmetic = false; - net->opt.use_vulkan_compute = false; // default - // setup Focus in yolov5 --> no Focus layer in yolov5 v6.0 - // net->register_custom_layer("YoloV5Focus", YoloV5Focus_layer_creator); - net->load_param(param_path); - net->load_model(bin_path); -#ifdef LITENCNN_DEBUG - this->print_debug_string(); -#endif -} - -NCNNYoloV5_V_6_0::~NCNNYoloV5_V_6_0() -{ - if (net) delete net; - net = nullptr; -} - -void NCNNYoloV5_V_6_0::transform(const cv::Mat &mat_rs, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNYoloV5_V_6_0::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YoloV5ScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void NCNNYoloV5_V_6_0::detect(const cv::Mat &mat, - std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - // resize & unscale - cv::Mat mat_rs; - YoloV5ScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - // 1. make input tensor - ncnn::Mat input; - this->transform(mat_rs, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("images", input); - // 4. rescale & fetch da|ll seg. - std::vector bbox_collection; - this->generate_bboxes(scale_params, extractor, bbox_collection, - score_threshold, img_height, img_width); - // 5. hard|blend nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void NCNNYoloV5_V_6_0::generate_anchors(unsigned int target_height, unsigned int target_width) -{ - if (center_anchors_is_update) return; - - for (auto stride : strides) - { - unsigned int num_grid_w = target_width / stride; - unsigned int num_grid_h = target_height / stride; - std::vector anchors; - - if (stride == 8) - { - // 0 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 10.f; - anchor.height = 13.f; - anchors.push_back(anchor); - } - } - // 1 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 16.f; - anchor.height = 30.f; - anchors.push_back(anchor); - } - } - // 2 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 33.f; - anchor.height = 23.f; - anchors.push_back(anchor); - } - } - } // 16 - else if (stride == 16) - { - // 0 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 30.f; - anchor.height = 61.f; - anchors.push_back(anchor); - } - } - // 1 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 62.f; - anchor.height = 45.f; - anchors.push_back(anchor); - } - } - // 2 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 59.f; - anchor.height = 119.f; - anchors.push_back(anchor); - } - } - } // 32 - else - { - // 0 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 116.f; - anchor.height = 90.f; - anchors.push_back(anchor); - } - } - // 1 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 156.f; - anchor.height = 198.f; - anchors.push_back(anchor); - } - } - // 2 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchor.width = 373.f; - anchor.height = 326.f; - anchors.push_back(anchor); - } - } - } - center_anchors[stride] = anchors; - } - - center_anchors_is_update = true; -} - -void NCNNYoloV5_V_6_0::generate_bboxes(const YoloV5ScaleParams &scale_params, - ncnn::Extractor &extractor, - std::vector &bbox_collection, - float score_threshold, float img_height, - float img_width) -{ - // (1,n,85=5+80=cxcy+cwch+obj_conf+cls_conf) - ncnn::Mat det_stride_8, det_stride_16, det_stride_32; - extractor.extract("det_stride_8", det_stride_8); - extractor.extract("det_stride_16", det_stride_16); - extractor.extract("det_stride_32", det_stride_32); - - this->generate_anchors(input_height, input_width); - - // generate bounding boxes. - bbox_collection.clear(); - this->generate_bboxes_single_stride(scale_params, det_stride_8, 8, score_threshold, - img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, det_stride_16, 16, score_threshold, - img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, det_stride_32, 32, score_threshold, - img_height, img_width, bbox_collection); -#if LITENCNN_DEBUG - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -// inner function -static inline float sigmoid(float x) -{ - return static_cast(1.f / (1.f + std::exp(-x))); -} - -void NCNNYoloV5_V_6_0::generate_bboxes_single_stride(const YoloV5ScaleParams &scale_params, - ncnn::Mat &det_pred, - unsigned int stride, - float score_threshold, - float img_height, float img_width, - std::vector &bbox_collection) -{ - unsigned int nms_pre_ = (stride / 8) * nms_pre; // 1 * 1000,2*1000,... - nms_pre_ = nms_pre_ >= nms_pre ? nms_pre_ : nms_pre; - - const unsigned int f_h = (unsigned int) input_height / stride; - const unsigned int f_w = (unsigned int) input_width / stride; - // e.g, 3*80*80 + 3*40*40 + 3*20*20 = 25200 - const unsigned int num_anchors = 3 * f_h * f_w; - const unsigned int num_classes = 80; - - float r_ = scale_params.r; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - // have c=3 indicate 3 anchors at one grid - unsigned int count = 0; - auto &stride_anchors = center_anchors[stride]; - - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *offset_obj_cls_ptr = (float *) det_pred.data + (i * (num_classes + 5)); - float obj_conf = sigmoid(offset_obj_cls_ptr[4]); - if (obj_conf < score_threshold) continue; // filter first. - - float cls_conf = sigmoid(offset_obj_cls_ptr[5]); - unsigned int label = 0; // 80 class - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = sigmoid(offset_obj_cls_ptr[j + 5]); - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - - float conf = obj_conf * cls_conf; // cls_conf (0.,1.) - if (conf < score_threshold) continue; // filter - - int grid0 = stride_anchors.at(i).grid0; // w - int grid1 = stride_anchors.at(i).grid1; // h - float anchor_w = stride_anchors.at(i).width; - float anchor_h = stride_anchors.at(i).height; - - float dx = sigmoid(offset_obj_cls_ptr[0]); - float dy = sigmoid(offset_obj_cls_ptr[1]); - float dw = sigmoid(offset_obj_cls_ptr[2]); - float dh = sigmoid(offset_obj_cls_ptr[3]); - - float cx = (dx * 2.f - 0.5f + (float) grid0) * (float) stride; - float cy = (dy * 2.f - 0.5f + (float) grid1) * (float) stride; - float w = std::pow(dw * 2.f, 2) * anchor_w; - float h = std::pow(dh * 2.f, 2) * anchor_h; - - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - - types::Boxf box; - // de-padding & rescaling - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(x2, (float) img_width - 1.f); - box.y2 = std::min(y2, (float) img_height - 1.f); - box.score = conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - - if (bbox_collection.size() > nms_pre_) - { - std::sort(bbox_collection.begin(), bbox_collection.end(), - [](const types::Boxf &a, const types::Boxf &b) - { return a.score > b.score; }); // sort inplace - // trunc - bbox_collection.resize(nms_pre_); - } -} - -void NCNNYoloV5_V_6_0::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - -void NCNNYoloV5_V_6_0::print_debug_string() -{ - std::cout << "LITENCNN_DEBUG LogId: " << log_id << "\n"; - input_indexes = net->input_indexes(); - output_indexes = net->output_indexes(); -#ifdef NCNN_STRING - input_names = net->input_names(); - output_names = net->output_names(); -#endif - std::cout << "=============== Input-Dims ==============\n"; - for (int i = 0; i < input_indexes.size(); ++i) - { - std::cout << "Input: "; - auto tmp_in_blob = net->blobs().at(input_indexes.at(i)); -#ifdef NCNN_STRING - std::cout << input_names.at(i) << ": "; -#endif - std::cout << "shape: c=" << tmp_in_blob.shape.c - << " h=" << tmp_in_blob.shape.h << " w=" << tmp_in_blob.shape.w << "\n"; - } - - std::cout << "=============== Output-Dims ==============\n"; - for (int i = 0; i < output_indexes.size(); ++i) - { - auto tmp_out_blob = net->blobs().at(output_indexes.at(i)); - std::cout << "Output: "; -#ifdef NCNN_STRING - std::cout << output_names.at(i) << ": "; -#endif - std::cout << "shape: c=" << tmp_out_blob.shape.c - << " h=" << tmp_out_blob.shape.h << " w=" << tmp_out_blob.shape.w << "\n"; - } - std::cout << "========================================\n"; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_yolov5_v6.0.h b/lite/ncnn/cv/ncnn_yolov5_v6.0.h deleted file mode 100644 index 32c64ecc..00000000 --- a/lite/ncnn/cv/ncnn_yolov5_v6.0.h +++ /dev/null @@ -1,131 +0,0 @@ -// -// Created by DefTruth on 2021/11/10. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOV5_V6_0_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOV5_V6_0_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNYoloV5_V_6_0 - { - private: - ncnn::Net *net = nullptr; - const char *log_id = nullptr; - const char *param_path = nullptr; - const char *bin_path = nullptr; - std::vector input_names; - std::vector output_names; - std::vector input_indexes; - std::vector output_indexes; - - private: - // nested classes - typedef struct - { - int grid0; - int grid1; - int stride; - float width; - float height; - } YoloV5Anchor; - - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } YoloV5ScaleParams; - - public: - explicit NCNNYoloV5_V_6_0(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1, - int _input_height = 640, - int _input_width = 640); // - ~NCNNYoloV5_V_6_0(); - - private: - const unsigned int num_threads; // initialize at runtime. - // target image size after resize - const int input_height; // 640/320/1280 - const int input_width; // 640/320/1280 - - const float mean_vals[3] = {0.f, 0.f, 0.f}; // RGB - const float norm_vals[3] = {1.0 / 255.f, 1.0 / 255.f, 1.0 / 255.f}; - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int nms_pre = 1000; - static constexpr const unsigned int max_nms = 30000; - - std::vector strides = {8, 16, 32}; - std::unordered_map> center_anchors; - bool center_anchors_is_update = false; - - protected: - NCNNYoloV5_V_6_0(const NCNNYoloV5_V_6_0 &) = delete; // - NCNNYoloV5_V_6_0(NCNNYoloV5_V_6_0 &&) = delete; // - NCNNYoloV5_V_6_0 &operator=(const NCNNYoloV5_V_6_0 &) = delete; // - NCNNYoloV5_V_6_0 &operator=(NCNNYoloV5_V_6_0 &&) = delete; // - - private: - void print_debug_string(); - - void transform(const cv::Mat &mat_rs, ncnn::Mat &in); - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YoloV5ScaleParams &scale_params); - - // only generate once - void generate_anchors(unsigned int target_height, unsigned int target_width); - - void generate_bboxes_single_stride(const YoloV5ScaleParams &scale_params, - ncnn::Mat &det_pred, - unsigned int stride, - float score_threshold, - float img_height, - float img_width, - std::vector &bbox_collection); - - void generate_bboxes(const YoloV5ScaleParams &scale_params, - ncnn::Extractor &extractor, - std::vector &bbox_collection, - float score_threshold, float img_height, - float img_width); - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, - std::vector &detected_boxes, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - - }; - -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOV5_V6_0_H diff --git a/lite/ncnn/cv/ncnn_yolov5_v6.0_p6.cpp b/lite/ncnn/cv/ncnn_yolov5_v6.0_p6.cpp deleted file mode 100644 index 129a6c6d..00000000 --- a/lite/ncnn/cv/ncnn_yolov5_v6.0_p6.cpp +++ /dev/null @@ -1,573 +0,0 @@ -// -// Created by DefTruth on 2021/11/11. -// - -#include "ncnn_yolov5_v6.0_p6.h" -#include "lite/utils.h" - -using ncnncv::NCNNYoloV5_V_6_0_P6; - -NCNNYoloV5_V_6_0_P6::NCNNYoloV5_V_6_0_P6(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads, - int _input_height, - int _input_width) : - log_id(_param_path.data()), param_path(_param_path.data()), - bin_path(_bin_path.data()), num_threads(_num_threads), - input_height(_input_height), input_width(_input_width) -{ - net = new ncnn::Net(); - // init net, change this setting for better performance. - net->opt.use_fp16_arithmetic = false; - net->opt.use_vulkan_compute = false; // default - // setup Focus in yolov5 --> no Focus layer in yolov5 v6.0 - // net->register_custom_layer("YoloV5Focus", YoloV5Focus_layer_creator); - net->load_param(param_path); - net->load_model(bin_path); -#ifdef LITENCNN_DEBUG - this->print_debug_string(); -#endif -} - -NCNNYoloV5_V_6_0_P6::~NCNNYoloV5_V_6_0_P6() -{ - if (net) delete net; - net = nullptr; -} - -void NCNNYoloV5_V_6_0_P6::transform(const cv::Mat &mat_rs, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNYoloV5_V_6_0_P6::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YoloV5ScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat = mat.clone(); - cv::resize(new_unpad_mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void NCNNYoloV5_V_6_0_P6::detect(const cv::Mat &mat, - std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - // resize & unscale - cv::Mat mat_rs; - YoloV5ScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - // 1. make input tensor - ncnn::Mat input; - this->transform(mat_rs, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("images", input); - // 4. rescale & fetch da|ll seg. - std::vector bbox_collection; - this->generate_bboxes(scale_params, extractor, bbox_collection, - score_threshold, img_height, img_width); - // 5. hard|blend nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void NCNNYoloV5_V_6_0_P6::generate_anchors(unsigned int target_height, unsigned int target_width) -{ - if (center_anchors_is_update) return; - bool is_p6_1280 = target_height == 1280 ? true : false; - - for (auto stride : strides) - { - unsigned int num_grid_w = target_width / stride; - unsigned int num_grid_h = target_height / stride; - std::vector anchors; - - if (stride == 8) - { - // 0 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - if (is_p6_1280) - { - anchor.width = 19.f; - anchor.height = 27.f; - } // p6_640 - else - { - anchor.width = 9.f; - anchor.height = 11.f; - } - anchors.push_back(anchor); - } - } - // 1 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - if (is_p6_1280) - { - anchor.width = 44.f; - anchor.height = 40.f; - } // p6_640 - else - { - anchor.width = 21.f; - anchor.height = 19.f; - } - anchors.push_back(anchor); - } - } - // 2 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - if (is_p6_1280) - { - anchor.width = 38.f; - anchor.height = 94.f; - } // p6_640 - else - { - anchor.width = 17.f; - anchor.height = 41.f; - } - anchors.push_back(anchor); - } - } - } // 16 - else if (stride == 16) - { - // 0 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - if (is_p6_1280) - { - anchor.width = 96.f; - anchor.height = 68.f; - } // p6_640 - else - { - anchor.width = 43.f; - anchor.height = 32.f; - } - anchors.push_back(anchor); - } - } - // 1 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - if (is_p6_1280) - { - anchor.width = 86.f; - anchor.height = 152.f; - } // p6_640 - else - { - anchor.width = 39.f; - anchor.height = 70.f; - } - anchors.push_back(anchor); - } - } - // 2 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - if (is_p6_1280) - { - anchor.width = 180.f; - anchor.height = 137.f; - } // p6_640 - else - { - anchor.width = 86.f; - anchor.height = 64.f; - } - anchors.push_back(anchor); - } - } - } // 32 - else if (stride == 32) - { - // 0 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - if (is_p6_1280) - { - anchor.width = 140.f; - anchor.height = 301.f; - } // p6_640 - else - { - anchor.width = 65.f; - anchor.height = 131.f; - } - anchors.push_back(anchor); - } - } - // 1 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - if (is_p6_1280) - { - anchor.width = 303.f; - anchor.height = 264.f; - } // p6_640 - else - { - anchor.width = 134.f; - anchor.height = 130.f; - } - anchors.push_back(anchor); - } - } - // 2 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - if (is_p6_1280) - { - anchor.width = 238.f; - anchor.height = 542.f; - } // p6_640 - else - { - anchor.width = 120.f; - anchor.height = 265.f; - } - anchors.push_back(anchor); - } - } - } // 64 - else - { - // 0 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - if (is_p6_1280) - { - anchor.width = 436.f; - anchor.height = 615.f; - } // p6_640 - else - { - anchor.width = 282.f; - anchor.height = 180.f; - } - anchors.push_back(anchor); - } - } - // 1 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - if (is_p6_1280) - { - anchor.width = 739.f; - anchor.height = 380.f; - } // p6_640 - else - { - anchor.width = 247.f; - anchor.height = 354.f; - } - anchors.push_back(anchor); - } - } - // 2 anchor - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - YoloV5Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - if (is_p6_1280) - { - anchor.width = 925.f; - anchor.height = 792.f; - } // p6_640 - else - { - anchor.width = 512.f; - anchor.height = 387.f; - } - anchors.push_back(anchor); - } - } - } - center_anchors[stride] = anchors; - } - - center_anchors_is_update = true; -} - -void NCNNYoloV5_V_6_0_P6::generate_bboxes(const YoloV5ScaleParams &scale_params, - ncnn::Extractor &extractor, - std::vector &bbox_collection, - float score_threshold, float img_height, - float img_width) -{ - // (1,n,85=5+80=cxcy+cwch+obj_conf+cls_conf) - ncnn::Mat det_stride_8, det_stride_16, det_stride_32, det_stride_64; - extractor.extract("det_stride_8", det_stride_8); - extractor.extract("det_stride_16", det_stride_16); - extractor.extract("det_stride_32", det_stride_32); - extractor.extract("det_stride_64", det_stride_64); - - this->generate_anchors(input_height, input_width); - - // generate bounding boxes. - bbox_collection.clear(); - this->generate_bboxes_single_stride(scale_params, det_stride_8, 8, score_threshold, - img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, det_stride_16, 16, score_threshold, - img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, det_stride_32, 32, score_threshold, - img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, det_stride_64, 64, score_threshold, - img_height, img_width, bbox_collection); -#if LITENCNN_DEBUG - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -// inner function -static inline float sigmoid(float x) -{ - return static_cast(1.f / (1.f + std::exp(-x))); -} - -void NCNNYoloV5_V_6_0_P6::generate_bboxes_single_stride(const YoloV5ScaleParams &scale_params, - ncnn::Mat &det_pred, - unsigned int stride, - float score_threshold, - float img_height, float img_width, - std::vector &bbox_collection) -{ - unsigned int nms_pre_ = (stride / 8) * nms_pre; // 1 * 1000,2*1000,... - nms_pre_ = nms_pre_ >= nms_pre ? nms_pre_ : nms_pre; - - const unsigned int f_h = (unsigned int) input_height / stride; - const unsigned int f_w = (unsigned int) input_width / stride; - // e.g, 3*80*80 + 3*40*40 + 3*20*20 = 25200 - const unsigned int num_anchors = 3 * f_h * f_w; - const unsigned int num_classes = 80; - - float r_ = scale_params.r; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - // have c=3 indicate 3 anchors at one grid - unsigned int count = 0; - auto &stride_anchors = center_anchors[stride]; - - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *offset_obj_cls_ptr = (float *) det_pred.data + (i * (num_classes + 5)); - float obj_conf = sigmoid(offset_obj_cls_ptr[4]); - if (obj_conf < score_threshold) continue; // filter first. - - float cls_conf = sigmoid(offset_obj_cls_ptr[5]); - unsigned int label = 0; // 80 class - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = sigmoid(offset_obj_cls_ptr[j + 5]); - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - - float conf = obj_conf * cls_conf; // cls_conf (0.,1.) - if (conf < score_threshold) continue; // filter - - int grid0 = stride_anchors.at(i).grid0; // w - int grid1 = stride_anchors.at(i).grid1; // h - float anchor_w = stride_anchors.at(i).width; - float anchor_h = stride_anchors.at(i).height; - - float dx = sigmoid(offset_obj_cls_ptr[0]); - float dy = sigmoid(offset_obj_cls_ptr[1]); - float dw = sigmoid(offset_obj_cls_ptr[2]); - float dh = sigmoid(offset_obj_cls_ptr[3]); - - float cx = (dx * 2.f - 0.5f + (float) grid0) * (float) stride; - float cy = (dy * 2.f - 0.5f + (float) grid1) * (float) stride; - float w = std::pow(dw * 2.f, 2) * anchor_w; - float h = std::pow(dh * 2.f, 2) * anchor_h; - - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - - types::Boxf box; - // de-padding & rescaling - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(x2, (float) img_width); - box.y2 = std::min(y2, (float) img_height); - box.score = conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - - if (bbox_collection.size() > nms_pre_) - { - std::sort(bbox_collection.begin(), bbox_collection.end(), - [](const types::Boxf &a, const types::Boxf &b) - { return a.score > b.score; }); // sort inplace - // trunc - bbox_collection.resize(nms_pre_); - } -} - -void NCNNYoloV5_V_6_0_P6::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - -void NCNNYoloV5_V_6_0_P6::print_debug_string() -{ - std::cout << "LITENCNN_DEBUG LogId: " << log_id << "\n"; - input_indexes = net->input_indexes(); - output_indexes = net->output_indexes(); -#ifdef NCNN_STRING - input_names = net->input_names(); - output_names = net->output_names(); -#endif - std::cout << "=============== Input-Dims ==============\n"; - for (int i = 0; i < input_indexes.size(); ++i) - { - std::cout << "Input: "; - auto tmp_in_blob = net->blobs().at(input_indexes.at(i)); -#ifdef NCNN_STRING - std::cout << input_names.at(i) << ": "; -#endif - std::cout << "shape: c=" << tmp_in_blob.shape.c - << " h=" << tmp_in_blob.shape.h << " w=" << tmp_in_blob.shape.w << "\n"; - } - - std::cout << "=============== Output-Dims ==============\n"; - for (int i = 0; i < output_indexes.size(); ++i) - { - auto tmp_out_blob = net->blobs().at(output_indexes.at(i)); - std::cout << "Output: "; -#ifdef NCNN_STRING - std::cout << output_names.at(i) << ": "; -#endif - std::cout << "shape: c=" << tmp_out_blob.shape.c - << " h=" << tmp_out_blob.shape.h << " w=" << tmp_out_blob.shape.w << "\n"; - } - std::cout << "========================================\n"; -} \ No newline at end of file diff --git a/lite/ncnn/cv/ncnn_yolov5_v6.0_p6.h b/lite/ncnn/cv/ncnn_yolov5_v6.0_p6.h deleted file mode 100644 index d8730bcc..00000000 --- a/lite/ncnn/cv/ncnn_yolov5_v6.0_p6.h +++ /dev/null @@ -1,131 +0,0 @@ -// -// Created by DefTruth on 2021/11/11. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOV5_V6_0_P6_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOV5_V6_0_P6_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNYoloV5_V_6_0_P6 - { - private: - ncnn::Net *net = nullptr; - const char *log_id = nullptr; - const char *param_path = nullptr; - const char *bin_path = nullptr; - std::vector input_names; - std::vector output_names; - std::vector input_indexes; - std::vector output_indexes; - - private: - // nested classes - typedef struct - { - int grid0; - int grid1; - int stride; - float width; - float height; - } YoloV5Anchor; - - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } YoloV5ScaleParams; - - public: - explicit NCNNYoloV5_V_6_0_P6(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1, - int _input_height = 640, - int _input_width = 640); // - ~NCNNYoloV5_V_6_0_P6(); - - private: - const unsigned int num_threads; // initialize at runtime. - // target image size after resize - const int input_height; // 640/1280 - const int input_width; // 640/1280 - - const float mean_vals[3] = {0.f, 0.f, 0.f}; // RGB - const float norm_vals[3] = {1.0 / 255.f, 1.0 / 255.f, 1.0 / 255.f}; - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int nms_pre = 1000; - static constexpr const unsigned int max_nms = 30000; - - std::vector strides = {8, 16, 32, 64}; // p6 have 4 detection heads - std::unordered_map> center_anchors; - bool center_anchors_is_update = false; - - protected: - NCNNYoloV5_V_6_0_P6(const NCNNYoloV5_V_6_0_P6 &) = delete; // - NCNNYoloV5_V_6_0_P6(NCNNYoloV5_V_6_0_P6 &&) = delete; // - NCNNYoloV5_V_6_0_P6 &operator=(const NCNNYoloV5_V_6_0_P6 &) = delete; // - NCNNYoloV5_V_6_0_P6 &operator=(NCNNYoloV5_V_6_0_P6 &&) = delete; // - - private: - void print_debug_string(); - - void transform(const cv::Mat &mat_rs, ncnn::Mat &in); - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YoloV5ScaleParams &scale_params); - - // only generate once - void generate_anchors(unsigned int target_height, unsigned int target_width); - - void generate_bboxes_single_stride(const YoloV5ScaleParams &scale_params, - ncnn::Mat &det_pred, - unsigned int stride, - float score_threshold, - float img_height, - float img_width, - std::vector &bbox_collection); - - void generate_bboxes(const YoloV5ScaleParams &scale_params, - ncnn::Extractor &extractor, - std::vector &bbox_collection, - float score_threshold, float img_height, - float img_width); - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, - std::vector &detected_boxes, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - - }; - -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOV5_V6_0_P6_H diff --git a/lite/ncnn/cv/ncnn_yolov6.cpp b/lite/ncnn/cv/ncnn_yolov6.cpp deleted file mode 100644 index e9c1336e..00000000 --- a/lite/ncnn/cv/ncnn_yolov6.cpp +++ /dev/null @@ -1,269 +0,0 @@ -// -// Created by DefTruth on 2022/6/25. -// - -#include "ncnn_yolov6.h" -#include "lite/utils.h" - -using ncnncv::NCNNYOLOv6; - - -NCNNYOLOv6::NCNNYOLOv6(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads, - int _input_height, - int _input_width) : - log_id(_param_path.data()), param_path(_param_path.data()), - bin_path(_bin_path.data()), num_threads(_num_threads), - input_height(_input_height), input_width(_input_width) -{ - net = new ncnn::Net(); - // init net, change this setting for better performance. - net->opt.use_fp16_arithmetic = false; - net->opt.use_vulkan_compute = false; // default - // setup Focus in yolov5 - // net->register_custom_layer("YoloV5Focus", YoloV5Focus_layer_creator); - net->load_param(param_path); - net->load_model(bin_path); -#ifdef LITENCNN_DEBUG - this->print_debug_string(); -#endif -} - -NCNNYOLOv6::~NCNNYOLOv6() -{ - if (net) delete net; - net = nullptr; -} - -void NCNNYOLOv6::transform(const cv::Mat &mat_rs, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -// letterbox -void NCNNYOLOv6::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YOLOv6ScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void NCNNYOLOv6::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - // resize & unscale - cv::Mat mat_rs; - YOLOv6ScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat_rs, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("image_arrays", input); - // 3.rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, extractor, score_threshold, img_height, img_width); - // 4. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void NCNNYOLOv6::generate_anchors(const int target_height, - const int target_width, - std::vector &strides, - std::vector &anchors) -{ - for (auto stride: strides) - { - int num_grid_w = target_width / stride; - int num_grid_h = target_height / stride; - for (int g1 = 0; g1 < num_grid_h; ++g1) - { - for (int g0 = 0; g0 < num_grid_w; ++g0) - { - YOLOv6Anchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchors.push_back(anchor); - } - } - } -} - -static inline float sigmoid(float x) -{ - return static_cast(1.f / (1.f + std::exp(-x))); -} - -void NCNNYOLOv6::generate_bboxes(const YOLOv6ScaleParams &scale_params, - std::vector &bbox_collection, - ncnn::Extractor &extractor, - float score_threshold, int img_height, - int img_width) -{ - ncnn::Mat outputs; - extractor.extract("outputs", outputs); // (1,n=?,85=5+80=cxcy+cwch+obj_conf+cls_conf) - - const unsigned int num_anchors = outputs.h; - const unsigned int num_classes = outputs.w - 5; - - std::vector anchors; - std::vector strides = {8, 16, 32}; // might have stride=64 - this->generate_anchors(input_height, input_width, strides, anchors); - - float r_ = scale_params.r; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - bbox_collection.clear(); - unsigned int count = 0; - - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *offset_obj_cls_ptr = - (float *) outputs.data + (i * (num_classes + 5)); // row ptr - float obj_conf = sigmoid(offset_obj_cls_ptr[4]); - if (obj_conf < score_threshold) continue; // filter first. - - float cls_conf = sigmoid(offset_obj_cls_ptr[5]); - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = sigmoid(offset_obj_cls_ptr[j + 5]); - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - - float conf = obj_conf * cls_conf; // cls_conf (0.,1.) - if (conf < score_threshold) continue; // filter - - const int grid0 = anchors.at(i).grid0; - const int grid1 = anchors.at(i).grid1; - const int stride = anchors.at(i).stride; - - float dx = offset_obj_cls_ptr[0]; - float dy = offset_obj_cls_ptr[1]; - float dw = offset_obj_cls_ptr[2]; - float dh = offset_obj_cls_ptr[3]; - - float cx = (dx + (float) grid0) * (float) stride; - float cy = (dy + (float) grid1) * (float) stride; - float w = std::exp(dw) * (float) stride; - float h = std::exp(dh) * (float) stride; - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - - types::Boxf box; - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(x2, (float) img_width - 1.f); - box.y2 = std::min(y2, (float) img_height - 1.f); - box.score = conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITENCNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void NCNNYOLOv6::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - - -void NCNNYOLOv6::print_debug_string() -{ - std::cout << "LITENCNN_DEBUG LogId: " << log_id << "\n"; - input_indexes = net->input_indexes(); - output_indexes = net->output_indexes(); -#ifdef NCNN_STRING - input_names = net->input_names(); - output_names = net->output_names(); -#endif - std::cout << "=============== Input-Dims ==============\n"; - for (int i = 0; i < input_indexes.size(); ++i) - { - std::cout << "Input: "; - auto tmp_in_blob = net->blobs().at(input_indexes.at(i)); -#ifdef NCNN_STRING - std::cout << input_names.at(i) << ": "; -#endif - std::cout << "shape: c=" << tmp_in_blob.shape.c - << " h=" << tmp_in_blob.shape.h << " w=" << tmp_in_blob.shape.w << "\n"; - } - - std::cout << "=============== Output-Dims ==============\n"; - for (int i = 0; i < output_indexes.size(); ++i) - { - auto tmp_out_blob = net->blobs().at(output_indexes.at(i)); - std::cout << "Output: "; -#ifdef NCNN_STRING - std::cout << output_names.at(i) << ": "; -#endif - std::cout << "shape: c=" << tmp_out_blob.shape.c - << " h=" << tmp_out_blob.shape.h << " w=" << tmp_out_blob.shape.w << "\n"; - } - std::cout << "========================================\n"; -} - - diff --git a/lite/ncnn/cv/ncnn_yolov6.h b/lite/ncnn/cv/ncnn_yolov6.h deleted file mode 100644 index 4a4cbc91..00000000 --- a/lite/ncnn/cv/ncnn_yolov6.h +++ /dev/null @@ -1,113 +0,0 @@ -// -// Created by DefTruth on 2022/6/25. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOV6_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOV6_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNYOLOv6 - { - private: - ncnn::Net *net = nullptr; - const char *log_id = nullptr; - const char *param_path = nullptr; - const char *bin_path = nullptr; - std::vector input_names; - std::vector output_names; - std::vector input_indexes; - std::vector output_indexes; - - public: - explicit NCNNYOLOv6(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1, - int _input_height = 640, - int _input_width = 640); // - ~NCNNYOLOv6(); - - private: - // nested classes - typedef struct GridAndStride - { - int grid0; - int grid1; - int stride; - } YOLOv6Anchor; - - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } YOLOv6ScaleParams; - - private: - const unsigned int num_threads; // initialize at runtime. - const int input_height; // 640/320 - const int input_width; // 640/320 - - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - const float mean_vals[3] = {0.f, 0.f, 0.f}; // RGB - const float norm_vals[3] = {1.0 / 255.f, 1.0 / 255.f, 1.0 / 255.f}; - static constexpr const unsigned int max_nms = 30000; - - protected: - NCNNYOLOv6(const NCNNYOLOv6 &) = delete; // - NCNNYOLOv6(NCNNYOLOv6 &&) = delete; // - NCNNYOLOv6 &operator=(const NCNNYOLOv6 &) = delete; // - NCNNYOLOv6 &operator=(NCNNYOLOv6 &&) = delete; // - - private: - void print_debug_string(); - - void transform(const cv::Mat &mat_rs, ncnn::Mat &in); - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YOLOv6ScaleParams &scale_params); - - void generate_anchors(const int target_height, - const int target_width, - std::vector &strides, - std::vector &anchors); - - void generate_bboxes(const YOLOv6ScaleParams &scale_params, - std::vector &bbox_collection, - ncnn::Extractor &extractor, - float score_threshold, int img_height, - int img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOV6_H diff --git a/lite/ncnn/cv/ncnn_yolox.cpp b/lite/ncnn/cv/ncnn_yolox.cpp deleted file mode 100644 index 70b7c637..00000000 --- a/lite/ncnn/cv/ncnn_yolox.cpp +++ /dev/null @@ -1,276 +0,0 @@ -// -// Created by DefTruth on 2021/10/18. -// - -#include "ncnn_yolox.h" -#include "lite/utils.h" - -using ncnncv::NCNNYoloX; - - -NCNNYoloX::NCNNYoloX(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads, - int _input_height, - int _input_width) : - log_id(_param_path.data()), param_path(_param_path.data()), - bin_path(_bin_path.data()), num_threads(_num_threads), - input_height(_input_height), input_width(_input_width) -{ - net = new ncnn::Net(); - // init net, change this setting for better performance. - net->opt.use_fp16_arithmetic = false; - net->opt.use_vulkan_compute = false; // default - // setup Focus in yolov5 - net->register_custom_layer("YoloV5Focus", YoloV5Focus_layer_creator); - net->load_param(param_path); - net->load_model(bin_path); -#ifdef LITENCNN_DEBUG - this->print_debug_string(); -#endif -} - -NCNNYoloX::~NCNNYoloX() -{ - if (net) delete net; - net = nullptr; -} - -void NCNNYoloX::transform(const cv::Mat &mat_rs, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_BGR2RGB, input_width, input_height); - in.substract_mean_normalize(mean_vals, norm_vals); -} - -void NCNNYoloX::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YoloXScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void NCNNYoloX::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - // resize & unscale - cv::Mat mat_rs; - YoloXScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat_rs, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("inputs", input); - // 3.rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, extractor, score_threshold, img_height, img_width); - // 4. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void NCNNYoloX::generate_anchors(const int target_height, - const int target_width, - std::vector &strides, - std::vector &anchors) -{ - for (auto stride : strides) - { - int num_grid_w = target_width / stride; - int num_grid_h = target_height / stride; - for (int g1 = 0; g1 < num_grid_h; ++g1) - { - for (int g0 = 0; g0 < num_grid_w; ++g0) - { -#ifdef LITE_WIN32 - YoloXAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchors.push_back(anchor); -#else - anchors.push_back((YoloXAnchor) {g0, g1, stride}); -#endif - } - } - } -} - -void NCNNYoloX::generate_bboxes(const YoloXScaleParams &scale_params, - std::vector &bbox_collection, - ncnn::Extractor &extractor, - float score_threshold, int img_height, - int img_width) -{ - ncnn::Mat outputs; - extractor.extract("outputs", outputs); // (1,n=?,85=5+80=cxcy+cwch+obj_conf+cls_conf) - - const unsigned int num_anchors = outputs.h; - const unsigned int num_classes = outputs.w - 5; - - std::vector anchors; - std::vector strides = {8, 16, 32}; // might have stride=64 - this->generate_anchors(input_height, input_width, strides, anchors); - - float r_ = scale_params.r; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - bbox_collection.clear(); - unsigned int count = 0; - - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *offset_obj_cls_ptr = - (float *) outputs.data + (i * (num_classes + 5)); // row ptr - float obj_conf = offset_obj_cls_ptr[4]; - if (obj_conf < score_threshold) continue; // filter first. - - float cls_conf = offset_obj_cls_ptr[5]; - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = offset_obj_cls_ptr[j + 5]; - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - - float conf = obj_conf * cls_conf; // cls_conf (0.,1.) - if (conf < score_threshold) continue; // filter - - const int grid0 = anchors.at(i).grid0; - const int grid1 = anchors.at(i).grid1; - const int stride = anchors.at(i).stride; - - float dx = offset_obj_cls_ptr[0]; - float dy = offset_obj_cls_ptr[1]; - float dw = offset_obj_cls_ptr[2]; - float dh = offset_obj_cls_ptr[3]; - - float cx = (dx + (float) grid0) * (float) stride; - float cy = (dy + (float) grid1) * (float) stride; - float w = std::exp(dw) * (float) stride; - float h = std::exp(dh) * (float) stride; - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - - types::Boxf box; - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(x2, (float) img_width - 1.f); - box.y2 = std::min(y2, (float) img_height - 1.f); - box.score = conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITENCNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void NCNNYoloX::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - - -void NCNNYoloX::print_debug_string() -{ - std::cout << "LITENCNN_DEBUG LogId: " << log_id << "\n"; - input_indexes = net->input_indexes(); - output_indexes = net->output_indexes(); -#ifdef NCNN_STRING - input_names = net->input_names(); - output_names = net->output_names(); -#endif - std::cout << "=============== Input-Dims ==============\n"; - for (int i = 0; i < input_indexes.size(); ++i) - { - std::cout << "Input: "; - auto tmp_in_blob = net->blobs().at(input_indexes.at(i)); -#ifdef NCNN_STRING - std::cout << input_names.at(i) << ": "; -#endif - std::cout << "shape: c=" << tmp_in_blob.shape.c - << " h=" << tmp_in_blob.shape.h << " w=" << tmp_in_blob.shape.w << "\n"; - } - - std::cout << "=============== Output-Dims ==============\n"; - for (int i = 0; i < output_indexes.size(); ++i) - { - auto tmp_out_blob = net->blobs().at(output_indexes.at(i)); - std::cout << "Output: "; -#ifdef NCNN_STRING - std::cout << output_names.at(i) << ": "; -#endif - std::cout << "shape: c=" << tmp_out_blob.shape.c - << " h=" << tmp_out_blob.shape.h << " w=" << tmp_out_blob.shape.w << "\n"; - } - std::cout << "========================================\n"; -} - - - - - - - - - - - diff --git a/lite/ncnn/cv/ncnn_yolox.h b/lite/ncnn/cv/ncnn_yolox.h deleted file mode 100644 index 38dd99f4..00000000 --- a/lite/ncnn/cv/ncnn_yolox.h +++ /dev/null @@ -1,115 +0,0 @@ -// -// Created by DefTruth on 2021/10/18. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOX_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOX_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNYoloX - { - private: - ncnn::Net *net = nullptr; - const char *log_id = nullptr; - const char *param_path = nullptr; - const char *bin_path = nullptr; - std::vector input_names; - std::vector output_names; - std::vector input_indexes; - std::vector output_indexes; - - public: - explicit NCNNYoloX(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1, - int _input_height = 640, - int _input_width = 640); // - ~NCNNYoloX(); - - private: - // nested classes - typedef struct GridAndStride - { - int grid0; - int grid1; - int stride; - } YoloXAnchor; - - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } YoloXScaleParams; - - private: - const unsigned int num_threads; // initialize at runtime. - // target image size after resize, might use 416 for small model(nano/tiny) - const int input_height; // 640(s/m/l/x), 416(nano/tiny) - const int input_width; // 640(s/m/l/x), 416(nano/tiny) - - const float mean_vals[3] = {255.f * 0.485f, 255.f * 0.456, 255.f * 0.406f}; - const float norm_vals[3] = {1.f / (255.f * 0.229f), 1.f / (255.f * 0.224f), 1.f / (255.f * 0.225f)}; - - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - protected: - NCNNYoloX(const NCNNYoloX &) = delete; // - NCNNYoloX(NCNNYoloX &&) = delete; // - NCNNYoloX &operator=(const NCNNYoloX &) = delete; // - NCNNYoloX &operator=(NCNNYoloX &&) = delete; // - - private: - void print_debug_string(); - - void transform(const cv::Mat &mat_rs, ncnn::Mat &in); - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YoloXScaleParams &scale_params); - - void generate_anchors(const int target_height, - const int target_width, - std::vector &strides, - std::vector &anchors); - - void generate_bboxes(const YoloXScaleParams &scale_params, - std::vector &bbox_collection, - ncnn::Extractor &extractor, - float score_threshold, int img_height, - int img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - }; -} - -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOX_H diff --git a/lite/ncnn/cv/ncnn_yolox_v0.1.1.cpp b/lite/ncnn/cv/ncnn_yolox_v0.1.1.cpp deleted file mode 100644 index 8e9b9d70..00000000 --- a/lite/ncnn/cv/ncnn_yolox_v0.1.1.cpp +++ /dev/null @@ -1,275 +0,0 @@ -// -// Created by DefTruth on 2021/11/6. -// - -#include "ncnn_yolox_v0.1.1.h" -#include "lite/utils.h" - -using ncnncv::NCNNYoloX_V_0_1_1; - - -NCNNYoloX_V_0_1_1::NCNNYoloX_V_0_1_1(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads, - int _input_height, - int _input_width) : - log_id(_param_path.data()), param_path(_param_path.data()), - bin_path(_bin_path.data()), num_threads(_num_threads), - input_height(_input_height), input_width(_input_width) -{ - net = new ncnn::Net(); - // init net, change this setting for better performance. - net->opt.use_fp16_arithmetic = false; - net->opt.use_vulkan_compute = false; // default - // setup Focus in yolov5 - net->register_custom_layer("YoloV5Focus", YoloV5Focus_layer_creator); - net->load_param(param_path); - net->load_model(bin_path); -#ifdef LITENCNN_DEBUG - this->print_debug_string(); -#endif -} - -NCNNYoloX_V_0_1_1::~NCNNYoloX_V_0_1_1() -{ - if (net) delete net; - net = nullptr; -} - -void NCNNYoloX_V_0_1_1::transform(const cv::Mat &mat_rs, ncnn::Mat &in) -{ - // BGR NHWC -> RGB NCHW - in = ncnn::Mat::from_pixels(mat_rs.data, ncnn::Mat::PIXEL_RGB, input_width, input_height); -} - -void NCNNYoloX_V_0_1_1::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YoloXScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void NCNNYoloX_V_0_1_1::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - // resize & unscale - cv::Mat mat_rs; - YoloXScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - ncnn::Mat input; - this->transform(mat_rs, input); - // 2. inference & extract - auto extractor = net->create_extractor(); - extractor.set_light_mode(false); // default - extractor.set_num_threads(num_threads); - extractor.input("images", input); - // 3.rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, extractor, score_threshold, img_height, img_width); - // 4. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void NCNNYoloX_V_0_1_1::generate_anchors(const int target_height, - const int target_width, - std::vector &strides, - std::vector &anchors) -{ - for (auto stride : strides) - { - int num_grid_w = target_width / stride; - int num_grid_h = target_height / stride; - for (int g1 = 0; g1 < num_grid_h; ++g1) - { - for (int g0 = 0; g0 < num_grid_w; ++g0) - { -#ifdef LITE_WIN32 - YoloXAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchors.push_back(anchor); -#else - anchors.push_back((YoloXAnchor) {g0, g1, stride}); -#endif - } - } - } -} - -void NCNNYoloX_V_0_1_1::generate_bboxes(const YoloXScaleParams &scale_params, - std::vector &bbox_collection, - ncnn::Extractor &extractor, - float score_threshold, int img_height, - int img_width) -{ - ncnn::Mat outputs; - extractor.extract("output", outputs); // (1,n=?,85=5+80=cxcy+cwch+obj_conf+cls_conf) - - const unsigned int num_anchors = outputs.h; - const unsigned int num_classes = outputs.w - 5; - - std::vector anchors; - std::vector strides = {8, 16, 32}; // might have stride=64 - this->generate_anchors(input_height, input_width, strides, anchors); - - float r_ = scale_params.r; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - bbox_collection.clear(); - unsigned int count = 0; - - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *offset_obj_cls_ptr = - (float *) outputs.data + (i * (num_classes + 5)); // row ptr - float obj_conf = offset_obj_cls_ptr[4]; - if (obj_conf < score_threshold) continue; // filter first. - - float cls_conf = offset_obj_cls_ptr[5]; - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = offset_obj_cls_ptr[j + 5]; - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - - float conf = obj_conf * cls_conf; // cls_conf (0.,1.) - if (conf < score_threshold) continue; // filter - - const int grid0 = anchors.at(i).grid0; - const int grid1 = anchors.at(i).grid1; - const int stride = anchors.at(i).stride; - - float dx = offset_obj_cls_ptr[0]; - float dy = offset_obj_cls_ptr[1]; - float dw = offset_obj_cls_ptr[2]; - float dh = offset_obj_cls_ptr[3]; - - float cx = (dx + (float) grid0) * (float) stride; - float cy = (dy + (float) grid1) * (float) stride; - float w = std::exp(dw) * (float) stride; - float h = std::exp(dh) * (float) stride; - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - - types::Boxf box; - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(x2, (float) img_width - 1.f); - box.y2 = std::min(y2, (float) img_height - 1.f); - box.score = conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITENCNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void NCNNYoloX_V_0_1_1::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - - -void NCNNYoloX_V_0_1_1::print_debug_string() -{ - std::cout << "LITENCNN_DEBUG LogId: " << log_id << "\n"; - input_indexes = net->input_indexes(); - output_indexes = net->output_indexes(); -#ifdef NCNN_STRING - input_names = net->input_names(); - output_names = net->output_names(); -#endif - std::cout << "=============== Input-Dims ==============\n"; - for (int i = 0; i < input_indexes.size(); ++i) - { - std::cout << "Input: "; - auto tmp_in_blob = net->blobs().at(input_indexes.at(i)); -#ifdef NCNN_STRING - std::cout << input_names.at(i) << ": "; -#endif - std::cout << "shape: c=" << tmp_in_blob.shape.c - << " h=" << tmp_in_blob.shape.h << " w=" << tmp_in_blob.shape.w << "\n"; - } - - std::cout << "=============== Output-Dims ==============\n"; - for (int i = 0; i < output_indexes.size(); ++i) - { - auto tmp_out_blob = net->blobs().at(output_indexes.at(i)); - std::cout << "Output: "; -#ifdef NCNN_STRING - std::cout << output_names.at(i) << ": "; -#endif - std::cout << "shape: c=" << tmp_out_blob.shape.c - << " h=" << tmp_out_blob.shape.h << " w=" << tmp_out_blob.shape.w << "\n"; - } - std::cout << "========================================\n"; -} - - - - - - - - - - - diff --git a/lite/ncnn/cv/ncnn_yolox_v0.1.1.h b/lite/ncnn/cv/ncnn_yolox_v0.1.1.h deleted file mode 100644 index aea7b459..00000000 --- a/lite/ncnn/cv/ncnn_yolox_v0.1.1.h +++ /dev/null @@ -1,111 +0,0 @@ -// -// Created by DefTruth on 2021/11/6. -// - -#ifndef LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOX_V0_1_1_H -#define LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOX_V0_1_1_H - -#include "lite/ncnn/core/ncnn_core.h" - -namespace ncnncv -{ - class LITE_EXPORTS NCNNYoloX_V_0_1_1 - { - private: - ncnn::Net *net = nullptr; - const char *log_id = nullptr; - const char *param_path = nullptr; - const char *bin_path = nullptr; - std::vector input_names; - std::vector output_names; - std::vector input_indexes; - std::vector output_indexes; - - public: - explicit NCNNYoloX_V_0_1_1(const std::string &_param_path, - const std::string &_bin_path, - unsigned int _num_threads = 1, - int _input_height = 640, - int _input_width = 640); // - ~NCNNYoloX_V_0_1_1(); - - private: - // nested classes - typedef struct GridAndStride - { - int grid0; - int grid1; - int stride; - } YoloXAnchor; - - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } YoloXScaleParams; - - private: - const unsigned int num_threads; // initialize at runtime. - // target image size after resize, might use 416 for small model(nano/tiny) - const int input_height; // 640(s/m/l/x), 416(nano/tiny) - const int input_width; // 640(s/m/l/x), 416(nano/tiny) - - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - protected: - NCNNYoloX_V_0_1_1(const NCNNYoloX_V_0_1_1 &) = delete; // - NCNNYoloX_V_0_1_1(NCNNYoloX_V_0_1_1 &&) = delete; // - NCNNYoloX_V_0_1_1 &operator=(const NCNNYoloX_V_0_1_1 &) = delete; // - NCNNYoloX_V_0_1_1 &operator=(NCNNYoloX_V_0_1_1 &&) = delete; // - - private: - void print_debug_string(); - - void transform(const cv::Mat &mat_rs, ncnn::Mat &in); - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YoloXScaleParams &scale_params); - - void generate_anchors(const int target_height, - const int target_width, - std::vector &strides, - std::vector &anchors); - - void generate_bboxes(const YoloXScaleParams &scale_params, - std::vector &bbox_collection, - ncnn::Extractor &extractor, - float score_threshold, int img_height, - int img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - }; -} -#endif //LITE_AI_TOOLKIT_NCNN_CV_NCNN_YOLOX_V0_1_1_H diff --git a/lite/tnn/core/tnn_config.h b/lite/tnn/core/tnn_config.h deleted file mode 100644 index 52170cd9..00000000 --- a/lite/tnn/core/tnn_config.h +++ /dev/null @@ -1,22 +0,0 @@ -// -// Created by DefTruth on 2021/10/17. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CORE_TNN_CONFIG_H -#define LITE_AI_TOOLKIT_TNN_CORE_TNN_CONFIG_H - -#include "tnn_defs.h" -#include "lite/lite.ai.headers.h" - -#ifdef ENABLE_TNN -#include "tnn/core/macro.h" -#include "tnn/core/tnn.h" -#include "tnn/core/mat.h" -#include "tnn/utils/blob_converter.h" -#include "tnn/utils/mat_utils.h" -#include "tnn/utils/dims_vector_utils.h" -#endif - -namespace tnncore {} - -#endif //LITE_AI_TOOLKIT_TNN_CORE_TNN_CONFIG_H diff --git a/lite/tnn/core/tnn_core.h b/lite/tnn/core/tnn_core.h deleted file mode 100644 index e5c40cef..00000000 --- a/lite/tnn/core/tnn_core.h +++ /dev/null @@ -1,104 +0,0 @@ -// -// Created by DefTruth on 2021/10/17. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CORE_TNN_CORE_H -#define LITE_AI_TOOLKIT_TNN_CORE_TNN_CORE_H - -#include "tnn_config.h" -#include "tnn_handler.h" -#include "tnn_types.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNNanoDet; // [0] * reference: https://github.com/RangiLyu/nanodet - class LITE_EXPORTS TNNNanoDetEfficientNetLite; // [1] * reference: https://github.com/RangiLyu/nanodet - class LITE_EXPORTS TNNRobustVideoMatting; // [2] * reference: https://github.com/PeterL1n/RobustVideoMatting - class LITE_EXPORTS TNNYoloX; // [3] * reference: https://github.com/Megvii-BaseDetection/YOLOX - class LITE_EXPORTS TNNYOLOP; // [4] * reference: https://github.com/hustvl/YOLOP - class LITE_EXPORTS TNNYoloV5; // [5] * reference: https://github.com/ultralytics/yolov5 - class LITE_EXPORTS TNNYoloX_V_0_1_1; // [6] * reference: https://github.com/Megvii-BaseDetection/YOLOX - class LITE_EXPORTS TNNYoloR; // [7] * reference: https://github.com/WongKinYiu/yolor - class LITE_EXPORTS TNNYoloV5_V_6_0; // [8] * reference: https://github.com/ultralytics/yolov5 - class LITE_EXPORTS TNNGlintArcFace; // [9] * reference: https://github.com/deepinsight/insightface/tree/master/recognition/arcface_torch - class LITE_EXPORTS TNNGlintCosFace; // [10] * reference: https://github.com/deepinsight/insightface/tree/master/recognition/arcface_torch - class LITE_EXPORTS TNNGlintPartialFC; // [11] * reference: https://github.com/deepinsight/insightface/tree/master/recognition/partial_fc - class LITE_EXPORTS TNNFaceNet; // [12] * reference: https://github.com/timesler/facenet-pytorch - class LITE_EXPORTS TNNFocalArcFace; // [13] * reference: https://github.com/ZhaoJ9014/face.evoLVe.PyTorch - class LITE_EXPORTS TNNFocalAsiaArcFace; // [14] * reference: https://github.com/ZhaoJ9014/face.evoLVe.PyTorch - class LITE_EXPORTS TNNTencentCurricularFace; // [15] * reference: https://github.com/Tencent/TFace/tree/master/tasks/distfc - class LITE_EXPORTS TNNTencentCifpFace; // [16] * reference: https://github.com/Tencent/TFace/tree/master/tasks/cifp - class LITE_EXPORTS TNNCenterLossFace; // [17] * reference: https://github.com/louis-she/center-loss.pytorch - class LITE_EXPORTS TNNSphereFace; // [18] * reference: https://github.com/clcarwin/sphereface_pytorch - class LITE_EXPORTS TNNMobileFaceNet; // [19] * reference: https://github.com/Xiaoccer/MobileFaceNet_Pytorch - class LITE_EXPORTS TNNCavaGhostArcFace; // [20] * reference: https://github.com/cavalleria/cavaface.pytorch - class LITE_EXPORTS TNNCavaCombinedFace; // [21] * reference: https://github.com/cavalleria/cavaface.pytorch - class LITE_EXPORTS TNNMobileSEFocalFace; // [22] * reference: https://github.com/grib0ed0v/face_recognition.pytorch - class LITE_EXPORTS TNNUltraFace; // [23] * reference: https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB - class LITE_EXPORTS TNNRetinaFace; // [24] * reference: https://github.com/biubug6/Pytorch_Retinaface - class LITE_EXPORTS TNNFaceBoxes; // [25] * reference: https://github.com/zisianw/FaceBoxes.PyTorch - class LITE_EXPORTS TNNPFLD; // [26] * reference: https://github.com/Hsintao/pfld_106_face_landmarks - class LITE_EXPORTS TNNPFLD98; // [27] * reference: https://github.com/polarisZhao/PFLD-pytorch - class LITE_EXPORTS TNNMobileNetV268; // [28] * reference: https://github.com/cunjian/pytorch_face_landmark - class LITE_EXPORTS TNNMobileNetV2SE68; // [29] * reference: https://github.com/cunjian/pytorch_face_landmark - class LITE_EXPORTS TNNPFLD68; // [30] * reference: https://github.com/cunjian/pytorch_face_landmark - class LITE_EXPORTS TNNFaceLandmark1000; // [31] * reference: https://github.com/Single430/FaceLandmark1000 - class LITE_EXPORTS TNNFSANet; // [32] * reference: https://github.com/omasaht/headpose-fsanet-pytorch - class LITE_EXPORTS TNNAgeGoogleNet; // [33] * reference: https://github.com/onnx/models/tree/master/vision/body_analysis/age_gender - class LITE_EXPORTS TNNGenderGoogleNet; // [34] * reference: https://github.com/onnx/models/tree/master/vision/body_analysis/age_gender - class LITE_EXPORTS TNNEmotionFerPlus; // [35] * reference: https://github.com/onnx/models/blob/master/vision/body_analysis/emotion_ferplus - class LITE_EXPORTS TNNSSRNet; // [36] * reference: https://github.com/oukohou/SSR_Net_Pytorch - class LITE_EXPORTS TNNEfficientEmotion7; // [37] * reference: https://github.com/HSE-asavchenko/face-emotion-recognition - class LITE_EXPORTS TNNEfficientEmotion8; // [38] * reference: https://github.com/HSE-asavchenko/face-emotion-recognition - class LITE_EXPORTS TNNMobileEmotion7; // [39] * reference: https://github.com/HSE-asavchenko/face-emotion-recognition - class LITE_EXPORTS TNNReXNetEmotion7; // [40] * reference: https://github.com/HSE-asavchenko/face-emotion-recognition - class LITE_EXPORTS TNNEfficientNetLite4; // [41] * reference: https://github.com/onnx/models/blob/master/vision/classification/efficientnet-lite4 - class LITE_EXPORTS TNNShuffleNetV2; // [42] * reference: https://github.com/onnx/models/blob/master/vision/classification/shufflenet - class LITE_EXPORTS TNNDenseNet; // [43] * reference: https://pytorch.org/hub/pytorch_vision_densenet/ - class LITE_EXPORTS TNNGhostNet; // [44] * reference：https://pytorch.org/hub/pytorch_vision_ghostnet/ - class LITE_EXPORTS TNNHdrDNet; // [45] * reference: https://pytorch.org/hub/pytorch_vision_hardnet/ - class LITE_EXPORTS TNNIBNNet; // [46] * reference: https://pytorch.org/hub/pytorch_vision_ibnnet/ - class LITE_EXPORTS TNNMobileNetV2; // [47] * reference: https://pytorch.org/hub/pytorch_vision_mobilenet_v2/ - class LITE_EXPORTS TNNResNet; // [48] * reference: https://pytorch.org/hub/pytorch_vision_resnet/ - class LITE_EXPORTS TNNResNeXt; // [49] * reference: https://pytorch.org/hub/pytorch_vision_resnext/ - class LITE_EXPORTS TNNFastStyleTransfer; // [50] * reference: https://github.com/onnx/models/blob/master/vision/style_transfer/fast_neural_style - class LITE_EXPORTS TNNColorizer; // [51] * reference: https://github.com/richzhang/colorization - class LITE_EXPORTS TNNSubPixelCNN; // [52] * reference: https://github.com/niazwazir/SUB_PIXEL_CNN - class LITE_EXPORTS TNNDeepLabV3ResNet101; // [53] * reference: https://pytorch.org/hub/pytorch_vision_deeplabv3_resnet101/ - class LITE_EXPORTS TNNFCNResNet101; // [54] * reference: https://pytorch.org/hub/pytorch_vision_fcn_resnet101/ - class LITE_EXPORTS TNNMGMatting; // [55] * reference: https://github.com/yucornetto/MGMatting - class LITE_EXPORTS TNNNanoDetPlus; // [56] * reference: https://github.com/RangiLyu/nanodet - class LITE_EXPORTS TNNSCRFD; // [57] * reference: https://github.com/deepinsight/insightface/tree/master/detection/scrfd - class LITE_EXPORTS TNNYOLO5Face; // [58] * reference: https://github.com/deepcam-cn/yolov5-face - class LITE_EXPORTS TNNFaceBoxesV2; // [59] * reference: https://github.com/jhb86253817/FaceBoxesV2 - class LITE_EXPORTS TNNPIPNet19; // [60] * reference: https://github.com/jhb86253817/PIPNet - class LITE_EXPORTS TNNPIPNet29; // [61] * reference: https://github.com/jhb86253817/PIPNet - class LITE_EXPORTS TNNPIPNet68; // [62] * reference: https://github.com/jhb86253817/PIPNet - class LITE_EXPORTS TNNPIPNet98; // [63] * reference: https://github.com/jhb86253817/PIPNet - class LITE_EXPORTS TNNInsectDet; // [64] * reference: https://github.com/quarrying/quarrying-insect-id - class LITE_EXPORTS TNNInsectID; // [65] * reference: https://github.com/quarrying/quarrying-insect-id - class LITE_EXPORTS TNNPlantID; // [66] * reference: https://github.com/quarrying/quarrying-plant-id - class LITE_EXPORTS TNNMODNet; // [67] * reference: https://github.com/ZHKKKe/MODNet - class LITE_EXPORTS TNNBackgroundMattingV2; // [68] * reference: https://github.com/PeterL1n/BackgroundMattingV2 - class LITE_EXPORTS TNNHeadSeg; // [69] * reference: https://github.com/minivision-ai/photo2cartoon - class LITE_EXPORTS TNNFemalePhoto2Cartoon; // [70] * reference: https://github.com/minivision-ai/photo2cartoon - class LITE_EXPORTS TNNYOLOv6; // [71] * reference: https://github.com/meituan/YOLOv6 - class LITE_EXPORTS TNNFaceParsingBiSeNet; // [72] * reference: https://github.com/zllrunning/face-parsing.PyTorch -} - -namespace tnncv -{ - using tnncore::BasicTNNHandler; -} - -namespace tnnnlp -{ - using tnncore::BasicTNNHandler; -} - -namespace tnnasr -{ - using tnncore::BasicTNNHandler; -} - -#endif //LITE_AI_TOOLKIT_TNN_CORE_TNN_CORE_H diff --git a/lite/tnn/core/tnn_defs.h b/lite/tnn/core/tnn_defs.h deleted file mode 100644 index 67f8f7d3..00000000 --- a/lite/tnn/core/tnn_defs.h +++ /dev/null @@ -1,23 +0,0 @@ -// -// Created by DefTruth on 2021/10/17. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CORE_TNN_DEFS_H -#define LITE_AI_TOOLKIT_TNN_CORE_TNN_DEFS_H - -#include "lite/config.h" -#include "lite/lite.ai.defs.h" - -#ifdef ENABLE_DEBUG_STRING -# define LITETNN_DEBUG 1 -#else -# define LITETNN_DEBUG 0 -#endif - -#ifdef LITE_WIN32 -# ifndef NOMINMAX -# define NOMINMAX -# endif -#endif - -#endif //LITE_AI_TOOLKIT_TNN_CORE_TNN_DEFS_H diff --git a/lite/tnn/core/tnn_handler.cpp b/lite/tnn/core/tnn_handler.cpp deleted file mode 100644 index 2f278546..00000000 --- a/lite/tnn/core/tnn_handler.cpp +++ /dev/null @@ -1,377 +0,0 @@ -// -// Created by DefTruth on 2021/10/17. -// - -#include "tnn_handler.h" - -using tnncore::BasicTNNHandler; - -BasicTNNHandler::BasicTNNHandler( - const std::string &_proto_path, const std::string &_model_path, - unsigned int _num_threads) : proto_path(_proto_path.data()), - model_path(_model_path.data()), - log_id(_proto_path.data()), - num_threads(_num_threads) -{ - initialize_handler(); -} - -BasicTNNHandler::~BasicTNNHandler() -{ - net = nullptr; - instance = nullptr; - input_mat = nullptr; -} - -void BasicTNNHandler::initialize_handler() -{ - std::string proto_content_buffer, model_content_buffer; - proto_content_buffer = BasicTNNHandler::content_buffer_from(proto_path); - model_content_buffer = BasicTNNHandler::content_buffer_from(model_path); - - tnn::ModelConfig model_config; - model_config.model_type = tnn::MODEL_TYPE_TNN; - model_config.params = {proto_content_buffer, model_content_buffer}; - - // 1. init TNN net - tnn::Status status; - net = std::make_shared(); - status = net->Init(model_config); - if (status != tnn::TNN_OK || !net) - { -#ifdef LITETNN_DEBUG - std::cout << "net->Init failed!\n"; -#endif - return; - } - // 2. init device type, change this default setting - // for better performance. such as CUDA/OPENCL/... -#ifdef __ANDROID__ - network_device_type = tnn::DEVICE_ARM; // CPU,GPU - input_device_type = tnn::DEVICE_ARM; // CPU only - output_device_type = tnn::DEVICE_ARM; -#else - network_device_type = tnn::DEVICE_X86; // CPU,GPU - input_device_type = tnn::DEVICE_X86; // CPU only - output_device_type = tnn::DEVICE_X86; -#endif - // 3. init instance - tnn::NetworkConfig network_config; - network_config.library_path = {""}; - network_config.device_type = network_device_type; - - instance = net->CreateInst(network_config, status); - if (status != tnn::TNN_OK || !instance) - { -#ifdef LITETNN_DEBUG - std::cout << "CreateInst failed!" << status.description().c_str() << "\n"; -#endif - return; - } - // 4. setting up num_threads - instance->SetCpuNumThreads((int) num_threads); - // 5. init input information. - input_name = this->get_input_names().front(); - input_shape = this->get_input_shape(input_name); - if (input_shape.size() != 4) - { -#ifdef LITETNN_DEBUG - throw std::runtime_error("Found input_shape.size()!=4, but " - "BasicTNNHandler only support 4 dims." - "Such as NCHW, NHWC ..."); -#else - return; -#endif - } - input_mat_type = this->get_input_mat_type(input_name); - input_data_format = this->get_input_data_format(input_name); - // This BasicTNNHandler only support NC_INT32 & NCHW_FLOAT - if (input_data_format == tnn::DATA_FORMAT_NCHW) - { - input_batch = input_shape.at(0); - input_channel = input_shape.at(1); - input_height = input_shape.at(2); - input_width = input_shape.at(3); - } // NHWC - else if (input_data_format == tnn::DATA_FORMAT_NHWC) - { - input_batch = input_shape.at(0); - input_height = input_shape.at(1); - input_width = input_shape.at(2); - input_channel = input_shape.at(3); - } // unsupport - else - { -#ifdef LITETNN_DEBUG - std::cout << "BasicTNNHandler only support NCHW and NHWC " - "input_data_format, but found others.\n"; -#endif - return; - } - // 6. init input_mat - input_value_size = input_batch * input_channel * input_height * input_width; - // 7. init output information, debug only. - output_names = this->get_output_names(); - num_outputs = output_names.size(); - for (auto &name: output_names) - output_shapes[name] = this->get_output_shape(name); -#ifdef LITETNN_DEBUG - this->print_debug_string(); -#endif -} - -inline tnn::DimsVector BasicTNNHandler::get_input_shape(std::string name) -{ - return BasicTNNHandler::get_input_shape(instance, name); -} - -inline tnn::DimsVector BasicTNNHandler::get_output_shape(std::string name) -{ - return BasicTNNHandler::get_output_shape(instance, name); -} - -inline std::vector BasicTNNHandler::get_input_names() -{ - return BasicTNNHandler::get_input_names(instance); -} - -inline std::vector BasicTNNHandler::get_output_names() -{ - return BasicTNNHandler::get_output_names(instance); -} - -inline tnn::MatType BasicTNNHandler::get_output_mat_type(std::string name) -{ - return BasicTNNHandler::get_output_mat_type(instance, name); -} - -inline tnn::DataFormat BasicTNNHandler::get_output_data_format(std::string name) -{ - return BasicTNNHandler::get_output_data_format(instance, name); -} - -inline tnn::MatType BasicTNNHandler::get_input_mat_type(std::string name) -{ - return BasicTNNHandler::get_input_mat_type(instance, name); -} - -inline tnn::DataFormat BasicTNNHandler::get_input_data_format(std::string name) -{ - return BasicTNNHandler::get_input_data_format(instance, name); -} - -void BasicTNNHandler::print_debug_string() -{ - std::cout << "LITETNN_DEBUG LogId: " << log_id << "\n"; - std::cout << "=============== Input-Dims ==============\n"; - BasicTNNHandler::print_name_shape(input_name, input_shape); - std::string data_format_string = - (input_data_format == tnn::DATA_FORMAT_NCHW) ? "NCHW" : "NHWC"; - std::cout << "Input Data Format: " << data_format_string << "\n"; - std::cout << "=============== Output-Dims ==============\n"; - for (auto &out: output_shapes) - BasicTNNHandler::print_name_shape(out.first, out.second); - std::cout << "========================================\n"; -} - -// static methods. -void BasicTNNHandler::print_name_shape(std::string name, tnn::DimsVector &shape) -{ - std::cout << name << ": ["; - for (const auto &d: shape) std::cout << d << " "; - std::cout << "]\n"; -} - -// static methods. -// reference: https://github.com/Tencent/TNN/blob/master/examples/base/utils/utils.cc -std::string BasicTNNHandler::content_buffer_from(const char *proto_or_model_path) -{ - std::ifstream file(proto_or_model_path, std::ios::binary); - if (file.is_open()) - { - file.seekg(0, file.end); - int size = file.tellg(); - char *content = new char[size]; - file.seekg(0, file.beg); - file.read(content, size); - std::string file_content; - file_content.assign(content, size); - delete[] content; - file.close(); - return file_content; - } // empty buffer - else - { -#ifdef LITETNN_DEBUG - std::cout << "Can not open " << proto_or_model_path << "\n"; -#endif - return ""; - } -} - -// static methods. -tnn::DimsVector BasicTNNHandler::get_input_shape( - const std::shared_ptr &_instance, - std::string name) -{ - tnn::DimsVector shape = {}; - tnn::BlobMap blob_map = {}; - if (_instance) - { - _instance->GetAllInputBlobs(blob_map); - } - - if (name == "" && blob_map.size() > 0) - if (blob_map.begin()->second) - shape = blob_map.begin()->second->GetBlobDesc().dims; - - if (blob_map.find(name) != blob_map.end() - && blob_map[name]) - { - shape = blob_map[name]->GetBlobDesc().dims; - } - - return shape; -} - -// static methods. -tnn::DimsVector BasicTNNHandler::get_output_shape( - const std::shared_ptr &_instance, - std::string name) -{ - tnn::DimsVector shape = {}; - tnn::BlobMap blob_map = {}; - if (_instance) - { - _instance->GetAllOutputBlobs(blob_map); - } - - if (name == "" && blob_map.size() > 0) - if (blob_map.begin()->second) - shape = blob_map.begin()->second->GetBlobDesc().dims; - - if (blob_map.find(name) != blob_map.end() - && blob_map[name]) - { - shape = blob_map[name]->GetBlobDesc().dims; - } - - return shape; -} - -// static methods. -std::vector BasicTNNHandler::get_input_names( - const std::shared_ptr &_instance) -{ - std::vector names; - if (_instance) - { - tnn::BlobMap blob_map; - _instance->GetAllInputBlobs(blob_map); - for (const auto &item : blob_map) - { - names.push_back(item.first); - } - } - return names; -} - -// static method -std::vector BasicTNNHandler::get_output_names( - const std::shared_ptr &_instance) -{ - std::vector names; - if (_instance) - { - tnn::BlobMap blob_map; - _instance->GetAllOutputBlobs(blob_map); - for (const auto &item : blob_map) - { - names.push_back(item.first); - } - } - return names; -} - -// static method -tnn::MatType BasicTNNHandler::get_output_mat_type( - const std::shared_ptr &_instance, - std::string name) -{ - if (_instance) - { - tnn::BlobMap output_blobs; - _instance->GetAllOutputBlobs(output_blobs); - auto blob = (name == "") ? output_blobs.begin()->second : output_blobs[name]; - if (blob->GetBlobDesc().data_type == tnn::DATA_TYPE_INT32) - { - return tnn::NC_INT32; - } - } - return tnn::NCHW_FLOAT; -} - -// static method -tnn::DataFormat BasicTNNHandler::get_output_data_format( - const std::shared_ptr &_instance, - std::string name) -{ - if (_instance) - { - tnn::BlobMap output_blobs; - _instance->GetAllOutputBlobs(output_blobs); - auto blob = (name == "") ? output_blobs.begin()->second : output_blobs[name]; - return blob->GetBlobDesc().data_format; - } - return tnn::DATA_FORMAT_NCHW; -} - -// static method -tnn::MatType BasicTNNHandler::get_input_mat_type( - const std::shared_ptr &_instance, - std::string name) -{ - if (_instance) - { - tnn::BlobMap input_blobs; - _instance->GetAllInputBlobs(input_blobs); - auto blob = (name == "") ? input_blobs.begin()->second : input_blobs[name]; - if (blob->GetBlobDesc().data_type == tnn::DATA_TYPE_INT32) - { - return tnn::NC_INT32; - } - } - return tnn::NCHW_FLOAT; -} - -// static method -tnn::DataFormat BasicTNNHandler::get_input_data_format( - const std::shared_ptr &_instance, - std::string name) -{ - if (_instance) - { - tnn::BlobMap input_blobs; - _instance->GetAllInputBlobs(input_blobs); - auto blob = (name == "") ? input_blobs.begin()->second : input_blobs[name]; - return blob->GetBlobDesc().data_format; - } - return tnn::DATA_FORMAT_NCHW; -} - - - - - - - - - - - - - - - - - diff --git a/lite/tnn/core/tnn_handler.h b/lite/tnn/core/tnn_handler.h deleted file mode 100644 index f21a777c..00000000 --- a/lite/tnn/core/tnn_handler.h +++ /dev/null @@ -1,103 +0,0 @@ -// -// Created by DefTruth on 2021/10/17. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CORE_TNN_HANDLER_H -#define LITE_AI_TOOLKIT_TNN_CORE_TNN_HANDLER_H - -#include "tnn_config.h" - -namespace tnncore -{ - class LITE_EXPORTS BasicTNNHandler - { - protected: - const char *log_id = nullptr; - const char *proto_path = nullptr; - const char *model_path = nullptr; - // Note, tnn:: actually is TNN_NS::, I prefer the first one. - std::shared_ptr net; - std::shared_ptr instance; - std::shared_ptr input_mat; // assume single input. - - protected: - const unsigned int num_threads; // initialize at runtime. - int input_batch; - int input_channel; - int input_height; - int input_width; - int num_outputs = 1; - unsigned int input_value_size; - tnn::DataFormat input_data_format; // e.g DATA_FORMAT_NHWC - tnn::MatType input_mat_type; // e.g NCHW_FLOAT - tnn::DeviceType input_device_type; // only CPU, namely ARM or X86 - tnn::DeviceType output_device_type; // only CPU, namely ARM or X86 - tnn::DeviceType network_device_type; // e.g DEVICE_X86 DEVICE_NAIVE DEVICE_ARM - // Actually, i prefer to hardcode the input/output names - // into subclasses, but we just let the auto detection here - // to make sure the debug information can show more details. - std::string input_name; // assume single input only. - std::vector output_names; // assume >= 1 outputs. - tnn::DimsVector input_shape; // vector - std::map output_shapes; - - protected: - explicit BasicTNNHandler(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); - - virtual ~BasicTNNHandler(); - - // un-copyable - protected: - BasicTNNHandler(const BasicTNNHandler &) = delete; // - BasicTNNHandler(BasicTNNHandler &&) = delete; // - BasicTNNHandler &operator=(const BasicTNNHandler &) = delete; // - BasicTNNHandler &operator=(BasicTNNHandler &&) = delete; // - - private: - virtual void transform(const cv::Mat &mat) = 0; - - private: - void initialize_handler(); // init net & instance - void print_debug_string(); // debug information - - protected: - // helper functions. - tnn::DimsVector get_input_shape(std::string name); - tnn::DimsVector get_output_shape(std::string name); - tnn::MatType get_output_mat_type(std::string name); - tnn::DataFormat get_output_data_format(std::string name); - tnn::MatType get_input_mat_type(std::string name); - tnn::DataFormat get_input_data_format(std::string name); - std::vector get_input_names(); - std::vector get_output_names(); - - public: - // helper functions. override for user firendly - static tnn::DimsVector get_input_shape( - const std::shared_ptr &_instance, std::string name); - static tnn::DimsVector get_output_shape( - const std::shared_ptr &_instance, std::string name); - static tnn::MatType get_output_mat_type( - const std::shared_ptr &_instance, std::string name); - static tnn::DataFormat get_output_data_format( - const std::shared_ptr &_instance, std::string name); - static tnn::MatType get_input_mat_type( - const std::shared_ptr &_instance, std::string name); - static tnn::DataFormat get_input_data_format( - const std::shared_ptr &_instance, std::string name); - static std::vector get_input_names( - const std::shared_ptr &_instance); - static std::vector get_output_names( - const std::shared_ptr &_instance); - - public: - static std::string content_buffer_from( - const char *proto_or_model_path); - static void print_name_shape(std::string name, tnn::DimsVector &shape); - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CORE_TNN_HANDLER_H diff --git a/lite/tnn/core/tnn_types.h b/lite/tnn/core/tnn_types.h deleted file mode 100644 index 4c4cd9e0..00000000 --- a/lite/tnn/core/tnn_types.h +++ /dev/null @@ -1,15 +0,0 @@ -// -// Created by DefTruth on 2021/10/17. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CORE_TNN_TYPES_H -#define LITE_AI_TOOLKIT_TNN_CORE_TNN_TYPES_H - -#include "lite/types.h" - -namespace tnncv -{ - namespace types = lite::types; -} - -#endif //LITE_AI_TOOLKIT_TNN_CORE_TNN_TYPES_H diff --git a/lite/tnn/core/tnn_utils.cpp b/lite/tnn/core/tnn_utils.cpp deleted file mode 100644 index 73311a5e..00000000 --- a/lite/tnn/core/tnn_utils.cpp +++ /dev/null @@ -1,5 +0,0 @@ -// -// Created by DefTruth on 2021/10/17. -// - -#include "tnn_utils.h" \ No newline at end of file diff --git a/lite/tnn/core/tnn_utils.h b/lite/tnn/core/tnn_utils.h deleted file mode 100644 index 272e4769..00000000 --- a/lite/tnn/core/tnn_utils.h +++ /dev/null @@ -1,13 +0,0 @@ -// -// Created by DefTruth on 2021/10/17. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CORE_TNN_UTILS_H -#define LITE_AI_TOOLKIT_TNN_CORE_TNN_UTILS_H - -namespace tnncv -{ - // no specific utils for TNN now. -} - -#endif //LITE_AI_TOOLKIT_TNN_CORE_TNN_UTILS_H diff --git a/lite/tnn/cv/tnn_age_googlenet.cpp b/lite/tnn/cv/tnn_age_googlenet.cpp deleted file mode 100644 index 52ca3748..00000000 --- a/lite/tnn/cv/tnn_age_googlenet.cpp +++ /dev/null @@ -1,90 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#include "tnn_age_googlenet.h" -#include "lite/utils.h" - -using tnncv::TNNAgeGoogleNet; - -TNNAgeGoogleNet::TNNAgeGoogleNet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNAgeGoogleNet::transform(const cv::Mat &mat_rs) -{ - // be carefully, no deepcopy inside this tnn::Mat constructor, - // so, we can not pass a local cv::Mat to this constructor. - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNAgeGoogleNet::detect(const cv::Mat &mat, types::Age &age) -{ - if (mat.empty()) return; - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch. - tnn::MatConvertParam cvt_param; - std::shared_ptr age_logits; // (1,8) - status = instance->GetOutputMat(age_logits, cvt_param, "loss3/loss3_Y", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto age_dims = age_logits->GetDims(); - unsigned int interval = 0; - const unsigned int num_intervals = age_dims.at(1); // 8 - const float *pred_logits_ptr = (float *) age_logits->GetData(); - - auto softmax_probs = lite::utils::math::softmax(pred_logits_ptr, num_intervals, interval); - const float pred_age = static_cast(age_intervals[interval][0] + age_intervals[interval][1]) / 2.0f; - - age.age = pred_age; - age.age_interval[0] = age_intervals[interval][0]; - age.age_interval[1] = age_intervals[interval][1]; - age.interval_prob = softmax_probs[interval]; - age.flag = true; -} \ No newline at end of file diff --git a/lite/tnn/cv/tnn_age_googlenet.h b/lite/tnn/cv/tnn_age_googlenet.h deleted file mode 100644 index 381a94a3..00000000 --- a/lite/tnn/cv/tnn_age_googlenet.h +++ /dev/null @@ -1,43 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_AGE_GOOGLENET_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_AGE_GOOGLENET_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNAgeGoogleNet : public BasicTNNHandler - { - public: - explicit TNNAgeGoogleNet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNAgeGoogleNet() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.0f, 1.0f, 1.0f}; - std::vector bias_vals = {-104.0f, -117.0f, -123.0f}; - const unsigned int age_intervals[8][2] = { - {0, 2}, - {4, 6}, - {8, 12}, - {15, 20}, - {25, 32}, - {38, 43}, - {48, 53}, - {60, 100} - }; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::Age &age); - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_AGE_GOOGLENET_H diff --git a/lite/tnn/cv/tnn_backgroundmattingv2.cpp b/lite/tnn/cv/tnn_backgroundmattingv2.cpp deleted file mode 100644 index a187ff9b..00000000 --- a/lite/tnn/cv/tnn_backgroundmattingv2.cpp +++ /dev/null @@ -1,296 +0,0 @@ -// -// Created by DefTruth on 2022/4/9. -// - -#include "tnn_backgroundmattingv2.h" -#include "lite/utils.h" - -using tnncv::TNNBackgroundMattingV2; - -TNNBackgroundMattingV2::TNNBackgroundMattingV2( - const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads -) : proto_path(_proto_path.data()), - model_path(_model_path.data()), - log_id(_proto_path.data()), - num_threads(_num_threads) -{ - initialize_instance(); -} - -TNNBackgroundMattingV2::~TNNBackgroundMattingV2() -{ - net = nullptr; - src_mat = nullptr; - bgr_mat = nullptr; - instance = nullptr; -} - -void TNNBackgroundMattingV2::initialize_instance() -{ - std::string proto_content_buffer, model_content_buffer; - proto_content_buffer = BasicTNNHandler::content_buffer_from(proto_path); - model_content_buffer = BasicTNNHandler::content_buffer_from(model_path); - - tnn::ModelConfig model_config; - model_config.model_type = tnn::MODEL_TYPE_TNN; - model_config.params = {proto_content_buffer, model_content_buffer}; - - // 1. init TNN net - tnn::Status status; - net = std::make_shared(); - status = net->Init(model_config); - if (status != tnn::TNN_OK || !net) - { -#ifdef LITETNN_DEBUG - std::cout << "net->Init failed!\n"; -#endif - return; - } - // 2. init device type, change this default setting - // for better performance. such as CUDA/OPENCL/... -#ifdef __ANDROID__ - network_device_type = tnn::DEVICE_ARM; // CPU,GPU - input_device_type = tnn::DEVICE_ARM; // CPU only - output_device_type = tnn::DEVICE_ARM; -#else - network_device_type = tnn::DEVICE_X86; // CPU,GPU - input_device_type = tnn::DEVICE_X86; // CPU only - output_device_type = tnn::DEVICE_X86; -#endif - // 3. init instance - tnn::NetworkConfig network_config; - network_config.library_path = {""}; - network_config.device_type = network_device_type; - - instance = net->CreateInst(network_config, status); - if (status != tnn::TNN_OK || !instance) - { -#ifdef LITETNN_DEBUG - std::cout << "CreateInst failed!" << status.description().c_str() << "\n"; -#endif - return; - } - // 4. setting up num_threads - instance->SetCpuNumThreads((int) num_threads); - // 5. init input information. - for (auto &name: input_names) - input_shapes[name] = BasicTNNHandler::get_input_shape(instance, name); - auto src_shape = input_shapes.at("src"); - if (src_shape.size() != 4) - { -#ifdef LITETNN_DEBUG - throw std::runtime_error("Found src_shape.size()!=4, but " - "src input only support 4 dims." - "Such as NCHW, NHWC ..."); -#else - return; -#endif - } - input_mat_type = BasicTNNHandler::get_input_mat_type(instance, "src"); - input_data_format = BasicTNNHandler::get_input_data_format(instance, "src"); - if (input_data_format == tnn::DATA_FORMAT_NCHW) - { - input_height = src_shape.at(2); - input_width = src_shape.at(3); - } // NHWC - else if (input_data_format == tnn::DATA_FORMAT_NHWC) - { - input_height = src_shape.at(1); - input_width = src_shape.at(2); - } // unsupport - else - { -#ifdef LITETNN_DEBUG - std::cout << "src input only support NCHW and NHWC " - "input_data_format, but found others.\n"; -#endif - return; - } - // 6. init output information, debug only. - for (auto &name: output_names) - output_shapes[name] = BasicTNNHandler::get_output_shape(instance, name); -#ifdef LITETNN_DEBUG - this->print_debug_string(); -#endif -} - -void TNNBackgroundMattingV2::print_debug_string() -{ - std::cout << "LITETNN_DEBUG LogId: " << log_id << "\n"; - std::cout << "=============== Input-Dims ==============\n"; - for (auto &in: input_shapes) - BasicTNNHandler::print_name_shape(in.first, in.second); - std::string data_format_string = - (input_data_format == tnn::DATA_FORMAT_NCHW) ? "NCHW" : "NHWC"; - std::cout << "Input Data Format: " << data_format_string << "\n"; - std::cout << "=============== Output-Dims ==============\n"; - for (auto &out: output_shapes) - BasicTNNHandler::print_name_shape(out.first, out.second); - std::cout << "========================================\n"; -} - -void TNNBackgroundMattingV2::transform(const cv::Mat &mat_rs, const cv::Mat &bgr_rs) -{ - // push into src_mat - src_mat = std::make_shared( - input_device_type, - tnn::N8UC3, - input_shapes.at("src"), - (void *) mat_rs.data - ); - if (!src_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "src_mat == nullptr! transform failed\n"; -#endif - } - bgr_mat = std::make_shared( - input_device_type, - tnn::N8UC3, - input_shapes.at("bgr"), - (void *) bgr_rs.data - ); - if (!bgr_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "bgr_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNBackgroundMattingV2::detect(const cv::Mat &mat, const cv::Mat &bgr, - types::MattingContent &content, bool remove_noise, - bool minimum_post_process) -{ - if (mat.empty() || bgr.empty()) return; - cv::Mat mat_rs, bgr_rs; - // resize mat outside 'transform' to prevent memory overflow - // reference: https://github.com/DefTruth/lite.ai.toolkit/issues/240 - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::resize(bgr, bgr_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - cv::cvtColor(bgr_rs, bgr_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs, bgr_rs); - // 2. set input_mat - tnn::MatConvertParam cvt_param; - cvt_param.scale = scale_vals; - cvt_param.bias = bias_vals; - - auto status_src = instance->SetInputMat(src_mat, cvt_param, "src"); - auto status_bgr = instance->SetInputMat(bgr_mat, cvt_param, "bgr"); - if (status_src != tnn::TNN_OK || status_bgr != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status_src.description().c_str() << ": " - << status_bgr.description().c_str() << "\n"; -#endif - return; - } -// 3. forward - auto status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - // 4. generate matting - this->generate_matting(instance, mat, content, remove_noise, minimum_post_process); -} - -void TNNBackgroundMattingV2::generate_matting(std::shared_ptr &_instance, - const cv::Mat &mat, types::MattingContent &content, - bool remove_noise, bool minimum_post_process) -{ - std::shared_ptr fgr_mat; - std::shared_ptr pha_mat; - tnn::MatConvertParam cvt_param; - tnn::Status status_fgr, status_pha; - - status_fgr = _instance->GetOutputMat(fgr_mat, cvt_param, "fgr", output_device_type); - status_pha = _instance->GetOutputMat(pha_mat, cvt_param, "pha", output_device_type); - - if (status_fgr != tnn::TNN_OK || status_pha != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status_fgr.description().c_str() << ": " - << status_pha.description().c_str() << "\n"; -#endif - return; - } - const unsigned int h = mat.rows; - const unsigned int w = mat.cols; - const unsigned int out_h = input_height; - const unsigned int out_w = input_width; - - float *fgr_ptr = (float *) fgr_mat->GetData(); - float *pha_ptr = (float *) pha_mat->GetData(); - const unsigned int channel_step = out_h * out_w; - - // fast assign & channel transpose(CHW->HWC). - cv::Mat pmat(out_h, out_w, CV_32FC1, pha_ptr); - if (remove_noise) lite::utils::remove_small_connected_area(pmat, 0.05f); - - std::vector fgr_channel_mats; - cv::Mat rmat(out_h, out_w, CV_32FC1, fgr_ptr); - cv::Mat gmat(out_h, out_w, CV_32FC1, fgr_ptr + channel_step); - cv::Mat bmat(out_h, out_w, CV_32FC1, fgr_ptr + 2 * channel_step); - rmat *= 255.; - bmat *= 255.; - gmat *= 255.; - fgr_channel_mats.push_back(bmat); - fgr_channel_mats.push_back(gmat); - fgr_channel_mats.push_back(rmat); - - content.pha_mat = pmat; - cv::merge(fgr_channel_mats, content.fgr_mat); - content.fgr_mat.convertTo(content.fgr_mat, CV_8UC3); - - if (!minimum_post_process) - { - std::vector merge_channel_mats; - cv::Mat rest = 1. - pmat; - cv::Mat mbmat = bmat.mul(pmat) + rest * 153.; - cv::Mat mgmat = gmat.mul(pmat) + rest * 255.; - cv::Mat mrmat = rmat.mul(pmat) + rest * 120.; - merge_channel_mats.push_back(mbmat); - merge_channel_mats.push_back(mgmat); - merge_channel_mats.push_back(mrmat); - cv::merge(merge_channel_mats, content.merge_mat); - content.merge_mat.convertTo(content.merge_mat, CV_8UC3); - } - - // resize alpha - if (out_h != h || out_w != w) - { - cv::resize(content.pha_mat, content.pha_mat, cv::Size(w, h)); - cv::resize(content.fgr_mat, content.fgr_mat, cv::Size(w, h)); - if (!minimum_post_process) - cv::resize(content.merge_mat, content.merge_mat, cv::Size(w, h)); - } - - content.flag = true; -} - - - - - - - - - - - - - - - - - diff --git a/lite/tnn/cv/tnn_backgroundmattingv2.h b/lite/tnn/cv/tnn_backgroundmattingv2.h deleted file mode 100644 index 955e322e..00000000 --- a/lite/tnn/cv/tnn_backgroundmattingv2.h +++ /dev/null @@ -1,98 +0,0 @@ -// -// Created by DefTruth on 2022/4/9. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_BACKGROUNDMATTINGV2_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_BACKGROUNDMATTINGV2_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNBackgroundMattingV2 - { - public: - explicit TNNBackgroundMattingV2(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); - - ~TNNBackgroundMattingV2(); - - private: - const char *log_id = nullptr; - const char *proto_path = nullptr; - const char *model_path = nullptr; - // Note, tnn:: actually is TNN_NS::, I prefer the first one. - std::shared_ptr net; - std::shared_ptr instance; - - private: - std::vector scale_vals = {1.f / 255.f, 1.f / 255.f, 1.f / 255.f}; - std::vector bias_vals = {0.f, 0.f, 0.f}; // RGB - // hardcode input node names, hint only. - // downsample_ratio has been freeze while onnx exported - // and, the input size of each input has been freeze, also. - std::vector input_names = { - "src", - "bgr" - }; - // hardcode output node names, hint only. - std::vector output_names = { - "pha", - "fgr", - "pha_sm", - "fgr_sm", - "err_sm", - "ref_sm" - }; - - private: - const unsigned int num_threads; // initialize at runtime. - // multi inputs, rxi will be update inner video matting process. - std::shared_ptr src_mat; - std::shared_ptr bgr_mat; - int input_height; - int input_width; - tnn::DataFormat input_data_format; // e.g DATA_FORMAT_NHWC - tnn::MatType input_mat_type; // e.g NCHW_FLOAT - tnn::DeviceType input_device_type; // only CPU, namely ARM or X86 - tnn::DeviceType output_device_type; // only CPU, namely ARM or X86 - tnn::DeviceType network_device_type; // e.g DEVICE_X86 DEVICE_NAIVE DEVICE_ARM - std::map input_shapes; - std::map output_shapes; - - // un-copyable - protected: - TNNBackgroundMattingV2(const TNNBackgroundMattingV2 &) = delete; // - TNNBackgroundMattingV2(TNNBackgroundMattingV2 &&) = delete; // - TNNBackgroundMattingV2 &operator=(const TNNBackgroundMattingV2 &) = delete; // - TNNBackgroundMattingV2 &operator=(TNNBackgroundMattingV2 &&) = delete; // - - private: - void print_debug_string(); // debug information - - private: - void transform(const cv::Mat &mat_rs, const cv::Mat &bgr_rs); - - void initialize_instance(); // init net & instance - - void generate_matting(std::shared_ptr &_instance, - const cv::Mat &mat, types::MattingContent &content, - bool remove_noise = false, bool minimum_post_process = false); - - public: - /** - * @param mat cv::Mat input image with BGR format. - * @param bgr cv::Mat input background image with BGR format. - * @param content MattingContent output fgr, pha and merge_mat (if minimum_post_process is false) - * @param remove_noise bool, whether to remove small connected areas. - * @param minimum_post_process bool, will not return demo merge mat if True. - */ - void detect(const cv::Mat &mat, const cv::Mat &bgr, types::MattingContent &content, - bool remove_noise = false, bool minimum_post_process = false); - - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_BACKGROUNDMATTINGV2_H diff --git a/lite/tnn/cv/tnn_cava_combined_face.cpp b/lite/tnn/cv/tnn_cava_combined_face.cpp deleted file mode 100644 index 199da4de..00000000 --- a/lite/tnn/cv/tnn_cava_combined_face.cpp +++ /dev/null @@ -1,92 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "tnn_cava_combined_face.h" - -using tnncv::TNNCavaCombinedFace; - -TNNCavaCombinedFace::TNNCavaCombinedFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNCavaCombinedFace::transform(const cv::Mat &mat_rs) -{ - // cv::Mat canvas; - // cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // cv::cvtColor(canvas, canvas, cv::COLOR_BGR2RGB); - // be carefully, no deepcopy inside this tnn::Mat constructor, - // so, we can not pass a local cv::Mat to this constructor. - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNCavaCombinedFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 4. fetch output mat - std::shared_ptr embedding_mat; - tnn::MatConvertParam embed_cvt_param; // default - - status = instance->GetOutputMat(embedding_mat, embed_cvt_param, "embedding", output_device_type); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - auto embedding_dims = embedding_mat->GetDims(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = (float *) embedding_mat->GetData(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} - diff --git a/lite/tnn/cv/tnn_cava_combined_face.h b/lite/tnn/cv/tnn_cava_combined_face.h deleted file mode 100644 index 9b165385..00000000 --- a/lite/tnn/cv/tnn_cava_combined_face.h +++ /dev/null @@ -1,35 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_CAVA_COMBINED_FACE_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_CAVA_COMBINED_FACE_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNCavaCombinedFace : public BasicTNNHandler - { - public: - explicit TNNCavaCombinedFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNCavaCombinedFace() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f / 128.0f, 1.f / 128.0f, 1.f / 128.0f}; - std::vector bias_vals = {-127.5f / 128.0f, -127.5f / 128.0f, -127.5f / 128.0f}; // RGB - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - - }; -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_CAVA_COMBINED_FACE_H diff --git a/lite/tnn/cv/tnn_cava_ghost_arcface.cpp b/lite/tnn/cv/tnn_cava_ghost_arcface.cpp deleted file mode 100644 index b9e39b48..00000000 --- a/lite/tnn/cv/tnn_cava_ghost_arcface.cpp +++ /dev/null @@ -1,92 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "tnn_cava_ghost_arcface.h" - -using tnncv::TNNCavaGhostArcFace; - -TNNCavaGhostArcFace::TNNCavaGhostArcFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNCavaGhostArcFace::transform(const cv::Mat &mat_rs) -{ - // cv::Mat canvas; - // cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // cv::cvtColor(canvas, canvas, cv::COLOR_BGR2RGB); - // be carefully, no deepcopy inside this tnn::Mat constructor, - // so, we can not pass a local cv::Mat to this constructor. - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNCavaGhostArcFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 4. fetch output mat - std::shared_ptr embedding_mat; - tnn::MatConvertParam embed_cvt_param; // default - - status = instance->GetOutputMat(embedding_mat, embed_cvt_param, "embedding", output_device_type); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - auto embedding_dims = embedding_mat->GetDims(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = (float *) embedding_mat->GetData(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} - diff --git a/lite/tnn/cv/tnn_cava_ghost_arcface.h b/lite/tnn/cv/tnn_cava_ghost_arcface.h deleted file mode 100644 index 0413cdc4..00000000 --- a/lite/tnn/cv/tnn_cava_ghost_arcface.h +++ /dev/null @@ -1,34 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_CAVA_GHOST_ARCFACE_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_CAVA_GHOST_ARCFACE_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNCavaGhostArcFace : public BasicTNNHandler - { - public: - explicit TNNCavaGhostArcFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNCavaGhostArcFace() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f / 128.0f, 1.f / 128.0f, 1.f / 128.0f}; - std::vector bias_vals = {-127.5f / 128.0f, -127.5f / 128.0f, -127.5f / 128.0f}; // RGB - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_CAVA_GHOST_ARCFACE_H diff --git a/lite/tnn/cv/tnn_center_loss_face.cpp b/lite/tnn/cv/tnn_center_loss_face.cpp deleted file mode 100644 index ee65e4b9..00000000 --- a/lite/tnn/cv/tnn_center_loss_face.cpp +++ /dev/null @@ -1,88 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "tnn_center_loss_face.h" - -using tnncv::TNNCenterLossFace; - -TNNCenterLossFace::TNNCenterLossFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNCenterLossFace::transform(const cv::Mat &mat_rs) -{ - // be carefully, no deepcopy inside this tnn::Mat constructor, - // so, we can not pass a local cv::Mat to this constructor. - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNCenterLossFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 4. fetch output mat - std::shared_ptr embedding_mat; - tnn::MatConvertParam embed_cvt_param; // default - - status = instance->GetOutputMat(embedding_mat, embed_cvt_param, "embedding", output_device_type); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - auto embedding_dims = embedding_mat->GetDims(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = (float *) embedding_mat->GetData(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} diff --git a/lite/tnn/cv/tnn_center_loss_face.h b/lite/tnn/cv/tnn_center_loss_face.h deleted file mode 100644 index 1b15433e..00000000 --- a/lite/tnn/cv/tnn_center_loss_face.h +++ /dev/null @@ -1,34 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_CENTER_LOSS_FACE_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_CENTER_LOSS_FACE_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNCenterLossFace : public BasicTNNHandler - { - public: - explicit TNNCenterLossFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNCenterLossFace() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - std::vector bias_vals = {-1.f, -1.f, -1.f}; // RGB - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_CENTER_LOSS_FACE_H diff --git a/lite/tnn/cv/tnn_colorizer.cpp b/lite/tnn/cv/tnn_colorizer.cpp deleted file mode 100644 index 3c2ac6e4..00000000 --- a/lite/tnn/cv/tnn_colorizer.cpp +++ /dev/null @@ -1,137 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "tnn_colorizer.h" - -using tnncv::TNNColorizer; - -TNNColorizer::TNNColorizer(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNColorizer::transform(const cv::Mat &mat_l) -{ -// cv::Mat mat_l; // assume that input mat is L of Lab -// mat.convertTo(mat_l, CV_32FC1, 1.0f, 0.f); // (256,256,1) range (0.,100.) -// be carefully, no deepcopy inside this tnn::Mat constructor, -// so, we can not pass a local cv::Mat to this constructor. - input_mat = std::make_shared(input_device_type, tnn::NCHW_FLOAT, - input_shape, (void *) mat_l.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNColorizer::detect(const cv::Mat &mat, types::ColorizeContent &colorize_content) -{ - if (mat.empty()) return; - const unsigned int height = mat.rows; - const unsigned int width = mat.cols; - - cv::Mat mat_rs = mat.clone(); - cv::resize(mat_rs, mat_rs, cv::Size(input_width, input_height)); // (256,256,3) - cv::Mat mat_rs_norm, mat_orig_norm; - mat_rs.convertTo(mat_rs_norm, CV_32FC3, 1.0f / 255.0f, 0.f); // (0.,1.) BGR - mat.convertTo(mat_orig_norm, CV_32FC3, 1.0f / 255.0f, 0.f); // (0.,1.) BGR - if (mat_rs_norm.empty() || mat_orig_norm.empty()) return; - - cv::Mat mat_lab_orig, mat_lab_rs; - cv::cvtColor(mat_rs_norm, mat_lab_rs, cv::COLOR_BGR2Lab); - cv::cvtColor(mat_orig_norm, mat_lab_orig, cv::COLOR_BGR2Lab); - - cv::Mat mat_rs_l, mat_orig_l; - std::vector mats_rs_lab, mats_orig_lab; - cv::split(mat_lab_rs, mats_rs_lab); - cv::split(mat_lab_orig, mats_orig_lab); - - mat_rs_l = mats_rs_lab.at(0); - mat_orig_l = mats_orig_lab.at(0); - - // 1. make input tensor - cv::Mat mat_l; // assume that input mat is L of Lab - mat_rs_l.convertTo(mat_l, CV_32FC1, 1.0f, 0.f); // (256,256,1) range (0.,100.) - this->transform(mat_l); - - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch - tnn::MatConvertParam cvt_param; - std::shared_ptr pred_ab_mat; // (1,2,256,256) - status = instance->GetOutputMat(pred_ab_mat, cvt_param, "out_ab", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto pred_dims = pred_ab_mat->GetDims(); // (1,2,256,256) - const unsigned int rows = pred_dims.at(2); // H 256 - const unsigned int cols = pred_dims.at(3); // W 256 - const unsigned int pred_step = rows * cols; - - float *pred_ab_ptr = (float *) pred_ab_mat->GetData(); - - cv::Mat out_a_orig(rows, cols, CV_32FC1); - cv::Mat out_b_orig(rows, cols, CV_32FC1); - - for (unsigned int i = 0; i < rows; ++i) - { - float *pa = out_a_orig.ptr(i); - float *pb = out_b_orig.ptr(i); - for (unsigned int j = 0; j < cols; ++j) - { - pa[j] = pred_ab_ptr[0 * pred_step + i * cols + j]; - pb[j] = pred_ab_ptr[1 * pred_step + i * cols + j]; - } // CHW->HWC - } - - if (rows != height || cols != width) - { - cv::resize(out_a_orig, out_a_orig, cv::Size(width, height)); - cv::resize(out_b_orig, out_b_orig, cv::Size(width, height)); - } - - std::vector out_mats_lab; - out_mats_lab.push_back(mat_orig_l); - out_mats_lab.push_back(out_a_orig); - out_mats_lab.push_back(out_b_orig); - - cv::Mat merge_mat_lab, mat_bgr_norm; - cv::merge(out_mats_lab, merge_mat_lab); - if (merge_mat_lab.empty()) return; - cv::cvtColor(merge_mat_lab, mat_bgr_norm, cv::COLOR_Lab2BGR); // CV_32FC3 - mat_bgr_norm *= 255.0f; - - mat_bgr_norm.convertTo(colorize_content.mat, CV_8UC3); // uint8 - - colorize_content.flag = true; -} \ No newline at end of file diff --git a/lite/tnn/cv/tnn_colorizer.h b/lite/tnn/cv/tnn_colorizer.h deleted file mode 100644 index a112f3bd..00000000 --- a/lite/tnn/cv/tnn_colorizer.h +++ /dev/null @@ -1,29 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_COLORIZER_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_COLORIZER_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNColorizer : public BasicTNNHandler - { - public: - explicit TNNColorizer(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNColorizer() override = default; - - private: - void transform(const cv::Mat &mat_l) override; // - - public: - void detect(const cv::Mat &mat, types::ColorizeContent &colorize_content); - }; -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_COLORIZER_H diff --git a/lite/tnn/cv/tnn_deeplabv3_resnet101.cpp b/lite/tnn/cv/tnn_deeplabv3_resnet101.cpp deleted file mode 100644 index f070b76c..00000000 --- a/lite/tnn/cv/tnn_deeplabv3_resnet101.cpp +++ /dev/null @@ -1,307 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "tnn_deeplabv3_resnet101.h" - -using tnncv::TNNDeepLabV3ResNet101; - -TNNDeepLabV3ResNet101::TNNDeepLabV3ResNet101( - const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads -) : proto_path(_proto_path.data()), - model_path(_model_path.data()), - log_id(_proto_path.data()), - num_threads(_num_threads) -{ - initialize_instance(); -} - -TNNDeepLabV3ResNet101::~TNNDeepLabV3ResNet101() -{ - net = nullptr; - input_mat = nullptr; - instance = nullptr; -} - -void TNNDeepLabV3ResNet101::initialize_instance() -{ - std::string proto_content_buffer, model_content_buffer; - proto_content_buffer = BasicTNNHandler::content_buffer_from(proto_path); - model_content_buffer = BasicTNNHandler::content_buffer_from(model_path); - - tnn::ModelConfig model_config; - model_config.model_type = tnn::MODEL_TYPE_TNN; - model_config.params = {proto_content_buffer, model_content_buffer}; - - // 1. init TNN net - tnn::Status status; - net = std::make_shared(); - status = net->Init(model_config); - if (status != tnn::TNN_OK || !net) - { -#ifdef LITETNN_DEBUG - std::cout << "net->Init failed!\n"; -#endif - return; - } - // 2. init device type, change this default setting - // for better performance. such as CUDA/OPENCL/... -#ifdef __ANDROID__ - network_device_type = tnn::DEVICE_ARM; // CPU,GPU - input_device_type = tnn::DEVICE_ARM; // CPU only - output_device_type = tnn::DEVICE_ARM; -#else - network_device_type = tnn::DEVICE_X86; // CPU,GPU - input_device_type = tnn::DEVICE_X86; // CPU only - output_device_type = tnn::DEVICE_X86; -#endif - // 3. init instance - tnn::NetworkConfig network_config; - network_config.library_path = {""}; - network_config.device_type = network_device_type; - - instance = net->CreateInst(network_config, status); - if (status != tnn::TNN_OK || !instance) - { -#ifdef LITETNN_DEBUG - std::cout << "CreateInst failed!" << status.description().c_str() << "\n"; -#endif - return; - } - // 4. setting up num_threads - instance->SetCpuNumThreads((int) num_threads); - // 5. init input information. - input_shape = BasicTNNHandler::get_input_shape(instance, "input"); - - if (input_shape.size() != 4) - { -#ifdef LITETNN_DEBUG - throw std::runtime_error("Found input_shape.size()!=4, but " - "input only support 4 dims." - "Such as NCHW, NHWC ..."); -#else - return; -#endif - } - input_mat_type = BasicTNNHandler::get_input_mat_type(instance, "input"); - input_data_format = BasicTNNHandler::get_input_data_format(instance, "input"); - if (input_data_format == tnn::DATA_FORMAT_NCHW) - { - dynamic_input_height = input_shape.at(2); - dynamic_input_width = input_shape.at(3); - } // NHWC - else if (input_data_format == tnn::DATA_FORMAT_NHWC) - { - dynamic_input_height = input_shape.at(1); - dynamic_input_width = input_shape.at(2); - } // unsupport - else - { -#ifdef LITETNN_DEBUG - std::cout << "input only support NCHW and NHWC " - "input_data_format, but found others.\n"; -#endif - return; - } - // 6. init output information, debug only. - output_shape = BasicTNNHandler::get_output_shape(instance, "out"); -#ifdef LITETNN_DEBUG - this->print_debug_string(); -#endif -} - -void TNNDeepLabV3ResNet101::print_debug_string() -{ - std::cout << "LITETNN_DEBUG LogId: " << log_id << "\n"; - std::cout << "=============== Input-Dims ==============\n"; - BasicTNNHandler::print_name_shape("input", input_shape); - std::string data_format_string = - (input_data_format == tnn::DATA_FORMAT_NCHW) ? "NCHW" : "NHWC"; - std::cout << "Input Data Format: " << data_format_string << "\n"; - std::cout << "=============== Output-Dims ==============\n"; - BasicTNNHandler::print_name_shape("out", output_shape); - std::cout << "========================================\n"; -} - -void TNNDeepLabV3ResNet101::transform(const cv::Mat &mat_rs) -{ -// const int img_width = mat.cols; -// const int img_height = mat.rows; -// // update dynamic input dims -// dynamic_input_height = img_height; -// dynamic_input_width = img_width; -// if (input_data_format == tnn::DATA_FORMAT_NCHW) -// { -// input_shape.at(2) = dynamic_input_height; -// input_shape.at(3) = dynamic_input_width; -// } // NHWC -// else if (input_data_format == tnn::DATA_FORMAT_NHWC) -// { -// input_shape.at(1) = dynamic_input_height; -// input_shape.at(2) = dynamic_input_width; -// } -// -// // update input mat and reshape instance -// // reference: https://github.com/Tencent/TNN/blob/master/examples/base/ocr_text_recognizer.cc#L120 -// tnn::InputShapesMap input_shape_map; -// input_shape_map.insert({"input", input_shape}); -// -// auto status = instance->Reshape(input_shape_map); -// if (status != tnn::TNN_OK) -// { -//#ifdef LITETNN_DEBUG -// std::cout << "instance Reshape failed in TNNDeepLabV3ResNet101\n"; -//#endif -// } -// -// cv::Mat canvas; -// cv::cvtColor(mat, canvas, cv::COLOR_BGR2RGB); - -// cv::Mat canvas; -// cv::resize(mat, canvas, cv::Size(dynamic_input_width, dynamic_input_height)); -// cv::cvtColor(canvas, canvas, cv::COLOR_BGR2RGB); - -// be carefully, no deepcopy inside this tnn::Mat constructor, -// so, we can not pass a local cv::Mat to this constructor. -// push into input_mat - input_mat = std::make_shared( - input_device_type, - tnn::N8UC3, - input_shape, - (void *) mat_rs.data - ); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNDeepLabV3ResNet101::detect(const cv::Mat &mat, types::SegmentContent &content) -{ - if (mat.empty()) return; - const int img_width = mat.cols; - const int img_height = mat.rows; - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(dynamic_input_width, dynamic_input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch - tnn::MatConvertParam cvt_param; - std::shared_ptr scores_mat; // (1,21,h,w) - status = instance->GetOutputMat(scores_mat, cvt_param, "out", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto scores_dims = scores_mat->GetDims(); -#ifdef LITETNN_DEBUG - BasicTNNHandler::print_name_shape("out", scores_dims); -#endif - - const unsigned int output_classes = scores_dims.at(1); - const unsigned int output_height = scores_dims.at(2); - const unsigned int output_width = scores_dims.at(3); - - const float *scores_ptr = (float *) scores_mat->GetData(); - - // time cost! - content.names_map.clear(); - content.class_mat = cv::Mat(output_height, output_width, CV_8UC1, cv::Scalar(0)); - cv::resize(mat, content.color_mat, cv::Size(output_width, output_height)); // init color mat - - const unsigned int scores_step = output_height * output_width; // h x w - - for (unsigned int i = 0; i < output_height; ++i) - { - - uchar *p_class = content.class_mat.ptr(i); - cv::Vec3b *p_color = content.color_mat.ptr(i); - - for (unsigned int j = 0; j < output_width; ++j) - { - // argmax - unsigned int max_label = 0; - float max_conf = scores_ptr[0 * scores_step + i * output_width + j]; - - for (unsigned int l = 0; l < output_classes; ++l) - { - float conf = scores_ptr[l * scores_step + i * output_width + j]; - if (conf > max_conf) - { - max_conf = conf; - max_label = l; - } - } - - if (max_label == 0) continue; - - // assign label for pixel(i,j) - p_class[j] = cv::saturate_cast(max_label); - // assign color for detected class at pixel(i,j). - p_color[j][0] = cv::saturate_cast((max_label % 10) * 20); - p_color[j][1] = cv::saturate_cast((max_label % 5) * 40); - p_color[j][2] = cv::saturate_cast((max_label % 10) * 20); - // assign names map - content.names_map[max_label] = class_names[max_label - 1]; // max_label >= 1 - } - - } - - cv::resize(content.class_mat, content.class_mat, cv::Size(img_width, img_height)); - cv::resize(content.color_mat, content.color_mat, cv::Size(img_width, img_height)); - - content.flag = true; - -} - - - - - - - - - - - - - - - - - - diff --git a/lite/tnn/cv/tnn_deeplabv3_resnet101.h b/lite/tnn/cv/tnn_deeplabv3_resnet101.h deleted file mode 100644 index bee6f050..00000000 --- a/lite/tnn/cv/tnn_deeplabv3_resnet101.h +++ /dev/null @@ -1,76 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_DEEPLABV3_RESNET101_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_DEEPLABV3_RESNET101_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNDeepLabV3ResNet101 - { - public: - explicit TNNDeepLabV3ResNet101(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); - - ~TNNDeepLabV3ResNet101(); - - private: - const char *log_id = nullptr; - const char *proto_path = nullptr; - const char *model_path = nullptr; - // Note, tnn:: actually is TNN_NS::, I prefer the first one. - std::shared_ptr net; - std::shared_ptr instance; - std::shared_ptr input_mat; - - private: - std::vector scale_vals = {(1.f / 0.229f) * (1.f / 255.f), - (1.f / 0.224f) * (1.f / 255.f), - (1.f / 0.225f) * (1.f / 255.f)}; - std::vector bias_vals = {-0.485f * 255.f * (1.f / 0.229f) * (1.f / 255.f), - -0.456f * 255.f * (1.f / 0.224f) * (1.f / 255.f), - -0.406f * 255.f * (1.f / 0.225f) * (1.f / 255.f)}; // RGB - - private: - const unsigned int num_threads; // initialize at runtime. - int dynamic_input_height = 512; // init only, will change according to input mat. - int dynamic_input_width = 512; // init only, will change according to input mat. - tnn::DataFormat input_data_format; // e.g DATA_FORMAT_NHWC - tnn::MatType input_mat_type; // e.g NCHW_FLOAT - tnn::DeviceType input_device_type; // only CPU, namely ARM or X86 - tnn::DeviceType output_device_type; // only CPU, namely ARM or X86 - tnn::DeviceType network_device_type; // e.g DEVICE_X86 DEVICE_NAIVE DEVICE_ARM - tnn::DimsVector input_shape; // debug - tnn::DimsVector output_shape; - - const char *class_names[20] = { - "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", - "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", - "train", "tvmonitor" - }; // 20 classes - - // un-copyable - protected: - TNNDeepLabV3ResNet101(const TNNDeepLabV3ResNet101 &) = delete; // - TNNDeepLabV3ResNet101(TNNDeepLabV3ResNet101 &&) = delete; // - TNNDeepLabV3ResNet101 &operator=(const TNNDeepLabV3ResNet101 &) = delete; // - TNNDeepLabV3ResNet101 &operator=(TNNDeepLabV3ResNet101 &&) = delete; // - - private: - void print_debug_string(); // debug information - - private: - void transform(const cv::Mat &mat_rs); // - - void initialize_instance(); // init net & instance - - public: - void detect(const cv::Mat &mat, types::SegmentContent &content); - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_DEEPLABV3_RESNET101_H diff --git a/lite/tnn/cv/tnn_densenet.cpp b/lite/tnn/cv/tnn_densenet.cpp deleted file mode 100644 index 2ac3242a..00000000 --- a/lite/tnn/cv/tnn_densenet.cpp +++ /dev/null @@ -1,94 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "tnn_densenet.h" -#include "lite/utils.h" - -using tnncv::TNNDenseNet; - -TNNDenseNet::TNNDenseNet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNDenseNet::transform(const cv::Mat &mat_rs) -{ - // push into input_mat (1,3,224,224) - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNDenseNet::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch. - tnn::MatConvertParam cvt_param; - std::shared_ptr logits_mat; // (1,1000) - status = instance->GetOutputMat(logits_mat, cvt_param, "logits", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto logits_dims = logits_mat->GetDims(); - const unsigned int num_classes = logits_dims.at(1); // 1000 - const float *logits = (float *) logits_mat->GetData(); - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/tnn/cv/tnn_densenet.h b/lite/tnn/cv/tnn_densenet.h deleted file mode 100644 index e17f5254..00000000 --- a/lite/tnn/cv/tnn_densenet.h +++ /dev/null @@ -1,414 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_DENSENET_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_DENSENET_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNDenseNet : public BasicTNNHandler - { - public: - explicit TNNDenseNet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNDenseNet() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - std::vector bias_vals = {-0.485f * 255.f * (1.0f / 0.229f) * (1.0 / 255.f), - -0.456f * 255.f * (1.0f / 0.224f) * (1.0 / 255.f), - -0.406f * 255.f * (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - - }; -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_DENSENET_H diff --git a/lite/tnn/cv/tnn_efficient_emotion7.cpp b/lite/tnn/cv/tnn_efficient_emotion7.cpp deleted file mode 100644 index e4abe851..00000000 --- a/lite/tnn/cv/tnn_efficient_emotion7.cpp +++ /dev/null @@ -1,86 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#include "tnn_efficient_emotion7.h" -#include "lite/utils.h" - -using tnncv::TNNEfficientEmotion7; - -TNNEfficientEmotion7::TNNEfficientEmotion7(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNEfficientEmotion7::transform(const cv::Mat &mat_rs) -{ - // push into input_mat (1,3,224,224) - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNEfficientEmotion7::detect(const cv::Mat &mat, types::Emotions &emotions) -{ - if (mat.empty()) return; - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch. - tnn::MatConvertParam cvt_param; - std::shared_ptr emotion_logits; // (1,7) - status = instance->GetOutputMat(emotion_logits, cvt_param, "logits", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto emotion_dims = emotion_logits->GetDims(); - const unsigned int num_emotions = emotion_dims.at(1); // 7 - - unsigned int pred_label = 0; - const float *pred_logits_ptr = (float *) emotion_logits->GetData(); - - auto softmax_probs = lite::utils::math::softmax(pred_logits_ptr, num_emotions, pred_label); - emotions.label = pred_label; - emotions.score = softmax_probs[pred_label]; - emotions.text = emotion_texts[pred_label]; - emotions.flag = true; -} diff --git a/lite/tnn/cv/tnn_efficient_emotion7.h b/lite/tnn/cv/tnn_efficient_emotion7.h deleted file mode 100644 index 032b5349..00000000 --- a/lite/tnn/cv/tnn_efficient_emotion7.h +++ /dev/null @@ -1,41 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_EFFICIENT_EMOTION7_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_EFFICIENT_EMOTION7_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNEfficientEmotion7 : public BasicTNNHandler - { - public: - explicit TNNEfficientEmotion7(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNEfficientEmotion7() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f / (255.f * 0.229f), - 1.f / (255.f * 0.224f), - 1.f / (255.f * 0.225f)}; - std::vector bias_vals = {-255.f * 0.485f * 1.f / (255.f * 0.229f), - -255.f * 0.456f * 1.f / (255.f * 0.224f), - -255.f * 0.406f * 1.f / (255.f * 0.225f)}; - const char *emotion_texts[7] = { - "angry", "disgust", "fear", "happiness", "neutral", "sadness", "surprise" - }; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::Emotions &emotions); - }; -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_EFFICIENT_EMOTION7_H diff --git a/lite/tnn/cv/tnn_efficient_emotion8.cpp b/lite/tnn/cv/tnn_efficient_emotion8.cpp deleted file mode 100644 index 1e01871c..00000000 --- a/lite/tnn/cv/tnn_efficient_emotion8.cpp +++ /dev/null @@ -1,86 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#include "tnn_efficient_emotion8.h" -#include "lite/utils.h" - -using tnncv::TNNEfficientEmotion8; - -TNNEfficientEmotion8::TNNEfficientEmotion8(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNEfficientEmotion8::transform(const cv::Mat &mat_rs) -{ - // push into input_mat (1,3,224,224) - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNEfficientEmotion8::detect(const cv::Mat &mat, types::Emotions &emotions) -{ - if (mat.empty()) return; - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch. - tnn::MatConvertParam cvt_param; - std::shared_ptr emotion_logits; // (1,8) - status = instance->GetOutputMat(emotion_logits, cvt_param, "logits", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto emotion_dims = emotion_logits->GetDims(); - const unsigned int num_emotions = emotion_dims.at(1); // 8 - - unsigned int pred_label = 0; - const float *pred_logits_ptr = (float *) emotion_logits->GetData(); - - auto softmax_probs = lite::utils::math::softmax(pred_logits_ptr, num_emotions, pred_label); - emotions.label = pred_label; - emotions.score = softmax_probs[pred_label]; - emotions.text = emotion_texts[pred_label]; - emotions.flag = true; -} diff --git a/lite/tnn/cv/tnn_efficient_emotion8.h b/lite/tnn/cv/tnn_efficient_emotion8.h deleted file mode 100644 index e5f12a6f..00000000 --- a/lite/tnn/cv/tnn_efficient_emotion8.h +++ /dev/null @@ -1,41 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_EFFICIENT_EMOTION8_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_EFFICIENT_EMOTION8_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNEfficientEmotion8 : public BasicTNNHandler - { - public: - explicit TNNEfficientEmotion8(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNEfficientEmotion8() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f / (255.f * 0.229f), - 1.f / (255.f * 0.224f), - 1.f / (255.f * 0.225f)}; - std::vector bias_vals = {-255.f * 0.485f * 1.f / (255.f * 0.229f), - -255.f * 0.456f * 1.f / (255.f * 0.224f), - -255.f * 0.406f * 1.f / (255.f * 0.225f)}; - const char *emotion_texts[8] = { - "angry", "contempt", "disgust", "fear", "happiness", "neutral", "sadness", "surprise" - }; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::Emotions &emotions); - }; -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_EFFICIENT_EMOTION8_H diff --git a/lite/tnn/cv/tnn_efficientnet_lite4.cpp b/lite/tnn/cv/tnn_efficientnet_lite4.cpp deleted file mode 100644 index 360ef68e..00000000 --- a/lite/tnn/cv/tnn_efficientnet_lite4.cpp +++ /dev/null @@ -1,94 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "tnn_efficientnet_lite4.h" -#include "lite/utils.h" - -using tnncv::TNNEfficientNetLite4; - -TNNEfficientNetLite4::TNNEfficientNetLite4(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ - input_width = 224; - input_height = 224; -} - -void TNNEfficientNetLite4::transform(const cv::Mat &mat_rs) -{ - // push into input_mat (1,3,224,224) - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNEfficientNetLite4::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch. - tnn::MatConvertParam cvt_param; - std::shared_ptr scores_mat; // (1,1000) - status = instance->GetOutputMat(scores_mat, cvt_param, "Softmax:0", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto scores_dims = scores_mat->GetDims(); - const unsigned int num_classes = scores_dims.at(1); // 1000 - const float *scores = (float *) scores_mat->GetData(); - - std::vector sorted_indices = lite::utils::math::argsort(scores, num_classes); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/tnn/cv/tnn_efficientnet_lite4.h b/lite/tnn/cv/tnn_efficientnet_lite4.h deleted file mode 100644 index 30c0bded..00000000 --- a/lite/tnn/cv/tnn_efficientnet_lite4.h +++ /dev/null @@ -1,409 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_EFFICIENTNET_LITE4_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_EFFICIENTNET_LITE4_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNEfficientNetLite4 : public BasicTNNHandler - { - public: - explicit TNNEfficientNetLite4(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNEfficientNetLite4() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f / 128.f, 1.f / 128.f, 1.f / 128.f}; - std::vector bias_vals = {-127.f / 128.f, -127.f / 128.f, -127.f / 128.f}; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_EFFICIENTNET_LITE4_H diff --git a/lite/tnn/cv/tnn_emotion_ferplus.cpp b/lite/tnn/cv/tnn_emotion_ferplus.cpp deleted file mode 100644 index e0ed132e..00000000 --- a/lite/tnn/cv/tnn_emotion_ferplus.cpp +++ /dev/null @@ -1,86 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#include "tnn_emotion_ferplus.h" -#include "lite/utils.h" - -using tnncv::TNNEmotionFerPlus; - -TNNEmotionFerPlus::TNNEmotionFerPlus(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNEmotionFerPlus::transform(const cv::Mat &mat_rs) -{ - // push into input_mat (1,1,64,64) - input_mat = std::make_shared(input_device_type, tnn::NGRAY, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNEmotionFerPlus::detect(const cv::Mat &mat, types::Emotions &emotions) -{ - if (mat.empty()) return; - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2GRAY); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch. - tnn::MatConvertParam cvt_param; - std::shared_ptr emotion_logits; // (1,8) - status = instance->GetOutputMat(emotion_logits, cvt_param, "Plus692_Output_0", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto emotion_dims = emotion_logits->GetDims(); - const unsigned int num_emotions = emotion_dims.at(1); // 8 - - unsigned int pred_label = 0; - const float *pred_logits_ptr = (float *) emotion_logits->GetData(); - - auto softmax_probs = lite::utils::math::softmax(pred_logits_ptr, num_emotions, pred_label); - emotions.label = pred_label; - emotions.score = softmax_probs[pred_label]; - emotions.text = emotion_texts[pred_label]; - emotions.flag = true; -} diff --git a/lite/tnn/cv/tnn_emotion_ferplus.h b/lite/tnn/cv/tnn_emotion_ferplus.h deleted file mode 100644 index ad3e8b75..00000000 --- a/lite/tnn/cv/tnn_emotion_ferplus.h +++ /dev/null @@ -1,38 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_EMOTION_FERPLUS_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_EMOTION_FERPLUS_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNEmotionFerPlus : public BasicTNNHandler - { - public: - explicit TNNEmotionFerPlus(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNEmotionFerPlus() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.0f}; - std::vector bias_vals = {0.f}; - const char *emotion_texts[8] = { - "neutral", "happiness", "surprise", "sadness", "anger", - "disgust", "fear", "contempt" - }; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::Emotions &emotions); - }; -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_EMOTION_FERPLUS_H diff --git a/lite/tnn/cv/tnn_face_landmarks_1000.cpp b/lite/tnn/cv/tnn_face_landmarks_1000.cpp deleted file mode 100644 index 7e2e3f41..00000000 --- a/lite/tnn/cv/tnn_face_landmarks_1000.cpp +++ /dev/null @@ -1,93 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#include "tnn_face_landmarks_1000.h" - -using tnncv::TNNFaceLandmark1000; - -TNNFaceLandmark1000::TNNFaceLandmark1000(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNFaceLandmark1000::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::NGRAY, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNFaceLandmark1000::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2GRAY); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch landmarks. - tnn::MatConvertParam cvt_param; - std::shared_ptr landmarks_norm; // (1,1953) - status = instance->GetOutputMat(landmarks_norm, cvt_param, "output0", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - auto landmark_dims = landmarks_norm->GetDims(); - - unsigned int num_landmarks = landmark_dims.at(1); - if (num_landmarks > 1946) num_landmarks = 1946; - - const float *landmarks_ptr = (float *) landmarks_norm->GetData(); - - for (unsigned int i = 0; i < num_landmarks; i += 2) - { - float x = landmarks_ptr[i]; - float y = landmarks_ptr[i + 1]; - - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - landmarks.flag = true; -} diff --git a/lite/tnn/cv/tnn_face_landmarks_1000.h b/lite/tnn/cv/tnn_face_landmarks_1000.h deleted file mode 100644 index 042be58d..00000000 --- a/lite/tnn/cv/tnn_face_landmarks_1000.h +++ /dev/null @@ -1,34 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_FACE_LANDMARKS_1000_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_FACE_LANDMARKS_1000_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNFaceLandmark1000 : public BasicTNNHandler - { - public: - explicit TNNFaceLandmark1000(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNFaceLandmark1000() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.0f}; - std::vector bias_vals = {0.f}; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - }; -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_FACE_LANDMARKS_1000_H diff --git a/lite/tnn/cv/tnn_face_parsing_bisenet.cpp b/lite/tnn/cv/tnn_face_parsing_bisenet.cpp deleted file mode 100644 index 6256fb09..00000000 --- a/lite/tnn/cv/tnn_face_parsing_bisenet.cpp +++ /dev/null @@ -1,201 +0,0 @@ -// -// Created by DefTruth on 2022/7/2. -// - -#include "tnn_face_parsing_bisenet.h" - -using tnncv::TNNFaceParsingBiSeNet; - -TNNFaceParsingBiSeNet::TNNFaceParsingBiSeNet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNFaceParsingBiSeNet::transform(const cv::Mat &mat_rs) -{ - // push into input_mat (1,3,512,512) no deepcopy inside TNN - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNFaceParsingBiSeNet::detect(const cv::Mat &mat, types::FaceParsingContent &content, - bool minimum_post_process) -{ - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. generate mask - this->generate_mask(instance, mat, content, minimum_post_process); -} - -static inline uchar argmax(float *mutable_ptr, const unsigned int &step) -{ - std::vector logits(19, 0.f); - for (unsigned int i = 0; i < 19; ++i) - logits[i] = *(mutable_ptr + i * step); - uchar label = 0; - float max_logit = logits[0]; - for (unsigned int i = 1; i < 19; ++i) - { - if (logits[i] > max_logit) - { - max_logit = logits[i]; - label = (uchar) i; - } - } - return label; -} - -static const uchar part_colors[20][3] = { - {255, 0, 0}, - {255, 85, 0}, - {255, 170, 0}, - {255, 0, 85}, - {255, 0, 170}, - {0, 255, 0}, - {85, 255, 0}, - {170, 255, 0}, - {0, 255, 85}, - {0, 255, 170}, - {0, 0, 255}, - {85, 0, 255}, - {170, 0, 255}, - {0, 85, 255}, - {0, 170, 255}, - {255, 255, 0}, - {255, 255, 85}, - {255, 255, 170}, - {255, 0, 255}, - {255, 85, 255} -}; - -void TNNFaceParsingBiSeNet::generate_mask(std::shared_ptr &_instance, const cv::Mat &mat, - types::FaceParsingContent &content, - bool minimum_post_process) -{ - std::shared_ptr output_mat; - tnn::MatConvertParam cvt_param; - auto status = _instance->GetOutputMat(output_mat, cvt_param, "out", output_device_type); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - const unsigned int h = mat.rows; - const unsigned int w = mat.cols; - - auto output_dims = output_mat->GetDims(); - const unsigned int out_h = output_dims.at(2); - const unsigned int out_w = output_dims.at(3); - const unsigned int channel_step = out_h * out_w; - - float *output_ptr = (float *) output_mat->GetData(); - std::vector elements(channel_step, 0); // allocate - for (unsigned int i = 0; i < channel_step; ++i) - elements[i] = argmax(output_ptr + i, channel_step); - - cv::Mat label(out_h, out_w, CV_8UC1, elements.data()); - - if (!minimum_post_process) - { - // FaceParsingBiSeNet only predict integer label mask, - // no fgr. So, the fake fgr and merge mat may not need, - // let the fgr mat and merge mat empty to - // Speed up the post processes. - const uchar *label_ptr = label.data; - cv::Mat color_mat(out_h, out_w, CV_8UC3, cv::Scalar(255, 255, 255)); - for (unsigned int i = 0; i < color_mat.rows; ++i) - { - cv::Vec3b *p = color_mat.ptr(i); - for (unsigned int j = 0; j < color_mat.cols; ++j) - { - if (label_ptr[i * out_w + j] == 0) continue; - p[j][0] = part_colors[label_ptr[i * out_w + j]][0]; - p[j][1] = part_colors[label_ptr[i * out_w + j]][1]; - p[j][2] = part_colors[label_ptr[i * out_w + j]][2]; - } - } - if (out_h != h || out_w != w) - cv::resize(color_mat, color_mat, cv::Size(w, h)); - cv::addWeighted(mat, 0.4, color_mat, 0.6, 0., content.merge); - } - // already allocated a new continuous memory after resize. - if (out_h != h || out_w != w) cv::resize(label, label, cv::Size(w, h)); - // need clone to allocate a new continuous memory if not performed resize. - // The memory elements point to will release after return. - else label = label.clone(); - - content.label = label; // auto handle the memory inside ocv with smart ref. - content.flag = true; -} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lite/tnn/cv/tnn_face_parsing_bisenet.h b/lite/tnn/cv/tnn_face_parsing_bisenet.h deleted file mode 100644 index 557f0b54..00000000 --- a/lite/tnn/cv/tnn_face_parsing_bisenet.h +++ /dev/null @@ -1,45 +0,0 @@ -// -// Created by DefTruth on 2022/7/2. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_FACE_PARSING_BISENET_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_FACE_PARSING_BISENET_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNFaceParsingBiSeNet : public BasicTNNHandler - { - public: - explicit TNNFaceParsingBiSeNet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); - - ~TNNFaceParsingBiSeNet() override = default; - - private: - // In TNN: x*scale + bias - std::vector bias_vals = { - -0.485f * 255.f * (1.f / (0.229f * 255.f)), - -0.456f * 255.f * (1.f / (0.224f * 255.f)), - -0.406f * 255.f * (1.f / (0.225f * 255.f))}; // RGB - std::vector scale_vals = { - 1.f / (0.229f * 255.f), - 1.f / (0.224f * 255.f), - 1.f / (0.225f * 255.f)}; - - private: - void transform(const cv::Mat &mat_rs) override; // - - void generate_mask(std::shared_ptr &_instance, - const cv::Mat &mat, types::FaceParsingContent &content, - bool minimum_post_process = false); - - public: - void detect(const cv::Mat &mat, types::FaceParsingContent &content, - bool minimum_post_process = false); - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_FACE_PARSING_BISENET_H diff --git a/lite/tnn/cv/tnn_faceboxes.cpp b/lite/tnn/cv/tnn_faceboxes.cpp deleted file mode 100644 index a7b51574..00000000 --- a/lite/tnn/cv/tnn_faceboxes.cpp +++ /dev/null @@ -1,278 +0,0 @@ -// -// Created by DefTruth on 2021/11/20. -// - -#include "tnn_faceboxes.h" -#include "lite/utils.h" - -using tnncv::TNNFaceBoxes; - -TNNFaceBoxes::TNNFaceBoxes(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNFaceBoxes::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNFaceBoxes::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(bbox_collection, instance, score_threshold, img_height, img_width); - // 5. hard|blend nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void TNNFaceBoxes::generate_anchors(const int target_height, const int target_width, - std::vector &anchors) -{ - std::vector> feature_maps; - for (auto step: steps) - { - feature_maps.push_back( - { - (int) std::ceil((float) target_height / (float) step), - (int) std::ceil((float) target_width / (float) step) - } // ceil - ); - } - - anchors.clear(); - const int num_feature_map = feature_maps.size(); - - for (int k = 0; k < num_feature_map; ++k) - { - auto f_map = feature_maps.at(k); // e.g [640//32,640/32] - auto tmp_min_sizes = min_sizes.at(k); // e.g [32,64,128] - int f_h = f_map.at(0); - int f_w = f_map.at(1); - std::vector offset_32 = {0.f, 0.25f, 0.5f, 0.75f}; - std::vector offset_64 = {0.f, 0.5f}; - - for (int i = 0; i < f_h; ++i) - { - for (int j = 0; j < f_w; ++j) - { - for (auto min_size: tmp_min_sizes) - { - float s_kx = (float) min_size / (float) target_width; // e.g 32/w - float s_ky = (float) min_size / (float) target_height; // e.g 32/h - - // 32 anchor size - if (min_size == 32) - { - // range y offsets first and then x - for (auto offset_y: offset_32) - { - for (auto offset_x: offset_32) - { - // (x or y + offset) * step / w or h normalized loc mapping to input size. - float cx = ((float) j + offset_x) * (float) steps.at(k) / (float) target_width; - float cy = ((float) i + offset_y) * (float) steps.at(k) / (float) target_height; - anchors.push_back(FaceBoxesAnchor{cx, cy, s_kx, s_ky}); // without clip - } - } - - } // 64 anchor size - else if (min_size == 64) - { - // range y offsets first and then x - for (auto offset_y: offset_64) - { - for (auto offset_x: offset_64) - { - float cx = ((float) j + offset_x) * (float) steps.at(k) / (float) target_width; - float cy = ((float) i + offset_y) * (float) steps.at(k) / (float) target_height; - anchors.push_back(FaceBoxesAnchor{cx, cy, s_kx, s_ky}); // without clip - } - } - - } // other anchor size - else - { - float cx = ((float) j + 0.5f) * (float) steps.at(k) / (float) target_width; - float cy = ((float) i + 0.5f) * (float) steps.at(k) / (float) target_height; - anchors.push_back(FaceBoxesAnchor{cx, cy, s_kx, s_ky}); // without clip - } - } - } - } - } -} - -void TNNFaceBoxes::generate_bboxes(std::vector &bbox_collection, - std::shared_ptr &_instance, - float score_threshold, float img_height, - float img_width) -{ - std::shared_ptr bboxes; // (1,n,4) - std::shared_ptr probs; // (1,n,2) - tnn::MatConvertParam cvt_param; - tnn::Status status_bboxes; - tnn::Status status_probs; - - status_bboxes = _instance->GetOutputMat(bboxes, cvt_param, "bbox", output_device_type); - status_probs = _instance->GetOutputMat(probs, cvt_param, "conf", output_device_type); - - if (status_bboxes != tnn::TNN_OK || status_probs != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status_bboxes.description().c_str() << ": " - << status_probs.description().c_str() << "\n"; -#endif - return; - } - auto bbox_dims = bboxes->GetDims(); - const unsigned int bbox_num = bbox_dims.at(1); // n = ? - - std::vector anchors; - this->generate_anchors(input_height, input_width, anchors); - - const unsigned int num_anchors = anchors.size(); - if (num_anchors != bbox_num) - throw std::runtime_error("mismatch num_anchors != bbox_num"); - - const float *bboxes_ptr = (float *) bboxes->GetData(); - const float *probs_ptr = (float *) probs->GetData(); - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - float conf = probs_ptr[2 * i + 1]; - if (conf < score_threshold) continue; // filter first. - - float prior_cx = anchors.at(i).cx; - float prior_cy = anchors.at(i).cy; - float prior_s_kx = anchors.at(i).s_kx; - float prior_s_ky = anchors.at(i).s_ky; - - float dx = bboxes_ptr[4 * i + 0]; - float dy = bboxes_ptr[4 * i + 1]; - float dw = bboxes_ptr[4 * i + 2]; - float dh = bboxes_ptr[4 * i + 3]; - // ref: https://github.com/biubug6/Pytorch_Retinaface/blob/master/utils/box_utils.py - float cx = prior_cx + dx * variance[0] * prior_s_kx; - float cy = prior_cy + dy * variance[0] * prior_s_ky; - float w = prior_s_kx * std::exp(dw * variance[1]); - float h = prior_s_ky * std::exp(dh * variance[1]); // norm coor (0.,1.) - - types::Boxf box; - box.x1 = (cx - w / 2.f) * img_width; - box.y1 = (cy - h / 2.f) * img_height; - box.x2 = (cx + w / 2.f) * img_width; - box.y2 = (cy + h / 2.f) * img_height; - box.score = conf; - box.label = 1; - box.label_text = "face"; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITETNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void TNNFaceBoxes::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lite/tnn/cv/tnn_faceboxes.h b/lite/tnn/cv/tnn_faceboxes.h deleted file mode 100644 index af92d6c2..00000000 --- a/lite/tnn/cv/tnn_faceboxes.h +++ /dev/null @@ -1,75 +0,0 @@ -// -// Created by DefTruth on 2021/11/20. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_FACEBOXES_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_FACEBOXES_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNFaceBoxes : public BasicTNNHandler - { - public: - explicit TNNFaceBoxes(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNFaceBoxes() override = default; - - private: - // nested classes - struct FaceBoxesAnchor - { - float cx; - float cy; - float s_kx; - float s_ky; - }; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f, 1.f, 1.f}; - std::vector bias_vals = { - -104.f * 1.0f, - -117.f * 1.0f, - -123.f * 1.0f - }; // bgr order - const float variance[2] = {0.1f, 0.2f}; - std::vector steps = {32, 64, 128}; - std::vector> min_sizes = { - {32, 64, 128}, - {256}, - {512} - }; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void transform(const cv::Mat &mat_rs) override; // - - void generate_anchors(const int target_height, - const int target_width, - std::vector &anchors); - - void generate_bboxes(std::vector &bbox_collection, - std::shared_ptr &_instance, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.7f, float iou_threshold = 0.3f, - unsigned int topk = 300, unsigned int nms_type = 0); - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_FACEBOXES_H diff --git a/lite/tnn/cv/tnn_faceboxesv2.cpp b/lite/tnn/cv/tnn_faceboxesv2.cpp deleted file mode 100644 index 460aba95..00000000 --- a/lite/tnn/cv/tnn_faceboxesv2.cpp +++ /dev/null @@ -1,236 +0,0 @@ -// -// Created by DefTruth on 2022/3/19. -// - -#include "tnn_faceboxesv2.h" -#include "lite/utils.h" - -using tnncv::TNNFaceBoxesV2; - -TNNFaceBoxesV2::TNNFaceBoxesV2(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNFaceBoxesV2::transform(const cv::Mat &mat_rs) -{ - // cv::Mat mat_rs; - // cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNFaceBoxesV2::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - this->transform(mat_rs); // resize outside transform to prevent overflow - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(bbox_collection, instance, score_threshold, img_height, img_width); - // 5. hard|blend nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void TNNFaceBoxesV2::generate_anchors(const int target_height, const int target_width, - std::vector &anchors) -{ - std::vector> feature_maps; - for (auto step: steps) - { - feature_maps.push_back( - { - (int) std::ceil((float) target_height / (float) step), - (int) std::ceil((float) target_width / (float) step) - } // ceil - ); - } - - anchors.clear(); - const int num_feature_map = feature_maps.size(); - - for (int k = 0; k < num_feature_map; ++k) - { - auto f_map = feature_maps.at(k); // e.g [640//32,640/32] - auto tmp_min_sizes = min_sizes.at(k); // e.g [32,64,128] - int f_h = f_map.at(0); - int f_w = f_map.at(1); - std::vector offset_32 = {0.f, 0.25f, 0.5f, 0.75f}; - std::vector offset_64 = {0.f, 0.5f}; - - for (int i = 0; i < f_h; ++i) - { - for (int j = 0; j < f_w; ++j) - { - for (auto min_size: tmp_min_sizes) - { - float s_kx = (float) min_size / (float) target_width; // e.g 32/w - float s_ky = (float) min_size / (float) target_height; // e.g 32/h - - // 32 anchor size - if (min_size == 32) - { - // range y offsets first and then x - for (auto offset_y: offset_32) - { - for (auto offset_x: offset_32) - { - // (x or y + offset) * step / w or h normalized loc mapping to input size. - float cx = ((float) j + offset_x) * (float) steps.at(k) / (float) target_width; - float cy = ((float) i + offset_y) * (float) steps.at(k) / (float) target_height; - anchors.push_back(FaceBoxesAnchorV2{cx, cy, s_kx, s_ky}); // without clip - } - } - - } // 64 anchor size - else if (min_size == 64) - { - // range y offsets first and then x - for (auto offset_y: offset_64) - { - for (auto offset_x: offset_64) - { - float cx = ((float) j + offset_x) * (float) steps.at(k) / (float) target_width; - float cy = ((float) i + offset_y) * (float) steps.at(k) / (float) target_height; - anchors.push_back(FaceBoxesAnchorV2{cx, cy, s_kx, s_ky}); // without clip - } - } - - } // other anchor size - else - { - float cx = ((float) j + 0.5f) * (float) steps.at(k) / (float) target_width; - float cy = ((float) i + 0.5f) * (float) steps.at(k) / (float) target_height; - anchors.push_back(FaceBoxesAnchorV2{cx, cy, s_kx, s_ky}); // without clip - } - } - } - } - } -} - -void TNNFaceBoxesV2::generate_bboxes(std::vector &bbox_collection, - std::shared_ptr &_instance, - float score_threshold, float img_height, - float img_width) -{ - std::shared_ptr bboxes; // (1,n,4) - std::shared_ptr probs; // (1,n,2) - tnn::MatConvertParam cvt_param; - tnn::Status status_bboxes; - tnn::Status status_probs; - - status_bboxes = _instance->GetOutputMat(bboxes, cvt_param, "loc", output_device_type); - status_probs = _instance->GetOutputMat(probs, cvt_param, "conf", output_device_type); - - if (status_bboxes != tnn::TNN_OK || status_probs != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status_bboxes.description().c_str() << ": " - << status_probs.description().c_str() << "\n"; -#endif - return; - } - auto bbox_dims = bboxes->GetDims(); - const unsigned int bbox_num = bbox_dims.at(1); // n = ? - - std::vector anchors; - this->generate_anchors(input_height, input_width, anchors); - - const unsigned int num_anchors = anchors.size(); - if (num_anchors != bbox_num) - throw std::runtime_error("mismatch num_anchors != bbox_num"); - - const float *bboxes_ptr = (float *) bboxes->GetData(); - const float *probs_ptr = (float *) probs->GetData(); - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - float conf = probs_ptr[2 * i + 1]; - if (conf < score_threshold) continue; // filter first. - - float prior_cx = anchors.at(i).cx; - float prior_cy = anchors.at(i).cy; - float prior_s_kx = anchors.at(i).s_kx; - float prior_s_ky = anchors.at(i).s_ky; - - float dx = bboxes_ptr[4 * i + 0]; - float dy = bboxes_ptr[4 * i + 1]; - float dw = bboxes_ptr[4 * i + 2]; - float dh = bboxes_ptr[4 * i + 3]; - float cx = prior_cx + dx * variance[0] * prior_s_kx; - float cy = prior_cy + dy * variance[0] * prior_s_ky; - float w = prior_s_kx * std::exp(dw * variance[1]); - float h = prior_s_ky * std::exp(dh * variance[1]); // norm coor (0.,1.) - - types::Boxf box; - box.x1 = (cx - w / 2.f) * img_width; - box.y1 = (cy - h / 2.f) * img_height; - box.x2 = (cx + w / 2.f) * img_width; - box.y2 = (cy + h / 2.f) * img_height; - box.score = conf; - box.label = 1; - box.label_text = "face"; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITETNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void TNNFaceBoxesV2::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - diff --git a/lite/tnn/cv/tnn_faceboxesv2.h b/lite/tnn/cv/tnn_faceboxesv2.h deleted file mode 100644 index fca2c406..00000000 --- a/lite/tnn/cv/tnn_faceboxesv2.h +++ /dev/null @@ -1,76 +0,0 @@ -// -// Created by DefTruth on 2022/3/19. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_FACEBOXESV2_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_FACEBOXESV2_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNFaceBoxesV2 : public BasicTNNHandler - { - public: - explicit TNNFaceBoxesV2(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNFaceBoxesV2() override = default; - - private: - // nested classes - struct FaceBoxesAnchorV2 - { - float cx; - float cy; - float s_kx; - float s_ky; - }; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f, 1.f, 1.f}; - std::vector bias_vals = { - -104.f * 1.0f, - -117.f * 1.0f, - -123.f * 1.0f - }; // bgr order - const float variance[2] = {0.1f, 0.2f}; - std::vector steps = {32, 64, 128}; - std::vector> min_sizes = { - {32, 64, 128}, - {256}, - {512} - }; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void transform(const cv::Mat &mat_rs) override; // - - void generate_anchors(const int target_height, - const int target_width, - std::vector &anchors); - - void generate_bboxes(std::vector &bbox_collection, - std::shared_ptr &_instance, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.35f, float iou_threshold = 0.3f, - unsigned int topk = 300, unsigned int nms_type = 0); - - }; -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_FACEBOXESV2_H diff --git a/lite/tnn/cv/tnn_facenet.cpp b/lite/tnn/cv/tnn_facenet.cpp deleted file mode 100644 index d3092b06..00000000 --- a/lite/tnn/cv/tnn_facenet.cpp +++ /dev/null @@ -1,87 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "tnn_facenet.h" - -using tnncv::TNNFaceNet; - -TNNFaceNet::TNNFaceNet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNFaceNet::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNFaceNet::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 4. fetch output mat - std::shared_ptr embedding_mat; - tnn::MatConvertParam embed_cvt_param; // default - - status = instance->GetOutputMat(embedding_mat, embed_cvt_param, "embedding", output_device_type); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - auto embedding_dims = embedding_mat->GetDims(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = (float *) embedding_mat->GetData(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} - diff --git a/lite/tnn/cv/tnn_facenet.h b/lite/tnn/cv/tnn_facenet.h deleted file mode 100644 index ae4fd3c1..00000000 --- a/lite/tnn/cv/tnn_facenet.h +++ /dev/null @@ -1,34 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_FACENET_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_FACENET_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNFaceNet : public BasicTNNHandler - { - public: - explicit TNNFaceNet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNFaceNet() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f / 128.0f, 1.f / 128.0f, 1.f / 128.0f}; - std::vector bias_vals = {-127.5f / 128.0f, -127.5f / 128.0f, -127.5f / 128.0f}; // RGB - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_FACENET_H diff --git a/lite/tnn/cv/tnn_fast_style_transfer.cpp b/lite/tnn/cv/tnn_fast_style_transfer.cpp deleted file mode 100644 index 35aa77c1..00000000 --- a/lite/tnn/cv/tnn_fast_style_transfer.cpp +++ /dev/null @@ -1,95 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "tnn_fast_style_transfer.h" - -using tnncv::TNNFastStyleTransfer; - -TNNFastStyleTransfer::TNNFastStyleTransfer(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNFastStyleTransfer::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNFastStyleTransfer::detect(const cv::Mat &mat, types::StyleContent &style_content) -{ - if (mat.empty()) return; - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); // (1,224,224,3) - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch - tnn::MatConvertParam cvt_param; - std::shared_ptr pred_mat; // (1,3,224,224) - status = instance->GetOutputMat(pred_mat, cvt_param, "output1", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto pred_dims = pred_mat->GetDims(); // (1,3,224,224) - const unsigned int rows = pred_dims.at(2); // H - const unsigned int cols = pred_dims.at(3); // W - const unsigned int channel_step = rows * cols; - - float *pred_ptr = (float *) pred_mat->GetData(); - - // fast copy & channel transpose(CHW->HWC). - cv::Mat rmat(rows, cols, CV_32FC1, pred_ptr); // ref only, zero-copy. - cv::Mat gmat(rows, cols, CV_32FC1, pred_ptr + channel_step); - cv::Mat bmat(rows, cols, CV_32FC1, pred_ptr + 2 * channel_step); - std::vector channel_mats; - channel_mats.push_back(bmat); - channel_mats.push_back(gmat); - channel_mats.push_back(rmat); - - cv::merge(channel_mats, style_content.mat); // BGR - - style_content.mat.convertTo(style_content.mat, CV_8UC3); - - style_content.flag = true; -} \ No newline at end of file diff --git a/lite/tnn/cv/tnn_fast_style_transfer.h b/lite/tnn/cv/tnn_fast_style_transfer.h deleted file mode 100644 index f8e2de2a..00000000 --- a/lite/tnn/cv/tnn_fast_style_transfer.h +++ /dev/null @@ -1,34 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_FAST_STYLE_TRANSFER_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_FAST_STYLE_TRANSFER_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNFastStyleTransfer : public BasicTNNHandler - { - public: - explicit TNNFastStyleTransfer(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNFastStyleTransfer() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.0f, 1.0f, 1.0f}; - std::vector bias_vals = {0.f, 0.f, 0.f}; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::StyleContent &style_content); - }; -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_FAST_STYLE_TRANSFER_H diff --git a/lite/tnn/cv/tnn_fcn_resnet101.cpp b/lite/tnn/cv/tnn_fcn_resnet101.cpp deleted file mode 100644 index eaf4b0bc..00000000 --- a/lite/tnn/cv/tnn_fcn_resnet101.cpp +++ /dev/null @@ -1,289 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "tnn_fcn_resnet101.h" -#include "lite/utils.h" - -using tnncv::TNNFCNResNet101; - -TNNFCNResNet101::TNNFCNResNet101( - const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads -) : proto_path(_proto_path.data()), - model_path(_model_path.data()), - log_id(_proto_path.data()), - num_threads(_num_threads) -{ - initialize_instance(); -} - -TNNFCNResNet101::~TNNFCNResNet101() -{ - net = nullptr; - input_mat = nullptr; - instance = nullptr; -} - -void TNNFCNResNet101::initialize_instance() -{ - std::string proto_content_buffer, model_content_buffer; - proto_content_buffer = BasicTNNHandler::content_buffer_from(proto_path); - model_content_buffer = BasicTNNHandler::content_buffer_from(model_path); - - tnn::ModelConfig model_config; - model_config.model_type = tnn::MODEL_TYPE_TNN; - model_config.params = {proto_content_buffer, model_content_buffer}; - - // 1. init TNN net - tnn::Status status; - net = std::make_shared(); - status = net->Init(model_config); - if (status != tnn::TNN_OK || !net) - { -#ifdef LITETNN_DEBUG - std::cout << "net->Init failed!\n"; -#endif - return; - } - // 2. init device type, change this default setting - // for better performance. such as CUDA/OPENCL/... -#ifdef __ANDROID__ - network_device_type = tnn::DEVICE_ARM; // CPU,GPU - input_device_type = tnn::DEVICE_ARM; // CPU only - output_device_type = tnn::DEVICE_ARM; -#else - network_device_type = tnn::DEVICE_X86; // CPU,GPU - input_device_type = tnn::DEVICE_X86; // CPU only - output_device_type = tnn::DEVICE_X86; -#endif - // 3. init instance - tnn::NetworkConfig network_config; - network_config.library_path = {""}; - network_config.device_type = network_device_type; - - instance = net->CreateInst(network_config, status); - if (status != tnn::TNN_OK || !instance) - { -#ifdef LITETNN_DEBUG - std::cout << "CreateInst failed!" << status.description().c_str() << "\n"; -#endif - return; - } - // 4. setting up num_threads - instance->SetCpuNumThreads((int) num_threads); - // 5. init input information. - input_shape = BasicTNNHandler::get_input_shape(instance, "input"); - - if (input_shape.size() != 4) - { -#ifdef LITETNN_DEBUG - throw std::runtime_error("Found input_shape.size()!=4, but " - "input only support 4 dims." - "Such as NCHW, NHWC ..."); -#else - return; -#endif - } - input_mat_type = BasicTNNHandler::get_input_mat_type(instance, "input"); - input_data_format = BasicTNNHandler::get_input_data_format(instance, "input"); - if (input_data_format == tnn::DATA_FORMAT_NCHW) - { - dynamic_input_height = input_shape.at(2); - dynamic_input_width = input_shape.at(3); - } // NHWC - else if (input_data_format == tnn::DATA_FORMAT_NHWC) - { - dynamic_input_height = input_shape.at(1); - dynamic_input_width = input_shape.at(2); - } // unsupport - else - { -#ifdef LITETNN_DEBUG - std::cout << "input only support NCHW and NHWC " - "input_data_format, but found others.\n"; -#endif - return; - } - // 6. init output information, debug only. - output_shape = BasicTNNHandler::get_output_shape(instance, "out"); -#ifdef LITETNN_DEBUG - this->print_debug_string(); -#endif -} - -void TNNFCNResNet101::print_debug_string() -{ - std::cout << "LITETNN_DEBUG LogId: " << log_id << "\n"; - std::cout << "=============== Input-Dims ==============\n"; - BasicTNNHandler::print_name_shape("input", input_shape); - std::string data_format_string = - (input_data_format == tnn::DATA_FORMAT_NCHW) ? "NCHW" : "NHWC"; - std::cout << "Input Data Format: " << data_format_string << "\n"; - std::cout << "=============== Output-Dims ==============\n"; - BasicTNNHandler::print_name_shape("out", output_shape); - std::cout << "========================================\n"; -} - -void TNNFCNResNet101::transform(const cv::Mat &mat_rs) -{ -// const int img_width = mat.cols; -// const int img_height = mat.rows; -// // update dynamic input dims -// dynamic_input_height = img_height; -// dynamic_input_width = img_width; -// if (input_data_format == tnn::DATA_FORMAT_NCHW) -// { -// input_shape.at(2) = dynamic_input_height; -// input_shape.at(3) = dynamic_input_width; -// } // NHWC -// else if (input_data_format == tnn::DATA_FORMAT_NHWC) -// { -// input_shape.at(1) = dynamic_input_height; -// input_shape.at(2) = dynamic_input_width; -// } -// -// // update input mat and reshape instance -// // reference: https://github.com/Tencent/TNN/blob/master/examples/base/ocr_text_recognizer.cc#L120 -// tnn::InputShapesMap input_shape_map; -// input_shape_map.insert({"input", input_shape}); -// -// auto status = instance->Reshape(input_shape_map); -// if (status != tnn::TNN_OK) -// { -//#ifdef LITETNN_DEBUG -// std::cout << "instance Reshape failed in TNNDeepLabV3ResNet101\n"; -//#endif -// } -// -// cv::Mat canvas; -// cv::cvtColor(mat, canvas, cv::COLOR_BGR2RGB); - -// cv::Mat canvas; -// cv::resize(mat, canvas, cv::Size(dynamic_input_width, dynamic_input_height)); -// cv::cvtColor(canvas, canvas, cv::COLOR_BGR2RGB); -// - // push into input_mat - input_mat = std::make_shared( - input_device_type, - tnn::N8UC3, - input_shape, - (void *) mat_rs.data - ); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNFCNResNet101::detect(const cv::Mat &mat, types::SegmentContent &content) -{ - if (mat.empty()) return; - const int img_width = mat.cols; - const int img_height = mat.rows; - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat_rs, mat_rs, cv::Size(dynamic_input_width, dynamic_input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch - tnn::MatConvertParam cvt_param; - std::shared_ptr scores_mat; // (1,21,h,w) - status = instance->GetOutputMat(scores_mat, cvt_param, "out", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto scores_dims = scores_mat->GetDims(); -#ifdef LITETNN_DEBUG - BasicTNNHandler::print_name_shape("out", scores_dims); -#endif - - const unsigned int output_classes = scores_dims.at(1); - const unsigned int output_height = scores_dims.at(2); - const unsigned int output_width = scores_dims.at(3); - - const float *scores_ptr = (float *) scores_mat->GetData(); - - // time cost! - content.names_map.clear(); - content.class_mat = cv::Mat(output_height, output_width, CV_8UC1, cv::Scalar(0)); - cv::resize(mat, content.color_mat, cv::Size(output_width, output_height)); // init color mat - - const unsigned int scores_step = output_height * output_width; // h x w - - for (unsigned int i = 0; i < output_height; ++i) - { - - uchar *p_class = content.class_mat.ptr(i); - cv::Vec3b *p_color = content.color_mat.ptr(i); - - for (unsigned int j = 0; j < output_width; ++j) - { - // argmax - unsigned int max_label = 0; - float max_conf = scores_ptr[0 * scores_step + i * output_width + j]; - - for (unsigned int l = 0; l < output_classes; ++l) - { - float conf = scores_ptr[l * scores_step + i * output_width + j]; - if (conf > max_conf) - { - max_conf = conf; - max_label = l; - } - } - - if (max_label == 0) continue; - - // assign label for pixel(i,j) - p_class[j] = cv::saturate_cast(max_label); - // assign color for detected class at pixel(i,j). - p_color[j][0] = cv::saturate_cast((max_label % 10) * 20); - p_color[j][1] = cv::saturate_cast((max_label % 5) * 40); - p_color[j][2] = cv::saturate_cast((max_label % 10) * 20); - // assign names map - content.names_map[max_label] = class_names[max_label - 1]; // max_label >= 1 - } - - } - - cv::resize(content.class_mat, content.class_mat, cv::Size(img_width, img_height)); - cv::resize(content.color_mat, content.color_mat, cv::Size(img_width, img_height)); - - content.flag = true; - -} - diff --git a/lite/tnn/cv/tnn_fcn_resnet101.h b/lite/tnn/cv/tnn_fcn_resnet101.h deleted file mode 100644 index 7cc8f6be..00000000 --- a/lite/tnn/cv/tnn_fcn_resnet101.h +++ /dev/null @@ -1,76 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_FCN_RESNET101_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_FCN_RESNET101_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNFCNResNet101 - { - public: - explicit TNNFCNResNet101(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); - - ~TNNFCNResNet101(); - - private: - const char *log_id = nullptr; - const char *proto_path = nullptr; - const char *model_path = nullptr; - // Note, tnn:: actually is TNN_NS::, I prefer the first one. - std::shared_ptr net; - std::shared_ptr instance; - std::shared_ptr input_mat; - - private: - std::vector scale_vals = {(1.f / 0.229f) * (1.f / 255.f), - (1.f / 0.224f) * (1.f / 255.f), - (1.f / 0.225f) * (1.f / 255.f)}; - std::vector bias_vals = {-0.485f * 255.f * (1.f / 0.229f) * (1.f / 255.f), - -0.456f * 255.f * (1.f / 0.224f) * (1.f / 255.f), - -0.406f * 255.f * (1.f / 0.225f) * (1.f / 255.f)}; // RGB - - private: - const unsigned int num_threads; // initialize at runtime. - int dynamic_input_height = 512; // init only, will change according to input mat. - int dynamic_input_width = 512; // init only, will change according to input mat. - tnn::DataFormat input_data_format; // e.g DATA_FORMAT_NHWC - tnn::MatType input_mat_type; // e.g NCHW_FLOAT - tnn::DeviceType input_device_type; // only CPU, namely ARM or X86 - tnn::DeviceType output_device_type; // only CPU, namely ARM or X86 - tnn::DeviceType network_device_type; // e.g DEVICE_X86 DEVICE_NAIVE DEVICE_ARM - tnn::DimsVector input_shape; // debug - tnn::DimsVector output_shape; - - const char *class_names[20] = { - "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", - "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", - "train", "tvmonitor" - }; // 20 classes - - // un-copyable - protected: - TNNFCNResNet101(const TNNFCNResNet101 &) = delete; // - TNNFCNResNet101(TNNFCNResNet101 &&) = delete; // - TNNFCNResNet101 &operator=(const TNNFCNResNet101 &) = delete; // - TNNFCNResNet101 &operator=(TNNFCNResNet101 &&) = delete; // - - private: - void print_debug_string(); // debug information - - private: - void transform(const cv::Mat &mat_rs); // - - void initialize_instance(); // init net & instance - - public: - void detect(const cv::Mat &mat, types::SegmentContent &content); - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_FCN_RESNET101_H diff --git a/lite/tnn/cv/tnn_female_photo2cartoon.cpp b/lite/tnn/cv/tnn_female_photo2cartoon.cpp deleted file mode 100644 index 474bb0a7..00000000 --- a/lite/tnn/cv/tnn_female_photo2cartoon.cpp +++ /dev/null @@ -1,149 +0,0 @@ -// -// Created by DefTruth on 2022/6/12. -// - -#include "tnn_female_photo2cartoon.h" - -using tnncv::TNNFemalePhoto2Cartoon; - -TNNFemalePhoto2Cartoon::TNNFemalePhoto2Cartoon( - const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNFemalePhoto2Cartoon::transform(const cv::Mat &mat_merged_rs) -{ - // push into input_mat (1,3,256,256) - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_merged_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNFemalePhoto2Cartoon::detect( - const cv::Mat &mat, const cv::Mat &mask, - types::FemalePhoto2CartoonContent &content) -{ - if (mat.empty() || mask.empty()) return; - const unsigned int channels = mat.channels(); - if (channels != 3) return; - const unsigned int mask_channels = mask.channels(); - if (mask_channels != 1 && mask_channels != 3) return; - // model input size - const unsigned int input_h = input_height; // 256 - const unsigned int input_w = input_width; // 256 - // resize before merging mat and mask - cv::Mat mat_rs, mask_rs; - cv::resize(mat, mat_rs, cv::Size(input_w, input_h)); - cv::resize(mask, mask_rs, cv::Size(input_w, input_h)); // CV_32FC1 - if (mask_channels != 3) cv::cvtColor(mask_rs, mask_rs, cv::COLOR_GRAY2BGR); // CV_32FC3 - mat_rs.convertTo(mat_rs, CV_32FC3, 1.f, 0.f); // CV_32FC3 - // merge mat_rs and mask_rs - cv::Mat mat_merged_rs = mat_rs.mul(mask_rs) + (1.f - mask_rs) * 255.f; // CV_32FC3 - mat_merged_rs.convertTo(mat_merged_rs, CV_8UC3); // mapping -> tnn::N8UC3 - // 1. make input tensor - this->transform(mat_merged_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - input_cvt_param.reverse_channel = true; // BGR -> RGB - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward cartoon (1,3,256,256) - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. generate cartoon - this->generate_cartoon(instance, mask_rs, content); -} - -void TNNFemalePhoto2Cartoon::generate_cartoon( - std::shared_ptr &_instance, - const cv::Mat &mask_rs, types::FemalePhoto2CartoonContent &content) -{ - tnn::MatConvertParam cvt_param; - std::shared_ptr cartoon_pred; // (1,3,256,256) - auto status = _instance->GetOutputMat(cartoon_pred, cvt_param, "output", output_device_type); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - auto cartoon_dims = cartoon_pred->GetDims(); - const unsigned int out_h = cartoon_dims.at(2); - const unsigned int out_w = cartoon_dims.at(3); - const unsigned int channel_step = out_h * out_w; - const unsigned int mask_h = mask_rs.rows; - const unsigned int mask_w = mask_rs.cols; - // fast assign & channel transpose(CHW->HWC). - float *cartoon_ptr = (float *) cartoon_pred->GetData(); - std::vector cartoon_channel_mats; - cv::Mat rmat(out_h, out_w, CV_32FC1, cartoon_ptr); // R - cv::Mat gmat(out_h, out_w, CV_32FC1, cartoon_ptr + channel_step); // G - cv::Mat bmat(out_h, out_w, CV_32FC1, cartoon_ptr + 2 * channel_step); // B - rmat = (rmat + 1.f) * 127.5f; - gmat = (gmat + 1.f) * 127.5f; - bmat = (bmat + 1.f) * 127.5f; - cartoon_channel_mats.push_back(rmat); - cartoon_channel_mats.push_back(gmat); - cartoon_channel_mats.push_back(bmat); - cv::Mat cartoon; - cv::merge(cartoon_channel_mats, cartoon); // CV_32FC3 - if (out_h != mask_h || out_w != mask_w) - cv::resize(cartoon, cartoon, cv::Size(mask_w, mask_h)); - // combine & RGB -> BGR -> uint8 - cartoon = cartoon.mul(mask_rs) + (1.f - mask_rs) * 255.f; - cv::cvtColor(cartoon, cartoon, cv::COLOR_RGB2BGR); - cartoon.convertTo(cartoon, CV_8UC3); - - content.cartoon = cartoon; - content.flag = true; -} - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lite/tnn/cv/tnn_female_photo2cartoon.h b/lite/tnn/cv/tnn_female_photo2cartoon.h deleted file mode 100644 index 58dee3e1..00000000 --- a/lite/tnn/cv/tnn_female_photo2cartoon.h +++ /dev/null @@ -1,37 +0,0 @@ -// -// Created by DefTruth on 2022/6/12. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_FEMALE_PHOTO2CARTOON_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_FEMALE_PHOTO2CARTOON_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNFemalePhoto2Cartoon : public BasicTNNHandler - { - public: - explicit TNNFemalePhoto2Cartoon(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); - - ~TNNFemalePhoto2Cartoon() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - std::vector bias_vals = {-1.f, -1.f, -1.f}; - - private: - void transform(const cv::Mat &mat_merged_rs /*merged & resized mat*/) override; - - void generate_cartoon(std::shared_ptr &_instance, - const cv::Mat &mask_rs, types::FemalePhoto2CartoonContent &content); - - public: - void detect(const cv::Mat &mat, const cv::Mat &mask, types::FemalePhoto2CartoonContent &content); - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_FEMALE_PHOTO2CARTOON_H diff --git a/lite/tnn/cv/tnn_focal_arcface.cpp b/lite/tnn/cv/tnn_focal_arcface.cpp deleted file mode 100644 index bce3b358..00000000 --- a/lite/tnn/cv/tnn_focal_arcface.cpp +++ /dev/null @@ -1,88 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "tnn_focal_arcface.h" - -using tnncv::TNNFocalArcFace; - -TNNFocalArcFace::TNNFocalArcFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNFocalArcFace::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNFocalArcFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 4. fetch output mat - std::shared_ptr embedding_mat; - tnn::MatConvertParam embed_cvt_param; // default - - status = instance->GetOutputMat(embedding_mat, embed_cvt_param, "embedding", output_device_type); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - auto embedding_dims = embedding_mat->GetDims(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = (float *) embedding_mat->GetData(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} - - diff --git a/lite/tnn/cv/tnn_focal_arcface.h b/lite/tnn/cv/tnn_focal_arcface.h deleted file mode 100644 index 19bae6dd..00000000 --- a/lite/tnn/cv/tnn_focal_arcface.h +++ /dev/null @@ -1,34 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_FOCAL_ARCFACE_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_FOCAL_ARCFACE_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNFocalArcFace : public BasicTNNHandler - { - public: - explicit TNNFocalArcFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNFocalArcFace() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - std::vector bias_vals = {-1.f, -1.f, -1.f}; // RGB - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_FOCAL_ARCFACE_H diff --git a/lite/tnn/cv/tnn_focal_asia_arcface.cpp b/lite/tnn/cv/tnn_focal_asia_arcface.cpp deleted file mode 100644 index 655b52c3..00000000 --- a/lite/tnn/cv/tnn_focal_asia_arcface.cpp +++ /dev/null @@ -1,87 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "tnn_focal_asia_arcface.h" - -using tnncv::TNNFocalAsiaArcFace; - -TNNFocalAsiaArcFace::TNNFocalAsiaArcFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNFocalAsiaArcFace::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNFocalAsiaArcFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 4. fetch output mat - std::shared_ptr embedding_mat; - tnn::MatConvertParam embed_cvt_param; // default - - status = instance->GetOutputMat(embedding_mat, embed_cvt_param, "embedding", output_device_type); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - auto embedding_dims = embedding_mat->GetDims(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = (float *) embedding_mat->GetData(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} - diff --git a/lite/tnn/cv/tnn_focal_asia_arcface.h b/lite/tnn/cv/tnn_focal_asia_arcface.h deleted file mode 100644 index 5a989fc4..00000000 --- a/lite/tnn/cv/tnn_focal_asia_arcface.h +++ /dev/null @@ -1,34 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_FOCAL_ASIA_ARCFACE_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_FOCAL_ASIA_ARCFACE_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNFocalAsiaArcFace : public BasicTNNHandler - { - public: - explicit TNNFocalAsiaArcFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNFocalAsiaArcFace() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - std::vector bias_vals = {-1.f, -1.f, -1.f}; // RGB - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_FOCAL_ASIA_ARCFACE_H diff --git a/lite/tnn/cv/tnn_fsanet.cpp b/lite/tnn/cv/tnn_fsanet.cpp deleted file mode 100644 index 8a82ac1d..00000000 --- a/lite/tnn/cv/tnn_fsanet.cpp +++ /dev/null @@ -1,93 +0,0 @@ -// -// Created by DefTruth on 2021/11/25. -// - -#include "tnn_fsanet.h" - -using tnncv::TNNFSANet; - -TNNFSANet::TNNFSANet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNFSANet::transform(const cv::Mat &mat_padded) -{ - // push into input_mat - // be carefully, no deepcopy inside this tnn::Mat constructor, - // so, we can not pass a local cv::Mat to this constructor. - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_padded.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNFSANet::detect(const cv::Mat &mat, types::EulerAngles &euler_angles) -{ - if (mat.empty()) return; - - // 1. make input mat - cv::Mat mat_padded; - // 0. padding - const int h = mat.rows; - const int w = mat.cols; - const int nh = static_cast((static_cast(h) + pad * static_cast(h))); - const int nw = static_cast((static_cast(w) + pad * static_cast(w))); - - const int nx1 = std::max(0, static_cast((nw - w) / 2)); - const int ny1 = std::max(0, static_cast((nh - h) / 2)); - - mat_padded = cv::Mat(nh, nw, CV_8UC3, cv::Scalar(0, 0, 0)); - mat.copyTo(mat_padded(cv::Rect(nx1, ny1, w, h))); - cv::resize(mat_padded, mat_padded, cv::Size(input_width, input_height)); - - this->transform(mat_padded); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param, "input"); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch angles. - tnn::MatConvertParam cvt_param; - std::shared_ptr angles; // (1,3) - status = instance->GetOutputMat(angles, cvt_param, "output", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - const float *angles_ptr = (float *) angles->GetData(); - - euler_angles.yaw = angles_ptr[0]; - euler_angles.pitch = angles_ptr[1]; - euler_angles.roll = angles_ptr[2]; - euler_angles.flag = true; -} \ No newline at end of file diff --git a/lite/tnn/cv/tnn_fsanet.h b/lite/tnn/cv/tnn_fsanet.h deleted file mode 100644 index 294267de..00000000 --- a/lite/tnn/cv/tnn_fsanet.h +++ /dev/null @@ -1,34 +0,0 @@ -// -// Created by DefTruth on 2021/11/25. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_FSANET_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_FSANET_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNFSANet : public BasicTNNHandler - { - public: - explicit TNNFSANet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNFSANet() override = default; - - private: - // In TNN: x*scale + bias - static constexpr const float pad = 0.3f; - std::vector scale_vals = {1.0f / 127.5f, 1.0f / 127.5f, 1.0f / 127.5f}; - std::vector bias_vals = {-1.f, -1.f, -1.f}; - - private: - void transform(const cv::Mat &mat_padded) override; // - - public: - void detect(const cv::Mat &mat, types::EulerAngles &euler_angles); - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_FSANET_H diff --git a/lite/tnn/cv/tnn_gender_googlenet.cpp b/lite/tnn/cv/tnn_gender_googlenet.cpp deleted file mode 100644 index 048b1067..00000000 --- a/lite/tnn/cv/tnn_gender_googlenet.cpp +++ /dev/null @@ -1,86 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#include "tnn_gender_googlenet.h" -#include "lite/utils.h" - -using tnncv::TNNGenderGoogleNet; - -TNNGenderGoogleNet::TNNGenderGoogleNet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNGenderGoogleNet::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNGenderGoogleNet::detect(const cv::Mat &mat, types::Gender &gender) -{ - if (mat.empty()) return; - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch. - tnn::MatConvertParam cvt_param; - std::shared_ptr gender_logits; // (1,8) - status = instance->GetOutputMat(gender_logits, cvt_param, "loss3/loss3_Y", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto gender_dims = gender_logits->GetDims(); - const unsigned int num_genders = gender_dims.at(1); // 2 - const float *pred_logits_ptr = (float *) gender_logits->GetData(); - - unsigned int pred_gender = 0; - auto softmax_probs = lite::utils::math::softmax(pred_logits_ptr, num_genders, pred_gender); - unsigned int gender_label = pred_gender == 1 ? 0 : 1; - gender.label = gender_label; - gender.text = gender_texts[gender_label]; - gender.score = softmax_probs[pred_gender]; - gender.flag = true; -} \ No newline at end of file diff --git a/lite/tnn/cv/tnn_gender_googlenet.h b/lite/tnn/cv/tnn_gender_googlenet.h deleted file mode 100644 index 460ab5d2..00000000 --- a/lite/tnn/cv/tnn_gender_googlenet.h +++ /dev/null @@ -1,35 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_GENDER_GOOGLENET_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_GENDER_GOOGLENET_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNGenderGoogleNet : public BasicTNNHandler - { - public: - explicit TNNGenderGoogleNet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNGenderGoogleNet() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.0f, 1.0f, 1.0f}; - std::vector bias_vals = {-104.0f, -117.0f, -123.0f}; - const char *gender_texts[2] = {"female", "male"}; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::Gender &gender); - }; -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_GENDER_GOOGLENET_H diff --git a/lite/tnn/cv/tnn_ghostnet.cpp b/lite/tnn/cv/tnn_ghostnet.cpp deleted file mode 100644 index bdec835b..00000000 --- a/lite/tnn/cv/tnn_ghostnet.cpp +++ /dev/null @@ -1,94 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "tnn_ghostnet.h" -#include "lite/utils.h" - -using tnncv::TNNGhostNet; - -TNNGhostNet::TNNGhostNet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNGhostNet::transform(const cv::Mat &mat_rs) -{ - // push into input_mat (1,3,224,224) - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNGhostNet::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch. - tnn::MatConvertParam cvt_param; - std::shared_ptr logits_mat; // (1,1000) - status = instance->GetOutputMat(logits_mat, cvt_param, "logits", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto logits_dims = logits_mat->GetDims(); - const unsigned int num_classes = logits_dims.at(1); // 1000 - const float *logits = (float *) logits_mat->GetData(); - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/tnn/cv/tnn_ghostnet.h b/lite/tnn/cv/tnn_ghostnet.h deleted file mode 100644 index 1d43e000..00000000 --- a/lite/tnn/cv/tnn_ghostnet.h +++ /dev/null @@ -1,413 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_GHOSTNET_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_GHOSTNET_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNGhostNet : public BasicTNNHandler - { - public: - explicit TNNGhostNet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNGhostNet() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - std::vector bias_vals = {-0.485f * 255.f * (1.0f / 0.229f) * (1.0 / 255.f), - -0.456f * 255.f * (1.0f / 0.224f) * (1.0 / 255.f), - -0.406f * 255.f * (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_GHOSTNET_H diff --git a/lite/tnn/cv/tnn_glint_arcface.cpp b/lite/tnn/cv/tnn_glint_arcface.cpp deleted file mode 100644 index c380a95b..00000000 --- a/lite/tnn/cv/tnn_glint_arcface.cpp +++ /dev/null @@ -1,111 +0,0 @@ -// -// Created by DefTruth on 2021/11/13. -// - -#include "tnn_glint_arcface.h" - -using tnncv::TNNGlintArcFace; - -TNNGlintArcFace::TNNGlintArcFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNGlintArcFace::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNGlintArcFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 4. fetch output mat - std::shared_ptr embedding_mat; - tnn::MatConvertParam embed_cvt_param; // default - - status = instance->GetOutputMat(embedding_mat, embed_cvt_param, "embedding", output_device_type); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - auto embedding_dims = embedding_mat->GetDims(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = (float *) embedding_mat->GetData(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lite/tnn/cv/tnn_glint_arcface.h b/lite/tnn/cv/tnn_glint_arcface.h deleted file mode 100644 index 880b729e..00000000 --- a/lite/tnn/cv/tnn_glint_arcface.h +++ /dev/null @@ -1,34 +0,0 @@ -// -// Created by DefTruth on 2021/11/13. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_GLINT_ARCFACE_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_GLINT_ARCFACE_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNGlintArcFace : public BasicTNNHandler - { - public: - explicit TNNGlintArcFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNGlintArcFace() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - std::vector bias_vals = {-1.f, -1.f, -1.f}; // RGB - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_GLINT_ARCFACE_H diff --git a/lite/tnn/cv/tnn_glint_cosface.cpp b/lite/tnn/cv/tnn_glint_cosface.cpp deleted file mode 100644 index 045e75df..00000000 --- a/lite/tnn/cv/tnn_glint_cosface.cpp +++ /dev/null @@ -1,88 +0,0 @@ -// -// Created by DefTruth on 2021/11/13. -// - -#include "tnn_glint_cosface.h" - -using tnncv::TNNGlintCosFace; - -TNNGlintCosFace::TNNGlintCosFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNGlintCosFace::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNGlintCosFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 4. fetch output mat - std::shared_ptr embedding_mat; - tnn::MatConvertParam embed_cvt_param; // default - - status = instance->GetOutputMat(embedding_mat, embed_cvt_param, "embedding", output_device_type); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - auto embedding_dims = embedding_mat->GetDims(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = (float *) embedding_mat->GetData(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} - - diff --git a/lite/tnn/cv/tnn_glint_cosface.h b/lite/tnn/cv/tnn_glint_cosface.h deleted file mode 100644 index 0419d9c0..00000000 --- a/lite/tnn/cv/tnn_glint_cosface.h +++ /dev/null @@ -1,34 +0,0 @@ -// -// Created by DefTruth on 2021/11/13. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_GLINT_COSFACE_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_GLINT_COSFACE_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNGlintCosFace : public BasicTNNHandler - { - public: - explicit TNNGlintCosFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNGlintCosFace() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - std::vector bias_vals = {-1.f, -1.f, -1.f}; // RGB - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_GLINT_COSFACE_H diff --git a/lite/tnn/cv/tnn_glint_partial_fc.cpp b/lite/tnn/cv/tnn_glint_partial_fc.cpp deleted file mode 100644 index a7f2b6be..00000000 --- a/lite/tnn/cv/tnn_glint_partial_fc.cpp +++ /dev/null @@ -1,86 +0,0 @@ -// -// Created by DefTruth on 2021/11/13. -// - -#include "tnn_glint_partial_fc.h" - -using tnncv::TNNGlintPartialFC; - -TNNGlintPartialFC::TNNGlintPartialFC(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNGlintPartialFC::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNGlintPartialFC::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 4. fetch output mat - std::shared_ptr embedding_mat; - tnn::MatConvertParam embed_cvt_param; // default - - status = instance->GetOutputMat(embedding_mat, embed_cvt_param, "embedding", output_device_type); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - auto embedding_dims = embedding_mat->GetDims(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = (float *) embedding_mat->GetData(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} diff --git a/lite/tnn/cv/tnn_glint_partial_fc.h b/lite/tnn/cv/tnn_glint_partial_fc.h deleted file mode 100644 index d5115a68..00000000 --- a/lite/tnn/cv/tnn_glint_partial_fc.h +++ /dev/null @@ -1,34 +0,0 @@ -// -// Created by DefTruth on 2021/11/13. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_GLINT_PARTIAL_FC_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_GLINT_PARTIAL_FC_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNGlintPartialFC : public BasicTNNHandler - { - public: - explicit TNNGlintPartialFC(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNGlintPartialFC() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - std::vector bias_vals = {-1.f, -1.f, -1.f}; // RGB - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_GLINT_PARTIAL_FC_H diff --git a/lite/tnn/cv/tnn_hdrdnet.cpp b/lite/tnn/cv/tnn_hdrdnet.cpp deleted file mode 100644 index 734d8d0c..00000000 --- a/lite/tnn/cv/tnn_hdrdnet.cpp +++ /dev/null @@ -1,94 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "tnn_hdrdnet.h" -#include "lite/utils.h" - -using tnncv::TNNHdrDNet; - -TNNHdrDNet::TNNHdrDNet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNHdrDNet::transform(const cv::Mat &mat_rs) -{ - // push into input_mat (1,3,224,224) - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNHdrDNet::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch. - tnn::MatConvertParam cvt_param; - std::shared_ptr logits_mat; // (1,1000) - status = instance->GetOutputMat(logits_mat, cvt_param, "logits", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto logits_dims = logits_mat->GetDims(); - const unsigned int num_classes = logits_dims.at(1); // 1000 - const float *logits = (float *) logits_mat->GetData(); - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/tnn/cv/tnn_hdrdnet.h b/lite/tnn/cv/tnn_hdrdnet.h deleted file mode 100644 index 6a5950c2..00000000 --- a/lite/tnn/cv/tnn_hdrdnet.h +++ /dev/null @@ -1,414 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_HDRDNET_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_HDRDNET_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNHdrDNet : public BasicTNNHandler - { - public: - explicit TNNHdrDNet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNHdrDNet() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - std::vector bias_vals = {-0.485f * 255.f * (1.0f / 0.229f) * (1.0 / 255.f), - -0.456f * 255.f * (1.0f / 0.224f) * (1.0 / 255.f), - -0.406f * 255.f * (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - - }; -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_HDRDNET_H diff --git a/lite/tnn/cv/tnn_head_seg.cpp b/lite/tnn/cv/tnn_head_seg.cpp deleted file mode 100644 index 2890d387..00000000 --- a/lite/tnn/cv/tnn_head_seg.cpp +++ /dev/null @@ -1,217 +0,0 @@ -// -// Created by DefTruth on 2022/6/11. -// - -#include "tnn_head_seg.h" - -using tnncv::TNNHeadSeg; - -TNNHeadSeg::TNNHeadSeg( - const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads -) : proto_path(_proto_path.data()), - model_path(_model_path.data()), - log_id(_proto_path.data()), - num_threads(_num_threads) -{ - initialize_instance(); -} - -TNNHeadSeg::~TNNHeadSeg() -{ - net = nullptr; - input_mat = nullptr; - instance = nullptr; -} - -void TNNHeadSeg::initialize_instance() -{ - std::string proto_content_buffer, model_content_buffer; - proto_content_buffer = BasicTNNHandler::content_buffer_from(proto_path); - model_content_buffer = BasicTNNHandler::content_buffer_from(model_path); - - tnn::ModelConfig model_config; - model_config.model_type = tnn::MODEL_TYPE_TNN; - model_config.params = {proto_content_buffer, model_content_buffer}; - - // 1. init TNN net - tnn::Status status; - net = std::make_shared(); - status = net->Init(model_config); - if (status != tnn::TNN_OK || !net) - { -#ifdef LITETNN_DEBUG - std::cout << "net->Init failed!\n"; -#endif - return; - } - // 2. init device type, change this default setting - // for better performance. such as CUDA/OPENCL/... -#ifdef __ANDROID__ - network_device_type = tnn::DEVICE_ARM; // CPU,GPU - input_device_type = tnn::DEVICE_ARM; // CPU only - output_device_type = tnn::DEVICE_ARM; -#else - network_device_type = tnn::DEVICE_X86; // CPU,GPU - input_device_type = tnn::DEVICE_X86; // CPU only - output_device_type = tnn::DEVICE_X86; -#endif - // 3. init instance - tnn::NetworkConfig network_config; - network_config.library_path = {""}; - network_config.device_type = network_device_type; - network_config.data_format = tnn::DATA_FORMAT_NHWC; - - instance = net->CreateInst(network_config, status); - if (status != tnn::TNN_OK || !instance) - { -#ifdef LITETNN_DEBUG - std::cout << "CreateInst failed!" << status.description().c_str() << "\n"; -#endif - return; - } - // 4. setting up num_threads - instance->SetCpuNumThreads((int) num_threads); - // 5. init input information. - input_shape = BasicTNNHandler::get_input_shape(instance, "input_1_0"); - // hard code (NHWC) from pb -> ONNX -> TNN - input_batch = input_shape.at(0); - input_height = input_shape.at(1); - input_width = input_shape.at(2); - input_channel = input_shape.at(3); - - if (input_shape.size() != 4) - { -#ifdef LITETNN_DEBUG - throw std::runtime_error("Found input_shape.size()!=4, but " - "input only support 4 dims." - "Such as NCHW, NHWC ..."); -#else - return; -#endif - } - input_mat_type = BasicTNNHandler::get_input_mat_type(instance, "input_1_0"); - input_data_format = BasicTNNHandler::get_input_data_format(instance, "input_1_0"); - // 6. init output information, debug only. - output_shape = BasicTNNHandler::get_output_shape(instance, "sigmoid/Sigmoid:0"); -#ifdef LITETNN_DEBUG - this->print_debug_string(); -#endif -} - -void TNNHeadSeg::print_debug_string() -{ - std::cout << "LITETNN_DEBUG LogId: " << log_id << "\n"; - std::cout << "=============== Input-Dims ==============\n"; - BasicTNNHandler::print_name_shape("input_1_0", input_shape); - std::string data_format_string = - (input_data_format == tnn::DATA_FORMAT_NCHW) ? "NCHW" : "NHWC"; - std::cout << "Input Data Format: " << data_format_string << "\n"; - std::cout << "=============== Output-Dims ==============\n"; - BasicTNNHandler::print_name_shape("sigmoid/Sigmoid:0", output_shape); - std::cout << "========================================\n"; -} - -void TNNHeadSeg::transform(const cv::Mat &mat_rs) -{ - // be carefully, no deepcopy inside this tnn::Mat constructor, - // so, we can not pass a local cv::Mat to this constructor. - // push into input_mat - input_mat = std::make_shared( - input_device_type, - tnn::N8UC3, - input_shape, - (void *) mat_rs.data - ); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNHeadSeg::detect(const cv::Mat &mat, types::HeadSegContent &content) -{ - if (mat.empty()) return; - const unsigned int img_h = mat.rows; - const unsigned int img_w = mat.cols; - const unsigned int channels = mat.channels(); - if (channels != 3) return; - const unsigned int input_h = input_height; // 384 - const unsigned int input_w = input_width; // 384 - - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_w, input_h)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - // 1. make input tensor - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch - tnn::MatConvertParam cvt_param; - std::shared_ptr mask_pred; // (1,384,384,1) - status = instance->GetOutputMat(mask_pred, cvt_param, "sigmoid/Sigmoid:0", output_device_type); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto mask_dims = mask_pred->GetDims(); -#ifdef LITETNN_DEBUG - BasicTNNHandler::print_name_shape("sigmoid/Sigmoid:0", mask_dims); -#endif - - const unsigned int out_h = mask_dims.at(1); - const unsigned int out_w = mask_dims.at(2); - float *mask_ptr = (float *) mask_pred->GetData(); - - cv::Mat mask_adj; - cv::Mat mask_out(out_h, out_w, CV_32FC1, mask_ptr); - cv::resize(mask_out, mask_adj, cv::Size(img_w, img_h)); // (img_h,img_w,1) - - content.mask = mask_adj; - content.flag = true; -} - - - - - - - - - - - - - - - - - - diff --git a/lite/tnn/cv/tnn_head_seg.h b/lite/tnn/cv/tnn_head_seg.h deleted file mode 100644 index 9d658120..00000000 --- a/lite/tnn/cv/tnn_head_seg.h +++ /dev/null @@ -1,72 +0,0 @@ -// -// Created by DefTruth on 2022/6/11. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_HEAD_SEG_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_HEAD_SEG_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNHeadSeg - { - public: - explicit TNNHeadSeg(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); - - ~TNNHeadSeg(); - - private: - const char *log_id = nullptr; - const char *proto_path = nullptr; - const char *model_path = nullptr; - // Note, tnn:: actually is TNN_NS::, I prefer the first one. - std::shared_ptr net; - std::shared_ptr instance; - std::shared_ptr input_mat; - const unsigned int num_threads; // initialize at runtime. - - private: - // y = scale*x + bias - std::vector scale_vals = {1.f / 255.f, 1.f / 255.f, 1.f / 255.f}; - std::vector bias_vals = {0.f, 0.f, 0.f}; // RGB - - private: - // input size (1,384,384,3) - unsigned int input_batch = 1; - unsigned int input_channel = 3; - unsigned int input_height = 384; - unsigned int input_width = 384; - - private: - tnn::DataFormat input_data_format; // e.g DATA_FORMAT_NHWC - tnn::MatType input_mat_type; // e.g NCHW_FLOAT - tnn::DeviceType input_device_type; // only CPU, namely ARM or X86 - tnn::DeviceType output_device_type; // only CPU, namely ARM or X86 - tnn::DeviceType network_device_type; // e.g DEVICE_X86 DEVICE_NAIVE DEVICE_ARM - tnn::DimsVector input_shape; // debug - tnn::DimsVector output_shape; - - // un-copyable - protected: - TNNHeadSeg(const TNNHeadSeg &) = delete; // - TNNHeadSeg(TNNHeadSeg &&) = delete; // - TNNHeadSeg &operator=(const TNNHeadSeg &) = delete; // - TNNHeadSeg &operator=(TNNHeadSeg &&) = delete; // - - private: - void print_debug_string(); // debug information - - private: - void transform(const cv::Mat &mat_rs); // - - void initialize_instance(); // init net & instance - - public: - void detect(const cv::Mat &mat, types::HeadSegContent &content); - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_HEAD_SEG_H diff --git a/lite/tnn/cv/tnn_ibnnet.cpp b/lite/tnn/cv/tnn_ibnnet.cpp deleted file mode 100644 index 4a6767a8..00000000 --- a/lite/tnn/cv/tnn_ibnnet.cpp +++ /dev/null @@ -1,94 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "tnn_ibnnet.h" -#include "lite/utils.h" - -using tnncv::TNNIBNNet; - -TNNIBNNet::TNNIBNNet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNIBNNet::transform(const cv::Mat &mat_rs) -{ - // push into input_mat (1,3,224,224) - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNIBNNet::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch. - tnn::MatConvertParam cvt_param; - std::shared_ptr logits_mat; // (1,1000) - status = instance->GetOutputMat(logits_mat, cvt_param, "logits", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto logits_dims = logits_mat->GetDims(); - const unsigned int num_classes = logits_dims.at(1); // 1000 - const float *logits = (float *) logits_mat->GetData(); - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/tnn/cv/tnn_ibnnet.h b/lite/tnn/cv/tnn_ibnnet.h deleted file mode 100644 index f2fd2d8b..00000000 --- a/lite/tnn/cv/tnn_ibnnet.h +++ /dev/null @@ -1,413 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_IBNNET_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_IBNNET_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNIBNNet : public BasicTNNHandler - { - public: - explicit TNNIBNNet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNIBNNet() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - std::vector bias_vals = {-0.485f * 255.f * (1.0f / 0.229f) * (1.0 / 255.f), - -0.456f * 255.f * (1.0f / 0.224f) * (1.0 / 255.f), - -0.406f * 255.f * (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_IBNNET_H diff --git a/lite/tnn/cv/tnn_insectdet.cpp b/lite/tnn/cv/tnn_insectdet.cpp deleted file mode 100644 index 5925aeb5..00000000 --- a/lite/tnn/cv/tnn_insectdet.cpp +++ /dev/null @@ -1,189 +0,0 @@ -// -// Created by DefTruth on 2022/3/27. -// - -#include "tnn_insectdet.h" -#include "lite/utils.h" - -using tnncv::TNNInsectDet; - -TNNInsectDet::TNNInsectDet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNInsectDet::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - InsectDetScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.ratio = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.flag = true; -} - -void TNNInsectDet::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNInsectDet::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - // resize & unscale - cv::Mat mat_rs; - InsectDetScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - cv::Mat mat_rs_; - cv::cvtColor(mat_rs, mat_rs_, cv::COLOR_BGR2RGB); - this->transform(mat_rs_); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - // 5. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, instance, score_threshold, img_height, img_width); - // 6. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk); -} - -void TNNInsectDet::generate_bboxes(const InsectDetScaleParams &scale_params, - std::vector &bbox_collection, - std::shared_ptr &_instance, - float score_threshold, int img_height, - int img_width) -{ - tnn::MatConvertParam cvt_param; - std::shared_ptr output; - tnn::Status status; - - status = _instance->GetOutputMat(output, cvt_param, "output", output_device_type); // [1,N,6] - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" << status.description().c_str() << "\n"; -#endif - return; - } - - auto output_dims = output->GetDims(); - const unsigned int num_anchors = output_dims.at(1); // n = ? - const float *output_ptr = (float *) output->GetData(); - - float r_ = scale_params.ratio; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *row_ptr = output_ptr + i * 6; - float obj_conf = row_ptr[4]; - if (obj_conf < score_threshold) continue; // filter first. - float cls_conf = row_ptr[5]; - if (cls_conf < score_threshold) continue; // insect score. - - // bounding box - const float *offsets = row_ptr; - float cx = offsets[0]; - float cy = offsets[1]; - float w = offsets[2]; - float h = offsets[3]; - - types::Boxf box; - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min((float) img_width - 1.f, x2); - box.y2 = std::min((float) img_height - 1.f, y2); - box.score = cls_conf; - box.label = 1; - box.label_text = "insect"; - box.flag = true; - - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - -#if LITETNN_DEBUG - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void TNNInsectDet::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk) -{ - lite::utils::hard_nms(input, output, iou_threshold, topk); -} \ No newline at end of file diff --git a/lite/tnn/cv/tnn_insectdet.h b/lite/tnn/cv/tnn_insectdet.h deleted file mode 100644 index 51c98144..00000000 --- a/lite/tnn/cv/tnn_insectdet.h +++ /dev/null @@ -1,62 +0,0 @@ -// -// Created by DefTruth on 2022/3/27. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_INSECTDET_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_INSECTDET_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNInsectDet : public BasicTNNHandler - { - public: - explicit TNNInsectDet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); - - ~TNNInsectDet() override = default; - - private: - // nested classes - typedef struct - { - float ratio; - int dw; - int dh; - bool flag; - } InsectDetScaleParams; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f / 255.f, 1.f / 255.f, 1.f / 255.f}; - std::vector bias_vals = {0.f, 0.f, 0.f}; // RGB - static constexpr const unsigned int max_nms = 30000; - - private: - void transform(const cv::Mat &mat_rs) override; // without resize - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - InsectDetScaleParams &scale_params); - - void generate_bboxes(const InsectDetScaleParams &scale_params, - std::vector &bbox_collection, - std::shared_ptr &_instance, - float score_threshold, int img_height, - int img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.5f, float iou_threshold = 0.45f, - unsigned int topk = 100); - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_INSECTDET_H diff --git a/lite/tnn/cv/tnn_insectid.cpp b/lite/tnn/cv/tnn_insectid.cpp deleted file mode 100644 index 377dc468..00000000 --- a/lite/tnn/cv/tnn_insectid.cpp +++ /dev/null @@ -1,94 +0,0 @@ -// -// Created by DefTruth on 2022/3/27. -// - -#include "tnn_insectid.h" -#include "lite/utils.h" - -using tnncv::TNNInsectID; - -TNNInsectID::TNNInsectID(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNInsectID::transform(const cv::Mat &mat_rs) -{ - // push into input_mat (1,3,224,224) - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNInsectID::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch. - tnn::MatConvertParam cvt_param; - std::shared_ptr logits_mat; // (1,2037) - status = instance->GetOutputMat(logits_mat, cvt_param, "477", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto logits_dims = logits_mat->GetDims(); - const unsigned int num_classes = logits_dims.at(1); // 2037 - const float *logits = (float *) logits_mat->GetData(); - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/tnn/cv/tnn_insectid.h b/lite/tnn/cv/tnn_insectid.h deleted file mode 100644 index 6a44af7f..00000000 --- a/lite/tnn/cv/tnn_insectid.h +++ /dev/null @@ -1,376 +0,0 @@ -// -// Created by DefTruth on 2022/3/27. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_INSECTID_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_INSECTID_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNInsectID : public BasicTNNHandler - { - public: - explicit TNNInsectID(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNInsectID() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - std::vector bias_vals = {-0.485f * 255.f * (1.0f / 0.229f) * (1.0 / 255.f), - -0.456f * 255.f * (1.0f / 0.224f) * (1.0 / 255.f), - -0.406f * 255.f * (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[2037] = { - "Pseudoscorpiones", "Diplopoda", "Megymenum", "Cicadellidae", "Bothrogonia addita", "Bothrogonia ferruginea", "Cicadella viridis", - "Maiestas dorsalis", "Nephotettix cincticeps", "Mileewa", "Ledra", "Olidiana brevis", "Acanthosoma denticaudum", - "Sastragala esakii", "Neolethaeus dallasi", "Metochus uniguttatus", "Metochus abbreviatus", "Horridipamera inconspicua", - "Geocoris pallidipennis", "Geocoris varius", "Clovia", "Omalophora pectoralis", "Ricaniidae", "Ricaniidae", "Ricanula pulverosa", - "Ricania speculum", "Euricania facialis", "Ricania guttata", "Ricanula sublimata", "Euricania ocella", "Ricania taeniata", - "Euricania clara", "Ricania simulans", "Urochela quadrinotata", "Cercopidae", "Cosmoscarta", "Cosmoscarta abdominalis", - "Cosmoscarta exultans", "Cosmoscarta dimidiata", "Cosmoscarta dorsimacula", "Callitettix versicolor", "Reduviidae", - "Haematoloecha nigrorufa", "Platymeris", "Agriosphodrus dohrni", "Euagoras plagiatus", "Yolinus albopustulatus", - "Sycanus croceovittatus", "Sphedanolestes impressicollis", "Epidaus", "Epidaus sexspinus", "Vesbius sanguinosus", "Acanthaspis", - "Isyndus obscurus", "Sirthenea flavipes", "Ectrychotes andreae", "Sclomina erinacea", "Issidae", "Phymatidae", "Miridae", - "Eurystylus coelestialium", "Apolygus lucorum", "Helopeltis cinchonae", "Eucorysses grandis", "Hyperoncus lateritius", - "Poecilocoris nepalensis", "Poecilocoris sanszeusignatus", "Poecilocoris druraei", "Poecilocoris latus", "", "Poecilocoris lewisi", - "", "Tetrarthria variegata", "Sphaerocoris annulus", "Scutellera amethystina(Scutellera fasciata)", "Chrysocoris stollii", - "Lamprocoris lateralis", "Calliphara nobilis", "Cantao ocellatus", "Pyrrhocoridae", "Pyrrhocoris sibiricus", "Macrocheraia grandis", - "Physopelta quadriguttata", "Physopelta gutta", "", "Dysdercus decussatus", "Dysdercus cingulatus", "Dysdercus poecilus", - "Dindymus rubiginosus", "Dindymus brevis", "Antilochus coquebertii", "Coreidae", "Mictis tenebrosa", "Mictis gallina", - "Mictis serina", "Mictis fuscipes", "Paradasynus spinosus", "Homoeocerus unipunctatus", "Homoeocerus dilatatus", - "Homoeocerus striicornis", "Molipteryx", "Molipteryx lunata", "Cletus", "Acanthocoris scaber", "Riptortus", "Riptortus pedestris", - "Plinachtus bicoloripes", "Notobitus meleagris", "Tingidae", "Corythucha ciliata", "Corythucha marmorata", "Anthocoris confusus", - "Eurostus", "", "Tessaratoma papillosa", "", "Borysthenes maculatus", "Flatidae", "Cerynia maria", "Lawana imitata", - "Geisha distinctissima", "Salurnis marginella", "Pyrops", "Pyrops spinolae", "Pyrops watanabei", "Pyrops watanabei", - "Pyrops candelaria", "Penthicodes atomaria", "Lycorma delicatula", "Lycorma delicatula", "Penthicodes pulchella", "Saiva bullata", - "Cicadidae", "Cicadidae", "Talainga chinensis", "Meimuna", "Gaeana maculata", "Hyalessa maculaticollis", "Scieroptera", - "Sulphogaeana sulphurea", "Polymeura chenni", "Chremistica ochracea", "Platypleura kaempferi", "Tacua speciosa", - "Formotosena seebohmi", "Huechys sanguinea", "Cryptotympana atrata", "Nepidae", "Eysarcoris", "Eysarcoris guttigerus", - "Eysarcoris aeneus", "Eysarcoris ventralis", "Metonymia glandulosa", "Palomena viridissima", "Priassus spiniger", "Dalpada", - "Lelia decempunctata", "Dolycoris baccarum", "Eurydema gebleri", "Plautia", "Cazira", "Nezara", "Carpocoris purpureipennis", - "Menida violacea", "Palomena prasina", "Catacanthus incarnatus", "Alcimocoris", "Halyomorpha halys", "Eurydema dominulus", - "Zicrona caerulea", "", "Graphosoma rubrolineatum", "Erthesina fullo", "", "Derbidae", "Diostrombus politus", "Membracidae", - "Dictyopharidae", "Kirkaldyia deyrollei", "Berytidae", "Lygaeus equestris", "Spilostethus hospes", "Tropidothorax elegans", - "Lygaeus hanseni", "Graptostethus servus", "Gerridae", "Plataspidae", "Tipulidae", "", "Tephritidae", "Tachinidae", "Chironomidae", - "Stratiomyidae", "Ptecticus aurifer", "Hermetia illucens", "Liriomyza sativae", "Anthomyia illocata", "Culicidae", "Psychodidae", - "Bombyliidae", "Muscidae", "Asilidae", "Microstylum oberthurii", "Syrphidae", "Eupeodes nitens", "Eupeodes corollae", - "Eristalinus arvorum", "Eristalis cerealis", "Ischiodon scutellaris", "Eristalis arbustorum", "Phytomia zonata", "Phytomia errans", - "Syrphus torvus", "Paragus crenulatus", "Syrphus ribesii", "Eristalinus quinquestriatus", "Episyrphus balteatus", - "Helophilus pendulus", "Corydalidae", "", "Neochauliodes", "", "", "Trichoptera", "Opiliones", "Ornebius kanetataki", - "Eucriotettix oculatus", "Tetrix japonica", "Erianthus dohrni", "Acrida cinerea", "Oedaleus infernalis", "Chondracris rosea", - "Trilophidia annulata", "Xenocatantops brachycerus", "Oxya chinensis", "Shirakiacris", "Stauroderus scalaris", - "Aiolopus thalassinus tamulus", "Pseudoxya diminuta", "Ceracris nigricornis", "Locusta migratoria", "Aularches miliaris", "Patanga", - "", "Tettigoniidae", "Pseudophyllus titan", "", "Ducetia japonica", "Hexacentrus unicolor", "", "", "Conocephalus melaenus", "", - "Gampsocleis sedakovii", "Phaneroptera falcata", "Sanaa intermedia", "Gryllacrididae", "Xenogryllus marmoratus", - "Teleogryllus mitratus", "Gryllus bimaculatus", "Teleogryllus emma", "Atractomorpha sinensis", "", "", "", "Ixodida", "Phasmatodea", - "Porcellio", "Lepismatidae", "Nemopteridae", "Chrysopidae", "Myrmeleontidae", "Psychopsidae", "Ascalaphidae", - "Ascalaphus sibiricus", "Mantispidae", "Hemerobiidae", "Tenthredinidae", "Scolia superciliaris", "Ichneumonidae", "Megarhyssa", - "Xanthopimpla", "Brachymeria minuta", "Liris aurulentus", "", "Ampulex compressa", "Sphex argentatus", "Sceliphron madraspatanum", - "Sphex subtruncatus", "Sceliphron javanum", "Vespidae", "Parapolybia nodosa", "Parapolybia varia", "Polistes snelleni", - "Polistes japonicus", "Polistes gigas", "Polistes jokahamae", "Vespa velutina", "Vespa mandarinia", "Vespa affinis", "Polistinae", - "Vespula flaviceps", "Formicidae", "Pseudoneoponera rufipes", "Oecophylla smaragdina", "Mutillidae", "Pompilidae", "Apidae", - "Xylocopinae", "Bombus", "Bombus pyrosoma", "Bombus picipes", "Amegilla calceifera", "Delta esuriens", "Phimenes flavopictus", - "Oreumenes decoratus", "Delta pyriforme", "Chrysididae", "Scutigeridae", "Scolopendridae", "Ephemeroptera", "Araneae", "Araneidae", - "Araneus diadematus", "Araneus ventricosus", "Macracantha arcuata", "Neoscona mellotteei", "Gasteracantha hasselti", - "Gasteracantha kuhli", "Gasteracantha diadesmia", "Nephila pilipes", "", "Neoscona vigilans", "Argiope", "Argiope amoena", - "Araneus ejusmodi", "Araneus mitificus", "Heteropoda venatoria", "Pholcidae", "Macrothele raveni", "Agelenidae", "Lycosidae", - "Steatoda nobilis", "Latrodectus tredecimguttatus", "Tetragnathidae", "Leucauge tessellata", "", "Ebrechtella tricuspidata", - "Salticidae", "Thiania bhamoensis", "Telamonia caprina", "Plexippoides", "Siler semiglaucus", "Pancorius crassipes", "Epeus", - "Hasarius adansoni", "Phintella bifurcilinea", "Cheliceroides longipalpis", "Plexippus paykulli", "", "Eresidae", "Blattodea", - "Periplaneta australasiae", "Periplaneta americana", "Periplaneta fuliginosa", "Blattella germanica", "Corydidae", - "Indolestes peregrinus", "Indolestes cyaneus", "Chlorogomphus papilio", "", "Platycnemididae", "Copera annulata", - "Coeliccia cyanomelas", "Pseudolestes mirabilis", "Gomphidae", "Sinictinogomphus clavatus", "Ictinogomphus rapax", - "Gomphidia confluens", "", "Philoganga vetusta", "Euphaea decorata", "Calopterygidae", "Calopteryx splendens", - "Neurobasis chinensis", "Matrona basilaris", "Calopteryx virgo", "Mnais", "Mnais mneme", "Archineura incarnata", - "Atrocalopteryx atrata", "Anax guttatus", "Anax parthenope", "Anax immaculifrons", "Anax nigrofasciatus", "Gynacantha japonica", - "Gynacantha subinterrupta", "Aeshna mixta", "Rhyothemis", "Rhyothemis variegata", "Rhyothemis fuliginosa", "Tholymis tillarga", - "Palpopleura sexmaculata", "Tramea virginia", "Deielia phaon", "Tetrathemis platyptera", "Sympetrum vulgatum", - "Indothemis carnatica", "Potamarcha congener", "Orthetrum", "Orthetrum chrysis", "Orthetrum luzonicum", "Orthetrum melania", - "Orthetrum poecilops", "Orthetrum sabina", "Orthetrum albistylum", "Orthetrum cancellatum", "Orthetrum lineostigma", - "Orthetrum pruinosum", "Orthetrum glaucum", "Orthetrum triangulare", "Pseudothemis zonata", "Crocothemis servilia", - "Zyxomma petiolatum", "Neurothemis taiwanensis", "Neurothemis tullia", "Neurothemis fulvia", "Neurothemis intermedia", - "Diplacodes trivialis", "Brachydiplax chalybea", "Trithemis festiva", "Trithemis aurora", "Sympetrum croceolum", - "Sympetrum parvulum", "Sympetrum risi", "Sympetrum eroticum", "Sympetrum pedemontanum", "Sympetrum danae", "Acisoma panorpoides", - "Lyriothemis pachygastra", "Epophthalmia elegans", "Brachythemis contaminata", "Pantala flavescens", "Selysiothemis nigra", - "Pseudagrion rubriceps", "Ceriagrion fallax", "Ischnura asiatica", "Ischnura senegalensis", "Ischnura rufostigma", - "Ischnura aurora", "Agriocnemis femina", "Enallagma cyathigerum", "Paracercion calamorum", "Ceriagrion nipponicum", - "Agriocnemis pygmaea", "Chlorocyphidae", "Heliocypha perforata", "Scorpiones", "Heterometrus petersii", "Mantodea", - "Pseudocreobotra wahlbergi", "Phyllocrania paradoxa", "Acromantis japonica", "Creobroter", "Sibylla pretiosa", - "Hymenopus coronatus", "Tenodera sinensis", "Tenodera aridifolia", "Phyllothelys", "Hierodula patellifera", "Mantis religiosa", - "Statilia maculata", "Plecoptera", "Mecoptera", "", "Trictenotomidae", "Rutelidae", "Anomala", "Popillia", - "Eumorphus quadriguttatus", "Attelabidae", "Byctiscus betulae", "Paratrachelophorus nodicornis", "Tomapoderus ruficollis", - "Apoderus coryli", "Aspidobyctiscus lacunipennis", "Trachelophorus giraffa", "Elateridae", "Campsosternus", "Campsosternus gemma", - "Chrysomelidae", "Gallerucida bifasciata", "Monolepta quadriguttata", "Chrysomela populi", "Chrysomela vigintipunctata", - "Plagiodera versicolora", "Oides decempunctata", "Oides bowringii", "Colasposoma dauricum", "Leptinotarsa decemlineata", - "Sagra femorata", "Agasicles hygrophila", "Criocerinae", "", "Chrysolina polita", "Chaetocnema hortensis", "Aulacophora indica", - "Monolepta signata", "Phyllotreta striolata", "Diabrotica undecimpunctata", "Podontia lutea", "Aulacophora lewisii", - "Gastrolina thoracica", "Aulacophora nigripennis", "Buprestidae", "Chrysochroa fulgidissima", "Agrilus planipennis", "Chalcophora", - "Cerambycidae", "Thysia", "Monochamus saltuarius", "Leptura duodecimguttata", "Lamiomimus gottschei", "Moechotypa diphysis", - "Xystrocera globosa", "Mesosa myops", "Dorysthenes", "Monochamus alternatus", "Polyzonus fasciatus", "Agapanthia amurensis", - "Stenocorus meridianus", "Acanthocinus griseus", "Leptura thoracica", "Apomecyna saltator", "Anoplophora", "Anoplophora horsfieldi", - "Leptura annularis", "Rhytiphora bankii", "Semanotus bifasciatus", "Strangalia attenuata", "Neocerambyx raddei", - "Pterolophia annulata", "Glenea relicta", "Imantocera penicillata", "Eupromus ruber", "Aristobia horridula", - "Dicelosternus corallinus", "Batocera", "", "Batocera rubus", "Glenea cantor", "Oberea", "Olenecamptus", "Apriona rugicollis", - "Apriona swainson", "Purpuricenus temminckii", "Callidium violaceum", "Chlorophorus", "Chlorophorus douei", - "Chlorophorus annularis", "Chlorophorus signaticollis", "Eucomatocera vittata", "Xylotrechus", "Xylotrechus yanoi", - "Xylotrechus rusticus", "Asemum striatum", "Paraglenea fortunei", "Phytoecia rufiventris", "Xylorhiza", "", "Aegosoma", - "Arhopalus rusticus", "Stromatium longicorne", "Macrochenus guerini", "Euryphagus", "Saperda populnea", "Aromia bungii", - "Tetraopes tetrophthalmus", "Thyestilla gebleri", "Psacothea", "Paraleprodera diophthalma", "", "", "Tenebrionidae", "Lagriinae", - "Blaps rynchopetera", "", "", "Carabidae", "Therates fruhstorferi", "Pheropsophus", "Carabus lafossei", "Carabus elysii", - "Carabus smaragdinus", "Scarites", "Dolichus halensis", "Chlaenius", "Carabus brandti", "Dynastidae", "Allomyrina dichotoma", - "Oryctes rhinoceros", "Xylotrupes gideon", "", "Eupatorus gracilicornis", "Trichogomphus mongol", "Oryctes nasicornis", - "Dynastes hercules", "Coccinellidae", "Coccinellidae", "Coccinella septempunctata", "Aiolocaria hexaspilota", - "Cheilomenes sexmaculata", "Oenopia formosana", "Vibidia duodecimguttata", "Coccinula quatuordecimpustulata", - "Coelophora biplagiata", "Calvia muiri", "Propylaea quatuordecimpunctata", "Illeis koebelei", "Henosepilachna vigintioctopunctata", - "Oenopia conglobata", "Halmus chalybeus", "Henosepilachna vigintioctomaculata", "Propylea japonica", "Lasioderma serricorne", - "Geotrupidae", "Eumolpidae", "Platycorynus parryi", "Smaragdina nigrifrons", "Euchiridae", "Cheirotonus gestroi", - "Cheirotonus jansoni", "Meloidae", "Lytta caraganae", "Epicauta", "", "Themus", "Cetoniidae", "Euselates", "Goliathus", - "Gametis jucunda", "Pseudotorynorrhina japonica", "Protaetia", "Clinterocera mandarina", "Dicronorhina derbyana", - "Glycyphana horsfieldi", "Agestrata orichalca", "Rhomborhina", "Campsiura mirabilis", "Dicronocephalus adamsi", - "Dicronocephalus wallichii", "Dicronocephalus bowringi", "Pyrocoelia", "Pyrocoelia analis", "Silphidae", "Collyris", "Tricondyla", - "Cicindela", "Cicindela chinenesis", "Cicindela separata", "Cicindela gemmata", "Cicindela aurulenta", "Aphodius fimetarius", - "Bruchidae", "Curculionidae", "Cryptorhynchus lapathi", "Sipalinus gigas", "Eucryptorrhynchus", "Cylas formicarius", "", - "Sitophilus oryzae", "Rhynchophorus ferrugineus", "Hypomeces pulviger", "Pyrochroidae", "Cleridae", "Trichodes sinae", - "Scarabaeoidea", "Hispidae", "Cassida rubiginosa", "Chiridopsis bowringii", "Aspidimorpha miliaris", "Aspidimorpha furcata", - "Aspidimorpha sanctaecrucis", "Taiwania circumdata", "Laccoptera nepalensis(Laccoptera quadrimaculata)", "Cassida nebulosa", - "Lucanidae", "Dorcus titanus", "Dorcus hopei", "Neolucanus", "Neolucanus swinhoei", "", "Lucanus", "Prosopocoilus confucius", - "Prosopocoilus astacoides", "Prosopocoilus girafa", "Prosopocoilus biplagiatus", "Odontolabis cuvera", "Odontolabis siva", - "Eucorynus crassicornis", "Bolboceratidae", "Staphylinidae", "Melolonthidae", "Polyphylla", "Polyphylla decemlineata", - "Melolontha hippocastani", "Amphimallon solstitiale", "Dytiscidae", "Uropygi", "Heliodinidae", "Epicopeia mencia", - "Epicopeia hainesii", "Papilionidae", "Sericinus montelus", "Papilio krishna", "Papilio glaucus", "", "Papilio multicaudata", - "Papilio hermosanus", "Papilio ulysses", "Papilio nephelus", "Papilio paris", "Papilio dehaanii", "Papilio prexaspes", - "Papilio xuthus", "", "Papilio polytes", "Papilio helenus", "Papilio castor", "Papilio bianor", "Papilio dialis", - "Papilio arcturus", "Papilio alcmenor", "Papilio maackii", "Papilio memnon", "Papilio macilentus", "Papilio cresphontes", - "Papilio protenor", "Papilio demoleus", "Papilio hoppo", "Papilio machaon", "", "Papilio troilus", "Pazala", "Pazala eurous", - "Pazala mullah", "Teinopalpus imperialis", "Teinopalpus aureus", "Agehana elwesi", "Bhutanitis thaidina", "Bhutanitis ludlowi", - "Bhutanitis lidderdalii", "Chilasa clytia", "Chilasa clytia", "Iphiclides podalirius", "Atrophaneura horishana", - "Atrophaneura varuna", "Lamproptera curius", "Lamproptera meges", "Pachliopta aristolochiae", "Trogonoptera brookiana", - "Pathysa agetes", "Pathysa_antiphates", "Luehdorfia chinensis", "Troides magellanus", "Troides helena", "Troides aeacus", - "Meandrusa sciron", "Meandrusa payeni", "Losaria coon", "Graphium", "Graphium cloanthus", "Graphium doson", "Graphium chironides", - "Graphium nomius", "Graphium megarus", "Graphium agamemnon", "Graphium sarpedon", "Graphium leechi", "Eurytides marcellus", "Byasa", - "Byasa confusa", "Byasa hedistus", "Byasa polyeuctes", "Byasa mencius", "Byasa dasarada", "Byasa impediens", "Byasa alcinous", - "Limacodidae", "", "Chalcoscelides castaneipars", "Ceratonema", "Thosea", "Matsumurides", "Iragoides conjuncta", "", - "Narosoideus flavidorsalis", "Iraga rugosa", "Rhamnosa uniformis", "Scopelodes venosa", "Scopelodes contracta", "", "Narosa", - "Phocoderma velutina", "Parasa", "Parasa bicolor", "Parasa bicolor", "Parasa lepida", "", "Parasa darma", "Parasa consocia", "", - "Parasa pastoralis", "", "Belippa horrida", "Demonarosa rufotessellata", "Setora postornata", "", "Setora baibarana", - "Miresa bracteata", "Miresa fulgida", "Hyphorma minax", "Monema flavescens", "Monema flavescens", "Thosea sinensis", - "Thosea sinensis", "Tortricidae", "Gypsonoma minutana", "Loboschiza koenigiana", "Eupoecilia ambiguella", "Epiblema foenella", - "Eucosma campoliliana", "Cerace xanthocosma", "Grapholita delineana", "Libythea lepita", "Libythea myrrha", "Noctuidae", - "Chalciope geometrica", "Chalciope mygdon", "Chalciope hyppasia", "Anomis mesogona", "Hadjina chinensis", - "Thysanoplusia intermixta", "Sphragifera sigillata", "Chytonix segregata", "Anisoneura aluco", "Sarbanissa subflava", - "Daddala lucilla", "Cucullia fraterna", "Pericyma cruegeri", "Acronicta tridens", "Acronicta tridens", "Acronicta cuspis", - "Acronicta euphorbiae", "Acronicta euphorbiae", "Acronicta alni", "Acronicta alni", "Acronicta rumicis", "Acronicta rumicis", - "Acronicta hercules", "Acronicta denticulata", "Acronicta psi", "Acronicta psi", "Acronicta pruinosa", "Acronicta pruinosa", - "Acronicta megacephala", "Acronicta megacephala", "Supersypnoides simplex", "Conservula indica", "Hypopyra vespertilio", - "Mimeusemia vilemani", "Mimeusemia vilemani", "Asota heliconia", "Asota heliconia", "Hylophilodes tsukusensis", "Paracolax fentoni", - "Paracolax sugii", "Corgatha nitens", "Corgatha dictaria", "Ophiusa coronata", "Ophiusa tirhaca", "Protoschinia scutosa", - "Agrotis ipsilon", "Oruza albigutta", "Parallelia arctotaenia", "Parallelia stuposa", "Parallelia maturata", "Phyllodes imperialis", - "Staurophora celsia", "Episteme vetula", "Episteme lectrix", "Episteme adulatrix", "Lopharthrum comprimens", "Asota tortuosa", - "Mimeusemia persimilis", "Tiracola plagiata", "Callopistria nobilior", "Callopistria repleta", "Eligma narcissus", "", - "Spirama retorta", "Sphragifera biplagiata", "Lophoptera squamigera", "Ercheia cyllaria", "Axylia putris", "Ramadasa pavo", - "Adris tyrannus", "Hydrillodes lentalis", "Diarsia canescens", "Diarsia subtincta", "Brithys crini", "", "Mocis frugalis", - "Mocis undata", "Spodoptera depravata", "Macdunnoughia purissima", "Spodoptera picta", "Spodoptera litura", "Spodoptera pecten", - "Narangodes argyrostrigatus", "Athetis lepigone", "Xanthodes transversa", "", "Mamestra brassicae", "Spodoptera exigua", "Bocula", - "Cosmia restituta", "Aedia leucomelas", "Phlogophora albovittata", "Trachea auriplena", "Ctenoplusia albostriata", - "Pangrapta lunulata", "Edessena gentiusalis", "Erebus macrops", "Erebus pilosa", "Erebus albicincta", "Erebus caprimulgus", - "Erebus crepuscularis", "Erebus ephesperis", "Ommatophora luminosa", "Cruriopsis funebris", "Checupa stegeri", - "Ischyja ferrifracta", "Narangodes confluens", "Adris okurai", "Sarcopteron punctimargo", "Catocala fraxini", "Thyas honesta", - "Eudocima salaminia", "", "Eudocima phalonia", "Yepcalphis dilectissima", "Arcte coerula", "", "Spodoptera frugiperda", - "Xylostola indistincta", "Achaea janata", "Ischyja manlia", "Catocala electa", "Heliophobus dissectus", "Baorisa hieroglyphica", - "Scrobigera", "Sinna extrema", "Sinna floralis", "Apsarasa radians", "Thysanoplusia daubei", "Tiracola aureata", - "Anacronicta nitida", "Anacronicta horishana", "Edessena hamada", "Serrodes campana", "Gabala argentata", "Othreis homaena", "", - "Asota plana", "Asota plana", "Daseochaeta pulchra", "Diphtherocome", "Hypena", "Hypena trigonalis", "Hypena vestita", - "Hypena lignealis", "Hypena amica", "Hypena indicatalis", "Hypena albopunctalis", "Hypena strigatus", "Hypena perspicua", - "Hypena obesalis", "Hypena lividalis", "Hypena laceratalis", "Sympis rufibasis", "Saturniidae", "Attacus atlas", - "Graellsia isabellae", "Antheraea yamamai", "Actias sinensis", "Caligula simla", "Antheraea polyphemus", "Actias maenas", - "Cricula andrei", "", "Argema mittrei", "Actias luna", "Antheraea pernyi", "Samia", "", "Automeris io", "", "", "Saturnia thibeta", - "Loepa", "Loepa oberthuri", "Loepa megacore", "Antheraea assamensis", "Dictyoploca japonica(Caligula japonica)", "", "Sphingidae", - "Marumba saishiuana", "Marumba sperchius", "Marumba dyras", "Marumba cristata", "Meganoton analis", "Hayesiana triopus", - "Eupanacra mydon", "Theretra oldenlandiae", "", "Theretra alecto subsp. cretica", "Theretra latreillei", "Theretra silhetensis", "", - "Theretra tibetiana", "Theretra pallicosta", "Theretra japonica", "Theretra nessus", "Hippotion rafflesii", "Hippotion rosetta", - "Hippotion celerio", "Pergesa acteus", "", "Dolbina inexacta", "Dolbina tancrei", "Sphecodina caudata", "Parum colligata", "", - "Cypoides", "Callambulyx tatarinovii", "Agrius convolvuli", "", "Rhagastis", "Daphnis nerii", "", "Daphnis hypothous", - "Smerinthus caecus", "Smerinthus planus", "Phyllosphingia", "Deilephila elpenor", "Angonyx testacea", "Acosmeryx formosana", - "Acosmeryx castanea", "Acosmeryx naga", "Acosmeryx miskini", "Cechenena minor", "Cechenena lineosa", "Cechenena subangustata", - "Amplypterus panopus", "Ampelophaga rubiginosa", "Clanis", "Cephonodes hylas", "Nephele hespera", "Langia zenzeroides", - "Macroglossum", "Macroglossum fritzei", "Macroglossum stellatarum", "Macroglossum passalus", "", "Macroglossum bombylans", - "Macroglossum pyrrhosticta", "", "Psilogramma increta", "Psilogramma menephron", "Acherontia styx", "Acherontia atropos", "", - "Acherontia lachesis", "", "Ambulyx", "Haemorrhagiae", "Ethmia lineatonotella", "Labdia semicoccinea", "Geometridae", - "Mixochlora vittata", "Sarcinodes aequilinearia", "Abraxas suspecta", "Xanthabraxas hemionata", "Plutodes", "Plutodes flavescens", - "Plutodes exquisita", "Plutodes costatus", "Gandaritis fixseni", "Semiothisa emersaria", "Paramaxates", "Biston comitata", - "Megaspilates mundataria", "Neohipparchus vallata", "Cleora cinctaria", "Chlorodontopera discospilata", "Semiothisa intermediaria", - "Dalima patularia", "Terpna subtrita", "Ectropis excellens", "Percnia cordiforma", "Naxa seriaria", "Herochroma cristata", - "Herochroma supraviridaria", "Psyra conferta", "Jankowskia fuscaria", "Idaea muricata", "Hypomecis punctinalis", - "Ourapteryx sambucaria", "Ourapteryx nigrociliaris", "Ourapteryx clara", "Ourapteryx nivea", "Scopula yamanei", "Dindica taiwana", - "Dindica polyphaenaria", "Ophthalmitis cordularia", "Agnibesa pictaria", "Eucyclodes semialba", - "Eucyclodes gavissima(Chloromachia gavissima)", "Antipercnia albinigrata", "Plagodis dolabraria", "Telenomeuta punctimarginaria", - "Hemithea tritonaria", "Oxymacaria temeraria", "Dooabia lunifera", "Biston panterinaria", "Deileptenia ribeata", - "Percnia giraffata", "", "Erebomorpha fulguraria", "Ophthalmitis albosignaria", "Chiasmia hebesata", "Phthonandria atrilineata", - "Apochima excavata", "", "Abraxas sylvata", "Thalassodes antiquadraria", "Inurois membranaria", "Chiasmia defixaria", - "Catoria olivescens", "Myrteta angelica", "Hydrelia bicauliata", "Hydrelia bicolorata", "Hydrelia ulula", "Hydrelia enisaria", - "Hydrelia flammeolaria", "Evecliptopera decurrens", "Biston suppressaria", "Biston marginata", "Uliocnemis castalaria", - "Nycterosea obstipata", "Ninodes splendens", "Tyloptera bella", "Chartographa", "Ectropis bhurmitra", "Biston perclara", - "Myrteta tinagmaria", "Thalassodes immissaria", "Percnia suffusa", "Bizia aexaria", "Electrophaes zaphenges", - "Electrophaes corylata", "Xandrames latiferaria", "Xandrames dholaria", "Cyclothea disjuncta", "Stegania cararia", - "Lophomachia lalashana", "Abraxaphantes perampla", "Operophtera relegata", "Krananda latimarginaria", "Krananda semihyalina", - "Krananda lucidaria", "Colotois pennaria", "Amblychia angeronaria", "Dischidesia cinerea", "Problepsis", "Problepsis vulgaris", - "Problepsis superans", "Problepsis albidior", "Ennomos autumnaria", "Corymica", "Pingasa ruginaria", "Pingasa alba", "Idaea impexa", - "Fascellina chromataria", "", "Palpoctenidia phoenicosoma", "Berta rugosivalva", "Timandra dichela", "Timandra stueningi", - "Timandra convectaria", "Timandra synthaca", "Timandra comptaria", "Timandra recompta", "Comibaena", "Comibaena pictipennis", - "Comostola subtiliaria", "Comibaena nigromacularia", "Comibaena procumbaria", "Hemistola monotona", "Fascellina plagiata", - "Tanaoctenia haliaria", "Episothalma robustaria", "Aporandria specularia", "Hypochrosis hyadaria", "Capasa festivaria", - "Gnamptoloma aventiaria", "", "Timandromorpha discolor", "Laciniodes plurilinearia", "Ascotis selenaria", "Xenoplia trivialis", - "Agathia", "Agathia lycaenaria", "Agathia hilarata", "Agathia arcuata", "Agathia laetata", "Agathia diversiformis", - "Agathia carissima", "Milionia basalis", "Cystidia", "Pseudomiza aurata", "Chorodna creataria", "Hydatocapnia gemina", - "Tephrina inchoata", "Metallolophia arenaria", "Dysphania militaris", "Obeidia tigrata", "Obeidia gigantearia", "Obeidia lucifera", - "Odontopera insulata", "Odontopera bilinearia", "Culpinia diffusa", "Iotaphora", "Spilopera divaricata", "Plesiomorpha flaviceps", - "", "Acolutha pulchella subsp. semifulva", "Hyposidra aquilaria", "Heterolocha aristonaria", "Ophthalmitis herbidaria", - "Auaxa cesadaria", "Tanaorhinus viridiluteata", "Tanaorhinus kina", "Tanaorhinus rafflesii", "Tanaorhinus reciprocata", - "Sibatania arizana", "Eumelea ludovicata", "Alcis angulifera", "Alcis repandata", "Heterolocha coccinea", - "Trichopteryx polycommata", "Opisthograptis moelleri", "Garaeus specularis", "Zanclopera falcata", "Arichanna melanaria", - "Nothomiza flavicosta", "", "Thinopteryx crocoptera", "Eilicrinia flava", "Borbacha pardaria", "Hyposidra infixaria", - "Cleora fraterna", "Medasina corticaria", "Yponomeutidae", "Yponomeuta evonymella", "Yponomeuta padella", "Hesperiidae", - "Burara gomata", "Baoris farri", "Udaspes folus", "Polytremis lubricans", "Badamia exclamationis", "Isoteinon lamprospilus", - "Celaenorrhinus maculosus", "Mooreana trichoneura", "Matapa aria", "Erynnis montanus", "Erynnis tages", "Seseria dohertyi", - "Abraximorpha davidii", "Parnara naso", "Parnara ganga", "Parnara guttata", "Borbo cinnara", "Suastus gremius", "", - "Astictopterus jama", "Erionota torus", "Notocrypta curvifascia", "Tagiades litigiosa", "Tagiades menaka", "Pseudocoladenia dan", - "Odontoptilum angulatum", "Pelopidas", "Pelopidas agna", "Pelopidas conjuncta", "Pelopidas mathias", "Hasora badra", - "Hasora chromus", "Hasora anura", "Hasora vitta", "Halpe porus", "Ancistroides nigrita", "Telicota besta", "Telicota colon", - "Telicota ohara", "Iambrix salsala", "Potanthus confucius", "Potanthus trachala", "Ampittia virgata", "Daimio tethys", "Zygaenidae", - "", "Erasmia pulchella", "", "Pryeria sinica", "Pidorus", "Campylotes", "Phauda flammans", "", "Elcysma westwoodi", - "Thyrassia penangae", "", "Artona hainana", "Trypanophora semihyalina", "", "Eterusia aedea", "", "Clelea sapphirina", - "Cyclosia midama", "Cyclosia papilionaris", "Cyclosia papilionaris", "Cyclosia panthona", "Amesia sanguiflua", "Histia rhodope", - "Gynautocera papilionaria", "Soritia strandi", "Soritia strandi", "Rhodopsona rubiginosa", "Idea leuconoe", "Danaus genutia", - "Danaus chrysippus", "", "Danaus plexippus", "Ideopsis similis", "Ideopsis vulgaris", "Euploea", "Euploea sylvester", - "Euploea tulliolus", "Euploea core", "Euploea mulciber", "Euploea midamus", "Parantica", "Parantica sita", "Parantica swinhoei", - "Parantica aglea", "Parantica melaneus", "Tirumala septentrionis", "Tirumala limniace", "Cossidae", "Zeuzera coffeae", - "Zeuzera multistrigata", "Zeuzera pyrina", "Lasiocampidae", "Gastropacha quercifolia", "Gastropacha populifolia", "Trabala vishnou", - "", "Gastropacha pardale", "Lebeda nobilis", "", "Euthrix laeta", "Metanastria gemella", "", "Odonestis pruni", "Euthrix isocyma", - "Cosmotriche discitincta", "Lymantriidae", "Calliteara pudibunda", "Calliteara horsfieldii", "Calliteara horsfieldii", - "Calliteara grotei", "Calliteara grotei", "Arna bipunctapex", "Orgyia antiqua", "Orgyia antiqua", "Orgyia postica", - "Orgyia postica", "Olene mendosa", "Olene mendosa", "Leucoma salicis", "Lymantria mathura", "Lymantria mathura#幼虫", - "Lymantria concolor", "Lymantria dispar", "Lymantria dispar", "Lymantria marginata", "Dasychira suisharyonis", - "Dasychira suisharyonis", "Arctornis l-nigrum", "Laelia coenosa", "Olene dudgeoni", "Olene dudgeoni", "Cifuna locuples", - "Euproctis similis", "Euproctis similis", "Habrosyne pyritoides", "Parapsestis tomponis", "Thyatira batis", "Tethea consimilis", - "Arctiidae", "Phragmatobia luctifera", "Areas galactina", "Peridrome subfascia", "Phragmatobia fuliginosa", - "Phragmatobia fuliginosa", "Ammatho tairadiata", "Peridrome orbicularis", "Eilema costipuncta", "Nudaria ranruna", - "Aglaomorpha histrio", "Utetheisa lotrix", "Pericallia matronula", "Asota plaginota", "Spilosoma lubricipeda", "Asota ficus", - "Asota egens", "Pelosia muscerda", "Arctia flavia", "Arctia caja", "Eilema griseola", "Creatonotus transiens", "Creatonotos gangis", - "Stictane rectilinea", "Rhyparioides metelkana", "Agrisius fuliginosus", "Stigmatophora palmata", "Stigmatophora flava", - "Vamuna remelana", "Aloa lactinea", "Spilosoma subcarnea", "", "Tyria jacobaeae", "", "Macrobrochis gigas", "", "Hyphantria cunea", - "Hyphantria cunea", "Miltochrista", "Miltochrista sauteri(Barsine sauteri)", "Miltochrista ziczac", "Miltochrista convexa", - "Miltochrista fuscozonata", "Miltochrista miniata", "Mangina argus", "Teulisna tumida", "Eugoa grisea", "", "Nyctemera lacticinia", - "Nyctemera lacticinia", "Nyctemera baulus", "Nyctemera tripunctaria", "Nyctemera adversata", "Euplocia membliaria", - "Amerila astreus", "Chrysaeglia magnifica", "Neochera dominia", "Paraona staudingeri", "Cyana", "Cyana hamata", "Cyana propinqua", - "Spilosoma taiwanensis", "Lycaenidae", "Ticherra acte", "Amblopala_avidiena", "Miletus_chinensis", "Lampides boeticus", - "Creon cleobis", "Tajuria cippus", "Zizeeria karsandra", "Catochrysops strabo", "Catochrysops panormus", "Poritia erycinoides", - "Udara dilectus", "Udara albocaerulea", "Arhopala paramuta", "Arhopala bazala", "Arhopala rama", "Nacaduba kurava", - "Nacaduba berenice", "Plebejus orbitulus", "Ancema blanka", "Iraota timoleon", "Heliophorus", "Heliophorus brahma", - "Heliophorus epicles", "Heliophorus ila", "heliophorus saphir", "Caleta roxus", "Horaga onyx", "Horaga albimacula", - "Yasoda tripunctata", "Zizeeria otis", "Prosotas nora", "Lycaena dispar", "Lycaena phlaeas", "Neopithecops zalmora", "Rapala", - "Rapala suffusa", "Rapala nissa", "Tongeia potanini", "Tongeia filicaudis", "Tongeia fischeri", "Mahathala ameria", - "Deudorix epijarbas", "Pratapa deva", "Zeltus amasa", "Scolitantides orion", "Celastrina argiolus", "Sinthusa chandrana", - "Chilades pandava", "Tarucus plinius", "Artipe eryx", "Megisba malaya", "Remelana jangala", "Everes argiades", "Taraka hamada", - "Plebejus argyrognomon", "Ussuriana michaelis", "Pseudozizeeria maha", "Acytolepis puspa", "Teratozephyrus arisanus", - "Curetis acuta", "Spindasis", "Spindasis syama", "Allotinus_drumila", "Aeromachus pygmaeus", "Aeromachus inachus", "Zizula hylax", - "Jamides alecto", "Jamides celeno", "Jamides bochus", "Spialia galba", "Loxura atymnus", "Niphanda fusca", "Dysaethria erasaria", - "Urapteroides astheniata", "Orudiza protheclaria", "Lyssa zampa", "Acropteris leptaliata", "Acropteris iphiata", - "Warreniplema fumicosta", "Urania leilus", "Chrysiridia rhipheus", "Amathusiidae", "Faunis eumeus", "Faunis aerope", - "Faunis canens", "Thauria lathyi", "Thaumantis diores", "Discophora sondaica", "Stichophthalma howqua", "Aemona amathusia", - "Acraea violae", "Acraea terpsicore", "Acraea issoria", "", "Siglophora sanguinolenta", "Westermannia elliptica", - "Risoba prominens", "Blenina quinaria", "Blenina senex", "Iragaodes nobilis", "Carea varipes", "Satyridae", "Neorina patria", - "Mandarinia regalis", "Penthema formosanum", "Penthema darlisa", "Penthema adelma", "Melanitis leda", "Melanitis phedima", - "Coenonympha amaryllis", "Melanargia", "Melanargia galathea", "Mycalesis intermedia", "Mycalesis sangaica", "Mycalesis anaxias", - "Mycalesis mineus", "Mycalesis zonata", "Mycalesis francisca", "Mycalesis gotama", "Mycalesis perseus", "Ypthima", - "Ypthima motschulskyi", "Ypthima praenubila", "Ypthima baldus", "Callerebia", "Neope", "Neope bremeri", "Neope muirheadii", - "Neope pulaha", "Elymnias hypermnestra", "Aphantopus hyperantus", "Lethe", "Lethe mekara", "Lethe butleri", "Lethe gemina", - "Lethe sinorix", "Lethe vindhya", "Lethe chandica", "Lethe christophi", "Lethe rohria", "Lethe insana", "Lethe verma", - "Lethe confusa", "Lethe lanaris", "Lethe syrcis", "Lethe europa", "Lethe dura", "Brahmaeidae", "Brahmaea wallichii", - "Brahmaea porphyrio", "Brahmaea hearseyi", "Brahmaea certhia", "Pieridae", "Pontia daplidice", "Pontia chloridice", - "Leptidea sinapis", "Leptidea amurensis", "Leptidea morsei", "Appias libythea", "Appias lyncida", "Appias albina", "Appias nero", - "Delias hyparete", "Delias pasithoe", "Delias descombesi", "Delias acalis", "Delias belladonna", "Dercas verhuelli", "Ixias pyrene", - "Gandaca harina", "Pieris canidia", "Pieris napi", "Pieris rapae", "Pieris melete", "Leptosia nina", "Aporia", "Aporia agathon", - "Aporia crataegi", "Anthocharis bambusarum", "Anthocharis scolymus", "Colias erate", "Colias fieldii", "Colias hyale", - "Colias palaeno", "Catopsilia pyranthe", "Catopsilia pomona", "Catopsilia scylla", "Gonepteryx amintha", "Gonepteryx rhamni", - "Prioneris thestylis", "Pareronia valeria", "Hebomoia glaucippe", "Eurema mandarina", "Eurema andersoni", "Eurema hecabe", - "Eurema laeta", "Eurema brigitta", "Eurema blanda", "Cepora nerissa", "Promalactis suzukiella", "Scythris sinensis", - "Eretmocera impactella", "Parnassius", "Parnassius citrinarius", "Parnassius nomion", "Parnassius phoebus", "Parnassius bremeri", - "Parnassius apollonius", "Parnassius apollo", "Thyrididae", "Striglina scitaria", "Thyris fenestrella", "Pyrinioides sinuosa", - "Pterophoridae", "Saptha divitiosa", "Notodontidae", "Gazalina chrysolopha", "Cerura menciana", "Cerura vinula", "", - "Syntypistis subgeneris", "Shachihoka formosana", "Clostera anastomosis", "Formofentonia orbifer", "Quadricalcarifera viridipicta", - "Mimopydna", "Phalera", "Phalera grotei", "Phalera bucephala", "Phalera assimilis", "Phalera flavescens", "Pheosia rimosa", - "Clostera anachoreta", "Fentonia ocypete", "Netria viridescens", "Syntypistis comatus", "Clostera albosigma", "Rachia striata", - "Ptilodon saturata", "Uropyia meticulodina", "Spatalia doerriesi", "Stauropus fagi", "Syntypistis pallidifascia", - "Gonoclostera timoniorum", "Gangarides", "Euhampsonia splendida", "Ginshachia elongata", "Euhampsonia cristata", - "Dudusa sphingiformis", "Patania chlorophanta", "Paracymoriza cataclystalis", "Pycnarmon lactiferalis", "Heterocnephes lymphatalis", - "Pagyda quinquelineata", "Cotachena histricalis", "Anania funebris", "Talanga sexpunctalis", "Agathodes ostentalis", - "Syllepte taiwanalis", "Nagiella quadrimaculalis", "Glyphodes quadrimaculalis", "Cirrhochrista brizoalis", "Polythlipta liquidalis", - "Botyodes principalis", "Eoophyla gibbosalis", "Eoophyla conjunctalis", "Parapediasia teterrellus", "Syllepte iophanes", - "Glyphodes duplicalis", "Pleuroptya balteata", "Glyphodes pyloalis", "Syllepte derogata", "Ramila acciusalis", "Tyspanodes striata", - "Cotachena pubescens", "Herpetogramma licarsisalis", "Pachynoa sabelialis", "Pycnarmon cribrata", "Paracymoriza prodigalis", - "Diaphania indica", "Omphisa anastomosalis", "Botyodes asialis", "Cangetta rectilinea", "Agrioglypta itysalis", - "Cnaphalocrocis medinalis", "Crypsiptya coclesalis", "Parapoynx stagnalis", "Parapoynx fluctuosalis", "Parapoynx vittalis", - "Parapoynx crisonalis", "Parapoynx villidalis", "Parapoynx diminutalis", "Pleuroptya iopasalis", "Palpita", - "Palpita nigropunctalis", "Nevrina procopia", "Nosophora semitritalis", "Loxostege sticticalis", "Poliobotys ablactalis", - "Diplopseustis perieresalis", "Pagyda nebulosa", "Cyrtogramme turbata", "Agrotera scissalis", "Pleuroptya ruralis", - "Maruca vitrata", "Pycnarmon pantherata", "Pseudargyria interruptella", "Eumorphobotys eumorphalis", "Botyodes diniasalis", - "Goniorhynchus butyrosa", "Triuncina brunnea", "Bombyx mandarina", "Bombyx mandarina", "Rondotia menciana", "", "Riodinidae", - "Dodona", "Dodona egeon", "Dodona maculosa", "Dodona durga", "Dodona eugenes", "Zemeros flegyas", "Stiboges nymphidia", - "Abisara saturata", "Abisara fylloides", "Abisara burnii", "Abisara echerius", "Abisara bifasciata", "Abisara neophron", - "Abisara fylla", "Nymphalidae", "银纹红袖蝶 Agraulis vanillae", "Cyrestis cocles", "Cyrestis thyodamas", "Cyrestis nivea", - "Parthenos syvia", "Parasarpa dudu", "Chersonesia risa", "Chalinga", "Abrota ganga", "Siproeta stelenes", "Boloria titania", - "Brenthis daphne", "Polyura narcaea", "Polyura eudamippus", "Polyura nepenthes", "Polyura athamas", "Sephisa chandra", - "Sephisa princeps", "Pararge aegeria", "Terinos atlita", "Athyma", "Athyma cama", "Athyma zeroca", "Athyma selenophora", - "Athyma perius", "Athyma asura", "Athyma nefte", "Athyma ranga", "Athyma opalina", "Vagrans egista", "Lexias pardalis", - "Vindula erota", "Argyreus hyperbius", "Asterocampa celtis", "Hypolimnas bolina", "Hypolimnas missipus", "Kallima inachus", - "Euphaedra themis", "Ariadne ariadne", "Ariadne merione", "Diaethria", "Herona marathus", "Timelaea", "Timelaea albescens", - "Neptis", "Neptis hylas", "Neptis soma", "Neptis namba", "Neptis nata", "Neptis sappho", "Neptis miah", "Neptis sankara", - "Neptis clinia", "Neptis pryeri", "Tanaecia julii", "Tanaecia jahnu", "Clossiana freija", "Clossiana euphrosyne", "Clossiana dia", - "Phalanta phalantha", "Issoria eugenia", "Issoria lathonia", "Kaniska canace", "Prothoe franck", "Dichorragia nesimachus", - "Helcyra subalba", "Symbrenthia lilaea", "Symbrenthia brabira", "Junonia atlites", "Junonia almana", "Junonia orithya", - "Junonia lemonias", "Junonia iphita", "Junonia coenia", "Junonia coenia", "Junonia hierta", "Fabriciana adippe", - "Pseudergolis wedah", "Moduza procris", "Dilipa fenestra", "Sasakia charonda", "Sasakia funebris", "Vanessa atalanta", - "Vanessa indica", "Vanessa cardui", "Vanessa virginiensis", "Limenitis", "Limenitis doerriesi", "Limenitis sulpitia", - "Limenitis populi", "Calinaga buddha", "Dophla evelina", "Melitaea", "Rohana parisatis", "Euthalia", "Euthalia", "Euthalia phemius", - "Euthalia pratti", "Euthalia aconthea", "Euthalia lubentina", "Euthalia niepelti", "Argyronome laodice", "Bhagadatta austenia", - "Hestina persimilis", "Hestina nama", "Hestina assimilis", "Phaedyma columella", "Hamadryas", "Nymphalis xanthomelas", - "Nymphalis vau-album", "Nymphalis antiopa", "", "Araschnia doris", "Araschnia prorsoides", "Araschnia levana", "Charaxes bernardus", - "Charaxes bernardus", "Pantoporia hordonia", "Doleschallia bisaltide", "Heliconius erato", "Heliconius charithonia", - "Cupha erymanthis", "Cupha erymanthis", "Argynnis paphia", "Argynnis aglaja", "Mimathyma schrenckii", "Polygonia c-album", - "Polygonia c-aureum", "Proclossiana eunomia", "Chitoria ulupi", "Cethosia cyane", "Cethosia biblis", "Apatura ilia", "Apatura iris", - "Damora sagana", "Stibochiona nicea", "Aglais io", "Aglais urticae", "Lebadea martha", "Pyralidae", "Mabra charonialis", - "Plodia interpunctella", "Eurrhyparodes bracteolalis", "Aethaloessa calidalis", "Endotricha olivacealis", "Ostrinia palustralis", - "Spoladea recurvalis", "Bocchoris inspersalis", "Arippara indicator", "Ancylolomia japonica", "Circobotys aurealis", - "Oncocera semirubella", "Heortia vitessoides", "Locastra muscosalis", "Nosophora insignis", "Orybina regalis", - "Rhectothyris gratiosalis", "Leucinodes orbonalis", "Herpetogramma luctuosalis", "Conogethes punctiferalis", "Pyralis pictalis", - "Pyralis farinalis", "Pyralis regalis", "Diasemia accalis", "Apomyelois ceratoniae", "Omiodes indicata", "Orybina flaviplaga", - "Lista haraldusalis", "Eurrhyparodes tricoloralis", "Rehimena phrynealis", "Cydalima perspectalis", "", "Tyspanodes hypsalis", - "Lamprosema commixta", "Bocchoris onychinalis", "Ericeia inangulata", "Gesonia obeditalis", "Eublemma anachoresis", - "Nagadeba indecoralis", "Lagoptera juno", "Artena dotata", "Scoliopteryx libatrix", "Eublemma cochylioides", "Oruza glaucotorna", - "Autoba tristalis", "Paracolax pryeri", "Ercheia umbrosa", "Cruxoruza decorata", "Opogona nipponica", "Sesiidae", - "Paranthrene tabaniformis", "Drepanidae", "Drepana pallida", "Pseudalbara parvula", "Canucha miranda", "Callidrepana patrana", - "Oreta insignis", "Cyclidia substigmaria", "Cyclidia orciferaria", "Macrauzata maxima", "Oreta loochooana", "Nordstromia japonica", - "Ditrigona triangularia", "Macrocilix mysticata", "Deroca hidda", "Drepana curvatula", "Agnidra scabiosa", "Macrocilix maia", - "Drapetodes mitaria", "", "Petavia attenuata", "Tetragonus catamitus", "Adelidae", "Lepidotarphius perornatellus", "Ctenuchidae", - "Syntomoides imaon", "Amata sperbius", "Amata germana", "Amata fortunei", "Amata grotei", "Anacampsis populella", - "Dichomeris sandycitis" - }; - - }; -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_INSECTID_H diff --git a/lite/tnn/cv/tnn_mg_matting.cpp b/lite/tnn/cv/tnn_mg_matting.cpp deleted file mode 100644 index b7dd2a97..00000000 --- a/lite/tnn/cv/tnn_mg_matting.cpp +++ /dev/null @@ -1,530 +0,0 @@ -// -// Created by DefTruth on 2021/12/5. -// - -#include "tnn_mg_matting.h" -#include "lite/utils.h" - -using tnncv::TNNMGMatting; - -TNNMGMatting::TNNMGMatting( - const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads -) : proto_path(_proto_path.data()), - model_path(_model_path.data()), - log_id(_proto_path.data()), - num_threads(_num_threads) -{ - initialize_instance(); -} - -TNNMGMatting::~TNNMGMatting() -{ - net = nullptr; - image_mat = nullptr; - mask_mat = nullptr; - instance = nullptr; -} - -void TNNMGMatting::initialize_instance() -{ - std::string proto_content_buffer, model_content_buffer; - proto_content_buffer = BasicTNNHandler::content_buffer_from(proto_path); - model_content_buffer = BasicTNNHandler::content_buffer_from(model_path); - - tnn::ModelConfig model_config; - model_config.model_type = tnn::MODEL_TYPE_TNN; - model_config.params = {proto_content_buffer, model_content_buffer}; - - // 1. init TNN net - tnn::Status status; - net = std::make_shared(); - status = net->Init(model_config); - if (status != tnn::TNN_OK || !net) - { -#ifdef LITETNN_DEBUG - std::cout << "net->Init failed!\n"; -#endif - return; - } - // 2. init device type, change this default setting - // for better performance. such as CUDA/OPENCL/... -#ifdef __ANDROID__ - network_device_type = tnn::DEVICE_ARM; // CPU,GPU - input_device_type = tnn::DEVICE_ARM; // CPU only - output_device_type = tnn::DEVICE_ARM; -#else - network_device_type = tnn::DEVICE_X86; // CPU,GPU - input_device_type = tnn::DEVICE_X86; // CPU only - output_device_type = tnn::DEVICE_X86; -#endif - // 3. init instance - tnn::NetworkConfig network_config; - network_config.library_path = {""}; - network_config.device_type = network_device_type; - - instance = net->CreateInst(network_config, status); - if (status != tnn::TNN_OK || !instance) - { -#ifdef LITETNN_DEBUG - std::cout << "CreateInst failed!" << status.description().c_str() << "\n"; -#endif - return; - } - // 4. setting up num_threads - instance->SetCpuNumThreads((int) num_threads); - // 5. init input information. - image_shape = BasicTNNHandler::get_input_shape(instance, "image"); - mask_shape = BasicTNNHandler::get_input_shape(instance, "mask"); - - if (image_shape.size() != 4) - { -#ifdef LITETNN_DEBUG - throw std::runtime_error("Found input_shape.size()!=4, but " - "input only support 4 dims." - "Such as NCHW, NHWC ..."); -#else - return; -#endif - } - input_mat_type = BasicTNNHandler::get_input_mat_type(instance, "image"); - input_data_format = BasicTNNHandler::get_input_data_format(instance, "image"); - if (input_data_format == tnn::DATA_FORMAT_NCHW) - { - dynamic_input_height = image_shape.at(2); - dynamic_input_width = image_shape.at(3); - } // NHWC - else if (input_data_format == tnn::DATA_FORMAT_NHWC) - { - dynamic_input_height = image_shape.at(1); - dynamic_input_width = image_shape.at(2); - } // unsupport - else - { -#ifdef LITETNN_DEBUG - std::cout << "input only support NCHW and NHWC " - "input_data_format, but found others.\n"; -#endif - return; - } - // 6. init output information, debug only. - alpha_os1_shape = BasicTNNHandler::get_output_shape(instance, "alpha_os1"); - alpha_os4_shape = BasicTNNHandler::get_output_shape(instance, "alpha_os4"); - alpha_os8_shape = BasicTNNHandler::get_output_shape(instance, "alpha_os8"); -#ifdef LITETNN_DEBUG - this->print_debug_string(); -#endif -} - -void TNNMGMatting::print_debug_string() -{ - std::cout << "LITETNN_DEBUG LogId: " << log_id << "\n"; - std::cout << "=============== Input-Dims ==============\n"; - BasicTNNHandler::print_name_shape("image", image_shape); - BasicTNNHandler::print_name_shape("mask", mask_shape); - std::string data_format_string = - (input_data_format == tnn::DATA_FORMAT_NCHW) ? "NCHW" : "NHWC"; - std::cout << "Input Data Format: " << data_format_string << "\n"; - std::cout << "=============== Output-Dims ==============\n"; - BasicTNNHandler::print_name_shape("alpha_os1", alpha_os1_shape); - BasicTNNHandler::print_name_shape("alpha_os4", alpha_os4_shape); - BasicTNNHandler::print_name_shape("alpha_os8", alpha_os8_shape); - std::cout << "========================================\n"; -} - -void TNNMGMatting::transform(const cv::Mat &image_canvas, const cv::Mat &mask_canvas) -{ -// auto padded_mat = this->padding(mat); // 0-255 int8 -// auto padded_mask = this->padding(mask); // 0-1.0 float32 -// // update input mat and reshape instance -// // reference: https://github.com/Tencent/TNN/blob/master/examples/base/ocr_text_recognizer.cc#L120 -// tnn::InputShapesMap input_shape_map; -// BasicTNNHandler::print_name_shape("image", image_shape); -// BasicTNNHandler::print_name_shape("mask", mask_shape); -// std::cout << padded_mask.rows << "," << padded_mask.cols << std::endl; -// std::cout << padded_mat.rows << "," << padded_mat.cols << std::endl; -// -// input_shape_map.insert({"image", image_shape}); -// input_shape_map.insert({"mask", mask_shape}); -// -// auto status = instance->Reshape(input_shape_map); -// if (status != tnn::TNN_OK) -// { -//#ifdef LITETNN_DEBUG -// std::cout << "instance Reshape failed in TNNMGMatting\n"; -//#endif -// } -// std::cout << "Reshape done!" << std::endl; -// auto new_image_shape = BasicTNNHandler::get_input_shape(instance, "image"); -// auto new_mask_shape = BasicTNNHandler::get_input_shape(instance, "mask"); -// BasicTNNHandler::print_name_shape("image", new_image_shape); -// BasicTNNHandler::print_name_shape("mask", new_mask_shape); -// -// cv::cvtColor(padded_mat, padded_mat, cv::COLOR_BGR2RGB); - -// cv::Mat image_canvas, mask_canvas; -// cv::cvtColor(mat, image_canvas, cv::COLOR_BGR2RGB); -// cv::resize(image_canvas, image_canvas, cv::Size(dynamic_input_width, dynamic_input_height)); -// cv::resize(mask, mask_canvas, cv::Size(dynamic_input_width, dynamic_input_height)); - - // push into image_mat - // be carefully, no deepcopy inside this tnn::Mat constructor, - // so, we can not pass a local cv::Mat to this constructor. - image_mat = std::make_shared( - input_device_type, - tnn::N8UC3, - image_shape, - (void *) image_canvas.data - ); - if (!image_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "image_mat == nullptr! transform failed\n"; -#endif - } - - // push into mask_mat - mask_mat = std::make_shared( - input_device_type, - tnn::NCHW_FLOAT, - mask_shape, - (void *) mask_canvas.data - ); - if (!mask_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "mask_mat == nullptr! transform failed\n"; -#endif - } -} - -cv::Mat TNNMGMatting::padding(const cv::Mat &unpad_mat) -{ - const unsigned int h = unpad_mat.rows; - const unsigned int w = unpad_mat.cols; - - // aligned - if (h % align_val == 0 && w % align_val == 0) - { - unsigned int target_h = h + 2 * align_val; - unsigned int target_w = w + 2 * align_val; - cv::Mat pad_mat(target_h, target_w, unpad_mat.type()); - - cv::copyMakeBorder(unpad_mat, pad_mat, align_val, align_val, - align_val, align_val, cv::BORDER_REFLECT); - return pad_mat; - } // un-aligned - else - { - // align & padding - unsigned int align_h = align_val * ((h - 1) / align_val + 1); - unsigned int align_w = align_val * ((w - 1) / align_val + 1); - unsigned int pad_h = align_h - h; // >= 0 - unsigned int pad_w = align_w - w; // >= 0 - unsigned int target_h = h + align_val + (pad_h + align_val); - unsigned int target_w = w + align_val + (pad_w + align_val); - - cv::Mat pad_mat(target_h, target_w, unpad_mat.type()); - - cv::copyMakeBorder(unpad_mat, pad_mat, align_val, pad_h + align_val, - align_val, pad_w + align_val, cv::BORDER_REFLECT); - return pad_mat; - } -} - -void TNNMGMatting::update_guidance_mask(cv::Mat &mask, unsigned int guidance_threshold) -{ - if (mask.type() != CV_32FC1) mask.convertTo(mask, CV_32FC1); - const unsigned int h = mask.rows; - const unsigned int w = mask.cols; - if (mask.isContinuous()) - { - const unsigned int data_size = h * w * 1; - float *mutable_data_ptr = (float *) mask.data; - float guidance_threshold_ = (float) guidance_threshold; - for (unsigned int i = 0; i < data_size; ++i) - { - if (mutable_data_ptr[i] >= guidance_threshold_) - mutable_data_ptr[i] = 1.0f; - else - mutable_data_ptr[i] = 0.0f; - } - } // - else - { - float guidance_threshold_ = (float) guidance_threshold; - for (unsigned int i = 0; i < h; ++i) - { - float *p = mask.ptr(i); - for (unsigned int j = 0; j < w; ++j) - { - if (p[j] >= guidance_threshold_) - p[j] = 1.0; - else - p[j] = 0.; - } - } - } -} - -void TNNMGMatting::detect(const cv::Mat &mat, cv::Mat &mask, types::MattingContent &content, - bool remove_noise, unsigned int guidance_threshold, - bool minimum_post_process) -{ - if (mat.empty() || mask.empty()) return; - // const unsigned int img_height = mat.rows; - // const unsigned int img_width = mat.cols; - // this->update_dynamic_shape(img_height, img_width); - this->update_guidance_mask(mask, guidance_threshold); // -> float32 hw1 0~1.0 - - // 1. make input tensors, image, mask - cv::Mat image_canvas, mask_canvas; - cv::cvtColor(mat, image_canvas, cv::COLOR_BGR2RGB); - cv::resize(image_canvas, image_canvas, cv::Size(dynamic_input_width, dynamic_input_height)); - cv::resize(mask, mask_canvas, cv::Size(dynamic_input_width, dynamic_input_height)); - - this->transform(image_canvas, mask_canvas); - - // 2. set input_mat - tnn::MatConvertParam image_cvt_param, mask_cvt_param; - image_cvt_param.scale = scale_vals; - image_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(image_mat, image_cvt_param, "image"); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - status = instance->SetInputMat(mask_mat, mask_cvt_param, "mask"); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. generate matting - this->generate_matting(instance, mat, content, remove_noise, minimum_post_process); -} - -void TNNMGMatting::generate_matting( - std::shared_ptr &_instance, - const cv::Mat &mat, types::MattingContent &content, - bool remove_noise, bool minimum_post_process) -{ - std::shared_ptr alpha_os1_mat; - std::shared_ptr alpha_os4_mat; - std::shared_ptr alpha_os8_mat; - tnn::MatConvertParam cvt_param; - tnn::Status status_os1, status_os4, status_os8; - - // https://github.com/yucornetto/MGMatting/blob/main/code-base/infer.py - // e.g (1,1,h+2*pad_val,w+2*pad_val) - status_os1 = _instance->GetOutputMat(alpha_os1_mat, cvt_param, "alpha_os1", output_device_type); - status_os4 = _instance->GetOutputMat(alpha_os4_mat, cvt_param, "alpha_os4", output_device_type); - status_os8 = _instance->GetOutputMat(alpha_os8_mat, cvt_param, "alpha_os8", output_device_type); - if (status_os1 != tnn::TNN_OK || status_os4 != tnn::TNN_OK || status_os8 != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status_os1.description().c_str() << ": " - << status_os4.description().c_str() << ": " - << status_os8.description().c_str() << "\n"; -#endif - return; - } - - const unsigned int h = mat.rows; - const unsigned int w = mat.cols; - - auto output_dims = alpha_os1_mat->GetDims(); - const unsigned int out_h = output_dims.at(2); - const unsigned int out_w = output_dims.at(3); - float *alpha_os1_ptr = (float *) alpha_os1_mat->GetData(); - float *alpha_os4_ptr = (float *) alpha_os4_mat->GetData(); - float *alpha_os8_ptr = (float *) alpha_os8_mat->GetData(); - - cv::Mat alpha_os1_pred(out_h, out_w, CV_32FC1, alpha_os1_ptr); - cv::Mat alpha_os4_pred(out_h, out_w, CV_32FC1, alpha_os4_ptr); - cv::Mat alpha_os8_pred(out_h, out_w, CV_32FC1, alpha_os8_ptr); - - cv::Mat alpha_pred(out_h, out_w, CV_32FC1, alpha_os8_ptr); - cv::Mat weight_os4 = this->get_unknown_tensor_from_pred(alpha_pred, 30); - this->update_alpha_pred(alpha_pred, weight_os4, alpha_os4_pred); - cv::Mat weight_os1 = this->get_unknown_tensor_from_pred(alpha_pred, 15); - this->update_alpha_pred(alpha_pred, weight_os1, alpha_os1_pred); - if (remove_noise) lite::utils::remove_small_connected_area(alpha_pred, 0.05f); - - cv::Mat pmat = alpha_pred; - if (out_h != h || out_w != w) cv::resize(pmat, pmat, cv::Size(w, h)); - content.pha_mat = pmat; - - if (!minimum_post_process) - { - cv::Mat mat_copy; - mat.convertTo(mat_copy, CV_32FC3); - std::vector mat_channels; - cv::split(mat_copy, mat_channels); - cv::Mat bmat = mat_channels.at(0); - cv::Mat gmat = mat_channels.at(1); - cv::Mat rmat = mat_channels.at(2); // ref only, zero-copy. - bmat = bmat.mul(pmat); - gmat = gmat.mul(pmat); - rmat = rmat.mul(pmat); - cv::Mat rest = 1.f - pmat; - cv::Mat mbmat = bmat.mul(pmat) + rest * 153.f; - cv::Mat mgmat = gmat.mul(pmat) + rest * 255.f; - cv::Mat mrmat = rmat.mul(pmat) + rest * 120.f; - std::vector fgr_channel_mats, merge_channel_mats; - fgr_channel_mats.push_back(bmat); - fgr_channel_mats.push_back(gmat); - fgr_channel_mats.push_back(rmat); - merge_channel_mats.push_back(mbmat); - merge_channel_mats.push_back(mgmat); - merge_channel_mats.push_back(mrmat); - - cv::merge(fgr_channel_mats, content.fgr_mat); - cv::merge(merge_channel_mats, content.merge_mat); - content.fgr_mat.convertTo(content.fgr_mat, CV_8UC3); - content.merge_mat.convertTo(content.merge_mat, CV_8UC3); - } - - content.flag = true; -} - -// https://github.com/yucornetto/MGMatting/issues/11 -// https://github.com/yucornetto/MGMatting/blob/main/code-base/utils/util.py#L225 -cv::Mat TNNMGMatting::get_unknown_tensor_from_pred(const cv::Mat &alpha_pred, unsigned int rand_width) -{ - const unsigned int h = alpha_pred.rows; - const unsigned int w = alpha_pred.cols; - const unsigned int data_size = h * w; - cv::Mat uncertain_area(h, w, CV_32FC1, cv::Scalar(1.0f)); // continuous - const float *pred_ptr = (float *) alpha_pred.data; - float *uncertain_ptr = (float *) uncertain_area.data; - // threshold - if (alpha_pred.isContinuous() && uncertain_area.isContinuous()) - { - for (unsigned int i = 0; i < data_size; ++i) - if ((pred_ptr[i] < 1.0f / 255.0f) || (pred_ptr[i] > 1.0f - 1.0f / 255.0f)) - uncertain_ptr[i] = 0.f; - } // - else - { - for (unsigned int i = 0; i < h; ++i) - { - const float *pred_row_ptr = alpha_pred.ptr(i); - float *uncertain_row_ptr = uncertain_area.ptr(i); - for (unsigned int j = 0; j < w; ++j) - { - if ((pred_row_ptr[j] < 1.0f / 255.0f) || (pred_row_ptr[j] > 1.0f - 1.0f / 255.0f)) - uncertain_row_ptr[j] = 0.f; - } - } - } - // dilate - unsigned int size = rand_width / 2; - auto kernel = cv::getStructuringElement(cv::MORPH_ELLIPSE, cv::Size(size, size)); - cv::dilate(uncertain_area, uncertain_area, kernel); - - // weight - cv::Mat weight(h, w, CV_32FC1, uncertain_area.data); // ref only, zero copy. - float *weight_ptr = (float *) weight.data; - if (weight.isContinuous()) - { - for (unsigned int i = 0; i < data_size; ++i) - if (weight_ptr[i] != 1.0f) weight_ptr[i] = 0; - } // - else - { - for (unsigned int i = 0; i < h; ++i) - { - float *weight_row_ptr = weight.ptr(i); - for (unsigned int j = 0; j < w; ++j) - if (weight_row_ptr[j] != 1.0f) weight_row_ptr[j] = 0.f; - - } - } - - return weight; -} - -void TNNMGMatting::update_alpha_pred(cv::Mat &alpha_pred, const cv::Mat &weight, const cv::Mat &other_alpha_pred) -{ - const unsigned int h = alpha_pred.rows; - const unsigned int w = alpha_pred.cols; - const unsigned int data_size = h * w; - const float *weight_ptr = (float *) weight.data; - float *mutable_alpha_ptr = (float *) alpha_pred.data; - const float *other_alpha_ptr = (float *) other_alpha_pred.data; - - if (alpha_pred.isContinuous() && weight.isContinuous() && other_alpha_pred.isContinuous()) - { - for (unsigned int i = 0; i < data_size; ++i) - if (weight_ptr[i] > 0.f) mutable_alpha_ptr[i] = other_alpha_ptr[i]; - } // - else - { - for (unsigned int i = 0; i < h; ++i) - { - const float *weight_row_ptr = weight.ptr(i); - float *mutable_alpha_row_ptr = alpha_pred.ptr(i); - const float *other_alpha_row_ptr = other_alpha_pred.ptr(i); - for (unsigned int j = 0; j < w; ++j) - if (weight_row_ptr[j] > 0.f) mutable_alpha_row_ptr[j] = other_alpha_row_ptr[j]; - } - } -} - -void TNNMGMatting::update_dynamic_shape(unsigned int img_height, unsigned int img_width) -{ - // update dynamic input dims - unsigned int h = img_height; - unsigned int w = img_width; - // update dynamic input dims - if (h % align_val == 0 && w % align_val == 0) - { - // aligned - dynamic_input_height = h + 2 * align_val; - dynamic_input_width = w + 2 * align_val; - } // un-aligned - else - { - // align first - unsigned int align_h = align_val * ((h - 1) / align_val + 1); - unsigned int align_w = align_val * ((w - 1) / align_val + 1); - unsigned int pad_h = align_h - h; // >= 0 - unsigned int pad_w = align_w - w; // >= 0 - dynamic_input_height = h + align_val + (pad_h + align_val); - dynamic_input_width = w + align_val + (pad_w + align_val); - } - - if (input_data_format == tnn::DATA_FORMAT_NCHW) - { - image_shape.at(2) = dynamic_input_height; - image_shape.at(3) = dynamic_input_width; - mask_shape.at(2) = dynamic_input_height; - mask_shape.at(3) = dynamic_input_width; - } // NHWC - else if (input_data_format == tnn::DATA_FORMAT_NHWC) - { - image_shape.at(1) = dynamic_input_height; - image_shape.at(2) = dynamic_input_width; - mask_shape.at(1) = dynamic_input_height; - mask_shape.at(2) = dynamic_input_width; - } -} \ No newline at end of file diff --git a/lite/tnn/cv/tnn_mg_matting.h b/lite/tnn/cv/tnn_mg_matting.h deleted file mode 100644 index d7cc11cf..00000000 --- a/lite/tnn/cv/tnn_mg_matting.h +++ /dev/null @@ -1,98 +0,0 @@ -// -// Created by DefTruth on 2021/12/5. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_MG_MATTING_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_MG_MATTING_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNMGMatting - { - public: - explicit TNNMGMatting(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); - - ~TNNMGMatting(); - - private: - const char *log_id = nullptr; - const char *proto_path = nullptr; - const char *model_path = nullptr; - // Note, tnn:: actually is TNN_NS::, I prefer the first one. - std::shared_ptr net; - std::shared_ptr instance; - std::shared_ptr image_mat; - std::shared_ptr mask_mat; - - private: - std::vector scale_vals = {(1.f / 0.229f) * (1.f / 255.f), - (1.f / 0.224f) * (1.f / 255.f), - (1.f / 0.225f) * (1.f / 255.f)}; - std::vector bias_vals = {-0.485f * 255.f * (1.f / 0.229f) * (1.f / 255.f), - -0.456f * 255.f * (1.f / 0.224f) * (1.f / 255.f), - -0.406f * 255.f * (1.f / 0.225f) * (1.f / 255.f)}; // RGB - - private: - const unsigned int num_threads; // initialize at runtime. - int dynamic_input_height = 1024; // init only, will change according to input mat. - int dynamic_input_width = 1024; // init only, will change according to input mat. - tnn::DataFormat input_data_format; // e.g DATA_FORMAT_NHWC - tnn::MatType input_mat_type; // e.g NCHW_FLOAT - tnn::DeviceType input_device_type; // only CPU, namely ARM or X86 - tnn::DeviceType output_device_type; // only CPU, namely ARM or X86 - tnn::DeviceType network_device_type; // e.g DEVICE_X86 DEVICE_NAIVE DEVICE_ARM - tnn::DimsVector image_shape; // debug - tnn::DimsVector mask_shape; // debug - tnn::DimsVector alpha_os1_shape; // debug - tnn::DimsVector alpha_os4_shape; // debug - tnn::DimsVector alpha_os8_shape; // debug - static constexpr const unsigned int align_val = 32; - - // un-copyable - protected: - TNNMGMatting(const TNNMGMatting &) = delete; // - TNNMGMatting(TNNMGMatting &&) = delete; // - TNNMGMatting &operator=(const TNNMGMatting &) = delete; // - TNNMGMatting &operator=(TNNMGMatting &&) = delete; // - - private: - void print_debug_string(); - - private: - void transform(const cv::Mat &image_canvas, const cv::Mat &mask_canvas); - - void initialize_instance(); // init net & instance - - cv::Mat padding(const cv::Mat &unpad_mat); - - void update_guidance_mask(cv::Mat &mask, unsigned int guidance_threshold = 128); - - void update_dynamic_shape(unsigned int img_height, unsigned int img_width); - - void update_alpha_pred(cv::Mat &alpha_pred, const cv::Mat &weight, const cv::Mat &other_alpha_pred); - - cv::Mat get_unknown_tensor_from_pred(const cv::Mat &alpha_pred, unsigned int rand_width = 30); - - void generate_matting(std::shared_ptr &_instance, - const cv::Mat &mat, types::MattingContent &content, - bool remove_noise = false, bool minimum_post_process = false); - - public: - /** - * Image Matting Using MGMatting(https://github.com/yucornetto/MGMatting) - * @param mat: cv::Mat BGR HWC, source image - * @param mask: cv::Mat Gray, guidance mask. - * @param guidance_threshold: int, guidance threshold.. - * @param content: types::MattingContent to catch the detected results. - */ - void detect(const cv::Mat &mat, cv::Mat &mask, types::MattingContent &content, - bool remove_noise = false, unsigned int guidance_threshold = 128, - bool minimum_post_process = false); - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_MG_MATTING_H diff --git a/lite/tnn/cv/tnn_mobile_emotion7.cpp b/lite/tnn/cv/tnn_mobile_emotion7.cpp deleted file mode 100644 index ed1c8875..00000000 --- a/lite/tnn/cv/tnn_mobile_emotion7.cpp +++ /dev/null @@ -1,98 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#include "tnn_mobile_emotion7.h" - -using tnncv::TNNMobileEmotion7; - -TNNMobileEmotion7::TNNMobileEmotion7(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ - // TODO: pre-process bug fix - input_width = 224; - input_height = 224; -} - -void TNNMobileEmotion7::transform(const cv::Mat &mat_rs) -{ - // push into input_mat (1,3,224,224) - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNMobileEmotion7::detect(const cv::Mat &mat, types::Emotions &emotions) -{ - if (mat.empty()) return; - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch. - tnn::MatConvertParam cvt_param; - std::shared_ptr emotion_probs; // (1,7) - status = instance->GetOutputMat(emotion_probs, cvt_param, "emotion_preds", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto emotion_dims = emotion_probs->GetDims(); - const unsigned int num_emotions = emotion_dims.at(1); // 7 - - unsigned int pred_label = 0; - const float *pred_probs_ptr = (float *) emotion_probs->GetData(); - - float pred_score = pred_probs_ptr[0]; - - for (unsigned int i = 0; i < num_emotions; ++i) - { - if (pred_probs_ptr[i] > pred_score) - { - pred_score = pred_probs_ptr[i]; - pred_label = i; - } - } - - emotions.label = pred_label; - emotions.score = pred_score; - emotions.text = emotion_texts[pred_label]; - emotions.flag = true; -} diff --git a/lite/tnn/cv/tnn_mobile_emotion7.h b/lite/tnn/cv/tnn_mobile_emotion7.h deleted file mode 100644 index 5836e96f..00000000 --- a/lite/tnn/cv/tnn_mobile_emotion7.h +++ /dev/null @@ -1,36 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_MOBILE_EMOTION7_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_MOBILE_EMOTION7_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNMobileEmotion7 : public BasicTNNHandler - { - public: - explicit TNNMobileEmotion7(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNMobileEmotion7() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f, 1.f, 1.f}; - std::vector bias_vals = {-103.939f, -116.779f, -123.68f}; - const char *emotion_texts[7] = { - "angry", "disgust", "fear", "happiness", "neutral", "sadness", "surprise" - }; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::Emotions &emotions); - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_MOBILE_EMOTION7_H diff --git a/lite/tnn/cv/tnn_mobile_facenet.cpp b/lite/tnn/cv/tnn_mobile_facenet.cpp deleted file mode 100644 index 243e5312..00000000 --- a/lite/tnn/cv/tnn_mobile_facenet.cpp +++ /dev/null @@ -1,87 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "tnn_mobile_facenet.h" - -using tnncv::TNNMobileFaceNet; - -TNNMobileFaceNet::TNNMobileFaceNet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNMobileFaceNet::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNMobileFaceNet::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 4. fetch output mat - std::shared_ptr embedding_mat; - tnn::MatConvertParam embed_cvt_param; // default - - status = instance->GetOutputMat(embedding_mat, embed_cvt_param, "embedding", output_device_type); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - auto embedding_dims = embedding_mat->GetDims(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = (float *) embedding_mat->GetData(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} - diff --git a/lite/tnn/cv/tnn_mobile_facenet.h b/lite/tnn/cv/tnn_mobile_facenet.h deleted file mode 100644 index 523fc08a..00000000 --- a/lite/tnn/cv/tnn_mobile_facenet.h +++ /dev/null @@ -1,34 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_MOBILE_FACENET_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_MOBILE_FACENET_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNMobileFaceNet : public BasicTNNHandler - { - public: - explicit TNNMobileFaceNet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNMobileFaceNet() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f / 128.0f, 1.f / 128.0f, 1.f / 128.0f}; - std::vector bias_vals = {-127.5f / 128.0f, -127.5f / 128.0f, -127.5f / 128.0f}; // RGB - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_MOBILE_FACENET_H diff --git a/lite/tnn/cv/tnn_mobilenetv2.cpp b/lite/tnn/cv/tnn_mobilenetv2.cpp deleted file mode 100644 index 6880400c..00000000 --- a/lite/tnn/cv/tnn_mobilenetv2.cpp +++ /dev/null @@ -1,94 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "tnn_mobilenetv2.h" -#include "lite/utils.h" - -using tnncv::TNNMobileNetV2; - -TNNMobileNetV2::TNNMobileNetV2(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNMobileNetV2::transform(const cv::Mat &mat_rs) -{ - // push into input_mat (1,3,224,224) - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNMobileNetV2::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch. - tnn::MatConvertParam cvt_param; - std::shared_ptr logits_mat; // (1,1000) - status = instance->GetOutputMat(logits_mat, cvt_param, "logits", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto logits_dims = logits_mat->GetDims(); - const unsigned int num_classes = logits_dims.at(1); // 1000 - const float *logits = (float *) logits_mat->GetData(); - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/tnn/cv/tnn_mobilenetv2.h b/lite/tnn/cv/tnn_mobilenetv2.h deleted file mode 100644 index 3e1b40b5..00000000 --- a/lite/tnn/cv/tnn_mobilenetv2.h +++ /dev/null @@ -1,414 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_MOBILENETV2_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_MOBILENETV2_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNMobileNetV2 : public BasicTNNHandler - { - public: - explicit TNNMobileNetV2(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNMobileNetV2() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - std::vector bias_vals = {-0.485f * 255.f * (1.0f / 0.229f) * (1.0 / 255.f), - -0.456f * 255.f * (1.0f / 0.224f) * (1.0 / 255.f), - -0.406f * 255.f * (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - - }; -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_MOBILENETV2_H diff --git a/lite/tnn/cv/tnn_mobilenetv2_68.cpp b/lite/tnn/cv/tnn_mobilenetv2_68.cpp deleted file mode 100644 index 1ca1369e..00000000 --- a/lite/tnn/cv/tnn_mobilenetv2_68.cpp +++ /dev/null @@ -1,91 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#include "tnn_mobilenetv2_68.h" - -using tnncv::TNNMobileNetV268; - -TNNMobileNetV268::TNNMobileNetV268(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNMobileNetV268::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNMobileNetV268::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch landmarks. - tnn::MatConvertParam cvt_param; - std::shared_ptr landmarks_norm; // (1,68*2=136) - status = instance->GetOutputMat(landmarks_norm, cvt_param, "output", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto landmark_dims = landmarks_norm->GetDims(); - const unsigned int num_landmarks = landmark_dims.at(1); // 68*2=136 - const float *landmarks_ptr = (float *) landmarks_norm->GetData(); - - for (unsigned int i = 0; i < num_landmarks; i += 2) - { - float x = landmarks_ptr[i]; - float y = landmarks_ptr[i + 1]; - - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - landmarks.flag = true; -} diff --git a/lite/tnn/cv/tnn_mobilenetv2_68.h b/lite/tnn/cv/tnn_mobilenetv2_68.h deleted file mode 100644 index d1a01706..00000000 --- a/lite/tnn/cv/tnn_mobilenetv2_68.h +++ /dev/null @@ -1,41 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_MOBILENETV2_68_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_MOBILENETV2_68_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNMobileNetV268 : public BasicTNNHandler - { - public: - explicit TNNMobileNetV268(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNMobileNetV268() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = { - 1.0f / (255.f * 0.229f), - 1.0f / (255.f * 0.224f), - 1.0f / (255.f * 0.225f) - }; - std::vector bias_vals = { - -255.f * 0.485f * (1.0f / (255.f * 0.229f)), - -255.f * 0.456f * (1.0f / (255.f * 0.224f)), - -255.f * 0.406f * (1.0f / (255.f * 0.225f)) - }; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_MOBILENETV2_68_H diff --git a/lite/tnn/cv/tnn_mobilenetv2_se_68.cpp b/lite/tnn/cv/tnn_mobilenetv2_se_68.cpp deleted file mode 100644 index 5e99d15a..00000000 --- a/lite/tnn/cv/tnn_mobilenetv2_se_68.cpp +++ /dev/null @@ -1,91 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#include "tnn_mobilenetv2_se_68.h" - -using tnncv::TNNMobileNetV2SE68; - -TNNMobileNetV2SE68::TNNMobileNetV2SE68(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNMobileNetV2SE68::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNMobileNetV2SE68::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch landmarks. - tnn::MatConvertParam cvt_param; - std::shared_ptr landmarks_norm; // (1,68*2=136) - status = instance->GetOutputMat(landmarks_norm, cvt_param, "output", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto landmark_dims = landmarks_norm->GetDims(); - const unsigned int num_landmarks = landmark_dims.at(1); // 68*2=136 - const float *landmarks_ptr = (float *) landmarks_norm->GetData(); - - for (unsigned int i = 0; i < num_landmarks; i += 2) - { - float x = landmarks_ptr[i]; - float y = landmarks_ptr[i + 1]; - - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - landmarks.flag = true; -} diff --git a/lite/tnn/cv/tnn_mobilenetv2_se_68.h b/lite/tnn/cv/tnn_mobilenetv2_se_68.h deleted file mode 100644 index dc9ff4ed..00000000 --- a/lite/tnn/cv/tnn_mobilenetv2_se_68.h +++ /dev/null @@ -1,42 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_MOBILENETV2_SE_68_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_MOBILENETV2_SE_68_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNMobileNetV2SE68 : public BasicTNNHandler - { - public: - explicit TNNMobileNetV2SE68(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNMobileNetV2SE68() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = { - 1.0f / (255.f * 0.229f), - 1.0f / (255.f * 0.224f), - 1.0f / (255.f * 0.225f) - }; - std::vector bias_vals = { - -255.f * 0.485f * (1.0f / (255.f * 0.229f)), - -255.f * 0.456f * (1.0f / (255.f * 0.224f)), - -255.f * 0.406f * (1.0f / (255.f * 0.225f)) - }; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - }; -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_MOBILENETV2_SE_68_H diff --git a/lite/tnn/cv/tnn_mobilese_focal_face.cpp b/lite/tnn/cv/tnn_mobilese_focal_face.cpp deleted file mode 100644 index d81e40cd..00000000 --- a/lite/tnn/cv/tnn_mobilese_focal_face.cpp +++ /dev/null @@ -1,86 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "tnn_mobilese_focal_face.h" - -using tnncv::TNNMobileSEFocalFace; - -TNNMobileSEFocalFace::TNNMobileSEFocalFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNMobileSEFocalFace::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNMobileSEFocalFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 4. fetch output mat - std::shared_ptr embedding_mat; - tnn::MatConvertParam embed_cvt_param; // default - - status = instance->GetOutputMat(embedding_mat, embed_cvt_param, "embedding", output_device_type); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - auto embedding_dims = embedding_mat->GetDims(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = (float *) embedding_mat->GetData(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} \ No newline at end of file diff --git a/lite/tnn/cv/tnn_mobilese_focal_face.h b/lite/tnn/cv/tnn_mobilese_focal_face.h deleted file mode 100644 index 074aea70..00000000 --- a/lite/tnn/cv/tnn_mobilese_focal_face.h +++ /dev/null @@ -1,35 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_MOBILESE_FOCAL_FACE_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_MOBILESE_FOCAL_FACE_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNMobileSEFocalFace : public BasicTNNHandler - { - public: - explicit TNNMobileSEFocalFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNMobileSEFocalFace() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f / 255.0f, 1.f / 255.0f, 1.f / 255.0f}; - std::vector bias_vals = {0.f, 0.f, 0.f}; // RGB - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - - }; -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_MOBILESE_FOCAL_FACE_H diff --git a/lite/tnn/cv/tnn_modnet.cpp b/lite/tnn/cv/tnn_modnet.cpp deleted file mode 100644 index c6f32f80..00000000 --- a/lite/tnn/cv/tnn_modnet.cpp +++ /dev/null @@ -1,142 +0,0 @@ -// -// Created by DefTruth on 2022/3/27. -// - -#include "tnn_modnet.h" -#include "lite/utils.h" - -using tnncv::TNNMODNet; - - -TNNMODNet::TNNMODNet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNMODNet::transform(const cv::Mat &mat_rs) -{ - // push into input_mat (1,3,512,512) - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNMODNet::detect(const cv::Mat &mat, types::MattingContent &content, bool remove_noise, - bool minimum_post_process) -{ - if (mat.empty()) return; - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. generate matting - this->generate_matting(instance, mat, content, remove_noise, minimum_post_process); -} - -void TNNMODNet::generate_matting(std::shared_ptr &_instance, - const cv::Mat &mat, types::MattingContent &content, - bool remove_noise, bool minimum_post_process) -{ - std::shared_ptr output_mat; - tnn::MatConvertParam cvt_param; - auto status = _instance->GetOutputMat(output_mat, cvt_param, "output", output_device_type); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - const unsigned int h = mat.rows; - const unsigned int w = mat.cols; - - auto output_dims = output_mat->GetDims(); - const unsigned int out_h = output_dims.at(2); - const unsigned int out_w = output_dims.at(3); - - float *output_ptr = (float *) output_mat->GetData(); - - cv::Mat alpha_pred(out_h, out_w, CV_32FC1, output_ptr); - if (remove_noise) lite::utils::remove_small_connected_area(alpha_pred, 0.05f); - // resize alpha - if (out_h != h || out_w != w) - // already allocated a new continuous memory after resize. - cv::resize(alpha_pred, alpha_pred, cv::Size(w, h)); - // need clone to allocate a new continuous memory if not performed resize. - // The memory elements point to will release after return. - else alpha_pred = alpha_pred.clone(); - - cv::Mat pmat = alpha_pred; // ref - content.pha_mat = pmat; // auto handle the memory inside ocv with smart ref. - - if (!minimum_post_process) - { - // MODNet only predict Alpha, no fgr. So, - // the fake fgr and merge mat may not need, - // let the fgr mat and merge mat empty to - // speed up the post processes. - cv::Mat mat_copy; - mat.convertTo(mat_copy, CV_32FC3); - // merge mat and fgr mat may not need - std::vector mat_channels; - cv::split(mat_copy, mat_channels); - cv::Mat bmat = mat_channels.at(0); - cv::Mat gmat = mat_channels.at(1); - cv::Mat rmat = mat_channels.at(2); // ref only, zero-copy. - bmat = bmat.mul(pmat); - gmat = gmat.mul(pmat); - rmat = rmat.mul(pmat); - cv::Mat rest = 1.f - pmat; - cv::Mat mbmat = bmat.mul(pmat) + rest * 153.f; - cv::Mat mgmat = gmat.mul(pmat) + rest * 255.f; - cv::Mat mrmat = rmat.mul(pmat) + rest * 120.f; - std::vector fgr_channel_mats, merge_channel_mats; - fgr_channel_mats.push_back(bmat); - fgr_channel_mats.push_back(gmat); - fgr_channel_mats.push_back(rmat); - merge_channel_mats.push_back(mbmat); - merge_channel_mats.push_back(mgmat); - merge_channel_mats.push_back(mrmat); - - cv::merge(fgr_channel_mats, content.fgr_mat); - cv::merge(merge_channel_mats, content.merge_mat); - - content.fgr_mat.convertTo(content.fgr_mat, CV_8UC3); - content.merge_mat.convertTo(content.merge_mat, CV_8UC3); - } - - content.flag = true; -} diff --git a/lite/tnn/cv/tnn_modnet.h b/lite/tnn/cv/tnn_modnet.h deleted file mode 100644 index 0dd94d3d..00000000 --- a/lite/tnn/cv/tnn_modnet.h +++ /dev/null @@ -1,40 +0,0 @@ -// -// Created by DefTruth on 2022/3/27. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_MODNET_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_MODNET_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNMODNet : public BasicTNNHandler - { - public: - explicit TNNMODNet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); - - ~TNNMODNet() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - std::vector bias_vals = {-1.f, -1.f, -1.f}; - - private: - void transform(const cv::Mat &mat_rs) override; // - - void generate_matting(std::shared_ptr &_instance, - const cv::Mat &mat, types::MattingContent &content, - bool remove_noise = false, bool minimum_post_process = false); - - public: - void detect(const cv::Mat &mat, types::MattingContent &content, bool remove_noise = false, - bool minimum_post_process = false); - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_MODNET_H diff --git a/lite/tnn/cv/tnn_nanodet.cpp b/lite/tnn/cv/tnn_nanodet.cpp deleted file mode 100644 index 407e0eb9..00000000 --- a/lite/tnn/cv/tnn_nanodet.cpp +++ /dev/null @@ -1,293 +0,0 @@ -// -// Created by DefTruth on 2021/10/18. -// - -#include "tnn_nanodet.h" -#include "lite/utils.h" - -using tnncv::TNNNanoDet; - -TNNNanoDet::TNNNanoDet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNNanoDet::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - NanoScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(0, 0, 0)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.ratio = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.flag = true; -} - -void TNNNanoDet::transform(const cv::Mat &mat_rs) -{ - // push into input_mat, BGR - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNNanoDet::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - NanoScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - if ((!scale_params.flag) || mat_rs.empty()) return; - // 1. make input mat - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch bounding boxes - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, instance, score_threshold, img_height, img_width); - // 5. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void TNNNanoDet::generate_points(unsigned int target_height, unsigned int target_width) -{ - if (center_points_is_update) return; - - for (auto stride: strides) - { - unsigned int num_grid_w = target_width / stride; - unsigned int num_grid_h = target_height / stride; - std::vector points; - - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - float grid0 = (float) g0 + 0.5f; - float grid1 = (float) g1 + 0.5f; -#ifdef LITE_WIN32 - NanoCenterPoint point; - point.grid0 = grid0; - point.grid1 = grid1; - point.stride = (float) stride; - points.push_back(point); -#else - points.push_back((NanoCenterPoint) {grid0, grid1, (float) stride}); -#endif - } - } - center_points[stride] = points; - } - - center_points_is_update = true; -} - -void TNNNanoDet::generate_bboxes(const NanoScaleParams &scale_params, - std::vector &bbox_collection, - std::shared_ptr &_instance, - float score_threshold, float img_height, - float img_width) -{ - std::shared_ptr cls_pred_stride_8; - std::shared_ptr cls_pred_stride_16; - std::shared_ptr cls_pred_stride_32; - std::shared_ptr dis_pred_stride_8; - std::shared_ptr dis_pred_stride_16; - std::shared_ptr dis_pred_stride_32; - tnn::MatConvertParam cvt_param; - tnn::Status status_cls_8, status_dis_8; - tnn::Status status_cls_16, status_dis_16; - tnn::Status status_cls_32, status_dis_32; - - status_cls_8 = _instance->GetOutputMat( - cls_pred_stride_8, cvt_param, "cls_pred_stride_8", output_device_type); // e.g (1,1600,80) - status_cls_16 = _instance->GetOutputMat( - cls_pred_stride_16, cvt_param, "cls_pred_stride_16", output_device_type); // e.g (1,400,80) - status_cls_32 = _instance->GetOutputMat( - cls_pred_stride_32, cvt_param, "cls_pred_stride_32", output_device_type); // e.g (1,100,80) - status_dis_8 = _instance->GetOutputMat( - dis_pred_stride_8, cvt_param, "dis_pred_stride_8", output_device_type); // (1,1600,4) xyxy (l,t,r,b) - status_dis_16 = _instance->GetOutputMat( - dis_pred_stride_16, cvt_param, "dis_pred_stride_16", output_device_type); // (1,400,4) xyxy (l,t,r,b) - status_dis_32 = _instance->GetOutputMat( - dis_pred_stride_32, cvt_param, "dis_pred_stride_32", output_device_type); // (1,100,4) xyxy (l,t,r,b) - - if (status_cls_8 != tnn::TNN_OK || status_cls_16 != tnn::TNN_OK || status_cls_32 != tnn::TNN_OK || - status_dis_8 != tnn::TNN_OK || status_dis_16 != tnn::TNN_OK || status_dis_32 != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status_cls_8.description().c_str() << ": " - << status_cls_16.description().c_str() << ": " - << status_cls_32.description().c_str() << ": " - << status_dis_8.description().c_str() << ": " - << status_dis_16.description().c_str() << ": " - << status_dis_32.description().c_str() << "\n"; -#endif - return; - } - - this->generate_points(input_height, input_width); // e.g 320 320 - - bbox_collection.clear(); - // level 8 & 16 & 32 - this->generate_bboxes_single_stride(scale_params, cls_pred_stride_8, dis_pred_stride_8, 8, - score_threshold, img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, cls_pred_stride_16, dis_pred_stride_16, 16, - score_threshold, img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, cls_pred_stride_32, dis_pred_stride_32, 32, - score_threshold, img_height, img_width, bbox_collection); -#if LITETNN_DEBUG - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif - -} - -void TNNNanoDet::generate_bboxes_single_stride(const NanoScaleParams &scale_params, - const std::shared_ptr &cls_pred, - const std::shared_ptr &dis_pred, - unsigned int stride, float score_threshold, - float img_height, float img_width, - std::vector &bbox_collection) -{ - unsigned int nms_pre_ = (stride / 8) * nms_pre; // 1 * 1000,2*1000,... - nms_pre_ = nms_pre_ >= nms_pre ? nms_pre_ : nms_pre; - - auto cls_pred_dims = cls_pred->GetDims(); // e.g (1,1600,80) - const unsigned int num_points = cls_pred_dims.at(1); // e.g 1600 - const unsigned int num_classes = cls_pred_dims.at(2); // e.g 80 - - float ratio = scale_params.ratio; - int dw = scale_params.dw; - int dh = scale_params.dh; - - unsigned int count = 0; - auto &stride_points = center_points[stride]; - for (unsigned int i = 0; i < num_points; ++i) - { - const float *scores = (float *) cls_pred->GetData() + (i * num_classes); // row ptr - float cls_conf = scores[0]; - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = scores[j]; - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - if (cls_conf < score_threshold) continue; // filter - - auto &point = stride_points.at(i); - const float cx = point.grid0; // cx - const float cy = point.grid1; // cy - const float s = point.stride; // stride - - const float *offsets = (float *) dis_pred->GetData() + (i * 4); - - float l = offsets[0]; // left - float t = offsets[1]; // top - float r = offsets[2]; // right - float b = offsets[3]; // bottom - - types::Boxf box; - float x1 = ((cx - l) * s - (float) dw) / ratio; // cx - l x1 - float y1 = ((cy - t) * s - (float) dh) / ratio; // cy - t y1 - float x2 = ((cx + r) * s - (float) dw) / ratio; // cx + r x2 - float y2 = ((cy + b) * s - (float) dh) / ratio; // cy + b y2 - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(img_width - 1.f, x2); - box.y2 = std::min(img_height - 1.f, y2); - box.score = cls_conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - - if (bbox_collection.size() > nms_pre_) - { - std::sort(bbox_collection.begin(), bbox_collection.end(), - [](const types::Boxf &a, const types::Boxf &b) - { return a.score > b.score; }); // sort inplace - // trunc - bbox_collection.resize(nms_pre_); - } -} - -void TNNNanoDet::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - diff --git a/lite/tnn/cv/tnn_nanodet.h b/lite/tnn/cv/tnn_nanodet.h deleted file mode 100644 index 6e911d4f..00000000 --- a/lite/tnn/cv/tnn_nanodet.h +++ /dev/null @@ -1,110 +0,0 @@ -// -// Created by DefTruth on 2021/10/18. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_NANODET_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_NANODET_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNNanoDet : public BasicTNNHandler - { - public: - explicit TNNNanoDet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNNanoDet() override = default; - - private: - // nested classes - typedef struct - { - float grid0; - float grid1; - float stride; - } NanoCenterPoint; - - typedef struct - { - float ratio; - int dw; - int dh; - bool flag; - } NanoScaleParams; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {0.017429f, 0.017507f, 0.017125f}; // BGR - std::vector bias_vals = {-103.53f * 0.0174291f, -116.28f * 0.0175070f, -123.675f * 0.0171247f}; - - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int nms_pre = 1000; - static constexpr const unsigned int max_nms = 30000; - - // multi-levels center points - std::vector strides = {8, 16, 32}; - std::unordered_map> center_points; - bool center_points_is_update = false; - - private: - void transform(const cv::Mat &mat_rs) override; // without resize - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - NanoScaleParams &scale_params); - - // only generate once - void generate_points(unsigned int target_height, unsigned int target_width); - - void generate_bboxes_single_stride(const NanoScaleParams &scale_params, - const std::shared_ptr &cls_pred, - const std::shared_ptr &dis_pred, - unsigned int stride, - float score_threshold, - float img_height, - float img_width, - std::vector &bbox_collection); - - void generate_bboxes(const NanoScaleParams &scale_params, - std::vector &bbox_collection, - std::shared_ptr &_instance, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - /** - * @param mat cv::Mat BGR format - * @param detected_boxes vector of Boxf to catch detected boxes. - * @param score_threshold default 0.45f, only keep the result which >= score_threshold. - * @param iou_threshold default 0.3f, iou threshold for NMS. - * @param topk default 100, maximum output boxes after NMS. - * @param nms_type the method. - */ - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.45f, float iou_threshold = 0.3f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_NANODET_H diff --git a/lite/tnn/cv/tnn_nanodet_efficientnet_lite.cpp b/lite/tnn/cv/tnn_nanodet_efficientnet_lite.cpp deleted file mode 100644 index b29893e5..00000000 --- a/lite/tnn/cv/tnn_nanodet_efficientnet_lite.cpp +++ /dev/null @@ -1,291 +0,0 @@ -// -// Created by DefTruth on 2021/10/24. -// - -#include "tnn_nanodet_efficientnet_lite.h" -#include "lite/utils.h" - -using tnncv::TNNNanoDetEfficientNetLite; - -TNNNanoDetEfficientNetLite::TNNNanoDetEfficientNetLite(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNNanoDetEfficientNetLite::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - NanoLiteScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(0, 0, 0)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.ratio = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.flag = true; -} - -void TNNNanoDetEfficientNetLite::transform(const cv::Mat &mat_rs) -{ - // push into input_mat, BGR - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNNanoDetEfficientNetLite::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - NanoLiteScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - if ((!scale_params.flag) || mat_rs.empty()) return; - // 1. make input mat - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch bounding boxes - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, instance, score_threshold, img_height, img_width); - // 5. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void TNNNanoDetEfficientNetLite::generate_points(unsigned int target_height, unsigned int target_width) -{ - if (center_points_is_update) return; - - for (auto stride : strides) - { - unsigned int num_grid_w = target_width / stride; - unsigned int num_grid_h = target_height / stride; - std::vector points; - - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - float grid0 = (float) g0 + 0.5f; - float grid1 = (float) g1 + 0.5f; -#ifdef LITE_WIN32 - NanoLiteCenterPoint point; - point.grid0 = grid0; - point.grid1 = grid1; - point.stride = (float) stride; - points.push_back(point); -#else - points.push_back((NanoLiteCenterPoint) {grid0, grid1, (float) stride}); -#endif - } - } - center_points[stride] = points; - } - - center_points_is_update = true; -} - -void TNNNanoDetEfficientNetLite::generate_bboxes(const NanoLiteScaleParams &scale_params, - std::vector &bbox_collection, - std::shared_ptr &_instance, - float score_threshold, float img_height, - float img_width) -{ - std::shared_ptr cls_pred_stride_8; - std::shared_ptr cls_pred_stride_16; - std::shared_ptr cls_pred_stride_32; - std::shared_ptr dis_pred_stride_8; - std::shared_ptr dis_pred_stride_16; - std::shared_ptr dis_pred_stride_32; - tnn::MatConvertParam cvt_param; - tnn::Status status_cls_8, status_dis_8; - tnn::Status status_cls_16, status_dis_16; - tnn::Status status_cls_32, status_dis_32; - - status_cls_8 = _instance->GetOutputMat( - cls_pred_stride_8, cvt_param, "cls_pred_stride_8", output_device_type); // e.g (1,1600,80) - status_cls_16 = _instance->GetOutputMat( - cls_pred_stride_16, cvt_param, "cls_pred_stride_16", output_device_type); // e.g (1,400,80) - status_cls_32 = _instance->GetOutputMat( - cls_pred_stride_32, cvt_param, "cls_pred_stride_32", output_device_type); // e.g (1,100,80) - status_dis_8 = _instance->GetOutputMat( - dis_pred_stride_8, cvt_param, "dis_pred_stride_8", output_device_type); // (1,1600,4) xyxy (l,t,r,b) - status_dis_16 = _instance->GetOutputMat( - dis_pred_stride_16, cvt_param, "dis_pred_stride_16", output_device_type); // (1,400,4) xyxy (l,t,r,b) - status_dis_32 = _instance->GetOutputMat( - dis_pred_stride_32, cvt_param, "dis_pred_stride_32", output_device_type); // (1,100,4) xyxy (l,t,r,b) - - if (status_cls_8 != tnn::TNN_OK || status_cls_16 != tnn::TNN_OK || status_cls_32 != tnn::TNN_OK || - status_dis_8 != tnn::TNN_OK || status_dis_16 != tnn::TNN_OK || status_dis_32 != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status_cls_8.description().c_str() << ": " - << status_cls_16.description().c_str() << ": " - << status_cls_32.description().c_str() << ": " - << status_dis_8.description().c_str() << ": " - << status_dis_16.description().c_str() << ": " - << status_dis_32.description().c_str() << "\n"; -#endif - return; - } - - this->generate_points(input_height, input_width); // e.g 320 320 - - bbox_collection.clear(); - // level 8 & 16 & 32 - this->generate_bboxes_single_stride(scale_params, cls_pred_stride_8, dis_pred_stride_8, 8, - score_threshold, img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, cls_pred_stride_16, dis_pred_stride_16, 16, - score_threshold, img_height, img_width, bbox_collection); - this->generate_bboxes_single_stride(scale_params, cls_pred_stride_32, dis_pred_stride_32, 32, - score_threshold, img_height, img_width, bbox_collection); -#if LITETNN_DEBUG - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void TNNNanoDetEfficientNetLite::generate_bboxes_single_stride(const NanoLiteScaleParams &scale_params, - const std::shared_ptr &cls_pred, - const std::shared_ptr &dis_pred, - unsigned int stride, float score_threshold, - float img_height, float img_width, - std::vector &bbox_collection) -{ - unsigned int nms_pre_ = (stride / 8) * nms_pre; // 1 * 1000,2 * 1000,... - nms_pre_ = nms_pre_ >= nms_pre ? nms_pre_ : nms_pre; - - auto cls_pred_dims = cls_pred->GetDims(); // e.g (1,1600,80) - const unsigned int num_points = cls_pred_dims.at(1); // e.g 1600 - const unsigned int num_classes = cls_pred_dims.at(2); // e.g 80 - - float ratio = scale_params.ratio; - int dw = scale_params.dw; - int dh = scale_params.dh; - - unsigned int count = 0; - auto &stride_points = center_points[stride]; - for (unsigned int i = 0; i < num_points; ++i) - { - const float *scores = (float *) cls_pred->GetData() + (i * num_classes); // row ptr - float cls_conf = scores[0]; - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = scores[j]; - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - if (cls_conf < score_threshold) continue; // filter - - auto &point = stride_points.at(i); - const float cx = point.grid0; // cx - const float cy = point.grid1; // cy - const float s = point.stride; // stride - - const float *offsets = (float *) dis_pred->GetData() + (i * 4); - - float l = offsets[0]; // left - float t = offsets[1]; // top - float r = offsets[2]; // right - float b = offsets[3]; // bottom - - types::Boxf box; - float x1 = ((cx - l) * s - (float) dw) / ratio; // cx - l x1 - float y1 = ((cy - t) * s - (float) dh) / ratio; // cy - t y1 - float x2 = ((cx + r) * s - (float) dw) / ratio; // cx + r x2 - float y2 = ((cy + b) * s - (float) dh) / ratio; // cy + b y2 - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(img_width - 1.f, x2); - box.y2 = std::min(img_height - 1.f, y2); - box.score = cls_conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - - if (bbox_collection.size() > nms_pre_) - { - std::sort(bbox_collection.begin(), bbox_collection.end(), - [](const types::Boxf &a, const types::Boxf &b) - { return a.score > b.score; }); // sort inplace - // trunc - bbox_collection.resize(nms_pre_); - } -} - -void TNNNanoDetEfficientNetLite::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} diff --git a/lite/tnn/cv/tnn_nanodet_efficientnet_lite.h b/lite/tnn/cv/tnn_nanodet_efficientnet_lite.h deleted file mode 100644 index a79dfd1e..00000000 --- a/lite/tnn/cv/tnn_nanodet_efficientnet_lite.h +++ /dev/null @@ -1,111 +0,0 @@ -// -// Created by DefTruth on 2021/10/24. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_NANODET_EFFICIENTNET_LITE_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_NANODET_EFFICIENTNET_LITE_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNNanoDetEfficientNetLite : public BasicTNNHandler - { - public: - explicit TNNNanoDetEfficientNetLite(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNNanoDetEfficientNetLite() override = default; - - private: - typedef struct - { - float grid0; - float grid1; - float stride; - } NanoLiteCenterPoint; - - typedef struct - { - float ratio; - int dw; - int dh; - bool flag; - } NanoLiteScaleParams; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {0.0078125f, 0.0078125f, 0.0078125f}; // BGR (1/128) - std::vector bias_vals = {-0.9921875f, -0.9921875f, -0.9921875f}; // (1/128)*127 - - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int nms_pre = 1000; - static constexpr const unsigned int max_nms = 30000; - - // multi-levels center points - std::vector strides = {8, 16, 32}; - std::unordered_map> center_points; - bool center_points_is_update = false; - - private: - void transform(const cv::Mat &mat_rs) override; // without resize - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - NanoLiteScaleParams &scale_params); - - // only generate once - void generate_points(unsigned int target_height, unsigned int target_width); - - void generate_bboxes_single_stride(const NanoLiteScaleParams &scale_params, - const std::shared_ptr &cls_pred, - const std::shared_ptr &dis_pred, - unsigned int stride, - float score_threshold, - float img_height, - float img_width, - std::vector &bbox_collection); - - void generate_bboxes(const NanoLiteScaleParams &scale_params, - std::vector &bbox_collection, - std::shared_ptr &_instance, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - /** - * @param mat cv::Mat BGR format - * @param detected_boxes vector of Boxf to catch detected boxes. - * @param score_threshold default 0.45f, only keep the result which >= score_threshold. - * @param iou_threshold default 0.3f, iou threshold for NMS. - * @param topk default 100, maximum output boxes after NMS. - * @param nms_type the method. - */ - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.45f, float iou_threshold = 0.3f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - }; - -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_NANODET_EFFICIENTNET_LITE_H diff --git a/lite/tnn/cv/tnn_nanodet_plus.cpp b/lite/tnn/cv/tnn_nanodet_plus.cpp deleted file mode 100644 index 4fd66966..00000000 --- a/lite/tnn/cv/tnn_nanodet_plus.cpp +++ /dev/null @@ -1,256 +0,0 @@ -// -// Created by DefTruth on 2021/12/27. -// - -#include "tnn_nanodet_plus.h" -#include "lite/utils.h" - -using tnncv::TNNNanoDetPlus; - -TNNNanoDetPlus::TNNNanoDetPlus(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNNanoDetPlus::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - NanoPlusScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(0, 0, 0)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.ratio = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.flag = true; -} - -void TNNNanoDetPlus::transform(const cv::Mat &mat_rs) -{ - // push into input_mat, BGR - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNNanoDetPlus::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - NanoPlusScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - if ((!scale_params.flag) || mat_rs.empty()) return; - // 1. make input mat - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch bounding boxes - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, instance, score_threshold, img_height, img_width); - // 5. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void TNNNanoDetPlus::generate_points(unsigned int target_height, unsigned int target_width) -{ - if (center_points_is_update) return; - // 8, 16, 32, 64 - for (auto stride: strides) - { - unsigned int num_grid_w = target_width / stride; - unsigned int num_grid_h = target_height / stride; - - for (unsigned int g1 = 0; g1 < num_grid_h; ++g1) - { - for (unsigned int g0 = 0; g0 < num_grid_w; ++g0) - { - float grid0 = (float) g0; - float grid1 = (float) g1; -#ifdef LITE_WIN32 - NanoPlusCenterPoint point; - point.grid0 = grid0; - point.grid1 = grid1; - point.stride = (float) stride; - center_points.push_back(point); -#else - center_points.push_back((NanoPlusCenterPoint) {grid0, grid1, (float) stride}); -#endif - } - } - } - - center_points_is_update = true; -} - -void TNNNanoDetPlus::generate_bboxes(const NanoPlusScaleParams &scale_params, - std::vector &bbox_collection, - std::shared_ptr &_instance, - float score_threshold, float img_height, - float img_width) -{ - std::shared_ptr output_pred; - tnn::MatConvertParam cvt_param; - tnn::Status status; - - status = _instance->GetOutputMat(output_pred, cvt_param, "output", output_device_type); // e.g [1,2125,112] - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - this->generate_points(input_height, input_width); // e.g 320 320 - - auto output_pred_dims = output_pred->GetDims(); // e.g [1,2125,112] -#ifdef LITETNN_DEBUG - BasicTNNHandler::print_name_shape("output", output_pred_dims); -#endif - const unsigned int num_classes = 80; - const unsigned int num_cls_reg = output_pred_dims.at(2); // 112 - const unsigned int reg_max = (num_cls_reg - num_classes) / 4; // e.g 8=7+1 - const unsigned int num_points = center_points.size(); - const float *output_pred_ptr = (float *) output_pred->GetData(); - float ratio = scale_params.ratio; - int dw = scale_params.dw; - int dh = scale_params.dh; - - unsigned int count = 0; - - bbox_collection.clear(); - for (unsigned int i = 0; i < num_points; ++i) - { - const float *scores = output_pred_ptr + i * num_cls_reg; // row ptr - float cls_conf = scores[0]; - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = scores[j]; - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - if (cls_conf < score_threshold) continue; // filter - - auto &point = center_points.at(i); - const float cx = point.grid0; // cx - const float cy = point.grid1; // cy - const float s = point.stride; // stride - - const float *logits = output_pred_ptr + i * num_cls_reg + num_classes; // 32|44... - std::vector offsets(4); - for (unsigned int k = 0; k < 4; ++k) - { - float offset = 0.f; - unsigned int max_id; - auto probs = lite::utils::math::softmax( - logits + (k * reg_max), reg_max, max_id); - for (unsigned int l = 0; l < reg_max; ++l) - offset += (float) l * probs[l]; - offsets[k] = offset; - } - - float l = offsets[0]; // left - float t = offsets[1]; // top - float r = offsets[2]; // right - float b = offsets[3]; // bottom - - types::Boxf box; - float x1 = ((cx - l) * s - (float) dw) / ratio; // cx - l x1 - float y1 = ((cy - t) * s - (float) dh) / ratio; // cy - t y1 - float x2 = ((cx + r) * s - (float) dw) / ratio; // cx + r x2 - float y2 = ((cy + b) * s - (float) dh) / ratio; // cy + b y2 - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(img_width - 1.f, x2); - box.y2 = std::min(img_height - 1.f, y2); - box.score = cls_conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - -#if LITETNN_DEBUG - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif - -} - -void TNNNanoDetPlus::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - diff --git a/lite/tnn/cv/tnn_nanodet_plus.h b/lite/tnn/cv/tnn_nanodet_plus.h deleted file mode 100644 index 88cb1760..00000000 --- a/lite/tnn/cv/tnn_nanodet_plus.h +++ /dev/null @@ -1,102 +0,0 @@ -// -// Created by DefTruth on 2021/12/27. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_NANODET_PLUS_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_NANODET_PLUS_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNNanoDetPlus : public BasicTNNHandler - { - public: - explicit TNNNanoDetPlus(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNNanoDetPlus() override = default; - - private: - // nested classes - typedef struct - { - float grid0; - float grid1; - float stride; - } NanoPlusCenterPoint; - - typedef struct - { - float ratio; - int dw; - int dh; - bool flag; - } NanoPlusScaleParams; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {0.017429f, 0.017507f, 0.017125f}; // BGR - std::vector bias_vals = {-103.53f * 0.0174291f, -116.28f * 0.0175070f, -123.675f * 0.0171247f}; - - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - // multi-levels center points - std::vector strides = {8, 16, 32, 64}; - std::vector center_points; - bool center_points_is_update = false; - - private: - void transform(const cv::Mat &mat_rs) override; // without resize - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - NanoPlusScaleParams &scale_params); - - // only generate once - void generate_points(unsigned int target_height, unsigned int target_width); - - - void generate_bboxes(const NanoPlusScaleParams &scale_params, - std::vector &bbox_collection, - std::shared_ptr &_instance, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - /** - * @param mat cv::Mat BGR format - * @param detected_boxes vector of Boxf to catch detected boxes. - * @param score_threshold default 0.45f, only keep the result which >= score_threshold. - * @param iou_threshold default 0.3f, iou threshold for NMS. - * @param topk default 100, maximum output boxes after NMS. - * @param nms_type the method. - */ - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.45f, float iou_threshold = 0.3f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - }; -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_NANODET_PLUS_H diff --git a/lite/tnn/cv/tnn_pfld.cpp b/lite/tnn/cv/tnn_pfld.cpp deleted file mode 100644 index c294f75d..00000000 --- a/lite/tnn/cv/tnn_pfld.cpp +++ /dev/null @@ -1,90 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#include "tnn_pfld.h" - -using tnncv::TNNPFLD; - -TNNPFLD::TNNPFLD(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNPFLD::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNPFLD::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch landmarks. - tnn::MatConvertParam cvt_param; - std::shared_ptr landmarks_norm; // (1,106*2=212) - status = instance->GetOutputMat(landmarks_norm, cvt_param, "output", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto landmark_dims = landmarks_norm->GetDims(); - const unsigned int num_landmarks = landmark_dims.at(1); // 106*2=212 - const float *landmarks_ptr = (float *) landmarks_norm->GetData(); - - for (unsigned int i = 0; i < num_landmarks; i += 2) - { - float x = landmarks_ptr[i]; - float y = landmarks_ptr[i + 1]; - - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - landmarks.flag = true; -} diff --git a/lite/tnn/cv/tnn_pfld.h b/lite/tnn/cv/tnn_pfld.h deleted file mode 100644 index 0c1cd4ce..00000000 --- a/lite/tnn/cv/tnn_pfld.h +++ /dev/null @@ -1,33 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_PFLD_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_PFLD_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNPFLD : public BasicTNNHandler - { - public: - explicit TNNPFLD(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNPFLD() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.0f / 255.0f, 1.0f / 255.0f, 1.0f / 255.0f}; - std::vector bias_vals = {0.f, 0.f, 0.f}; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_PFLD_H diff --git a/lite/tnn/cv/tnn_pfld68.cpp b/lite/tnn/cv/tnn_pfld68.cpp deleted file mode 100644 index 565a380f..00000000 --- a/lite/tnn/cv/tnn_pfld68.cpp +++ /dev/null @@ -1,91 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#include "tnn_pfld68.h" - -using tnncv::TNNPFLD68; - -TNNPFLD68::TNNPFLD68(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNPFLD68::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNPFLD68::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch landmarks. - tnn::MatConvertParam cvt_param; - std::shared_ptr landmarks_norm; // (1,68*2=136) - status = instance->GetOutputMat(landmarks_norm, cvt_param, "output", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto landmark_dims = landmarks_norm->GetDims(); - const unsigned int num_landmarks = landmark_dims.at(1); // 68*2=136 - const float *landmarks_ptr = (float *) landmarks_norm->GetData(); - - for (unsigned int i = 0; i < num_landmarks; i += 2) - { - float x = landmarks_ptr[i]; - float y = landmarks_ptr[i + 1]; - - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - landmarks.flag = true; -} diff --git a/lite/tnn/cv/tnn_pfld68.h b/lite/tnn/cv/tnn_pfld68.h deleted file mode 100644 index 3772e8d3..00000000 --- a/lite/tnn/cv/tnn_pfld68.h +++ /dev/null @@ -1,33 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_PFLD68_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_PFLD68_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNPFLD68 : public BasicTNNHandler - { - public: - explicit TNNPFLD68(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNPFLD68() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.0f / 255.0f, 1.0f / 255.0f, 1.0f / 255.0f}; - std::vector bias_vals = {0.f, 0.f, 0.f}; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_PFLD68_H diff --git a/lite/tnn/cv/tnn_pfld98.cpp b/lite/tnn/cv/tnn_pfld98.cpp deleted file mode 100644 index ef1b9dc8..00000000 --- a/lite/tnn/cv/tnn_pfld98.cpp +++ /dev/null @@ -1,90 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#include "tnn_pfld98.h" - -using tnncv::TNNPFLD98; - -TNNPFLD98::TNNPFLD98(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNPFLD98::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNPFLD98::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch landmarks. - tnn::MatConvertParam cvt_param; - std::shared_ptr landmarks_norm; // // (1,98*2) - status = instance->GetOutputMat(landmarks_norm, cvt_param, "landmarks", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto landmark_dims = landmarks_norm->GetDims(); - const unsigned int num_landmarks = landmark_dims.at(1); // (1,98*2) - const float *landmarks_ptr = (float *) landmarks_norm->GetData(); - - for (unsigned int i = 0; i < num_landmarks; i += 2) - { - float x = landmarks_ptr[i]; - float y = landmarks_ptr[i + 1]; - - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - landmarks.flag = true; -} diff --git a/lite/tnn/cv/tnn_pfld98.h b/lite/tnn/cv/tnn_pfld98.h deleted file mode 100644 index eb8d982b..00000000 --- a/lite/tnn/cv/tnn_pfld98.h +++ /dev/null @@ -1,34 +0,0 @@ -// -// Created by DefTruth on 2021/11/21. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_PFLD98_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_PFLD98_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNPFLD98 : public BasicTNNHandler - { - public: - explicit TNNPFLD98(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNPFLD98() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.0f / 255.0f, 1.0f / 255.0f, 1.0f / 255.0f}; - std::vector bias_vals = {0.f, 0.f, 0.f}; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - }; -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_PFLD98_H diff --git a/lite/tnn/cv/tnn_pipnet19.cpp b/lite/tnn/cv/tnn_pipnet19.cpp deleted file mode 100644 index 5fd0d037..00000000 --- a/lite/tnn/cv/tnn_pipnet19.cpp +++ /dev/null @@ -1,229 +0,0 @@ -// -// Created by DefTruth on 2022/3/20. -// - -#include "tnn_pipnet19.h" - -using tnncv::TNNPIPNet19; - -TNNPIPNet19::TNNPIPNet19(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNPIPNet19::transform(const cv::Mat &mat_rs) -{ - // be carefully, no deepcopy inside this tnn::Mat constructor, - // so, we can not pass a local cv::Mat to this constructor. - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNPIPNet19::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - this->transform(mat_rs); // resize outside transform to prevent overflow - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. generate landmarks - this->generate_landmarks(landmarks, instance, img_height, img_width); -} - -void TNNPIPNet19::generate_landmarks(types::Landmarks &landmarks, - std::shared_ptr &_instance, - float img_height, float img_width) -{ - std::shared_ptr outputs_cls, outputs_x, outputs_y, outputs_nb_x, outputs_nb_y; - tnn::MatConvertParam cvt_param; - tnn::Status status_cls = _instance->GetOutputMat(outputs_cls, cvt_param, "outputs_cls", output_device_type); - tnn::Status status_x = _instance->GetOutputMat(outputs_x, cvt_param, "outputs_x", output_device_type); - tnn::Status status_y = _instance->GetOutputMat(outputs_y, cvt_param, "outputs_y", output_device_type); - tnn::Status status_nb_x = _instance->GetOutputMat(outputs_nb_x, cvt_param, "outputs_nb_x", output_device_type); - tnn::Status status_nb_y = _instance->GetOutputMat(outputs_nb_y, cvt_param, "outputs_nb_y", output_device_type); - - if (status_cls != tnn::TNN_OK || status_x != tnn::TNN_OK || status_y != tnn::TNN_OK - || status_nb_x != tnn::TNN_OK || status_nb_y != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status_cls.description().c_str() << ": " - << status_x.description().c_str() << ": " - << status_y.description().c_str() << ": " - << status_nb_x.description().c_str() << ": " - << status_nb_y.description().c_str() << "\n"; -#endif - return; - } - auto cls_shape = outputs_cls->GetDims(); - const unsigned int grid_h = cls_shape.at(2); // 8 - const unsigned int grid_w = cls_shape.at(3); // 8 - const unsigned int grid_length = grid_h * grid_w; // 8 * 8 = 64 - const unsigned int input_h = input_height; - const unsigned int input_w = input_width; - - // fetch data from pointers - const float *outputs_cls_ptr = (float *) outputs_cls->GetData(); - const float *outputs_x_ptr = (float *) outputs_x->GetData(); - const float *outputs_y_ptr = (float *) outputs_y->GetData(); - const float *outputs_nb_x_ptr = (float *) outputs_nb_x->GetData(); - const float *outputs_nb_y_ptr = (float *) outputs_nb_y->GetData(); - - // find max_ids - std::vector max_ids(num_lms); - for (unsigned int i = 0; i < num_lms; ++i) - { - const float *score_ptr = outputs_cls_ptr + i * grid_length; - unsigned int max_id = 0; - float max_score = score_ptr[0]; - for (unsigned int j = 0; j < grid_length; ++j) - { - if (score_ptr[j] > max_score) - { - max_score = score_ptr[j]; - max_id = j; - } - } - max_ids[i] = max_id; // range 0~64 - } - - // find x & y offsets - std::vector output_x_select(num_lms); - std::vector output_y_select(num_lms); - for (unsigned int i = 0; i < num_lms; ++i) - { - const float *offset_x_ptr = outputs_x_ptr + i * grid_length; - const float *offset_y_ptr = outputs_y_ptr + i * grid_length; - const unsigned int max_id = max_ids.at(i); - output_x_select[i] = offset_x_ptr[max_id]; - output_y_select[i] = offset_y_ptr[max_id]; - } - - // find nb_x & nb_y offsets - std::unordered_map> output_nb_x_select; - std::unordered_map> output_nb_y_select; - // initialize offsets map - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector nb_x_offset(num_nb); - std::vector nb_y_offset(num_nb); - output_nb_x_select[i] = nb_x_offset; - output_nb_y_select[i] = nb_y_offset; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - for (unsigned int j = 0; j < num_nb; ++j) - { - const float *offset_nb_x_ptr = outputs_nb_x_ptr + (i * num_nb + j) * grid_length; - const float *offset_nb_y_ptr = outputs_nb_y_ptr + (i * num_nb + j) * grid_length; - const unsigned int max_id = max_ids.at(i); - output_nb_x_select[i][j] = offset_nb_x_ptr[max_id]; - output_nb_y_select[i][j] = offset_nb_y_ptr[max_id]; - } - } - - // calculate coords - std::vector lms_pred_x(num_lms); // 19 - std::vector lms_pred_y(num_lms); // 19 - std::unordered_map> lms_pred_nb_x; // 19,10 - std::unordered_map> lms_pred_nb_y; // 19,10 - // initialize pred maps - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector nb_x_offset(num_nb); - std::vector nb_y_offset(num_nb); - lms_pred_nb_x[i] = nb_x_offset; - lms_pred_nb_y[i] = nb_y_offset; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - float cx = static_cast(max_ids.at(i) % grid_w); - float cy = static_cast(max_ids.at(i) / grid_w); - // calculate coords & normalize - lms_pred_x[i] = ((cx + output_x_select[i]) * (float) net_stride) / (float) input_w; - lms_pred_y[i] = ((cy + output_y_select[i]) * (float) net_stride) / (float) input_h; - for (unsigned int j = 0; j < num_nb; ++j) - { - lms_pred_nb_x[i][j] = ((cx + output_nb_x_select[i][j]) * (float) net_stride) / (float) input_w; - lms_pred_nb_y[i][j] = ((cy + output_nb_y_select[i][j]) * (float) net_stride) / (float) input_h; - } - } - - // reverse indexes - std::unordered_map> tmp_nb_x; // 19,max_len - std::unordered_map> tmp_nb_y; // 19,max_len - // initialize reverse maps - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector tmp_x(max_len); - std::vector tmp_y(max_len); - tmp_nb_x[i] = tmp_x; - tmp_nb_y[i] = tmp_y; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - for (unsigned int j = 0; j < max_len; ++j) - { - unsigned int ri = reverse_index1[i * max_len + j]; - unsigned int rj = reverse_index2[i * max_len + j]; - tmp_nb_x[i][j] = lms_pred_nb_x[ri][rj]; - tmp_nb_y[i][j] = lms_pred_nb_y[ri][rj]; - } - } - - // merge predictions - landmarks.points.clear(); - for (unsigned int i = 0; i < num_lms; ++i) - { - float total_x = lms_pred_x[i]; - float total_y = lms_pred_y[i]; - for (unsigned int j = 0; j < max_len; ++j) - { - total_x += tmp_nb_x[i][j]; - total_y += tmp_nb_y[i][j]; - } - float x = total_x / ((float) max_len + 1.f); - float y = total_y / ((float) max_len + 1.f); - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - - landmarks.flag = true; -} - - diff --git a/lite/tnn/cv/tnn_pipnet19.h b/lite/tnn/cv/tnn_pipnet19.h deleted file mode 100644 index 33041f0d..00000000 --- a/lite/tnn/cv/tnn_pipnet19.h +++ /dev/null @@ -1,69 +0,0 @@ -// -// Created by DefTruth on 2022/3/20. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_PIPNET19_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_PIPNET19_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNPIPNet19 : public BasicTNNHandler - { - public: - explicit TNNPIPNet19(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNPIPNet19() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - std::vector bias_vals = {-0.485f * 255.f * (1.0f / 0.229f) * (1.0 / 255.f), - -0.456f * 255.f * (1.0f / 0.224f) * (1.0 / 255.f), - -0.406f * 255.f * (1.0f / 0.225f) * (1.0 / 255.f)}; - static constexpr const unsigned int num_nb = 10; - static constexpr const unsigned int num_lms = 19; - static constexpr const unsigned int max_len = 18; - static constexpr const unsigned int net_stride = 32; - - private: - void transform(const cv::Mat &mat_rs) override; // - - void generate_landmarks(types::Landmarks &landmarks, - std::shared_ptr &_instance, - float img_height, float img_width); - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - - private: - const unsigned int reverse_index1[19 * 18] = { - 1, 2, 6, 7, 8, 1, 2, 6, 7, 8, 1, 2, 6, 7, 8, 1, 2, 6, 0, 2, 3, 4, 6, 7, 8, 0, 2, 3, 4, 6, 7, 8, 0, 2, 3, 4, 0, 1, 3, 4, 5, 6, 7, 8, - 9, 10, 11, 12, 0, 1, 3, 4, 5, 6, 0, 1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 14, 0, 1, 2, 4, 5, 6, 1, 2, 3, 5, 9, 10, 11, 1, 2, 3, 5, 9, 10, - 11, 1, 2, 3, 5, 3, 4, 9, 10, 11, 3, 4, 9, 10, 11, 3, 4, 9, 10, 11, 3, 4, 9, 0, 1, 2, 3, 7, 8, 12, 13, 15, 0, 1, 2, 3, 7, 8, 12, 13, - 15, 0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 15, 16, 18, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 0, - 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15, 16, 17, 18, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 13, 14, 16, 17, 18, 0, 1, 3, 4, 5, 9, - 10, 14, 17, 3, 4, 5, 9, 10, 14, 17, 3, 4, 5, 9, 0, 1, 6, 7, 8, 13, 14, 15, 16, 17, 18, 0, 1, 6, 7, 8, 13, 14, 0, 2, 5, 6, 7, 8, 9, - 10, 11, 12, 14, 15, 16, 17, 18, 0, 2, 5, 4, 5, 9, 10, 11, 12, 13, 15, 16, 17, 18, 4, 5, 9, 10, 11, 12, 13, 12, 13, 14, 16, 17, 18, - 12, 13, 14, 16, 17, 18, 12, 13, 14, 16, 17, 18, 12, 13, 14, 15, 17, 18, 12, 13, 14, 15, 17, 18, 12, 13, 14, 15, 17, 18, 12, 13, 14, - 15, 16, 18, 12, 13, 14, 15, 16, 18, 12, 13, 14, 15, 16, 18, 15, 16, 17, 15, 16, 17, 15, 16, 17, 15, 16, 17, 15, 16, 17, 15, 16, 17 - }; - const unsigned int reverse_index2[19 * 18] = { - 0, 6, 1, 4, 6, 0, 6, 1, 4, 6, 0, 6, 1, 4, 6, 0, 6, 1, 0, 1, 8, 7, 2, 2, 3, 0, 1, 8, 7, 2, 2, 3, 0, 1, 8, 7, 3, 1, 3, 5, 5, 4, 3, 1, - 5, 6, 6, 9, 3, 1, 3, 5, 5, 4, 5, 5, 3, 1, 3, 7, 5, 5, 1, 3, 4, 9, 5, 5, 3, 1, 3, 7, 7, 8, 1, 0, 3, 2, 2, 7, 8, 1, 0, 3, 2, 2, 7, 8, - 1, 0, 6, 0, 6, 4, 1, 6, 0, 6, 4, 1, 6, 0, 6, 4, 1, 6, 0, 6, 1, 3, 4, 9, 1, 2, 6, 9, 8, 1, 3, 4, 9, 1, 2, 6, 9, 8, 2, 2, 2, 7, 8, 9, - 0, 0, 9, 9, 9, 5, 7, 7, 8, 8, 2, 2, 4, 4, 0, 5, 6, 6, 3, 0, 4, 5, 7, 4, 3, 8, 6, 6, 9, 6, 7, 6, 5, 0, 4, 4, 8, 6, 4, 0, 3, 8, 4, 4, - 9, 7, 6, 7, 9, 8, 7, 2, 2, 2, 9, 9, 9, 0, 0, 8, 5, 9, 7, 9, 9, 8, 4, 3, 1, 2, 1, 6, 8, 4, 3, 1, 2, 1, 6, 8, 4, 3, 1, 2, 6, 9, 5, 7, - 8, 0, 2, 1, 3, 4, 4, 6, 9, 5, 7, 8, 0, 2, 8, 9, 8, 6, 8, 7, 7, 8, 8, 0, 0, 2, 2, 2, 5, 8, 9, 8, 9, 7, 8, 7, 5, 2, 1, 4, 4, 1, 3, 9, - 7, 8, 7, 5, 2, 1, 1, 5, 7, 0, 3, 1, 1, 5, 7, 0, 3, 1, 1, 5, 7, 0, 3, 1, 3, 2, 3, 0, 0, 0, 3, 2, 3, 0, 0, 0, 3, 2, 3, 0, 0, 0, 7, 6, - 1, 3, 1, 2, 7, 6, 1, 3, 1, 2, 7, 6, 1, 3, 1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 - }; - }; -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_PIPNET19_H diff --git a/lite/tnn/cv/tnn_pipnet29.cpp b/lite/tnn/cv/tnn_pipnet29.cpp deleted file mode 100644 index 9b7234eb..00000000 --- a/lite/tnn/cv/tnn_pipnet29.cpp +++ /dev/null @@ -1,229 +0,0 @@ -// -// Created by DefTruth on 2022/3/20. -// - -#include "tnn_pipnet29.h" - -using tnncv::TNNPIPNet29; - -TNNPIPNet29::TNNPIPNet29(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNPIPNet29::transform(const cv::Mat &mat_rs) -{ - // be carefully, no deepcopy inside this tnn::Mat constructor, - // so, we can not pass a local cv::Mat to this constructor. - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNPIPNet29::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - this->transform(mat_rs); // resize outside transform to prevent overflow - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. generate landmarks - this->generate_landmarks(landmarks, instance, img_height, img_width); -} - -void TNNPIPNet29::generate_landmarks(types::Landmarks &landmarks, - std::shared_ptr &_instance, - float img_height, float img_width) -{ - std::shared_ptr outputs_cls, outputs_x, outputs_y, outputs_nb_x, outputs_nb_y; - tnn::MatConvertParam cvt_param; - tnn::Status status_cls = _instance->GetOutputMat(outputs_cls, cvt_param, "outputs_cls", output_device_type); - tnn::Status status_x = _instance->GetOutputMat(outputs_x, cvt_param, "outputs_x", output_device_type); - tnn::Status status_y = _instance->GetOutputMat(outputs_y, cvt_param, "outputs_y", output_device_type); - tnn::Status status_nb_x = _instance->GetOutputMat(outputs_nb_x, cvt_param, "outputs_nb_x", output_device_type); - tnn::Status status_nb_y = _instance->GetOutputMat(outputs_nb_y, cvt_param, "outputs_nb_y", output_device_type); - - if (status_cls != tnn::TNN_OK || status_x != tnn::TNN_OK || status_y != tnn::TNN_OK - || status_nb_x != tnn::TNN_OK || status_nb_y != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status_cls.description().c_str() << ": " - << status_x.description().c_str() << ": " - << status_y.description().c_str() << ": " - << status_nb_x.description().c_str() << ": " - << status_nb_y.description().c_str() << "\n"; -#endif - return; - } - auto cls_shape = outputs_cls->GetDims(); - const unsigned int grid_h = cls_shape.at(2); // 8 - const unsigned int grid_w = cls_shape.at(3); // 8 - const unsigned int grid_length = grid_h * grid_w; // 8 * 8 = 64 - const unsigned int input_h = input_height; - const unsigned int input_w = input_width; - - // fetch data from pointers - const float *outputs_cls_ptr = (float *) outputs_cls->GetData(); - const float *outputs_x_ptr = (float *) outputs_x->GetData(); - const float *outputs_y_ptr = (float *) outputs_y->GetData(); - const float *outputs_nb_x_ptr = (float *) outputs_nb_x->GetData(); - const float *outputs_nb_y_ptr = (float *) outputs_nb_y->GetData(); - - // find max_ids - std::vector max_ids(num_lms); - for (unsigned int i = 0; i < num_lms; ++i) - { - const float *score_ptr = outputs_cls_ptr + i * grid_length; - unsigned int max_id = 0; - float max_score = score_ptr[0]; - for (unsigned int j = 0; j < grid_length; ++j) - { - if (score_ptr[j] > max_score) - { - max_score = score_ptr[j]; - max_id = j; - } - } - max_ids[i] = max_id; // range 0~64 - } - - // find x & y offsets - std::vector output_x_select(num_lms); - std::vector output_y_select(num_lms); - for (unsigned int i = 0; i < num_lms; ++i) - { - const float *offset_x_ptr = outputs_x_ptr + i * grid_length; - const float *offset_y_ptr = outputs_y_ptr + i * grid_length; - const unsigned int max_id = max_ids.at(i); - output_x_select[i] = offset_x_ptr[max_id]; - output_y_select[i] = offset_y_ptr[max_id]; - } - - // find nb_x & nb_y offsets - std::unordered_map> output_nb_x_select; - std::unordered_map> output_nb_y_select; - // initialize offsets map - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector nb_x_offset(num_nb); - std::vector nb_y_offset(num_nb); - output_nb_x_select[i] = nb_x_offset; - output_nb_y_select[i] = nb_y_offset; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - for (unsigned int j = 0; j < num_nb; ++j) - { - const float *offset_nb_x_ptr = outputs_nb_x_ptr + (i * num_nb + j) * grid_length; - const float *offset_nb_y_ptr = outputs_nb_y_ptr + (i * num_nb + j) * grid_length; - const unsigned int max_id = max_ids.at(i); - output_nb_x_select[i][j] = offset_nb_x_ptr[max_id]; - output_nb_y_select[i][j] = offset_nb_y_ptr[max_id]; - } - } - - // calculate coords - std::vector lms_pred_x(num_lms); // 29 - std::vector lms_pred_y(num_lms); // 29 - std::unordered_map> lms_pred_nb_x; // 29,10 - std::unordered_map> lms_pred_nb_y; // 29,10 - // initialize pred maps - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector nb_x_offset(num_nb); - std::vector nb_y_offset(num_nb); - lms_pred_nb_x[i] = nb_x_offset; - lms_pred_nb_y[i] = nb_y_offset; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - float cx = static_cast(max_ids.at(i) % grid_w); - float cy = static_cast(max_ids.at(i) / grid_w); - // calculate coords & normalize - lms_pred_x[i] = ((cx + output_x_select[i]) * (float) net_stride) / (float) input_w; - lms_pred_y[i] = ((cy + output_y_select[i]) * (float) net_stride) / (float) input_h; - for (unsigned int j = 0; j < num_nb; ++j) - { - lms_pred_nb_x[i][j] = ((cx + output_nb_x_select[i][j]) * (float) net_stride) / (float) input_w; - lms_pred_nb_y[i][j] = ((cy + output_nb_y_select[i][j]) * (float) net_stride) / (float) input_h; - } - } - - // reverse indexes - std::unordered_map> tmp_nb_x; // 29,max_len - std::unordered_map> tmp_nb_y; // 29,max_len - // initialize reverse maps - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector tmp_x(max_len); - std::vector tmp_y(max_len); - tmp_nb_x[i] = tmp_x; - tmp_nb_y[i] = tmp_y; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - for (unsigned int j = 0; j < max_len; ++j) - { - unsigned int ri = reverse_index1[i * max_len + j]; - unsigned int rj = reverse_index2[i * max_len + j]; - tmp_nb_x[i][j] = lms_pred_nb_x[ri][rj]; - tmp_nb_y[i][j] = lms_pred_nb_y[ri][rj]; - } - } - - // merge predictions - landmarks.points.clear(); - for (unsigned int i = 0; i < num_lms; ++i) - { - float total_x = lms_pred_x[i]; - float total_y = lms_pred_y[i]; - for (unsigned int j = 0; j < max_len; ++j) - { - total_x += tmp_nb_x[i][j]; - total_y += tmp_nb_y[i][j]; - } - float x = total_x / ((float) max_len + 1.f); - float y = total_y / ((float) max_len + 1.f); - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - - landmarks.flag = true; -} - - diff --git a/lite/tnn/cv/tnn_pipnet29.h b/lite/tnn/cv/tnn_pipnet29.h deleted file mode 100644 index db84ec27..00000000 --- a/lite/tnn/cv/tnn_pipnet29.h +++ /dev/null @@ -1,81 +0,0 @@ -// -// Created by DefTruth on 2022/3/20. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_PIPNET29_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_PIPNET29_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNPIPNet29 : public BasicTNNHandler - { - public: - explicit TNNPIPNet29(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNPIPNet29() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - std::vector bias_vals = {-0.485f * 255.f * (1.0f / 0.229f) * (1.0 / 255.f), - -0.456f * 255.f * (1.0f / 0.224f) * (1.0 / 255.f), - -0.406f * 255.f * (1.0f / 0.225f) * (1.0 / 255.f)}; - static constexpr const unsigned int num_nb = 10; - static constexpr const unsigned int num_lms = 29; - static constexpr const unsigned int max_len = 19; - static constexpr const unsigned int net_stride = 32; - - private: - void transform(const cv::Mat &mat_rs) override; // - - void generate_landmarks(types::Landmarks &landmarks, - std::shared_ptr &_instance, - float img_height, float img_width); - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - - private: - const unsigned int reverse_index1[29 * 19] = { - 2, 4, 5, 8, 12, 13, 16, 2, 4, 5, 8, 12, 13, 16, 2, 4, 5, 8, 12, 3, 6, 7, 9, 14, 15, 17, 3, 6, 7, 9, 14, 15, 17, 3, 6, 7, 9, 14, 0, - 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 16, 0, 3, 4, 5, 6, 7, 0, 1, 2, 4, 5, 6, 7, 9, 10, 11, 12, 14, 15, 17, 0, 1, 2, 4, 5, 0, 2, 5, - 8, 10, 12, 13, 16, 0, 2, 5, 8, 10, 12, 13, 16, 0, 2, 5, 0, 2, 4, 8, 10, 12, 13, 16, 0, 2, 4, 8, 10, 12, 13, 16, 0, 2, 4, 1, 3, 7, 9, - 11, 14, 15, 17, 1, 3, 7, 9, 11, 14, 15, 17, 1, 3, 7, 1, 3, 6, 9, 11, 14, 15, 17, 1, 3, 6, 9, 11, 14, 15, 17, 1, 3, 6, 0, 2, 4, 5, - 10, 12, 13, 16, 0, 2, 4, 5, 10, 12, 13, 16, 0, 2, 4, 1, 3, 6, 7, 11, 14, 15, 17, 1, 3, 6, 7, 11, 14, 15, 17, 1, 3, 6, 0, 2, 3, 4, 5, - 8, 12, 13, 16, 18, 20, 0, 2, 3, 4, 5, 8, 12, 13, 1, 2, 3, 6, 7, 9, 14, 15, 17, 19, 20, 21, 1, 2, 3, 6, 7, 9, 14, 0, 2, 4, 5, 8, 10, - 13, 16, 0, 2, 4, 5, 8, 10, 13, 16, 0, 2, 4, 0, 2, 4, 5, 8, 10, 12, 16, 18, 22, 0, 2, 4, 5, 8, 10, 12, 16, 18, 1, 3, 6, 7, 9, 11, 15, - 17, 1, 3, 6, 7, 9, 11, 15, 17, 1, 3, 6, 1, 3, 6, 7, 9, 11, 14, 17, 19, 23, 1, 3, 6, 7, 9, 11, 14, 17, 19, 0, 2, 4, 5, 8, 10, 12, 13, - 18, 0, 2, 4, 5, 8, 10, 12, 13, 18, 0, 1, 3, 6, 7, 9, 11, 14, 15, 19, 1, 3, 6, 7, 9, 11, 14, 15, 19, 1, 0, 4, 5, 8, 10, 12, 13, 16, - 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 0, 1, 6, 7, 9, 11, 14, 15, 17, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 1, 1, 8, 9, 10, 11, - 13, 15, 16, 17, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 18, 19, 20, 22, 23, 24, 25, 26, 27, 28, 18, 19, 20, 22, 23, 24, 25, 26, 27, - 18, 20, 21, 24, 25, 26, 27, 28, 18, 20, 21, 24, 25, 26, 27, 28, 18, 20, 21, 19, 21, 24, 25, 26, 27, 28, 19, 21, 24, 25, 26, 27, 28, - 19, 21, 24, 25, 26, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 18, 19, 20, 21, 22, 23, 25, 26, 27, 18, 19, 20, 21, 22, 23, 24, 26, 27, - 28, 18, 19, 20, 21, 22, 23, 24, 26, 27, 18, 19, 20, 21, 22, 23, 24, 25, 27, 28, 18, 19, 20, 21, 22, 23, 24, 25, 27, 20, 21, 22, 23, - 24, 25, 26, 28, 20, 21, 22, 23, 24, 25, 26, 28, 20, 21, 22, 22, 23, 24, 25, 26, 27, 22, 23, 24, 25, 26, 27, 22, 23, 24, 25, 26, 27, - 22 - }; - const unsigned int reverse_index2[29 * 19] = { - 9, 3, 5, 3, 7, 7, 7, 9, 3, 5, 3, 7, 7, 7, 9, 3, 5, 3, 7, 9, 3, 5, 3, 7, 7, 7, 9, 3, 5, 3, 7, 7, 7, 9, 3, 5, 3, 7, 7, 6, 6, 6, 8, 9, - 7, 0, 9, 6, 5, 9, 6, 7, 6, 6, 6, 8, 9, 9, 7, 6, 8, 9, 6, 6, 7, 8, 0, 9, 6, 6, 6, 9, 7, 6, 8, 9, 2, 5, 0, 5, 5, 3, 6, 5, 2, 5, 0, 5, - 5, 3, 6, 5, 2, 5, 0, 1, 3, 0, 4, 4, 2, 4, 2, 1, 3, 0, 4, 4, 2, 4, 2, 1, 3, 0, 2, 4, 0, 5, 5, 3, 5, 5, 2, 4, 0, 5, 5, 3, 5, 5, 2, 4, - 0, 1, 3, 0, 4, 4, 2, 4, 2, 1, 3, 0, 4, 4, 2, 4, 2, 1, 3, 0, 0, 7, 4, 3, 6, 5, 3, 4, 0, 7, 4, 3, 6, 5, 3, 4, 0, 7, 4, 0, 7, 4, 3, 6, - 5, 2, 4, 0, 7, 4, 3, 6, 5, 2, 4, 0, 7, 4, 6, 0, 8, 7, 7, 6, 4, 2, 3, 5, 6, 6, 0, 8, 7, 7, 6, 4, 2, 6, 8, 0, 7, 7, 6, 4, 3, 3, 5, 7, - 9, 6, 8, 0, 7, 7, 6, 4, 3, 1, 1, 1, 2, 3, 1, 0, 3, 1, 1, 1, 2, 3, 1, 0, 3, 1, 1, 5, 4, 5, 4, 0, 2, 1, 1, 6, 9, 5, 4, 5, 4, 0, 2, 1, - 1, 6, 3, 1, 1, 1, 2, 3, 1, 0, 3, 1, 1, 1, 2, 3, 1, 0, 3, 1, 1, 5, 5, 5, 4, 0, 2, 1, 1, 7, 9, 5, 5, 5, 4, 0, 2, 1, 1, 7, 4, 2, 2, 2, - 1, 1, 0, 0, 9, 4, 2, 2, 2, 1, 1, 0, 0, 9, 4, 4, 2, 2, 2, 1, 1, 0, 0, 9, 4, 2, 2, 2, 1, 1, 0, 0, 9, 4, 8, 9, 8, 8, 7, 8, 8, 8, 8, 1, - 3, 0, 8, 5, 8, 9, 9, 9, 8, 8, 9, 8, 8, 7, 8, 8, 8, 8, 2, 4, 8, 0, 6, 7, 8, 8, 7, 8, 9, 9, 9, 9, 8, 9, 9, 9, 9, 0, 0, 0, 6, 6, 4, 4, - 6, 7, 8, 1, 1, 0, 5, 5, 2, 3, 3, 4, 6, 1, 1, 0, 5, 5, 2, 3, 3, 4, 2, 8, 7, 7, 5, 4, 6, 5, 2, 8, 7, 7, 5, 4, 6, 5, 2, 8, 7, 2, 8, 8, - 6, 5, 5, 4, 2, 8, 8, 6, 5, 5, 4, 2, 8, 8, 6, 5, 3, 3, 3, 1, 2, 3, 0, 2, 2, 3, 3, 3, 3, 1, 2, 3, 0, 2, 2, 4, 4, 4, 2, 1, 1, 0, 0, 1, - 2, 4, 4, 4, 2, 1, 1, 0, 0, 1, 7, 6, 5, 5, 3, 2, 1, 1, 0, 1, 7, 6, 5, 5, 3, 2, 1, 1, 0, 9, 6, 4, 4, 3, 2, 1, 0, 9, 6, 4, 4, 3, 2, 1, - 0, 9, 6, 4, 7, 7, 9, 9, 7, 3, 7, 7, 9, 9, 7, 3, 7, 7, 9, 9, 7, 3, 7 - }; - }; -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_PIPNET29_H diff --git a/lite/tnn/cv/tnn_pipnet68.cpp b/lite/tnn/cv/tnn_pipnet68.cpp deleted file mode 100644 index 2f24da60..00000000 --- a/lite/tnn/cv/tnn_pipnet68.cpp +++ /dev/null @@ -1,229 +0,0 @@ -// -// Created by DefTruth on 2022/3/20. -// - -#include "tnn_pipnet68.h" - -using tnncv::TNNPIPNet68; - -TNNPIPNet68::TNNPIPNet68(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNPIPNet68::transform(const cv::Mat &mat_rs) -{ - // be carefully, no deepcopy inside this tnn::Mat constructor, - // so, we can not pass a local cv::Mat to this constructor. - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNPIPNet68::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - this->transform(mat_rs); // resize outside transform to prevent overflow - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. generate landmarks - this->generate_landmarks(landmarks, instance, img_height, img_width); -} - -void TNNPIPNet68::generate_landmarks(types::Landmarks &landmarks, - std::shared_ptr &_instance, - float img_height, float img_width) -{ - std::shared_ptr outputs_cls, outputs_x, outputs_y, outputs_nb_x, outputs_nb_y; - tnn::MatConvertParam cvt_param; - tnn::Status status_cls = _instance->GetOutputMat(outputs_cls, cvt_param, "outputs_cls", output_device_type); - tnn::Status status_x = _instance->GetOutputMat(outputs_x, cvt_param, "outputs_x", output_device_type); - tnn::Status status_y = _instance->GetOutputMat(outputs_y, cvt_param, "outputs_y", output_device_type); - tnn::Status status_nb_x = _instance->GetOutputMat(outputs_nb_x, cvt_param, "outputs_nb_x", output_device_type); - tnn::Status status_nb_y = _instance->GetOutputMat(outputs_nb_y, cvt_param, "outputs_nb_y", output_device_type); - - if (status_cls != tnn::TNN_OK || status_x != tnn::TNN_OK || status_y != tnn::TNN_OK - || status_nb_x != tnn::TNN_OK || status_nb_y != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status_cls.description().c_str() << ": " - << status_x.description().c_str() << ": " - << status_y.description().c_str() << ": " - << status_nb_x.description().c_str() << ": " - << status_nb_y.description().c_str() << "\n"; -#endif - return; - } - auto cls_shape = outputs_cls->GetDims(); - const unsigned int grid_h = cls_shape.at(2); // 8 - const unsigned int grid_w = cls_shape.at(3); // 8 - const unsigned int grid_length = grid_h * grid_w; // 8 * 8 = 64 - const unsigned int input_h = input_height; - const unsigned int input_w = input_width; - - // fetch data from pointers - const float *outputs_cls_ptr = (float *) outputs_cls->GetData(); - const float *outputs_x_ptr = (float *) outputs_x->GetData(); - const float *outputs_y_ptr = (float *) outputs_y->GetData(); - const float *outputs_nb_x_ptr = (float *) outputs_nb_x->GetData(); - const float *outputs_nb_y_ptr = (float *) outputs_nb_y->GetData(); - - // find max_ids - std::vector max_ids(num_lms); - for (unsigned int i = 0; i < num_lms; ++i) - { - const float *score_ptr = outputs_cls_ptr + i * grid_length; - unsigned int max_id = 0; - float max_score = score_ptr[0]; - for (unsigned int j = 0; j < grid_length; ++j) - { - if (score_ptr[j] > max_score) - { - max_score = score_ptr[j]; - max_id = j; - } - } - max_ids[i] = max_id; // range 0~64 - } - - // find x & y offsets - std::vector output_x_select(num_lms); - std::vector output_y_select(num_lms); - for (unsigned int i = 0; i < num_lms; ++i) - { - const float *offset_x_ptr = outputs_x_ptr + i * grid_length; - const float *offset_y_ptr = outputs_y_ptr + i * grid_length; - const unsigned int max_id = max_ids.at(i); - output_x_select[i] = offset_x_ptr[max_id]; - output_y_select[i] = offset_y_ptr[max_id]; - } - - // find nb_x & nb_y offsets - std::unordered_map> output_nb_x_select; - std::unordered_map> output_nb_y_select; - // initialize offsets map - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector nb_x_offset(num_nb); - std::vector nb_y_offset(num_nb); - output_nb_x_select[i] = nb_x_offset; - output_nb_y_select[i] = nb_y_offset; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - for (unsigned int j = 0; j < num_nb; ++j) - { - const float *offset_nb_x_ptr = outputs_nb_x_ptr + (i * num_nb + j) * grid_length; - const float *offset_nb_y_ptr = outputs_nb_y_ptr + (i * num_nb + j) * grid_length; - const unsigned int max_id = max_ids.at(i); - output_nb_x_select[i][j] = offset_nb_x_ptr[max_id]; - output_nb_y_select[i][j] = offset_nb_y_ptr[max_id]; - } - } - - // calculate coords - std::vector lms_pred_x(num_lms); // 68 - std::vector lms_pred_y(num_lms); // 68 - std::unordered_map> lms_pred_nb_x; // 68,10 - std::unordered_map> lms_pred_nb_y; // 68,10 - // initialize pred maps - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector nb_x_offset(num_nb); - std::vector nb_y_offset(num_nb); - lms_pred_nb_x[i] = nb_x_offset; - lms_pred_nb_y[i] = nb_y_offset; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - float cx = static_cast(max_ids.at(i) % grid_w); - float cy = static_cast(max_ids.at(i) / grid_w); - // calculate coords & normalize - lms_pred_x[i] = ((cx + output_x_select[i]) * (float) net_stride) / (float) input_w; - lms_pred_y[i] = ((cy + output_y_select[i]) * (float) net_stride) / (float) input_h; - for (unsigned int j = 0; j < num_nb; ++j) - { - lms_pred_nb_x[i][j] = ((cx + output_nb_x_select[i][j]) * (float) net_stride) / (float) input_w; - lms_pred_nb_y[i][j] = ((cy + output_nb_y_select[i][j]) * (float) net_stride) / (float) input_h; - } - } - - // reverse indexes - std::unordered_map> tmp_nb_x; // 68,max_len - std::unordered_map> tmp_nb_y; // 68,max_len - // initialize reverse maps - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector tmp_x(max_len); - std::vector tmp_y(max_len); - tmp_nb_x[i] = tmp_x; - tmp_nb_y[i] = tmp_y; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - for (unsigned int j = 0; j < max_len; ++j) - { - unsigned int ri = reverse_index1[i * max_len + j]; - unsigned int rj = reverse_index2[i * max_len + j]; - tmp_nb_x[i][j] = lms_pred_nb_x[ri][rj]; - tmp_nb_y[i][j] = lms_pred_nb_y[ri][rj]; - } - } - - // merge predictions - landmarks.points.clear(); - for (unsigned int i = 0; i < num_lms; ++i) - { - float total_x = lms_pred_x[i]; - float total_y = lms_pred_y[i]; - for (unsigned int j = 0; j < max_len; ++j) - { - total_x += tmp_nb_x[i][j]; - total_y += tmp_nb_y[i][j]; - } - float x = total_x / ((float) max_len + 1.f); - float y = total_y / ((float) max_len + 1.f); - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - - landmarks.flag = true; -} - - diff --git a/lite/tnn/cv/tnn_pipnet68.h b/lite/tnn/cv/tnn_pipnet68.h deleted file mode 100644 index e4962330..00000000 --- a/lite/tnn/cv/tnn_pipnet68.h +++ /dev/null @@ -1,130 +0,0 @@ -// -// Created by DefTruth on 2022/3/20. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_PIPNET68_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_PIPNET68_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNPIPNet68 : public BasicTNNHandler - { - public: - explicit TNNPIPNet68(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNPIPNet68() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - std::vector bias_vals = {-0.485f * 255.f * (1.0f / 0.229f) * (1.0 / 255.f), - -0.456f * 255.f * (1.0f / 0.224f) * (1.0 / 255.f), - -0.406f * 255.f * (1.0f / 0.225f) * (1.0 / 255.f)}; - static constexpr const unsigned int num_nb = 10; - static constexpr const unsigned int num_lms = 68; - static constexpr const unsigned int max_len = 22; - static constexpr const unsigned int net_stride = 32; - - private: - void transform(const cv::Mat &mat_rs) override; // - - void generate_landmarks(types::Landmarks &landmarks, - std::shared_ptr &_instance, - float img_height, float img_width); - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - - private: - const unsigned int reverse_index1[68 * 22] = { - 1, 2, 17, 18, 36, 1, 2, 17, 18, 36, 1, 2, 17, 18, 36, 1, 2, 17, 18, 36, 1, 2, 0, 2, 3, 17, 0, 2, 3, 17, 0, 2, 3, 17, 0, 2, 3, 17, 0, - 2, 3, 17, 0, 2, 0, 1, 3, 4, 0, 1, 3, 4, 0, 1, 3, 4, 0, 1, 3, 4, 0, 1, 3, 4, 0, 1, 1, 2, 4, 5, 1, 2, 4, 5, 1, 2, 4, 5, 1, 2, 4, 5, 1, - 2, 4, 5, 1, 2, 2, 3, 5, 6, 2, 3, 5, 6, 2, 3, 5, 6, 2, 3, 5, 6, 2, 3, 5, 6, 2, 3, 3, 4, 6, 7, 3, 4, 6, 7, 3, 4, 6, 7, 3, 4, 6, 7, 3, - 4, 6, 7, 3, 4, 3, 4, 5, 7, 8, 3, 4, 5, 7, 8, 3, 4, 5, 7, 8, 3, 4, 5, 7, 8, 3, 4, 5, 6, 8, 9, 5, 6, 8, 9, 5, 6, 8, 9, 5, 6, 8, 9, 5, - 6, 8, 9, 5, 6, 6, 7, 9, 10, 6, 7, 9, 10, 6, 7, 9, 10, 6, 7, 9, 10, 6, 7, 9, 10, 6, 7, 7, 8, 10, 11, 7, 8, 10, 11, 7, 8, 10, 11, 7, - 8, 10, 11, 7, 8, 10, 11, 7, 8, 8, 9, 11, 12, 13, 8, 9, 11, 12, 13, 8, 9, 11, 12, 13, 8, 9, 11, 12, 13, 8, 9, 9, 10, 12, 13, 9, 10, - 12, 13, 9, 10, 12, 13, 9, 10, 12, 13, 9, 10, 12, 13, 9, 10, 10, 11, 13, 14, 10, 11, 13, 14, 10, 11, 13, 14, 10, 11, 13, 14, 10, 11, - 13, 14, 10, 11, 11, 12, 14, 15, 11, 12, 14, 15, 11, 12, 14, 15, 11, 12, 14, 15, 11, 12, 14, 15, 11, 12, 12, 13, 15, 16, 12, 13, 15, - 16, 12, 13, 15, 16, 12, 13, 15, 16, 12, 13, 15, 16, 12, 13, 13, 14, 16, 26, 13, 14, 16, 26, 13, 14, 16, 26, 13, 14, 16, 26, 13, 14, - 16, 26, 13, 14, 14, 15, 25, 26, 45, 14, 15, 25, 26, 45, 14, 15, 25, 26, 45, 14, 15, 25, 26, 45, 14, 15, 0, 1, 2, 18, 19, 36, 37, 41, - 0, 1, 2, 18, 19, 36, 37, 41, 0, 1, 2, 18, 19, 36, 0, 1, 17, 19, 20, 36, 37, 38, 41, 0, 1, 17, 19, 20, 36, 37, 38, 41, 0, 1, 17, 19, - 0, 17, 18, 20, 21, 36, 37, 38, 40, 41, 0, 17, 18, 20, 21, 36, 37, 38, 40, 41, 0, 17, 17, 18, 19, 21, 36, 37, 38, 39, 40, 41, 17, 18, - 19, 21, 36, 37, 38, 39, 40, 41, 17, 18, 18, 19, 20, 22, 27, 28, 37, 38, 39, 40, 41, 18, 19, 20, 22, 27, 28, 37, 38, 39, 40, 41, 21, - 23, 24, 25, 27, 28, 42, 43, 44, 46, 47, 21, 23, 24, 25, 27, 28, 42, 43, 44, 46, 47, 22, 24, 25, 26, 42, 43, 44, 45, 46, 47, 22, 24, - 25, 26, 42, 43, 44, 45, 46, 47, 22, 24, 16, 22, 23, 25, 26, 43, 44, 45, 46, 47, 16, 22, 23, 25, 26, 43, 44, 45, 46, 47, 16, 22, 15, - 16, 23, 24, 26, 43, 44, 45, 46, 15, 16, 23, 24, 26, 43, 44, 45, 46, 15, 16, 23, 24, 14, 15, 16, 24, 25, 44, 45, 46, 14, 15, 16, 24, - 25, 44, 45, 46, 14, 15, 16, 24, 25, 44, 20, 21, 22, 23, 28, 29, 38, 39, 40, 42, 43, 47, 20, 21, 22, 23, 28, 29, 38, 39, 40, 42, 21, - 22, 27, 29, 30, 39, 40, 42, 47, 21, 22, 27, 29, 30, 39, 40, 42, 47, 21, 22, 27, 29, 27, 28, 30, 31, 35, 39, 42, 27, 28, 30, 31, 35, - 39, 42, 27, 28, 30, 31, 35, 39, 42, 27, 28, 29, 31, 32, 33, 34, 35, 28, 29, 31, 32, 33, 34, 35, 28, 29, 31, 32, 33, 34, 35, 28, 2, - 3, 29, 30, 32, 33, 48, 49, 2, 3, 29, 30, 32, 33, 48, 49, 2, 3, 29, 30, 32, 33, 29, 30, 31, 33, 34, 35, 49, 50, 29, 30, 31, 33, 34, - 35, 49, 50, 29, 30, 31, 33, 34, 35, 29, 30, 31, 32, 34, 35, 50, 51, 52, 29, 30, 31, 32, 34, 35, 50, 51, 52, 29, 30, 31, 32, 29, 30, - 31, 32, 33, 35, 52, 53, 29, 30, 31, 32, 33, 35, 52, 53, 29, 30, 31, 32, 33, 35, 13, 14, 29, 30, 32, 33, 34, 53, 54, 13, 14, 29, 30, - 32, 33, 34, 53, 54, 13, 14, 29, 30, 0, 1, 2, 17, 18, 19, 20, 37, 38, 39, 40, 41, 0, 1, 2, 17, 18, 19, 20, 37, 38, 39, 0, 1, 17, 18, - 19, 20, 21, 36, 38, 39, 40, 41, 0, 1, 17, 18, 19, 20, 21, 36, 38, 39, 0, 1, 17, 18, 19, 20, 21, 27, 28, 36, 37, 39, 40, 41, 0, 1, - 17, 18, 19, 20, 21, 27, 19, 20, 21, 27, 28, 29, 36, 37, 38, 40, 41, 19, 20, 21, 27, 28, 29, 36, 37, 38, 40, 41, 0, 1, 17, 18, 19, - 20, 21, 27, 28, 36, 37, 38, 39, 41, 0, 1, 17, 18, 19, 20, 21, 27, 0, 1, 2, 17, 18, 19, 20, 21, 36, 37, 38, 39, 40, 0, 1, 2, 17, 18, - 19, 20, 21, 36, 22, 23, 24, 27, 28, 29, 43, 44, 45, 46, 47, 22, 23, 24, 27, 28, 29, 43, 44, 45, 46, 47, 15, 16, 22, 23, 24, 25, 26, - 27, 42, 44, 45, 46, 47, 15, 16, 22, 23, 24, 25, 26, 27, 42, 15, 16, 22, 23, 24, 25, 26, 42, 43, 45, 46, 47, 15, 16, 22, 23, 24, 25, - 26, 42, 43, 45, 14, 15, 16, 23, 24, 25, 26, 42, 43, 44, 46, 47, 14, 15, 16, 23, 24, 25, 26, 42, 43, 44, 14, 15, 16, 22, 23, 24, 25, - 26, 42, 43, 44, 45, 47, 14, 15, 16, 22, 23, 24, 25, 26, 42, 15, 16, 22, 23, 24, 25, 26, 27, 28, 42, 43, 44, 45, 46, 15, 16, 22, 23, - 24, 25, 26, 27, 2, 3, 4, 5, 6, 49, 59, 60, 2, 3, 4, 5, 6, 49, 59, 60, 2, 3, 4, 5, 6, 49, 3, 4, 5, 31, 32, 48, 50, 51, 59, 60, 61, - 67, 3, 4, 5, 31, 32, 48, 50, 51, 59, 60, 30, 31, 32, 33, 34, 48, 49, 51, 52, 58, 59, 60, 61, 62, 66, 67, 30, 31, 32, 33, 34, 48, 30, - 31, 32, 33, 34, 35, 48, 49, 50, 52, 53, 54, 56, 58, 60, 61, 62, 63, 64, 65, 66, 67, 30, 32, 33, 34, 35, 50, 51, 53, 54, 55, 56, 62, - 63, 64, 65, 30, 32, 33, 34, 35, 50, 51, 11, 12, 13, 34, 35, 52, 54, 55, 63, 64, 65, 11, 12, 13, 34, 35, 52, 54, 55, 63, 64, 65, 10, - 11, 12, 13, 14, 53, 55, 64, 10, 11, 12, 13, 14, 53, 55, 64, 10, 11, 12, 13, 14, 53, 8, 9, 10, 11, 12, 13, 53, 54, 56, 57, 63, 64, - 65, 8, 9, 10, 11, 12, 13, 53, 54, 56, 7, 8, 9, 10, 11, 12, 54, 55, 57, 58, 63, 64, 65, 66, 7, 8, 9, 10, 11, 12, 54, 55, 6, 7, 8, 9, - 10, 55, 56, 58, 59, 62, 65, 66, 67, 6, 7, 8, 9, 10, 55, 56, 58, 59, 4, 5, 6, 7, 8, 9, 48, 56, 57, 59, 60, 61, 62, 66, 67, 4, 5, 6, - 7, 8, 9, 48, 3, 4, 5, 6, 7, 8, 48, 49, 57, 58, 60, 61, 67, 3, 4, 5, 6, 7, 8, 48, 49, 57, 2, 3, 4, 5, 6, 31, 48, 49, 59, 2, 3, 4, 5, - 6, 31, 48, 49, 59, 2, 3, 4, 5, 31, 32, 33, 48, 49, 50, 51, 52, 57, 58, 59, 60, 62, 63, 66, 67, 31, 32, 33, 48, 49, 50, 33, 34, 48, - 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 63, 64, 65, 66, 67, 33, 34, 35, 50, 51, 52, 53, 54, 55, 56, 57, 61, 62, 64, 65, - 66, 34, 35, 50, 51, 52, 53, 54, 10, 11, 12, 13, 14, 35, 53, 54, 55, 10, 11, 12, 13, 14, 35, 53, 54, 55, 10, 11, 12, 13, 9, 10, 11, - 12, 51, 52, 53, 54, 55, 56, 57, 58, 61, 62, 63, 64, 66, 67, 9, 10, 11, 12, 7, 8, 9, 50, 51, 52, 55, 56, 57, 58, 59, 60, 61, 62, 63, - 64, 65, 67, 7, 8, 9, 50, 4, 5, 6, 7, 48, 49, 50, 51, 56, 57, 58, 59, 60, 61, 62, 63, 65, 66, 4, 5, 6, 7 - }; - const unsigned int reverse_index2[68 * 22] = { - 0, 3, 1, 7, 8, 0, 3, 1, 7, 8, 0, 3, 1, 7, 8, 0, 3, 1, 7, 8, 0, 3, 1, 1, 4, 9, 1, 1, 4, 9, 1, 1, 4, 9, 1, 1, 4, 9, 1, 1, 4, 9, 1, 1, - 6, 1, 1, 5, 6, 1, 1, 5, 6, 1, 1, 5, 6, 1, 1, 5, 6, 1, 1, 5, 6, 1, 5, 0, 0, 6, 5, 0, 0, 6, 5, 0, 0, 6, 5, 0, 0, 6, 5, 0, 0, 6, 5, 0, - 2, 0, 1, 7, 2, 0, 1, 7, 2, 0, 1, 7, 2, 0, 1, 7, 2, 0, 1, 7, 2, 0, 2, 1, 1, 6, 2, 1, 1, 6, 2, 1, 1, 6, 2, 1, 1, 6, 2, 1, 1, 6, 2, 1, - 9, 4, 0, 1, 4, 9, 4, 0, 1, 4, 9, 4, 0, 1, 4, 9, 4, 0, 1, 4, 9, 4, 5, 0, 1, 3, 5, 0, 1, 3, 5, 0, 1, 3, 5, 0, 1, 3, 5, 0, 1, 3, 5, 0, - 4, 0, 0, 4, 4, 0, 0, 4, 4, 0, 0, 4, 4, 0, 0, 4, 4, 0, 0, 4, 4, 0, 3, 0, 0, 5, 3, 0, 0, 5, 3, 0, 0, 5, 3, 0, 0, 5, 3, 0, 0, 5, 3, 0, - 3, 1, 0, 4, 9, 3, 1, 0, 4, 9, 3, 1, 0, 4, 9, 3, 1, 0, 4, 9, 3, 1, 6, 1, 0, 2, 6, 1, 0, 2, 6, 1, 0, 2, 6, 1, 0, 2, 6, 1, 0, 2, 6, 1, - 7, 1, 0, 2, 7, 1, 0, 2, 7, 1, 0, 2, 7, 1, 0, 2, 7, 1, 0, 2, 7, 1, 6, 1, 1, 4, 6, 1, 1, 4, 6, 1, 1, 4, 6, 1, 1, 4, 6, 1, 1, 4, 6, 1, - 5, 1, 0, 6, 5, 1, 0, 6, 5, 1, 0, 6, 5, 1, 0, 6, 5, 1, 0, 6, 5, 1, 3, 0, 0, 9, 3, 0, 0, 9, 3, 0, 0, 9, 3, 0, 0, 9, 3, 0, 0, 9, 3, 0, - 3, 1, 7, 2, 8, 3, 1, 7, 2, 8, 3, 1, 7, 2, 8, 3, 1, 7, 2, 8, 3, 1, 0, 3, 9, 0, 4, 4, 8, 6, 0, 3, 9, 0, 4, 4, 8, 6, 0, 3, 9, 0, 4, 4, - 3, 8, 0, 0, 6, 5, 7, 9, 7, 3, 8, 0, 0, 6, 5, 7, 9, 7, 3, 8, 0, 0, 7, 4, 1, 1, 6, 6, 5, 7, 9, 5, 7, 4, 1, 1, 6, 6, 5, 7, 9, 5, 7, 4, - 8, 4, 1, 0, 9, 6, 4, 7, 6, 8, 8, 4, 1, 0, 9, 6, 4, 7, 6, 8, 8, 4, 9, 6, 0, 4, 2, 7, 9, 6, 5, 5, 9, 9, 6, 0, 4, 2, 7, 9, 6, 5, 5, 9, - 4, 1, 6, 9, 3, 8, 5, 6, 9, 9, 6, 4, 1, 6, 9, 3, 8, 5, 6, 9, 9, 6, 0, 1, 4, 8, 7, 5, 7, 9, 8, 5, 0, 1, 4, 8, 7, 5, 7, 9, 8, 5, 0, 1, - 7, 6, 0, 1, 4, 7, 5, 6, 6, 9, 7, 6, 0, 1, 4, 7, 5, 6, 6, 9, 7, 6, 8, 3, 5, 0, 0, 9, 6, 5, 7, 8, 3, 5, 0, 0, 9, 6, 5, 7, 8, 3, 5, 0, - 8, 3, 1, 4, 0, 8, 4, 5, 8, 3, 1, 4, 0, 8, 4, 5, 8, 3, 1, 4, 0, 8, 9, 1, 1, 9, 1, 2, 8, 4, 7, 2, 8, 7, 9, 1, 1, 9, 1, 2, 8, 4, 7, 2, - 8, 8, 0, 0, 6, 6, 8, 6, 8, 8, 8, 0, 0, 6, 6, 8, 6, 8, 8, 8, 0, 0, 5, 0, 0, 9, 9, 9, 9, 5, 0, 0, 9, 9, 9, 9, 5, 0, 0, 9, 9, 9, 9, 5, - 4, 1, 2, 2, 2, 2, 2, 4, 1, 2, 2, 2, 2, 2, 4, 1, 2, 2, 2, 2, 2, 4, 8, 8, 6, 5, 0, 7, 7, 9, 8, 8, 6, 5, 0, 7, 7, 9, 8, 8, 6, 5, 0, 7, - 4, 3, 0, 0, 4, 5, 8, 7, 4, 3, 0, 0, 4, 5, 8, 7, 4, 3, 0, 0, 4, 5, 7, 2, 1, 1, 1, 1, 5, 8, 5, 7, 2, 1, 1, 1, 1, 5, 8, 5, 7, 2, 1, 1, - 3, 1, 5, 4, 1, 0, 6, 9, 3, 1, 5, 4, 1, 0, 6, 9, 3, 1, 5, 4, 1, 0, 8, 9, 5, 4, 9, 6, 0, 8, 7, 8, 9, 5, 4, 9, 6, 0, 8, 7, 8, 9, 5, 4, - 2, 2, 4, 2, 3, 5, 8, 1, 5, 8, 4, 1, 2, 2, 4, 2, 3, 5, 8, 1, 5, 8, 5, 6, 3, 2, 2, 3, 7, 1, 1, 3, 3, 0, 5, 6, 3, 2, 2, 3, 7, 1, 1, 3, - 9, 9, 6, 6, 3, 2, 2, 7, 9, 3, 2, 1, 0, 3, 9, 9, 6, 6, 3, 2, 2, 7, 9, 4, 3, 4, 3, 9, 7, 4, 2, 1, 4, 9, 4, 3, 4, 3, 9, 7, 4, 2, 1, 4, - 8, 7, 7, 8, 8, 5, 5, 8, 5, 2, 3, 0, 0, 2, 8, 7, 7, 8, 8, 5, 5, 8, 4, 4, 5, 5, 5, 7, 7, 9, 0, 0, 3, 2, 2, 4, 4, 5, 5, 5, 7, 7, 9, 0, - 3, 4, 9, 1, 2, 8, 2, 4, 7, 4, 2, 3, 4, 9, 1, 2, 8, 2, 4, 7, 4, 2, 9, 9, 2, 2, 3, 6, 6, 6, 1, 2, 3, 3, 0, 9, 9, 2, 2, 3, 6, 6, 6, 1, - 6, 5, 7, 3, 2, 2, 3, 4, 1, 1, 1, 3, 6, 5, 7, 3, 2, 2, 3, 4, 1, 1, 4, 2, 2, 8, 5, 3, 1, 8, 4, 1, 0, 4, 4, 2, 2, 8, 5, 3, 1, 8, 4, 1, - 5, 5, 4, 9, 7, 7, 5, 5, 3, 3, 0, 0, 1, 5, 5, 4, 9, 7, 7, 5, 5, 3, 7, 8, 5, 6, 8, 8, 7, 9, 6, 0, 0, 3, 2, 2, 7, 8, 5, 6, 8, 8, 7, 9, - 6, 3, 2, 2, 5, 3, 3, 0, 6, 3, 2, 2, 5, 3, 3, 0, 6, 3, 2, 2, 5, 3, 6, 7, 8, 4, 6, 1, 3, 9, 4, 1, 5, 8, 6, 7, 8, 4, 6, 1, 3, 9, 4, 1, - 7, 3, 3, 4, 8, 5, 1, 1, 7, 9, 8, 5, 1, 6, 9, 5, 7, 3, 3, 4, 8, 5, 9, 6, 5, 3, 5, 6, 9, 6, 1, 1, 6, 9, 8, 8, 8, 3, 0, 3, 8, 6, 6, 6, - 8, 8, 5, 3, 3, 8, 2, 1, 5, 8, 9, 7, 1, 5, 4, 8, 8, 5, 3, 3, 8, 2, 8, 7, 6, 6, 4, 3, 1, 3, 5, 1, 8, 8, 7, 6, 6, 4, 3, 1, 3, 5, 1, 8, - 5, 2, 2, 4, 6, 2, 4, 0, 5, 2, 2, 4, 6, 2, 4, 0, 5, 2, 2, 4, 6, 2, 7, 5, 2, 3, 6, 7, 5, 2, 2, 9, 8, 2, 5, 7, 5, 2, 3, 6, 7, 5, 2, 2, - 7, 5, 2, 3, 7, 8, 6, 0, 1, 5, 7, 6, 3, 8, 7, 5, 2, 3, 7, 8, 6, 0, 8, 4, 2, 4, 8, 7, 0, 0, 7, 8, 7, 4, 7, 8, 4, 2, 4, 8, 7, 0, 0, 7, - 9, 7, 3, 2, 6, 7, 6, 5, 0, 0, 6, 7, 9, 7, 3, 9, 7, 3, 2, 6, 7, 6, 7, 6, 3, 2, 5, 8, 2, 5, 8, 2, 2, 8, 4, 7, 6, 3, 2, 5, 8, 2, 5, 8, - 7, 5, 3, 4, 6, 8, 0, 0, 1, 7, 5, 3, 4, 6, 8, 0, 0, 1, 7, 5, 3, 4, 7, 7, 9, 3, 2, 0, 3, 9, 6, 4, 5, 3, 2, 6, 3, 0, 7, 7, 9, 3, 2, 0, - 8, 9, 8, 7, 2, 0, 2, 7, 8, 9, 6, 5, 6, 9, 7, 2, 2, 7, 2, 0, 2, 8, 7, 7, 9, 4, 0, 3, 3, 5, 4, 7, 6, 3, 3, 0, 5, 7, 7, 9, 4, 0, 3, 3, - 6, 4, 3, 5, 7, 8, 0, 0, 1, 6, 4, 3, 5, 7, 8, 0, 0, 1, 6, 4, 3, 5, 8, 9, 9, 9, 7, 4, 4, 4, 2, 1, 4, 7, 9, 5, 0, 4, 2, 9, 8, 9, 9, 9, - 9, 9, 9, 6, 5, 8, 6, 3, 2, 3, 6, 9, 4, 1, 4, 9, 1, 1, 9, 9, 9, 6, 8, 9, 9, 8, 4, 4, 4, 6, 7, 3, 1, 2, 4, 0, 4, 9, 9, 1, 8, 9, 9, 8 - }; - - }; -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_PIPNET68_H diff --git a/lite/tnn/cv/tnn_pipnet98.cpp b/lite/tnn/cv/tnn_pipnet98.cpp deleted file mode 100644 index b44be484..00000000 --- a/lite/tnn/cv/tnn_pipnet98.cpp +++ /dev/null @@ -1,229 +0,0 @@ -// -// Created by DefTruth on 2022/3/20. -// - -#include "tnn_pipnet98.h" - -using tnncv::TNNPIPNet98; - -TNNPIPNet98::TNNPIPNet98(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNPIPNet98::transform(const cv::Mat &mat_rs) -{ - // be carefully, no deepcopy inside this tnn::Mat constructor, - // so, we can not pass a local cv::Mat to this constructor. - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNPIPNet98::detect(const cv::Mat &mat, types::Landmarks &landmarks) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - this->transform(mat_rs); // resize outside transform to prevent overflow - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. generate landmarks - this->generate_landmarks(landmarks, instance, img_height, img_width); -} - -void TNNPIPNet98::generate_landmarks(types::Landmarks &landmarks, - std::shared_ptr &_instance, - float img_height, float img_width) -{ - std::shared_ptr outputs_cls, outputs_x, outputs_y, outputs_nb_x, outputs_nb_y; - tnn::MatConvertParam cvt_param; - tnn::Status status_cls = _instance->GetOutputMat(outputs_cls, cvt_param, "outputs_cls", output_device_type); - tnn::Status status_x = _instance->GetOutputMat(outputs_x, cvt_param, "outputs_x", output_device_type); - tnn::Status status_y = _instance->GetOutputMat(outputs_y, cvt_param, "outputs_y", output_device_type); - tnn::Status status_nb_x = _instance->GetOutputMat(outputs_nb_x, cvt_param, "outputs_nb_x", output_device_type); - tnn::Status status_nb_y = _instance->GetOutputMat(outputs_nb_y, cvt_param, "outputs_nb_y", output_device_type); - - if (status_cls != tnn::TNN_OK || status_x != tnn::TNN_OK || status_y != tnn::TNN_OK - || status_nb_x != tnn::TNN_OK || status_nb_y != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status_cls.description().c_str() << ": " - << status_x.description().c_str() << ": " - << status_y.description().c_str() << ": " - << status_nb_x.description().c_str() << ": " - << status_nb_y.description().c_str() << "\n"; -#endif - return; - } - auto cls_shape = outputs_cls->GetDims(); - const unsigned int grid_h = cls_shape.at(2); // 8 - const unsigned int grid_w = cls_shape.at(3); // 8 - const unsigned int grid_length = grid_h * grid_w; // 8 * 8 = 64 - const unsigned int input_h = input_height; - const unsigned int input_w = input_width; - - // fetch data from pointers - const float *outputs_cls_ptr = (float *) outputs_cls->GetData(); - const float *outputs_x_ptr = (float *) outputs_x->GetData(); - const float *outputs_y_ptr = (float *) outputs_y->GetData(); - const float *outputs_nb_x_ptr = (float *) outputs_nb_x->GetData(); - const float *outputs_nb_y_ptr = (float *) outputs_nb_y->GetData(); - - // find max_ids - std::vector max_ids(num_lms); - for (unsigned int i = 0; i < num_lms; ++i) - { - const float *score_ptr = outputs_cls_ptr + i * grid_length; - unsigned int max_id = 0; - float max_score = score_ptr[0]; - for (unsigned int j = 0; j < grid_length; ++j) - { - if (score_ptr[j] > max_score) - { - max_score = score_ptr[j]; - max_id = j; - } - } - max_ids[i] = max_id; // range 0~64 - } - - // find x & y offsets - std::vector output_x_select(num_lms); - std::vector output_y_select(num_lms); - for (unsigned int i = 0; i < num_lms; ++i) - { - const float *offset_x_ptr = outputs_x_ptr + i * grid_length; - const float *offset_y_ptr = outputs_y_ptr + i * grid_length; - const unsigned int max_id = max_ids.at(i); - output_x_select[i] = offset_x_ptr[max_id]; - output_y_select[i] = offset_y_ptr[max_id]; - } - - // find nb_x & nb_y offsets - std::unordered_map> output_nb_x_select; - std::unordered_map> output_nb_y_select; - // initialize offsets map - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector nb_x_offset(num_nb); - std::vector nb_y_offset(num_nb); - output_nb_x_select[i] = nb_x_offset; - output_nb_y_select[i] = nb_y_offset; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - for (unsigned int j = 0; j < num_nb; ++j) - { - const float *offset_nb_x_ptr = outputs_nb_x_ptr + (i * num_nb + j) * grid_length; - const float *offset_nb_y_ptr = outputs_nb_y_ptr + (i * num_nb + j) * grid_length; - const unsigned int max_id = max_ids.at(i); - output_nb_x_select[i][j] = offset_nb_x_ptr[max_id]; - output_nb_y_select[i][j] = offset_nb_y_ptr[max_id]; - } - } - - // calculate coords - std::vector lms_pred_x(num_lms); // 98 - std::vector lms_pred_y(num_lms); // 98 - std::unordered_map> lms_pred_nb_x; // 98,10 - std::unordered_map> lms_pred_nb_y; // 98,10 - // initialize pred maps - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector nb_x_offset(num_nb); - std::vector nb_y_offset(num_nb); - lms_pred_nb_x[i] = nb_x_offset; - lms_pred_nb_y[i] = nb_y_offset; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - float cx = static_cast(max_ids.at(i) % grid_w); - float cy = static_cast(max_ids.at(i) / grid_w); - // calculate coords & normalize - lms_pred_x[i] = ((cx + output_x_select[i]) * (float) net_stride) / (float) input_w; - lms_pred_y[i] = ((cy + output_y_select[i]) * (float) net_stride) / (float) input_h; - for (unsigned int j = 0; j < num_nb; ++j) - { - lms_pred_nb_x[i][j] = ((cx + output_nb_x_select[i][j]) * (float) net_stride) / (float) input_w; - lms_pred_nb_y[i][j] = ((cy + output_nb_y_select[i][j]) * (float) net_stride) / (float) input_h; - } - } - - // reverse indexes - std::unordered_map> tmp_nb_x; // 98,max_len - std::unordered_map> tmp_nb_y; // 98,max_len - // initialize reverse maps - for (unsigned int i = 0; i < num_lms; ++i) - { - std::vector tmp_x(max_len); - std::vector tmp_y(max_len); - tmp_nb_x[i] = tmp_x; - tmp_nb_y[i] = tmp_y; - } - for (unsigned int i = 0; i < num_lms; ++i) - { - for (unsigned int j = 0; j < max_len; ++j) - { - unsigned int ri = reverse_index1[i * max_len + j]; - unsigned int rj = reverse_index2[i * max_len + j]; - tmp_nb_x[i][j] = lms_pred_nb_x[ri][rj]; - tmp_nb_y[i][j] = lms_pred_nb_y[ri][rj]; - } - } - - // merge predictions - landmarks.points.clear(); - for (unsigned int i = 0; i < num_lms; ++i) - { - float total_x = lms_pred_x[i]; - float total_y = lms_pred_y[i]; - for (unsigned int j = 0; j < max_len; ++j) - { - total_x += tmp_nb_x[i][j]; - total_y += tmp_nb_y[i][j]; - } - float x = total_x / ((float) max_len + 1.f); - float y = total_y / ((float) max_len + 1.f); - x = std::min(std::max(0.f, x), 1.0f); - y = std::min(std::max(0.f, y), 1.0f); - - landmarks.points.push_back(cv::Point2f(x * img_width, y * img_height)); - } - - landmarks.flag = true; -} - - diff --git a/lite/tnn/cv/tnn_pipnet98.h b/lite/tnn/cv/tnn_pipnet98.h deleted file mode 100644 index 8bfefa72..00000000 --- a/lite/tnn/cv/tnn_pipnet98.h +++ /dev/null @@ -1,140 +0,0 @@ -// -// Created by DefTruth on 2022/3/20. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_PIPNET98_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_PIPNET98_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNPIPNet98 : public BasicTNNHandler - { - public: - explicit TNNPIPNet98(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNPIPNet98() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - std::vector bias_vals = {-0.485f * 255.f * (1.0f / 0.229f) * (1.0 / 255.f), - -0.456f * 255.f * (1.0f / 0.224f) * (1.0 / 255.f), - -0.406f * 255.f * (1.0f / 0.225f) * (1.0 / 255.f)}; - static constexpr const unsigned int num_nb = 10; - static constexpr const unsigned int num_lms = 98; - static constexpr const unsigned int max_len = 17; - static constexpr const unsigned int net_stride = 32; - - private: - void transform(const cv::Mat &mat_rs) override; // - - void generate_landmarks(types::Landmarks &landmarks, - std::shared_ptr &_instance, - float img_height, float img_width); - - public: - void detect(const cv::Mat &mat, types::Landmarks &landmarks); - - private: - const unsigned int reverse_index1[98 * 17] = { - 1, 2, 3, 4, 5, 33, 1, 2, 3, 4, 5, 33, 1, 2, 3, 4, 5, 0, 2, 3, 4, 5, 6, 33, 0, 2, 3, 4, 5, 6, 33, 0, 2, 3, 0, 1, 3, 4, 5, 6, 0, 1, 3, - 4, 5, 6, 0, 1, 3, 4, 5, 0, 1, 2, 4, 5, 6, 7, 0, 1, 2, 4, 5, 6, 7, 0, 1, 2, 0, 1, 2, 3, 5, 6, 7, 8, 0, 1, 2, 3, 5, 6, 7, 8, 0, 1, 2, - 3, 4, 6, 7, 8, 9, 1, 2, 3, 4, 6, 7, 8, 9, 1, 2, 3, 4, 5, 7, 8, 9, 10, 2, 3, 4, 5, 7, 8, 9, 10, 2, 3, 4, 5, 6, 8, 9, 10, 3, 4, 5, 6, - 8, 9, 10, 3, 4, 5, 4, 5, 6, 7, 9, 10, 11, 4, 5, 6, 7, 9, 10, 11, 4, 5, 6, 4, 5, 6, 7, 8, 10, 11, 12, 4, 5, 6, 7, 8, 10, 11, 12, 4, - 5, 6, 7, 8, 9, 11, 12, 13, 76, 5, 6, 7, 8, 9, 11, 12, 13, 7, 8, 9, 10, 12, 13, 14, 76, 88, 7, 8, 9, 10, 12, 13, 14, 76, 8, 9, 10, - 11, 13, 14, 15, 8, 9, 10, 11, 13, 14, 15, 8, 9, 10, 10, 11, 12, 14, 15, 16, 10, 11, 12, 14, 15, 16, 10, 11, 12, 14, 15, 11, 12, 13, - 15, 16, 17, 11, 12, 13, 15, 16, 17, 11, 12, 13, 15, 16, 12, 13, 14, 16, 17, 18, 12, 13, 14, 16, 17, 18, 12, 13, 14, 16, 17, 13, 14, - 15, 17, 18, 19, 13, 14, 15, 17, 18, 19, 13, 14, 15, 17, 18, 14, 15, 16, 18, 19, 20, 14, 15, 16, 18, 19, 20, 14, 15, 16, 18, 19, 15, - 16, 17, 19, 20, 21, 15, 16, 17, 19, 20, 21, 15, 16, 17, 19, 20, 16, 17, 18, 20, 21, 22, 16, 17, 18, 20, 21, 22, 16, 17, 18, 20, 21, - 17, 18, 19, 21, 22, 23, 24, 17, 18, 19, 21, 22, 23, 24, 17, 18, 19, 18, 19, 20, 22, 23, 24, 25, 82, 18, 19, 20, 22, 23, 24, 25, 82, - 18, 19, 20, 21, 23, 24, 25, 26, 27, 19, 20, 21, 23, 24, 25, 26, 27, 19, 20, 21, 22, 24, 25, 26, 27, 28, 20, 21, 22, 24, 25, 26, 27, - 28, 20, 21, 22, 23, 25, 26, 27, 28, 21, 22, 23, 25, 26, 27, 28, 21, 22, 23, 21, 22, 23, 24, 26, 27, 28, 29, 21, 22, 23, 24, 26, 27, - 28, 29, 21, 22, 23, 24, 25, 27, 28, 29, 30, 22, 23, 24, 25, 27, 28, 29, 30, 22, 23, 24, 25, 26, 28, 29, 30, 31, 23, 24, 25, 26, 28, - 29, 30, 31, 23, 24, 25, 26, 27, 29, 30, 31, 32, 24, 25, 26, 27, 29, 30, 31, 32, 24, 25, 26, 27, 28, 30, 31, 32, 25, 26, 27, 28, 30, - 31, 32, 25, 26, 27, 26, 27, 28, 29, 31, 32, 26, 27, 28, 29, 31, 32, 26, 27, 28, 29, 31, 26, 27, 28, 29, 30, 32, 46, 26, 27, 28, 29, - 30, 32, 46, 26, 27, 28, 27, 28, 29, 30, 31, 46, 27, 28, 29, 30, 31, 46, 27, 28, 29, 30, 31, 0, 1, 2, 3, 34, 41, 60, 0, 1, 2, 3, 34, - 41, 60, 0, 1, 2, 0, 33, 35, 40, 41, 60, 0, 33, 35, 40, 41, 60, 0, 33, 35, 40, 41, 33, 34, 36, 37, 39, 40, 41, 60, 61, 62, 33, 34, - 36, 37, 39, 40, 41, 34, 35, 37, 38, 39, 40, 63, 64, 34, 35, 37, 38, 39, 40, 63, 64, 34, 36, 38, 39, 51, 64, 36, 38, 39, 51, 64, 36, - 38, 39, 51, 64, 36, 38, 36, 37, 39, 51, 52, 63, 64, 65, 36, 37, 39, 51, 52, 63, 64, 65, 36, 35, 36, 37, 38, 40, 62, 63, 64, 65, 66, - 67, 96, 35, 36, 37, 38, 40, 33, 34, 35, 36, 37, 38, 39, 41, 60, 61, 62, 63, 65, 66, 67, 96, 33, 0, 1, 2, 33, 34, 35, 40, 60, 61, 67, - 0, 1, 2, 33, 34, 35, 40, 43, 49, 50, 51, 68, 43, 49, 50, 51, 68, 43, 49, 50, 51, 68, 43, 49, 42, 44, 45, 48, 49, 50, 68, 69, 42, 44, - 45, 48, 49, 50, 68, 69, 42, 42, 43, 45, 46, 47, 48, 49, 70, 42, 43, 45, 46, 47, 48, 49, 70, 42, 32, 44, 46, 47, 48, 71, 72, 73, 32, - 44, 46, 47, 48, 71, 72, 73, 32, 29, 30, 31, 32, 45, 47, 72, 29, 30, 31, 32, 45, 47, 72, 29, 30, 31, 30, 31, 32, 44, 45, 46, 48, 71, - 72, 73, 30, 31, 32, 44, 45, 46, 48, 42, 43, 44, 45, 46, 47, 49, 50, 69, 70, 71, 72, 73, 74, 75, 97, 42, 42, 43, 44, 48, 50, 68, 69, - 70, 74, 75, 97, 42, 43, 44, 48, 50, 68, 42, 43, 49, 51, 52, 68, 69, 75, 42, 43, 49, 51, 52, 68, 69, 75, 42, 37, 38, 42, 50, 52, 53, - 64, 68, 37, 38, 42, 50, 52, 53, 64, 68, 37, 51, 53, 54, 51, 53, 54, 51, 53, 54, 51, 53, 54, 51, 53, 54, 51, 53, 51, 52, 54, 55, 56, - 57, 59, 51, 52, 54, 55, 56, 57, 59, 51, 52, 54, 52, 53, 55, 56, 57, 58, 59, 52, 53, 55, 56, 57, 58, 59, 52, 53, 55, 53, 54, 56, 57, - 76, 77, 78, 88, 53, 54, 56, 57, 76, 77, 78, 88, 53, 53, 54, 55, 57, 58, 77, 78, 79, 88, 53, 54, 55, 57, 58, 77, 78, 79, 53, 54, 55, - 56, 58, 59, 78, 79, 80, 90, 53, 54, 55, 56, 58, 59, 78, 53, 54, 56, 57, 59, 79, 80, 81, 82, 92, 53, 54, 56, 57, 59, 79, 80, 53, 54, - 57, 58, 80, 81, 82, 92, 53, 54, 57, 58, 80, 81, 82, 92, 53, 0, 1, 2, 3, 4, 33, 34, 41, 61, 62, 66, 67, 96, 0, 1, 2, 3, 0, 1, 33, 34, - 35, 40, 41, 60, 62, 63, 65, 66, 67, 96, 0, 1, 33, 33, 34, 35, 36, 37, 38, 39, 40, 41, 60, 61, 63, 64, 65, 66, 67, 96, 35, 36, 37, - 38, 39, 40, 51, 52, 61, 62, 64, 65, 66, 67, 96, 35, 36, 36, 37, 38, 39, 51, 52, 53, 63, 65, 66, 96, 36, 37, 38, 39, 51, 52, 36, 37, - 38, 39, 52, 61, 62, 63, 64, 66, 67, 96, 36, 37, 38, 39, 52, 41, 60, 61, 62, 63, 64, 65, 67, 96, 41, 60, 61, 62, 63, 64, 65, 67, 0, - 1, 2, 3, 33, 34, 35, 40, 41, 60, 61, 62, 65, 66, 96, 0, 1, 42, 43, 49, 50, 51, 52, 53, 69, 74, 75, 97, 42, 43, 49, 50, 51, 52, 42, - 43, 44, 48, 49, 50, 51, 68, 70, 71, 73, 74, 75, 97, 42, 43, 44, 42, 43, 44, 45, 46, 47, 48, 49, 50, 68, 69, 71, 72, 73, 74, 75, 97, - 31, 32, 44, 45, 46, 47, 48, 69, 70, 72, 73, 74, 75, 97, 31, 32, 44, 28, 29, 30, 31, 32, 45, 46, 47, 70, 71, 73, 74, 97, 28, 29, 30, - 31, 29, 30, 31, 32, 44, 45, 46, 47, 48, 70, 71, 72, 74, 75, 97, 29, 30, 47, 68, 69, 70, 71, 72, 73, 75, 97, 47, 68, 69, 70, 71, 72, - 73, 75, 42, 43, 49, 50, 52, 68, 69, 70, 71, 72, 73, 74, 97, 42, 43, 49, 50, 6, 7, 8, 9, 10, 11, 12, 55, 77, 87, 88, 89, 95, 6, 7, 8, - 9, 55, 56, 76, 78, 86, 87, 88, 89, 95, 55, 56, 76, 78, 86, 87, 88, 89, 54, 55, 56, 57, 58, 76, 77, 79, 80, 85, 86, 87, 88, 89, 90, - 94, 95, 54, 55, 56, 57, 58, 59, 77, 78, 80, 81, 84, 85, 86, 89, 90, 91, 94, 54, 57, 58, 59, 78, 79, 81, 82, 83, 84, 85, 90, 91, 92, - 93, 94, 54, 58, 59, 80, 82, 83, 84, 91, 92, 93, 58, 59, 80, 82, 83, 84, 91, 92, 20, 21, 22, 23, 24, 25, 26, 59, 81, 83, 91, 92, 93, - 20, 21, 22, 23, 17, 18, 19, 20, 21, 22, 23, 81, 82, 84, 91, 92, 93, 17, 18, 19, 20, 16, 17, 18, 19, 20, 81, 82, 83, 85, 91, 92, 93, - 94, 16, 17, 18, 19, 14, 15, 16, 17, 18, 83, 84, 86, 87, 90, 93, 94, 95, 14, 15, 16, 17, 11, 12, 13, 14, 15, 16, 76, 77, 85, 87, 88, - 89, 94, 95, 11, 12, 13, 9, 10, 11, 12, 13, 14, 76, 77, 86, 88, 89, 95, 9, 10, 11, 12, 13, 7, 8, 9, 10, 11, 12, 13, 55, 76, 77, 86, - 87, 89, 95, 7, 8, 9, 55, 56, 76, 77, 78, 79, 86, 87, 88, 90, 95, 55, 56, 76, 77, 78, 79, 56, 57, 58, 78, 79, 80, 83, 84, 85, 86, 87, - 89, 91, 92, 93, 94, 95, 58, 59, 79, 80, 81, 82, 83, 84, 85, 90, 92, 93, 94, 58, 59, 79, 80, 19, 20, 21, 22, 23, 24, 25, 59, 81, 82, - 83, 84, 91, 93, 19, 20, 21, 18, 19, 79, 80, 81, 82, 83, 84, 85, 90, 91, 92, 94, 18, 19, 79, 80, 15, 16, 17, 78, 79, 80, 83, 84, 85, - 86, 87, 89, 90, 91, 93, 95, 15, 13, 14, 15, 76, 77, 78, 85, 86, 87, 88, 89, 90, 94, 13, 14, 15, 76, 34, 35, 36, 38, 39, 40, 41, 60, - 61, 62, 63, 64, 65, 66, 67, 34, 35, 43, 44, 45, 47, 48, 49, 50, 68, 69, 70, 71, 72, 73, 74, 75, 43, 44 - }; - const unsigned int reverse_index2[98 * 17] = { - 0, 2, 4, 6, 8, 4, 0, 2, 4, 6, 8, 4, 0, 2, 4, 6, 8, 0, 0, 2, 4, 6, 8, 8, 0, 0, 2, 4, 6, 8, 8, 0, 0, 2, 1, 1, 0, 2, 4, 6, 1, 1, 0, 2, - 4, 6, 1, 1, 0, 2, 4, 3, 2, 1, 0, 2, 4, 6, 3, 2, 1, 0, 2, 4, 6, 3, 2, 1, 6, 3, 3, 1, 0, 2, 4, 7, 6, 3, 3, 1, 0, 2, 4, 7, 6, 6, 4, 3, - 1, 0, 2, 4, 8, 6, 4, 3, 1, 0, 2, 4, 8, 6, 7, 5, 3, 1, 0, 2, 4, 9, 7, 5, 3, 1, 0, 2, 4, 9, 7, 6, 5, 3, 1, 0, 2, 4, 6, 5, 3, 1, 0, 2, - 4, 6, 5, 3, 7, 5, 3, 1, 0, 2, 4, 7, 5, 3, 1, 0, 2, 4, 7, 5, 3, 9, 7, 5, 3, 1, 0, 2, 5, 9, 7, 5, 3, 1, 0, 2, 5, 9, 9, 7, 5, 3, 1, 0, - 2, 5, 8, 9, 7, 5, 3, 1, 0, 2, 5, 7, 5, 3, 1, 0, 2, 5, 9, 9, 7, 5, 3, 1, 0, 2, 5, 9, 9, 5, 3, 1, 0, 2, 4, 9, 5, 3, 1, 0, 2, 4, 9, 5, - 3, 6, 3, 1, 0, 2, 6, 6, 3, 1, 0, 2, 6, 6, 3, 1, 0, 2, 7, 3, 1, 0, 3, 7, 7, 3, 1, 0, 3, 7, 7, 3, 1, 0, 3, 6, 3, 1, 1, 3, 6, 6, 3, 1, - 1, 3, 6, 6, 3, 1, 1, 3, 7, 3, 1, 1, 3, 7, 7, 3, 1, 1, 3, 7, 7, 3, 1, 1, 3, 6, 3, 0, 1, 3, 6, 6, 3, 0, 1, 3, 6, 6, 3, 0, 1, 3, 7, 2, - 0, 1, 3, 5, 7, 2, 0, 1, 3, 5, 7, 2, 0, 1, 3, 5, 2, 0, 1, 3, 5, 5, 2, 0, 1, 3, 5, 5, 2, 0, 1, 3, 4, 2, 0, 1, 3, 5, 8, 4, 2, 0, 1, 3, - 5, 8, 4, 2, 0, 5, 2, 0, 1, 3, 5, 7, 9, 5, 2, 0, 1, 3, 5, 7, 9, 5, 4, 2, 0, 1, 3, 5, 7, 9, 4, 2, 0, 1, 3, 5, 7, 9, 4, 4, 2, 0, 1, 3, - 5, 7, 9, 4, 2, 0, 1, 3, 5, 7, 9, 4, 4, 2, 0, 1, 3, 5, 7, 4, 2, 0, 1, 3, 5, 7, 4, 2, 0, 9, 4, 2, 0, 1, 3, 5, 6, 9, 4, 2, 0, 1, 3, 5, - 6, 9, 9, 4, 2, 0, 1, 3, 5, 6, 9, 4, 2, 0, 1, 3, 5, 6, 9, 8, 4, 2, 0, 1, 3, 4, 6, 8, 4, 2, 0, 1, 3, 4, 6, 8, 6, 4, 2, 0, 1, 3, 3, 5, - 6, 4, 2, 0, 1, 3, 3, 5, 6, 6, 4, 2, 0, 1, 2, 3, 6, 4, 2, 0, 1, 2, 3, 6, 4, 2, 6, 4, 2, 0, 1, 1, 6, 4, 2, 0, 1, 1, 6, 4, 2, 0, 1, 8, - 6, 4, 2, 0, 0, 9, 8, 6, 4, 2, 0, 0, 9, 8, 6, 4, 8, 6, 4, 2, 0, 6, 8, 6, 4, 2, 0, 6, 8, 6, 4, 2, 0, 2, 4, 5, 8, 3, 1, 6, 2, 4, 5, 8, - 3, 1, 6, 2, 4, 5, 7, 1, 1, 5, 0, 8, 7, 1, 1, 5, 0, 8, 7, 1, 1, 5, 0, 7, 1, 2, 8, 6, 0, 5, 9, 8, 8, 7, 1, 2, 8, 6, 0, 5, 8, 2, 1, 4, - 0, 6, 7, 9, 8, 2, 1, 4, 0, 6, 7, 9, 8, 1, 0, 5, 5, 7, 1, 0, 5, 5, 7, 1, 0, 5, 5, 7, 1, 0, 4, 0, 2, 2, 6, 6, 2, 8, 4, 0, 2, 2, 6, 6, - 2, 8, 4, 4, 0, 2, 1, 4, 7, 4, 4, 5, 9, 9, 7, 4, 0, 2, 1, 4, 5, 2, 0, 3, 9, 9, 4, 2, 7, 5, 4, 8, 9, 8, 6, 6, 5, 5, 7, 9, 0, 0, 3, 3, - 2, 6, 7, 5, 7, 9, 0, 0, 3, 3, 2, 5, 0, 6, 7, 2, 5, 0, 6, 7, 2, 5, 0, 6, 7, 2, 5, 1, 1, 8, 5, 0, 4, 9, 7, 1, 1, 8, 5, 0, 4, 9, 7, 1, - 8, 1, 1, 7, 4, 0, 6, 9, 8, 1, 1, 7, 4, 0, 6, 9, 8, 7, 2, 1, 0, 6, 9, 8, 9, 7, 2, 1, 0, 6, 9, 8, 9, 7, 8, 5, 4, 2, 2, 1, 6, 8, 5, 4, - 2, 2, 1, 6, 8, 5, 4, 9, 7, 6, 3, 0, 0, 3, 6, 2, 7, 9, 7, 6, 3, 0, 0, 3, 7, 3, 0, 3, 5, 2, 2, 9, 8, 4, 5, 7, 6, 7, 9, 6, 7, 2, 0, 4, - 2, 1, 3, 2, 7, 9, 5, 8, 2, 0, 4, 2, 1, 3, 0, 4, 3, 1, 5, 2, 6, 8, 0, 4, 3, 1, 5, 2, 6, 8, 0, 5, 6, 5, 5, 1, 5, 8, 8, 5, 6, 5, 5, 1, - 5, 8, 8, 5, 0, 1, 9, 0, 1, 9, 0, 1, 9, 0, 1, 9, 0, 1, 9, 0, 1, 7, 0, 1, 9, 9, 9, 9, 7, 0, 1, 9, 9, 9, 9, 7, 0, 1, 4, 0, 5, 2, 0, 2, - 4, 4, 0, 5, 2, 0, 2, 4, 4, 0, 5, 6, 5, 0, 8, 6, 6, 9, 6, 6, 5, 0, 8, 6, 6, 9, 6, 6, 3, 2, 0, 2, 7, 7, 5, 7, 8, 3, 2, 0, 2, 7, 7, 5, - 7, 2, 0, 2, 1, 1, 2, 4, 3, 5, 7, 2, 0, 2, 1, 1, 2, 4, 4, 3, 7, 1, 0, 5, 4, 8, 8, 8, 4, 3, 7, 1, 0, 5, 4, 7, 4, 7, 0, 9, 6, 6, 6, 7, - 4, 7, 0, 9, 6, 6, 6, 7, 4, 5, 6, 7, 8, 2, 5, 4, 1, 9, 6, 1, 9, 4, 5, 6, 7, 8, 9, 3, 4, 6, 2, 3, 1, 2, 9, 7, 4, 0, 5, 8, 9, 3, 9, 6, - 5, 6, 7, 7, 3, 1, 7, 4, 2, 3, 6, 4, 1, 4, 0, 8, 5, 3, 3, 1, 8, 8, 9, 7, 3, 1, 0, 5, 8, 3, 8, 5, 8, 4, 2, 8, 4, 3, 9, 1, 1, 7, 8, 8, - 4, 2, 8, 4, 3, 9, 6, 5, 9, 7, 9, 6, 0, 0, 3, 5, 2, 9, 6, 5, 9, 7, 9, 3, 4, 1, 5, 5, 3, 2, 1, 9, 3, 4, 1, 5, 5, 3, 2, 9, 8, 8, 9, 6, - 7, 9, 9, 6, 0, 0, 5, 6, 2, 4, 9, 8, 4, 8, 8, 2, 3, 2, 8, 1, 8, 1, 9, 4, 8, 8, 2, 3, 2, 3, 5, 8, 8, 1, 3, 9, 0, 3, 7, 8, 5, 0, 5, 3, - 5, 8, 9, 6, 5, 6, 8, 6, 1, 4, 7, 6, 4, 2, 5, 4, 2, 4, 0, 9, 8, 6, 4, 3, 3, 4, 9, 1, 1, 0, 4, 7, 2, 9, 8, 6, 8, 7, 7, 5, 4, 5, 2, 5, - 8, 1, 1, 6, 7, 8, 7, 7, 5, 9, 8, 8, 9, 9, 7, 4, 7, 9, 5, 0, 0, 1, 6, 3, 9, 8, 9, 5, 5, 2, 4, 3, 2, 3, 1, 9, 5, 5, 2, 4, 3, 2, 3, 6, - 9, 9, 6, 8, 1, 0, 6, 8, 9, 5, 3, 4, 6, 9, 9, 6, 9, 8, 6, 6, 5, 6, 7, 8, 4, 2, 0, 8, 7, 9, 8, 6, 6, 1, 5, 2, 7, 5, 3, 2, 0, 3, 1, 5, - 2, 7, 5, 3, 2, 0, 7, 4, 3, 4, 9, 7, 5, 1, 3, 7, 7, 6, 7, 2, 2, 3, 4, 6, 7, 4, 3, 4, 6, 9, 0, 0, 9, 9, 6, 9, 7, 0, 7, 2, 8, 5, 3, 3, - 3, 2, 5, 7, 6, 7, 8, 3, 2, 7, 4, 4, 8, 5, 1, 6, 2, 3, 5, 0, 2, 3, 5, 1, 6, 2, 3, 5, 0, 2, 7, 6, 6, 6, 7, 8, 9, 8, 4, 2, 8, 0, 8, 7, - 6, 6, 6, 8, 7, 6, 5, 7, 8, 9, 3, 1, 1, 3, 1, 2, 8, 7, 6, 5, 7, 5, 4, 5, 9, 7, 5, 5, 1, 4, 5, 1, 5, 7, 5, 4, 5, 8, 5, 4, 6, 8, 8, 2, - 2, 8, 4, 9, 0, 9, 8, 5, 4, 6, 9, 8, 4, 4, 6, 8, 5, 8, 2, 5, 5, 4, 6, 1, 9, 8, 4, 9, 8, 5, 4, 6, 7, 1, 3, 1, 1, 3, 2, 9, 8, 5, 4, 6, - 9, 8, 7, 7, 8, 9, 9, 6, 0, 2, 8, 1, 5, 5, 9, 8, 7, 3, 6, 3, 0, 2, 8, 3, 4, 3, 6, 0, 3, 6, 3, 0, 2, 8, 8, 6, 8, 1, 0, 1, 9, 6, 3, 6, - 9, 6, 6, 9, 7, 1, 8, 6, 5, 6, 2, 0, 3, 4, 3, 9, 5, 3, 0, 9, 6, 5, 6, 2, 9, 8, 8, 7, 7, 9, 9, 7, 2, 0, 1, 8, 5, 5, 9, 8, 8, 9, 8, 9, - 8, 1, 4, 0, 0, 4, 8, 1, 4, 7, 9, 8, 9, 8, 8, 9, 9, 6, 4, 7, 7, 4, 0, 4, 7, 9, 1, 9, 6, 6, 8, 8, 9, 9, 4, 1, 8, 5, 0, 0, 4, 1, 9, 8, - 8, 9, 9, 4, 9, 7, 7, 8, 7, 7, 8, 5, 3, 0, 2, 3, 2, 0, 3, 9, 7, 7, 7, 9, 8, 7, 7, 8, 4, 3, 0, 3, 4, 3, 0, 2, 7, 7 - }; - - }; -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_PIPNET98_H diff --git a/lite/tnn/cv/tnn_plantid.cpp b/lite/tnn/cv/tnn_plantid.cpp deleted file mode 100644 index 45a82696..00000000 --- a/lite/tnn/cv/tnn_plantid.cpp +++ /dev/null @@ -1,94 +0,0 @@ -// -// Created by DefTruth on 2022/3/27. -// - -#include "tnn_plantid.h" -#include "lite/utils.h" - -using tnncv::TNNPlantID; - -TNNPlantID::TNNPlantID(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNPlantID::transform(const cv::Mat &mat_rs) -{ - // push into input_mat (1,3,224,224) - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNPlantID::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch. - tnn::MatConvertParam cvt_param; - std::shared_ptr logits_mat; // (1,4066) - status = instance->GetOutputMat(logits_mat, cvt_param, "477", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto logits_dims = logits_mat->GetDims(); - const unsigned int num_classes = logits_dims.at(1); // 4066 - const float *logits = (float *) logits_mat->GetData(); - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/tnn/cv/tnn_plantid.h b/lite/tnn/cv/tnn_plantid.h deleted file mode 100644 index ea739c7e..00000000 --- a/lite/tnn/cv/tnn_plantid.h +++ /dev/null @@ -1,820 +0,0 @@ -// -// Created by DefTruth on 2022/3/27. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_PLANTID_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_PLANTID_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNPlantID : public BasicTNNHandler - { - public: - explicit TNNPlantID(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNPlantID() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - std::vector bias_vals = {-0.485f * 255.f * (1.0f / 0.229f) * (1.0 / 255.f), - -0.456f * 255.f * (1.0f / 0.224f) * (1.0 / 255.f), - -0.406f * 255.f * (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[4066] = { - "Saururus chinensis", "Houttuynia cordata", "Aucuba chinensis", "Aucuba japonica var. variegata", "Aucuba obcordata", - "Blechnum novae-zelandiae", "Woodwardia fimbriata", "Woodwardia prolifera", "Pentaphylax euryoides", "Ternstroemia gymnanthera", - "Eurya chinensis", "Eurya distichophylla", "Eurya emarginata", "Eurya japonica", "Eurya macartneyi", "Eurya muricata", - "Eurya rubiginosa var. attenuata", "Eurya saxicola", "Cleyera japonica", "Anneslea fragrans", "Eleutherococcus nodiflorus", - "Eleutherococcus senticosus", "Eleutherococcus trifoliatus", "Panax ginseng", "Fatsia japonica", "Kalopanax septemlobus", - "Trevesia palmata", "Schefflera arboricola", "Schefflera elegantissima", "Schefflera heptaphylla", "Schefflera macrostachya", - "Hydrocotyle sibthorpioides", "Hydrocotyle sibthorpioides var. batrachium", "Hydrocotyle verticillata", "Hydrocotyle wilfordii", - "Hedera helix", "Hedera nepalensis var. sinensis", "Metapanax davidii", "Aralia cordata", "Aralia elata", "Aralia nudicaulis", - "Schisandra chinensis", "Schisandra sphenanthera", "Illicium angustisepalum", "Illicium dunnianum", "Illicium lanceolatum", - "Illicium verum", "Kadsura coccinea", "Kadsura heteroclita", "Kadsura longipedunculata", "Dillenia turbinata", - "Tetracera sarmentosa", "Adoxa moschatellina", "Sambucus adnata", "Sambucus javanica", "Sambucus nigra", "Sambucus nigra caerulea", - "Sambucus racemosa", "Sambucus williamsii", "Viburnum acerifolium", "Viburnum betulifolium", "Viburnum chinshanense", - "Viburnum dilatatum", "Viburnum foetidum var. rectangulatum", "Viburnum fordiae", "Viburnum lantanoides", "Viburnum macrocephalum", - "Viburnum macrocephalum f. keteleeri", "Viburnum melanocarpum", "Viburnum odoratissimum", "Viburnum odoratissimum var. awabuki", - "Viburnum opulus", "Viburnum opulus subsp. calvescens", "Viburnum plicatum", "Viburnum plicatum f. tomentosum", - "Viburnum prunifolium", "Viburnum setigerum", "Viburnum tinus", "Linum usitatissimum&perenne", "Reinwardtia indica", - "Lophophora williamsii", "Schlumbergera truncata", "Opuntia basilaris", "Opuntia ficus-indica", "Opuntia humifusa", - "Opuntia littoralis", "Opuntia microdasys", "Echinopsis chamaecereus", "Nopalxochia ackermannii", "Cylindropuntia imbricata", - "Cylindropuntia leptocaulis", "Ferocactus peninsulae", "Epiphyllum oxypetalum", "Astrophytum myriostigma", "Pereskia bleo", - "Cleistocactus colademononis", "Hylocereus undatus", "Echinocactus grusonii", "Aporocactus flagelliformis", "Curculigo capitulata", - "Hypoxis hirsuta", "Hypoxis juncea", "Pauridia capensis", "Eryngium leavenworthii", "Eryngium planum", "Eryngium yuccifolium", - "Sanicula lamelligera", "Sanicula orthacantha", "Angelica dahurica", "Angelica decursiva", "Angelica polymorpha", - "Changium smyrnioides", "Astrantia major", "Bupleurum smithii", "Pastinaca sativa", "Conium maculatum", "Oenanthe javanica", - "Heracleum maximum", "Glehnia littoralis", "Centella asiatica", "Torilis arvensis", "Torilis scabra", "Daucus carota", - "Daucus carota var. sativa", "Coriandrum sativum", "Apium graveolens", "Foeniculum vulgare", "Cnidium monnieri", "Zizia aurea", - "Quisqualis indica", "Terminalia arjuna", "Terminalia catappa", "Terminalia nigrovenulosa", "Combretum alfredii", - "Combretum constrictum", "", "", "Alstroemeria hybrida", "Isotria verticillata", "Sacoila lanceolata", "Limodorum abortivum", - "Anacamptis coriophora", "Anacamptis laxiflora", "Anacamptis morio", "Anacamptis palustris", "Anacamptis papilionacea", - "Anacamptis pyramidalis", "Eriochilus cucullatus", "Paphiopedilum", "Paphiopedilum emersonii", "Paphiopedilum hirsutissimum", - "Paphiopedilum purpuratum", "Neottianthe cucullata", "Cymbidium ensifolium", "Cymbidium faberi", "Cymbidium floribundum", - "Cymbidium goeringii", "Cymbidium kanran", "Cymbidium lancifolium", "Cymbidium serratum", "Cymbidium sinense", "Cattleya hybrida", - "Epigeneium fargesii", "Malaxis monophyllos", "Malaxis unifolia", "Cheirostylis yunnanensis", "Dipodium roseum", - "Chiloglottis valida", "Encyclia tampensis", "Polystachya concreta", "Cephalanthera damasonium", "Cephalanthera falcata", - "Cephalanthera longifolia", "Cephalanthera rubra", "Cryptochilus roseus", "Robiquetia succisa", "Oberonioides microtatantha", - "Ponerorchis brevicalcarata", "Dracula simia", "Oreorchis nana", "Galeola lindleyana", "Calypso bulbosa var. speciosa", - "Tainia dunnii", "Tainia hongkongensis", "Anoectochilus roxburghii", "Gymnadenia nigra", "Gymnadenia odoratissima", - "Gymnadenia rhellicani", "Bletia purpurea", "Aerides rosea", "Dactylorhiza fuchsii", "Dactylorhiza majalis", - "Dactylorhiza traunsteineri", "Dactylorhiza viridis", "Oncidium", "Goodyera foliosa", "Goodyera oblongifolia", "Goodyera procera", - "Goodyera pubescens", "Goodyera repens", "Goodyera schlechtendaliana", "Goodyera tesselata", "Goodyera viridiflora", - "Neotinea maculata", "Neotinea tridentata", "Amitostigma monanthum", "Amitostigma pinguicula", "Dienia ophrydis", - "Cypripedium acaule", "Cypripedium bardolphianum", "Cypripedium calceolus", "Cypripedium calcicola", "Cypripedium candidum", - "Cypripedium flavum", "Cypripedium franchetii", "Cypripedium guttatum", "Cypripedium henryi", "Cypripedium japonicum", - "Cypripedium lichiangense", "Cypripedium macranthos", "Cypripedium montanum", "Cypripedium parviflorum", - "Cypripedium plectrochilum", "Cypripedium reginae", "Cypripedium shanxiense", "Cypripedium tibeticum", "Cypripedium wardii", - "Cypripedium yunnanense", "Cypripedium × ventricosum", "Cremastra appendiculata", "Thelymitra antennifera", "Thelymitra longifolia", - "Epidendrum radicans", "Eria corneri", "Calopogon tuberosus", "Epipactis atrorubens", "Epipactis gigantea", "Epipactis helleborine", - "Epipactis mairei", "Epipactis microphylla", "Epipactis palustris", "Renanthera coccinea", "Appendicula cornuta", - "Pleione bulbocodioides", "Pleione formosana", "Habenaria ciliolaris", "Habenaria dentata", "Habenaria floribunda", - "Habenaria glaucifolia", "Habenaria leptoloba", "Habenaria limprichtii", "Habenaria monorrhiza", "Habenaria petelotii", - "Habenaria repens", "Habenaria rhodocheila", "Habenaria schindleri", "Corallorhiza maculata", "Corallorhiza mertensiana", - "Corallorhiza striata", "Corallorhiza trifida", "Corallorhiza wisteriana", "Bletilla ochracea", "Bletilla striata", - "Pseudorchis albida", "Pseudorchis straminea", "Thrixspermum centipeda", "Pecteilis susannae", "Gastrochilus calceolaris", - "Galearis rotundifolia", "Chamorchis alpina", "Pholidota articulata", "Pholidota cantonensis", "Pholidota chinensis", - "Dendrobium chrysotoxum", "Dendrobium crepidatum", "Dendrobium cucullatum", "Dendrobium densiflorum", "Dendrobium hancockii", - "Dendrobium henryi", "Dendrobium hercoglossum", "Dendrobium loddigesii", "Dendrobium moniliforme", "Dendrobium moschatum", - "Dendrobium officinale", "Dendrobium sinominutiflorum", "Dendrobium thyrsiflorum", "Bulbophyllum ambrosia", - "Bulbophyllum kwangtungense", "Bulbophyllum levinei", "Bulbophyllum odoratissimum", "Bulbophyllum orientale", - "Bulbophyllum pecten-veneris", "Bulbophyllum retusiusculum", "Prosthechea cochleata", "Arundina graminifolia", - "Orchis anthropophora", "Orchis italica", "Orchis mascula", "Orchis militaris", "Orchis pallens", "Orchis provincialis", - "Orchis simia", "Zeuxine parvifolia", "Zeuxine strateumatica", "Dendrolirium lasiopetalum", "Spiranthes cernua", - "Spiranthes lacera", "Spiranthes lucida", "Spiranthes magnicamporum", "Spiranthes praecox", "Spiranthes sinensis", - "Spiranthes spiralis", "Spiranthes tuberosa", "Spiranthes vernalis", "Liparis bootanensis", "Liparis nervosa", - "Liparis stricklandiana", "Liparis viridiflora", "Eulophia alta", "Eulophia cucullata", "Eulophia graminea", "Eulophia zollingeri", - "Arethusa bulbosa", "Pterostylis banksii", "Pterostylis nana", "Pterostylis nutans", "Acampe rigida", "Platanthera aquilonis", - "Platanthera blephariglottis", "Platanthera clavellata", "Platanthera dilatata", "Platanthera elegans", "Platanthera flava", - "Platanthera grandiflora", "Platanthera huronensis", "Platanthera hyperborea", "Platanthera lacera", "Platanthera minor", - "Platanthera obtusata", "Platanthera orbiculata", "Platanthera psycodes", "Platanthera sparsiflora", "Platanthera stricta", - "Platanthera ussuriensis", "Hemipilia flabellata", "Spathoglottis plicata", "Spathoglottis pubescens", "Disa bracteata", - "Microtis unifolia", "Traunsteinera globosa", "Ponthieva racemosa", "Epipogium aphyllum", "Epipogium roseum", "Calanthe brevicornu", - "Calanthe clavata", "Calanthe graciliflora", "Calanthe sylvatica", "Calanthe tricarinata", "Calanthe triplicata", - "Diploprora championii", "Conchidium pusillum", "Ophrys apifera", "Ophrys bertolonii", "Ophrys bombyliflora", "Ophrys fuciflora", - "Ophrys fusca", "Ophrys insectifera", "Ophrys lutea", "Ophrys scolopax", "Ophrys speculum", "Ophrys sphegodes", - "Ophrys tenthredinifera", "Arachnis labrosa", "Phalaenopsis aphrodite", "Ludisia discolor", "Caladenia caerulea", - "Caladenia carnea", "Caladenia flava", "Caladenia fuscata", "Caladenia major", "Caladenia tentaculata", "Herminium monorchis", - "Ansellia africana", "Coelogyne corymbosa", "Coelogyne fimbriata", "Acianthus exsertus", "Erythrodes blumei", "Corybas taliensis", - "Serapias cordigera", "Serapias lingua", "Serapias vomeracea", "Cleisostoma paniculatum", "Cleisostoma rostratum", - "Cleisostoma simondii var. guangdongense", "Neofinetia falcata", "Caleana major", "Neottia banksiana", "Neottia convallarioides", - "Neottia nidus-avis", "Neottia ovata", "Satyrium yunnanense", "Phaius", "Phaius flavus", "Phaius tancarvilleae", - "Cephalantheropsis obcordata", "Ilex aculeolata", "Ilex asprella", "Ilex centrochinensis", "Ilex cornuta", - "Ilex cornuta 'National'", "Ilex decidua", "Ilex latifolia", "Ilex macrocarpa", "Ilex opaca", "Ilex pubescens", "Ilex rotunda", - "Ilex verticillata", "Ilex vomitoria", "Impatiens arguta", "Impatiens balsamina", "Impatiens blepharosepala", "Impatiens capensis", - "Impatiens chekiangensis", "Impatiens chinensis", "Impatiens commelinoides", "Impatiens hawkeri", "Impatiens hongkongensis", - "Impatiens macrovexilla", "Impatiens niamniamensis", "Impatiens noli-tangere", "Impatiens pallida", "Impatiens platychlaena", - "Impatiens platysepala", "Impatiens tubulosa", "Impatiens walleriana", "Pellaea andromedifolia", "Adiantum aleuticum", - "Adiantum capillus-veneris", "Adiantum nelumboides", "Adiantum pedatum", "Aechmea fulgens", "Ananas comosus", "Cryptanthus acaulis", - "Billbergia pyramidalis", "Tillandsia cyanea", "Tillandsia recurvata", "Tillandsia usneoides", "Rehmannia chingii", - "Rehmannia glutinosa", "Cymbaria mongolica", "Euphrasia pectinata", "Euphrasia regelii", "Melampyrum laxum", "Melampyrum roseum", - "Brandisia hancei", "Phtheirospermum japonicum", "Phtheirospermum tenuisectum", "Castilleja exserta", "Castilleja indivisa", - "Striga asiatica", "Cistanche deserticola", "Conopholis americana", "Boschniakia himalaica", "Aeginetia indica", - "Siphonostegia chinensis", "Siphonostegia laeta", "Pedicularis cheilanthifolia", "Pedicularis chinensis", "Pedicularis cranolopha", - "Pedicularis davidii", "Pedicularis densiflora", "Pedicularis densispica", "Pedicularis kansuensis", "Pedicularis muscicola", - "Pedicularis rhinanthoides subsp. labellata", "Monochasma sheareri", "Portulacaria afra", "Portulacaria afra 'Variegata'", - "Solms-laubachia pulcherrima", "Pegaeophyton scapiflorum", "Iberis amara", "Barbarea orthoceras", "Barbarea vulgaris", - "Descurainia sophia", "Cakile maritima", "Lepidium apetalum", "Lepidium latifolium", "Lepidium virginicum", "Cardamine californica", - "Cardamine concatenata", "Cardamine diphylla", "Cardamine hirsuta", "Cardamine impatiens", "Cardamine leucantha", - "Cardamine lyrata", "Cardamine purpurascens", "Erysimum amurense", "Erysimum capitatum", "Erysimum × cheiri", "Matthiola incana", - "Eruca vesicaria subsp. sativa", "Dontostemon dentatus", "Dontostemon glandulosus", "Dontostemon tibeticus", "Brassica juncea", - "Brassica juncea var. gemmifera", "Brassica juncea var. multicep", "Brassica oleracea", "Brassica oleracea var. acephala", - "Brassica oleracea var. botrytis", "Brassica oleracea var. capitata", "Brassica oleracea var. gemmifera", - "Brassica oleracea var. gongylodes", "Brassica oleracea var. italica", "Brassica rapa var. chinensis", "Brassica rapa var. glabra", - "Brassica rapa var. oleifera", "Capsella bursa-pastoris", "Thlaspi arvense", "Raphanus raphanistrum", "Raphanus sativus", - "Alliaria petiolata", "Rorippa globosa", "Rorippa indica", "Orychophragmus violaceus", "Nasturtium officinale", - "Yinshania fumarioides", "Hesperis matronalis", "Lobularia maritima", "Megacarpaea delavayi", "Duabanga grandiflora", - "Lythrum salicaria", "Lawsonia inermis", "Sonneratia apetala", "Sonneratia caseolaris", "Punica granatum", - "Punica granatum 'Albescens'", "Lagerstroemia fordii", "Lagerstroemia indica", "Lagerstroemia indica f. alba", - "Lagerstroemia limii", "Lagerstroemia speciosa", "Lagerstroemia subcostata", "Rotala rotundifolia", "Trapa natans", - "Cuphea hookeriana", "Cuphea hyssopifolia", "Woodfordia fruticosa", "Heimia myrtifolia", "Celastrus monospermus", - "Celastrus orbiculatus", "Euonymus alatus", "Euonymus carnosus", "Euonymus centidens", "Euonymus cornutus", "Euonymus fortunei", - "Euonymus japonicus", "Euonymus japonicus 'Aurea-marginatus'", "Euonymus laxiflorus", "Euonymus maackii", "Euonymus myrianthus", - "Euonymus nitidus", "Euonymus phellomanus", "Euonymus schensianus", "Euonymus semenovii", "Parnassia wightiana", - "Brexia madagascariensis", "Tripterygium wilfordii", "Selaginella uncinata", "Bretschneidera sinensis", "", "", - "Erythroxylum sinense", "Antidesma bunius", "Antidesma japonicum", "Phyllanthus acidus", "Phyllanthus chekiangensis", - "Phyllanthus emblica", "Phyllanthus flexuosus", "Phyllanthus glaucus", "Phyllanthus hainanensis", "Phyllanthus pulcher", - "Phyllanthus sootepensis", "Phyllanthus urinaria", "Phyllanthus ussuriensis", "Actephila collinsiae", "Baccaurea ramiflora", - "Flueggea suffruticosa", "Bischofia polycarpa", "Glochidion eriocarpum", "Glochidion puberum", "Glochidion wrightii", - "Glochidion zeylanicum", "Aporosa dioica", "Cleistanthus sumatranus", "Breynia disticha", "Breynia fruticosa", "Rotheca myricoides", - "Petraeovitex wolfei", "Paraphlomis javanica", "Paraphlomis javanica var. angustifolia", "Paraphlomis javanica var. coronata", - "Physostegia virginiana", "Holmskioldia sanguinea", "Mesona chinensis", "Perovskia abrotanoides", "Pogostemon auricularius", - "Hanceola exserta", "Lycopus lucidus", "Lycopus lucidus var. hirtus", "Prunella hispida", "Prunella vulgaris", "Lagopsis supina", - "Clerodendrum bungei", "Clerodendrum canescens", "Clerodendrum chinense", "Clerodendrum chinense var. simplex", - "Clerodendrum cyrtophyllum", "Clerodendrum fortunatum", "Clerodendrum inerme", "Clerodendrum japonicum", "Clerodendrum lindleyi", - "Clerodendrum paniculatum", "Clerodendrum quadriloculare", "Clerodendrum serratum", "Clerodendrum speciosum", - "Clerodendrum splendens", "Clerodendrum thomsoniae", "Clerodendrum trichotomum", "Clerodendrum wallichii", "Galeobdolon chinense", - "Anisomeles indica", "Tectona grandis", "Phlomis fruticosa", "Phlomis mongolica", "Marrubium vulgare", "Stachys byzantina", - "Stachys geobombycis", "Stachys japonica", "Stachys oblongifolia", "Glechoma hederacea", "Glechoma longituba", - "Colquhounia seguinii", "Origanum vulgare", "Vitex agnus-castus", "Vitex negundo", "Vitex negundo var. cannabifolia", - "Vitex negundo var. heterophylla", "Vitex rotundifolia", "Vitex trifolia", "Lamiophlomis rotata", "Leonotis leonurus", - "Leonotis nepetifolia", "Leonurus japonicus", "Leonurus sibiricus", "Gmelina asiatica", "Gmelina hainanensis", - "Gmelina philippensis", "Mosla dianthera", "Mosla scabra", "Mosla soochowensis", "Karomia speciosa", "Ajuga ciliata", - "Ajuga decumbens", "Ajuga lupulina", "Ajuga reptans", "Callicarpa americana", "Callicarpa bodinieri&dichotoma", - "Callicarpa cathayana", "Callicarpa formosana", "Callicarpa giraldii", "Callicarpa rubella", "Perilla frutescens", - "Eriophyton wallichii", "Ocimum basilicum", "Monarda citriodora", "Monarda didyma", "Monarda fistulosa", "Monarda punctata", - "Clerodendranthus spicatus", "Nepeta cataria", "Nepeta × faassenii 'Six Hills Giant'", "Caryopteris incana", - "Caryopteris nepetifolia", "Caryopteris × clandonensis", "Mentha canadensis", "Lavandula dentata", "Lavandula stoechas", - "Agastache rugosa", "Premna microphylla", "Moluccella laevis", "Rosmarinus officinalis", "Lamium amplexicaule", "Lamium barbatum", - "Lamium purpureum", "Gomphostemma chinense", "Gomphostemma lucidum", "Dracocephalum heterophyllum", - "Coleus hybridu&scutellarioides", "Clinopodium chinense", "Clinopodium confine", "Clinopodium megalanthum", "Teucrium canadense", - "Teucrium fruticans", "Teucrium viscidum", "Keiskea elsholtzioides", "Isodon adenanthus", "Isodon amethystoides", - "Isodon lophanthoides", "Isodon sculponeatus", "Isodon serra", "Elsholtzia argyi", "Elsholtzia ciliata", "Elsholtzia fruticosa", - "Elsholtzia stauntonii", "Plectranthus ecklonii", "Plectranthus glabratus", "Plectranthus hadiensis var. tomentosus", - "Plectranthus prostratus", "Scutellaria baicalensis", "Scutellaria barbata", "Scutellaria indica", "Scutellaria viscidula", - "Scutellaria wongkei", "Salvia", "Salvia apiana", "Salvia bowleyana", "Salvia chinensis", "Salvia coccinea", "Salvia columbariae", - "Salvia farinacea", "Salvia greggii", "Salvia guaranitica 'Black and Blue'", "Salvia leucantha", "Salvia liguliloba", - "Salvia lyrata", "Salvia mellifera", "Salvia miltiorrhiza", "Salvia nemorosa", "Salvia plebeia", "Salvia pratensis", - "Salvia splendens", "Salvia uliginosa", "Meehania fargesii", "Meehania montis-koyae", "Phytolacca acinosa", "Phytolacca americana", - "Talinum paniculatum", "Marchantia polymorpha", "Rinorea bengalensis", "Viola acuminata", "Viola arcuata", "Viola betonicifolia", - "Viola cornuta", "Viola delavayi", "Viola diffusa", "Viola fargesii", "Viola grypoceras", "Viola inconspicua", "Viola japonica", - "Viola mongolica", "Viola philippica", "Viola sororia", "Viola stewardiana", "Viola tricolor", "Melicytus ramiflorus", - "Notholithocarpus densiflorus", "Lithocarpus corneus", "Lithocarpus glaber", "Lithocarpus hancei", "Quercus acutissima", - "Quercus agrifolia", "Quercus alba", "Quercus aliena", "Quercus kelloggii", "Quercus lobata", "Quercus macrocarpa", - "Quercus palustris", "Quercus phellos", "Quercus robur", "Quercus rubra", "Quercus stellata", "Quercus variabilis", - "Castanea dentata", "Castanea mollissima", "Castanea seguinii", "Fagus grandifolia", "Castanopsis fargesii", "Castanopsis fissa", - "Castanopsis lamontii", "Cyclobalanopsis fleuryi", "Trigonostemon chinensis", "Trigonostemon flavidus", "Triadica cochinchinensis", - "Triadica sebifera", "Codiaeum variegatum", "Codiaeum variegatum 'Excellent'", "Hura crepitans", "Euphorbia antiquorum", - "Euphorbia bicolor", "Euphorbia characias", "Euphorbia cotinifolia", "Euphorbia cyathophora", "Euphorbia dentata", - "Euphorbia helioscopia&esula", "Euphorbia humifusa", "Euphorbia hypericifolia", "Euphorbia kansuensis", "Euphorbia lathyris", - "Euphorbia leucocephala", "Euphorbia maculata", "Euphorbia marginata", "Euphorbia milii", "Euphorbia milii var. alba", - "Euphorbia neorubella", "Euphorbia obesa", "Euphorbia prostrata", "Euphorbia pulcherrima", "Euphorbia resinifera", - "Euphorbia tirucalli", "Euphorbia viguieri", "Sauropus androgynus", "Strophioblachia fimbricalyx", "Alchornea davidii", - "Alchornea trewioides", "Croton capitatus", "Croton setiger", "Croton tiglium", "Plukenetia volubilis", "Manihot esculenta", - "Garcia nutans", "Vernicia fordii&montana", "Excoecaria acerifolia", "Excoecaria agallocha", "Excoecaria cochinchinensis", - "Aleurites moluccana", "Pedilanthus tithymaloides", "Cnidoscolus texanus", "Ricinus communis", "Macaranga tanarius var. tomentosa", - "Mallotus apelta", "Mallotus barbatus", "Mallotus japonicus", "Mallotus paniculatus", "Mallotus philippensis", "Mallotus repandus", - "Mallotus repandus var. chrysocarpus", "Mallotus tenuifolius", "Acalypha australis", "Acalypha hispida", "Acalypha reptans", - "Jatropha curcas", "Jatropha integerrima", "Jatropha podagrica", "Cannabis sativa", "Trema cannabina var. dielsiana", - "Celtis biondii", "Celtis sinensis", "Humulus lupulus", "Humulus scandens", "Pteroceltis tatarinowii", "Caladium bicolor", - "Pinellia cordata", "Pinellia pedatisecta", "Pinellia ternata", "Syngonium podophyllum", "Philodendron erubescens", - "Philodendron selloum", "Pistia stratiotes", "Arisaema bockii", "Arisaema erubescens", "Arisaema heterophyllum", - "Arisaema hunanense", "Arisaema silvestrii", "Arisaema triphyllum", "Aglaonema modestum", "Lysichiton americanus", "Lemna minor", - "Alocasia 'Amazonica'", "Alocasia odora", "Typhonium blumei", "Spathiphyllum kochii", "Symplocarpus foetidus", - "Colocasia antiquorum", "Colocasia esculenta", "Anthurium andraeanum", "Zamioculcas zamiifolia", "Zantedeschia", - "Amorphophallus dunnii", "Amorphophallus kiusianus", "Amorphophallus konjac", "Amorphophallus paeoniifolius", "Epipremnum aureum", - "Dieffenbachia seguine", "Monstera deliciosa", "Yucca gloriosa", "Paradisea liliastrum", "Ruscus aculeatus", "Eucomis comosa", - "Chlorophytum comosum", "Albuca namaquensis", "Hesperocallis undulata", "Asparagus cochinchinensis", "Asparagus densiflorus", - "Asparagus officinalis", "Asparagus setaceus", "Liriope muscari", "Liriope spicata", "Campylandra delavayi", "Thysanotus chinensis", - "Triteleia laxa", "Ornithogalum caudatum", "Ornithogalum divergens", "Ornithogalum dubium", "Ornithogalum narbonense", - "Ornithogalum umbellatum", "Cordyline australis", "Cordyline fruticosa", "Ledebouria socialis", "Ophiopogon bodinieri", - "Ophiopogon chingii", "Ophiopogon japonicus", "Hosta albomarginata", "Hosta plantaginea", "Hosta ventricosa", "Speirantha gardenii", - "Chlorogalum pomeridianum", "Disporopsis aspersa", "Disporopsis fuscopicta", "Disporopsis longifolia", "Disporopsis pernyi", - "Dichopogon strictus", "Camassia leichtlinii", "Camassia quamash", "Camassia scilloides", "Lachenalia viridiflora", - "Barnardia japonica", "Maianthemum bifolium", "Maianthemum canadense", "Maianthemum henryi", "Maianthemum japonicum", - "Maianthemum racemosum", "Maianthemum stellatum", "Muscari botryoides", "Dichelostemma capitatum", "Scilla bifolia", - "Scilla luciliae", "Scilla siberica", "Scilla verna", "Hyacinthoides hispanica", "Hyacinthoides non-scripta", - "Sansevieria gracilis", "Sansevieria trifasciata", "Sansevieria trifasciata var. laurentii", "Puschkinia scilloides", - "Aspidistra fimbriata", "Aspidistra grandiflora", "Aspidistra minutiflora", "Hesperoyucca whipplei", "Beaucarnea recurvata", - "Convallaria majalis", "Hyacinthus orientalis", "Polygonatum cyrtonema", "Polygonatum filipes", "Polygonatum hookeri", - "Polygonatum odoratum", "Polygonatum sibiricum", "Polygonatum verticillatum", "Agave americana", "Dracaena cambodiana", - "Dracaena draco", "Dracaena fragrans", "Dracaena reflexa", "Dracaena sanderiana", "Dracaena surculosa var. maculata", - "Wrightia laevis", "Wrightia pubescens", "Wrightia religiosa", "Carissa macrocarpa", "Pseudolithos migiurtinus", - "Gymnema sylvestre", "Dregea sinensis", "Dregea volubilis", "Dregea yunnanensis", "Ceropegia trichantha", "Ceropegia woodii", - "Parsonsia alboflavescens", "Telosma cordata", "Graphistemma pictum", "Nerium oleander", "Nerium oleander 'Paihua'", - "Tylophora ovata", "Tylophora silvestris", "Melodinus suaveolens", "Tabernaemontana divaricata", "Periploca sepium", - "Cryptostegia grandiflora", "Pachypodium lamerei", "Urceola rosea", "Amsonia tabernaemontana", "Adenium obesum", "Cerbera manghas", - "Beaumontia brevituba", "Beaumontia grandiflora", "Calotropis gigantea", "Stapelia", "Hoya carnosa", "Hoya multiflora", - "Cryptolepis buchananii", "Dischidia chinensis", "Dischidia ruscifolia&nummularia", "Pentasachme caudatum", "Vallaris indecora", - "Trachelospermum axillare", "Trachelospermum jasminoides", "Trachelospermum jasminoides 'Flame'", "Apocynum androsaemifolium", - "Apocynum venetum", "Strophanthus divaricatus", "Strophanthus gratus", "Stephanotis floribunda", "Metaplexis japonica", - "Vinca major", "Vinca major 'Variegata'", "Vinca minor", "Kopsia arborea", "Kopsia fruticosa", "Heterostemma brownii", - "Gomphocarpus fruticosus", "Gomphocarpus physocarpus", "Catharanthus roseus", "Catharanthus roseus 'Albus'", "Mandevilla sanderi", - "Asclepias asperula", "Asclepias curassavica", "Asclepias curassavica 'Flaviflora'", "Asclepias fascicularis", - "Asclepias incarnata", "Asclepias oenotheroides", "Asclepias speciosa", "Asclepias syriaca", "Asclepias tuberosa", - "Asclepias verticillata", "Asclepias viridiflora", "Asclepias viridis", "Merrillanthus hainanensis", "Anodendron affine", - "Plumeria obtusa", "Plumeria pudica", "Plumeria rubra", "Plumeria rubra 'Acutifolia'", "Alstonia scholaris", - "Cynanchum acuminatifolium", "Cynanchum atratum", "Cynanchum auriculatum", "Cynanchum chekiangense", "Cynanchum chinense", - "Cynanchum corymbosum", "Cynanchum stauntonii", "Cynanchum thesioides", "Chonemorpha eriostylis", "Thevetia peruviana", - "Thevetia peruviana 'Aurantiaca'", "Allamanda blanchetii", "Allamanda schottii&cathartica", "Jasminanthes mucronata", - "Zingiber cochleariforme", "Zingiber mioga", "Zingiber officinale", "Zingiber striolatum", "Zingiber zerumbet", - "Hedychium coccineum", "Hedychium coronarium", "Hedychium flavescens", "Hedychium flavum", "Hedychium yunnanense", - "Curcuma alismatifolia", "Curcuma longa", "Curcuma phaeocaulis", "Curcuma wenyujin", "Alpinia hainanensis", "Alpinia japonica", - "Alpinia oblongifolia", "Alpinia officinarum", "Alpinia zerumbet", "Alpinia zerumbet 'Variegata'", "Kaempferia elegans", - "Kaempferia galanga", "Kaempferia rotunda", "Globba schomburgkii", "Etlingera elatior", "Amomum tsaoko", "Amomum villosum", - "Roscoea schneideriana", "Cheilocostus speciosus", "Styrax chinensis", "Styrax confusus", "Styrax faberi", "Styrax japonicus", - "Styrax odoratissimus", "Styrax suberifolius", "Huodendron biaristatum var. parviflorum", "Rehderodendron kwangtungense", - "Pterostyrax corymbosus", "Sinojackia xylocarpa", "Alniphyllum fortunei", "Halesia macgregorii", "Melliodendron xylocarpum", - "Myriophyllum aquaticum", "Myriophyllum verticillatum", "Podophyllum peltatum", "Mahonia bealei", "Mahonia fortunei", - "Mahonia oiwakensis", "Mahonia shenii", "Nandina domestica", "Berberis diaphana", "Berberis jamesiana", "Berberis julianae", - "Berberis lempergiana", "Berberis pruinosa", "Berberis thunbergii", "Berberis thunbergii 'Atropurpurea'", "Berberis trifoliolata", - "Berberis vulgaris", "Berberis wilsoniae", "Diphylleia grayi", "Sinopodophyllum hexandrum", "Epimedium brevicornu", - "Epimedium davidii", "Epimedium sagittatum", "Epimedium wushanense", "Gymnospermium kiangnanense", "Dysosma pleiantha", - "Dysosma versipellis", "Microdesmis caseariifolia", "Capparis acutifolia", "Capparis bodinieri", "Crateva formosensis", - "Crateva religiosa", "Crateva unilocularis", "Pouteria caimito", "Pouteria campechiana", "Synsepalum dulcificum", - "Madhuca pasquieri", "Chrysophyllum cainito", "Sinosideroxylon wightianum", "Manilkara zapota", "Mimusops elengi", - "Symplocos cochinchinensis", "Symplocos congesta", "Symplocos lancifolia", "Symplocos lucida", "Symplocos paniculata", - "Symplocos stellaris", "Symplocos sumuntia", "Alangium chinense", "Alangium kurzii", "Alangium platanifolium", - "Alangium salviifolium", "Cornus alba", "Cornus canadensis", "Cornus capitata", "Cornus controversa", "Cornus drummondii", - "Cornus florida", "Cornus hongkongensis", "Cornus hongkongensis subsp. elegans", "Cornus kousa subsp. chinensis", "Cornus mas", - "Cornus officinalis", "Cornus quinquenervis", "Cornus sanguinea", "Cornus sericea", "Polyspora axillaris", "Camellia amplexicaulis", - "Camellia azalea", "Camellia chekiangoleosa", "Camellia crapnelliana", "Camellia cuspidata", "Camellia grijsii", - "Camellia japonica", "Camellia oleifera", "Camellia petelotii", "Camellia pitardii", "Camellia salicifolia", "Camellia saluenensis", - "Camellia sasanqua", "Camellia sinensis", "Camellia sinensis var. assamica", "Camellia uraku", "Camellia yunnanensis", - "Schima superba", "Pyrenaria microcarpa", "Pyrenaria spectabilis", "Stewartia sinensis", "Helicia reticulata", "Protea cynaroides", - "Buckinghamia celsissima", "Macadamia integrifolia", "Leucospermum nutans", "Grevillea banksii", "Diapensia purpurea", - "Heptacodium miconioides", "Zabelia biflora", "Zabelia dielsii", "Acanthocalyx alba", "Linnaea borealis", "Dipsacus asper", - "Dipsacus fullonum", "Lonicera caerulea", "Lonicera chrysantha", "Lonicera elisae", "Lonicera ferdinandi", - "Lonicera fragrantissima", "Lonicera fragrantissima var. lancifolia", "Lonicera hispida", "Lonicera hispidula", - "Lonicera involucrata", "Lonicera japonica", "Lonicera japonica var. chinensis", "Lonicera korolkowi", "Lonicera maackii", - "Lonicera modesta", "Lonicera praeflorens", "Lonicera sempervirens", "Lonicera tangutica", "Lonicera tatarica", - "Lonicera tatarica 'Lutea'", "Lonicera trichosantha", "Symphoricarpos albus", "Symphoricarpos orbiculatus", "Kolkwitzia amabilis", - "Abelia chinensis", "Abelia macrotera", "Abelia uniflora", "Abelia × grandiflora", "Triosteum himalayanum", "Scabiosa atropurpurea", - "Scabiosa comosa", "Patrinia monandra", "Patrinia villosa", "Centranthus ruber", "Weigela coraeensis", "Weigela florida", - "Weigela florida 'Red Prince'", "Weigela florida 'Variegata'", "Weigela japonica var. sinica", "Leycesteria formosa", "Platanus", - "Platanus occidentalis", "Platanus racemosa", "Penthorum chinense", "Trientalis borealis", "Trientalis europaea", - "Trientalis latifolia", "Cyclamen persicum", "Stimpsonia chamaedryoides", "Primula acaulis", "Primula agleniana", - "Primula beesiana", "Primula bella", "Primula blinii", "Primula chionantha", "Primula cicutariifolia", "Primula denticulata", - "Primula denticulata subsp. sinodenticulata", "Primula dryadifolia subsp. jonardunii", "Primula hendersonii", - "Primula maximowiczii", "Primula nutans", "Primula obconica", "Primula palmata", "Primula pelargoniifolia", "Primula pinnatifida", - "Primula poissonii", "Primula polyneura", "Primula pseudodenticulata", "Primula pulverulenta", "Primula saxatilis", - "Primula secundiflora", "Primula sikkimensis", "Primula sinolisteri", "Primula sonchifolia", "Primula stenocalyx", - "Primula tangutica", "Primula valentiniana", "Primula vialii", "Maesa montana", "Maesa perlarius", "Androsace henryi", - "Androsace mariae", "Androsace rigida", "Androsace spinulifera", "Androsace umbellata", "Androsace wardii", - "Androsace yargongensis", "Omphalogramma vinciflorum", "Lysimachia alfredii", "Lysimachia barystachys", "Lysimachia candida", - "Lysimachia christiniae", "Lysimachia ciliata", "Lysimachia clethroides", "Lysimachia congestiflora", "Lysimachia fortunei", - "Lysimachia grammica", "Lysimachia hemsleyana", "Lysimachia heterogenea", "Lysimachia klattiana", "Lysimachia longipes", - "Lysimachia nanpingensis", "Lysimachia nummularia", "Lysimachia nummularia 'Aurea'", "Lysimachia paridiformis var. stenophylla", - "Lysimachia patungensis", "Lysimachia pseudohenryi", "Lysimachia pumila", "Lysimachia punctata", "Anagallis arvensis", - "Anagallis arvensis f. coerulea", "Ardisia crenata", "Ardisia crispa", "Ardisia elliptica", "Ardisia humilis", "Ardisia japonica", - "Ardisia mamillata", "Ardisia obtusa", "Ardisia villosa", "Aegiceras corniculatum", "Embelia parviflora", "Embelia ribes", - "Myrsine africana", "Myrsine seguinii", "Muntingia calabura", "Erycibe expansa", "Evolvulus alsinoides", "Evolvulus nuttallianus", - "Xenostegia tridentata", "Calystegia hederacea", "Calystegia sepium", "Calystegia soldanella", "Convolvulus arvensis", - "Convolvulus tragacanthoides", "Pharbitis limbata", "Operculina turpethum", "Quamoclit coccinea", "Quamoclit pennata", - "Quamoclit × sloteri", "Cuscuta japonica", "Ipomoea alba", "Ipomoea aquatica", "Ipomoea batatas", "Ipomoea biflora", - "Ipomoea cairica", "Ipomoea carnea subsp. fistulosa", "Ipomoea cordatotriloba", "Ipomoea indica", "Ipomoea lacunosa", - "Ipomoea nil&purpurea", "Ipomoea obscura", "Ipomoea pes-caprae", "Ipomoea triloba", "Dinetus racemosus", "Dichondra micrantha", - "Merremia dissecta", "Merremia hederacea", "Merremia sibirica", "Merremia tuberosa", "Merremia vitifolia", "Stachyurus chinensis", - "Stachyurus himalaicus", "Aesculus californica", "Aesculus chinensis", "Aesculus glabra", "Aesculus hippocastanum", - "Aesculus pavia", "Cardiospermum grandiflorum", "Cardiospermum halicacabum", "Blighia sapida", "Xanthoceras sorbifolium", - "Sapindus saponaria", "Koelreuteria bipinnata", "Koelreuteria paniculata", "Acer buergerianum", "Acer cordatum", "Acer davidii", - "Acer fabri", "Acer henryi", "Acer macrophyllum", "Acer negundo", "Acer palmatum", "Acer pensylvanicum", "Acer platanoides", - "Acer pubinerve", "Acer rubrum", "Acer saccharinum", "Acer saccharum", "Acer spicatum", "Acer tataricum subsp. ginnala", - "Acer tataricum subsp. theiferum", "Acer truncatum", "Arytera littoralis", "Delavaya toxocarpa", "Litchi chinensis", - "Dodonaea viscosa", "Nephelium lappaceum", "Dimocarpus longan", "Tropaeolum majus", "Kingdonia uniflora", "Kalanchoe beauverdii", - "Kalanchoe blossfeldiana", "Kalanchoe delagoensis", "Kalanchoe fedtschenkoi", "Kalanchoe marmorata", "Kalanchoe porphyrocalyx", - "Kalanchoe tomentosa", "Hylotelephium spectabile", "Hylotelephium tatarinowii", "× Pachyveria 'Powder Puff'", - "Adromischus cristatus var. clavifolius", "Greenovia", "Sedum acre 'Aurea'", "Sedum alfredii", "Sedum bulbiferum", - "Sedum drymarioides", "Sedum emarginatum", "Sedum lineare", "Sedum sarmentosum", "Sedum sediforme", "Sedum spathulifolium", - "Orostachys fimbriata", "Orostachys malacophylla", "Echeveria 'Neon Breakers'", "Echeveria 'Perle von Nurnberg'", - "Echeveria lilacina", "Echeveria pulidonis", "Echeveria pulvinata", "Echeveria runyonii 'Topsy Turvy'", "Rhodiola rosea", - "Rhodiola yunnanensis", "Aeonium 'Zwartkop'", "Bryophyllum pinnatum", "Phedimus aizoon", "Cotyledon tomentosa", - "Sempervivum arachnoideum subsp. tomentosum", "Crassula arborescens", "Crassula corymbulosa", "Crassula marnieriana", - "Crassula obliqua 'Gollum'", "Graptopetalum amethystinum", "Magnolia grandiflora", "Magnolia tripetala", "Houpoea officinalis", - "Michelia champaca", "Michelia chapensis", "Michelia crassipes", "Michelia figo", "Michelia foveolata", "Michelia guangdongensis", - "Michelia macclurei", "Michelia maudiae", "Michelia skinneriana", "Michelia × alba", "Oyama sieboldii", "Manglietia fordiana", - "Manglietia insignis", "Woonyoungia septentrionalis", "Yulania biondii", "Yulania denudata", "Yulania denudata 'Fei Huang'", - "Yulania liliiflora", "Yulania stellata", "Yulania zenii", "Yulania × soulangeana", "Talauma hodgsonii", "Lirianthe championii", - "Lirianthe coco", "Liriodendron chinense", "Liriodendron tulipifera", "Syringa", "Syringa meyeri", "Syringa oblata", - "Syringa pubescens", "Syringa reticulata subsp. amurensis", "Syringa reticulata subsp. pekinensis", - "Syringa reticulata subsp. pekinensis 'Jinyuan'", "Syringa vulgaris", "Syringa yunnanensis", "Syringa × persica", - "Ligustrum lucidum", "Ligustrum obtusifolium", "Ligustrum quihoui", "Ligustrum sinense", "Ligustrum × vicaryi", "Osmanthus armatus", - "Osmanthus fragrans", "Olea europaea", "Fraxinus chinensis", "Fraxinus pennsylvanica", "Fraxinus sieboldiana", - "Chionanthus retusus", "Jasminum elongatum", "Jasminum floridum", "Jasminum grandiflorum", "Jasminum humile", - "Jasminum lanceolaria", "Jasminum mesnyi", "Jasminum multiflorum", "Jasminum nervosum", "Jasminum nudiflorum", - "Jasminum odoratissimum", "Jasminum officinale", "Jasminum pentaneurum", "Jasminum polyanthum", "Jasminum sambac", - "Jasminum sinense", "Jasminum subhumile", "Forsythia suspensa", "Forsythia viridissima", "Fontanesia phillyreoides subsp. fortunei", - "", "Equisetum arvense", "Equisetum hyemale", "Equisetum ramosissimum", "Equisetum ramosissimum subsp. debile", "Akebia quinata", - "Akebia trifoliata", "Stauntonia chinensis", "Stauntonia obovatifoliola subsp. urophylla", "Eucommia ulmoides", - "Elaeocarpus apiculatus", "Elaeocarpus decipiens", "Elaeocarpus glabripetalus", "Elaeocarpus hainanensis", "Elaeocarpus serratus", - "Sloanea sinensis", "Monotropastrum humile", "Enkianthus campanulatus", "Enkianthus chinensis", "Enkianthus deflexus", - "Enkianthus quinqueflorus", "Enkianthus serrulatus", "Chimaphila maculata", "Kalmia latifolia", "Cassiope selaginoides", - "Diplarche multiflora", "Rhododendron argyrophyllum", "Rhododendron bachii", "Rhododendron campylogynum", "Rhododendron capitatum", - "Rhododendron championiae", "Rhododendron charitopes subsp. tsangpoense", "Rhododendron florulentum", "Rhododendron hongkongense", - "Rhododendron kwangtungense", "Rhododendron latoucheae", "Rhododendron lepidotum", "Rhododendron maculiferum subsp. anwheiense", - "Rhododendron micranthum", "Rhododendron molle", "Rhododendron mucronatum", "Rhododendron oreodoxa", "Rhododendron ovatum", - "Rhododendron rhuyuenense", "Rhododendron rivulare", "Rhododendron seniavinii", "Rhododendron simiarum", "Rhododendron simsii", - "Rhododendron stamineum", "Rhododendron yunnanense", "Rhododendron × pulchrum", "Pterospora andromedea", "Agapetes burmanica", - "Agapetes lacei", "Monotropa hypopitys", "Monotropa uniflora", "Moneses uniflora", "Lyonia ovalifolia var. hebecarpa", - "Gaultheria procumbens", "Gaultheria shallon", "Gaultheria trichophylla", "Arbutus menziesii", "Sarcodes sanguinea", - "Vaccinium bracteatum", "Vaccinium corymbosum", "Vaccinium macrocarpon", "Vaccinium mandarinorum", "Vaccinium ovatum", - "Vaccinium uliginosum", "Pieris formosa", "Pieris japonica", "Pyrola calliantha", "Homalium ceylanicum", "Homalium cochinchinense", - "Idesia polycarpa", "Populus alba", "Populus deltoides", "Populus simonii var. przewalskii", "Salix", "Salix integra", - "Salix integra 'Hakuro Nishiki'", "Salix wallichiana", "Itoa orientalis", "Casearia velutina", "Myrica rubra", "Picea abies", - "Picea likiangensis var. linzhiensis", "Abies balsamea", "Pinus densiflora", "Pinus massoniana", "Pinus palustris", - "Pinus parviflora", "Pinus ponderosa", "Pinus strobus", "Pinus sylvestris", "Pinus taeda", "Larix gmelinii", "Larix kaempferi", - "Pseudolarix amabilis", "Tsuga canadensis", "Pseudotsuga menziesii", "Platycladus orientalis", "Juniperus chinensis", - "Juniperus communis", "Juniperus virginiana", "Sequoia sempervirens", "Thuja occidentalis", "Taxodium distichum", - "Taxodium mucronatum", "Ludwigia adscendens", "Ludwigia octovalvis", "Ludwigia peploides subsp. stipulacea", "Ludwigia sedioides", - "Clarkia amoena", "Clarkia unguiculata", "Fuchsia hybrida", "Gaura lindheimeri", "Gaura parviflora", - "Oenothera biennis&glazioviana", "Oenothera drummondii", "Oenothera laciniata", "Oenothera macrocarpa", "Oenothera rosea", - "Oenothera speciosa", "Oenothera tetraptera", "Chamerion angustifolium", "Epilobium canum", "Epilobium hirsutum", - "Epilobium pyrricholophum", "Circaea cordata", "Tamarix chinensis", "Tamarix ramosissima", "Myricaria squamosa", "Diospyros armata", - "Diospyros cathayensis", "Diospyros japonica", "Diospyros kaki", "Diospyros lotus", "Diospyros nitida", "Diospyros rhombifolia", - "Diospyros vaccinioides", "Diospyros virginiana", "Corymbia ptychocarpa", "Plinia cauliflora", "Rhodomyrtus tomentosa", - "Eucalyptus cinerea", "Eugenia brasiliensis", "Eugenia uniflora", "Psidium guajava", "Melaleuca cajuputi subsp. cumingiana", - "Callistemon citrinus", "Callistemon rigidus", "Syzygium acuminatissimum", "Syzygium australe", "Syzygium cumini", - "Syzygium fluviatile", "Syzygium grijsii", "Syzygium jambos", "Syzygium malaccense", "Syzygium rehderianum", - "Syzygium samarangense", "Acca sellowiana", "Xanthostemon chrysanthus", "Chamelaucium uncinatum", "Myrtus communis", - "Leptospermum scoparium", "Dendrophthoe pentandra", "Scurrula parasitica", "Taxillus chinensis", "Broussonetia kaempferi", - "Broussonetia kaempferi var. australis", "Broussonetia kazinoki", "Broussonetia papyrifera", "Morus alba", "Morus australis", - "Ficus altissima", "Ficus auriculata", "Ficus carica", "Ficus deltoidea", "Ficus elastica", "Ficus erecta", - "Ficus gasparriniana var. laceratifolia", "Ficus hispida", "Ficus pandurata", "Ficus pumila", "Ficus racemosa", "Ficus religiosa", - "Ficus subpisocarpa", "Ficus vaccinioides", "Ficus virens", "Maclura cochinchinensis", "Maclura pomifera", "Maclura tricuspidata", - "Artocarpus communis", "Artocarpus heterophyllus", "Artocarpus hypargyreus", "Dorstenia elata", "Codonopsis lanceolata", - "Codonopsis subglobosa", "Lobelia cardinalis", "Lobelia chinensis", "Lobelia davidii", "Lobelia erinus", "Lobelia melliana", - "Lobelia nummularia", "Lobelia sessilifolia", "Lobelia siphilitica", "Lobelia zeylanica", "Triodanis perfoliata", - "Triodanis perfoliata subsp. biflora", "Platycodon grandiflorus", "Adenophora himalayana", "Adenophora petiolata subsp. hunanensis", - "Adenophora polyantha", "Adenophora potaninii", "Adenophora stricta", "Adenophora trachelioides", "Wahlenbergia marginata", - "Cyananthus formosus", "Cyananthus incanus", "Cyananthus macrocalyx", "Cyclocodon lancifolius", "Campanumoea javanica", - "Lithotoma axillaris", "Campanula", "Campanula glomerata subsp. speciosa", "Campanula punctata", "Campanula rotundifolia", - "Hippobroma longiflora", "Clethra barbinervis", "Clethra delavayi", "Alnus trabeculosa", "Corylus avellana", "Ostrya rehderiana", - "Washingtonia filifera", "Washingtonia robusta", "Chrysalidocarpus lutescens", "Trachycarpus fortunei", "Cocos nucifera", - "Areca catechu", "Phoenix sylvestris", "Wodyetia bifurcata", "Calamus thysanolepis", "Sabal minor", "Livistona chinensis", - "Salacca edulis", "Caryota maxima", "Aphanamixis polystachya", "Swietenia macrophylla", "Melia azedarach", "Aglaia odorata", - "Toona sinensis", "Heynea trijuga", "Chukrasia tabularis", "Ulmus americana", "Ulmus pumila", "Salvinia molesta", - "Azolla pinnata subsp. asiatica", "Umbellularia californica", "Lindera aggregata", "Lindera benzoin", "Lindera communis", - "Lindera megaphylla", "Laurus nobilis", "Litsea cubeba", "Litsea glutinosa", "Phoebe bournei", "Phoebe chekiangensis", - "Phoebe sheareri", "Cinnamomum burmannii", "Cinnamomum camphora", "Cinnamomum cassia", "Cinnamomum japonicum", - "Cinnamomum kotoense", "Sassafras albidum", "Sassafras tzumu", "Machilus grijsii", "Machilus leptophylla", "Machilus thunbergii", - "Machilus velutina", "Persea americana", "Canarium album", "Torenia concolor", "Torenia fournieri", "Torenia violacea", - "Lindernia anagallis", "Lindernia crustacea", "Lindernia ruellioides", "Aconitum barbatum var. puberulum", "Aconitum coreanum", - "Aconitum gymnandrum", "Aconitum hemsleyanum", "Aconitum kusnezoffii", "Aconitum tanguticum", "Dichocarpum dalzielii", "Adonis", - "Thalictrum acutifolium", "Thalictrum aquilegiifolium var. sibiricum", "Thalictrum delavayi", "Thalictrum fargesii", - "Thalictrum fortunei", "Thalictrum ichangense", "Thalictrum petaloideum", "Thalictrum thalictroides", "Semiaquilegia adoxoides", - "Paraquilegia microphylla", "Ficaria verna", "Ranunculus asiaticus", "Ranunculus cantoniensis", "Ranunculus muricatus", - "Ranunculus repens", "Ranunculus sceleratus", "Ranunculus ternatus", "Batrachium bungei", "Batrachium pekinense", - "Pulsatilla chinensis", "Actaea erythrocarpa", "Actaea pachypoda", "Actaea rubra", "Anemoclema glaucifolium", - "Delphinium anthriscifolium", "Delphinium anthriscifolium var. majus", "Delphinium anthriscifolium var. savatieri", - "Delphinium elatum", "Delphinium grandiflorum", "Aquilegia canadensis", "Aquilegia chrysantha", "Aquilegia ecalcarata", - "Aquilegia formosa", "Aquilegia oxysepala", "Aquilegia oxysepala var. oxysepala f. pallidiflora", "Aquilegia viridiflora", - "Aquilegia viridiflora var. atropurpurea", "Aquilegia vulgaris", "Aquilegia yabeana", "Anemonopsis macrophylla", - "Trollius chinensis", "Trollius yunnanensis", "Helleborus thibetanus", "Clematis 'Rooguchi'", "Clematis acerifolia", - "Clematis apiifolia", "Clematis apiifolia var. argentilucida", "Clematis armandii", "Clematis brevicaudata", "Clematis chinensis", - "Clematis chrysocoma", "Clematis courtoisii", "Clematis crassifolia", "Clematis finetiana", "Clematis florida", - "Clematis fruticosa", "Clematis fusca var. violacea", "Clematis henryi", "Clematis heracleifolia", "Clematis hexapetala", - "Clematis integrifolia", "Clematis lasiandra", "Clematis macropetala", "Clematis meyeniana", "Clematis montana", - "Clematis montana var. sterilis", "Clematis nannophylla", "Clematis peterae", "Clematis potaninii", "Clematis pseudootophora", - "Clematis pseudopogonandra", "Clematis ranunculoides", "Clematis rehderiana", "Clematis repens", "Clematis sibirica", - "Clematis sibirica var. ochotensis", "Clematis tangutica", "Clematis terniflora", "Clematis terniflora var. mandshurica", - "Clematis uncinata", "Clematis virginiana", "Anemone acutiloba", "Anemone americana", "Anemone coronaria", "Anemone demissa", - "Anemone flaccida", "Anemone geum subsp. ovalifolia", "Anemone hupehensis", "Anemone obtusiloba", "Anemone rivularis", - "Anemone rivularis var. flore-minore", "Anemone rupicola", "Anemone tomentosa&vitifolia", "Consolida ajacis", "Caltha palustris", - "Caltha sinogracilis", "Oxygraphis glacialis", "Souliea vaginata", "Nigella damascena", "Claytonia caroliniana", - "Claytonia perfoliata", "Claytonia virginica", "Lewisia cotyledon", "Burmannia disticha", "Burmannia itoana", - "Burmannia nepalensis", "Egeria densa", "Ottelia acuminata", "Ottelia acuminata var. crispa", "Ottelia alismoides", - "Hydrocharis dubia", "Polypodium virginianum", "Microsorum pustulatum", "Platycerium bifurcatum", "Platycerium wallichii", - "Aletris scopulorum", "Aletris spicata", "Paulownia", "Paulownia tomentosa", "Sagittaria latifolia", "Sagittaria montevidensis", - "Sagittaria pygmaea", "Sagittaria sagittifolia", "Sagittaria trifolia", "Hydrocleys nymphoides", "Alisma canaliculatum", - "Alisma plantago-aquatica", "Echinodorus grisebachii", "Limnocharis flava", "Pittosporum illicioides", "Pittosporum tobira", - "Lygodium japonicum", "Meliosma flexuosa", "Meliosma rigida", "Meliosma rigida var. pannosa", "Meliosma squamulata", - "Sabia campanulata subsp. ritchieae", "Sabia discolor", "Sabia japonica", "Sabia limoniacea", "Sabia swinhoei", "Malosma laurina", - "Choerospondias axillaris", "Mangifera indica", "Toxicodendron diversilobum", "Toxicodendron radicans", "Toxicodendron succedaneum", - "Rhus aromatica", "Rhus chinensis", "Rhus glabra", "Rhus integrifolia", "Rhus ovata", "Rhus typhina", "Anacardium occidentale", - "Cotinus coggygria", "Pistacia vera", "Juncus allioides", "Juncus effusus", "Juncus prismatocarpus", "Barleria cristata", - "Barleria lupulina", "Asystasia gangetica", "Asystasia gangetica subsp. micrantha", "Asystasia neesiana", - "Crossandra infundibuliformis", "Aphelandra sinclairiana", "Aphelandra squarrosa", "Eranthemum pulchellum", "Rungia densiflora", - "Pseuderanthemum carruthersii", "Pseuderanthemum crenulatum", "Pseuderanthemum laxiflorum", - "Pseuderanthemum reticulatum var. ovarifolium", "Thunbergia alata", "Thunbergia coccinea", "Thunbergia erecta", - "Thunbergia fragrans", "Thunbergia grandiflora", "Thunbergia laurifolia", "Thunbergia mysorensis", "Hygrophila ringens", - "Rhinacanthus nasutus", "Justicia adhatoda", "Justicia austrosinensis", "Justicia betonica", "Justicia brandegeeana", - "Justicia brasiliana", "Justicia procumbens", "Justicia quadrifaria", "Dicliptera chinensis", "Cyrtanthera carnea", - "Andrographis paniculata", "Fittonia albivenis", "Acanthus ilicifolius", "Acanthus mollis", "Perilepta dyeriana", "Ruellia elegans", - "Ruellia simplex", "Ruellia venusta", "Peristrophe hyssopifolia 'Aureo-variegata'", "Peristrophe japonica", - "Megaskepasma erythrochlamys", "Brillantaisia owariensis", "Pachystachys lutea", "Codonacanthus pauciflorus", - "Strobilanthes aprica", "Strobilanthes cusia", "Strobilanthes dimorphotricha", "Strobilanthes hamiltoniana", - "Strobilanthes sarcorrhiza", "Strobilanthes schomburgkii", "Strobilanthes tetrasperma", "Clinacanthus nutans", - "Cystacanthus pyramidalis", "Odontonema strictum", "Sanchezia speciosa", "Rourea microphylla", "Pelargonium graveolens", - "Pelargonium hortorum", "Pelargonium peltatum", "Pelargonium zonale", "Erodium cicutarium", "Erodium stephanianum", - "Geranium carolinianum", "Geranium maculatum", "Geranium nepalense", "Geranium pratense", "Geranium pylzowianum", - "Geranium refractum", "Geranium robertianum", "Geranium sibiricum", "Geranium sinense", "Geranium wilfordii", - "Geranium wlassovianum", "Pinguicula alpina", "Utricularia aurea", "Utricularia australis", "Utricularia bifida", - "Utricularia caerulea", "Utricularia striatula", "Utricularia warburgii", "Saurauia tristyla", "Actinidia arguta", - "Actinidia callosa var. discolor", "Actinidia chinensis", "Actinidia eriantha", "Actinidia lanceolata", "Actinidia latifolia", - "Actinidia macrosperma", "Actinidia rubricaulis var. coriacea", "Nepenthes mirabilis", "Diascia barberae", "Verbascum blattaria", - "Verbascum thapsus", "Scrophularia californica", "Scrophularia ningpoensis", "Leucophyllum frutescens", "Buddleja asiatica", - "Buddleja davidii", "Buddleja fallowiana", "Buddleja lindleyana", "Buddleja officinalis", "Nemesia strumosa", - "Couroupita guianensis", "Barringtonia acutangula", "Barringtonia asiatica", "Barringtonia racemosa", "Onoclea sensibilis", - "Matteuccia struthiopteris", "Aquilaria sinensis", "Stellera chamaejasme", "Daphne aurantiaca", "Daphne championii", - "Daphne genkwa", "Daphne giraldii", "Daphne kiusiana var. atrocaulis", "Daphne longilobata", "Daphne odora", "Daphne papyracea", - "Daphne tangutica", "Edgeworthia chrysantha", "Wikstroemia indica", "Wikstroemia monnula", "Wikstroemia nutans", - "Wikstroemia pilosa", "Sarracenia purpurea", "Eriodictyon californicum", "Hydrophyllum virginianum", "Philydrum lanuginosum", - "Carica papaya", "Mesembryanthemum cordifolium", "Mesembryanthemum crystallinum", "Lampranthus spectabilis", "Carpobrotus edulis", - "Lithops pseudotruncatella subsp. archerae", "Fenestraria aurantiaca", "Glottiphyllum longum", "Rhombophyllum nelii", - "Astridia velutina", "Cananga odorata", "Cananga odorata var. fruticosa", "Desmos chinensis", "Asimina triloba", "Polyalthia laui", - "Polyalthia longifolia", "Polyalthia suberosa", "Fissistigma oldhamii", "Fissistigma polyanthum", "Annona glabra", "Annona montana", - "Annona muricata", "Annona squamosa", "Uvaria boniana", "Uvaria grandiflora", "Uvaria macrophylla", "Uvaria tonkinensis", - "Chieniodendron hainanense", "Mitrephora tomentosa", "Artabotrys hainanensis", "Artabotrys hexapetalus", "Artabotrys hongkongensis", - "Peganum harmala", "Armeria maritima", "Plumbago auriculata", "Plumbago indica", "Plumbago zeylanica", "Limonium bicolor", - "Limonium sinense", "Limonium tenellum", "Peritoma arborea", "Tarenaya hassleriana", "Clintonia borealis", "Calochortus albus", - "Calochortus amabilis", "Calochortus leichtlinii", "Calochortus luteus", "Calochortus plummerae", "Calochortus pulchellus", - "Calochortus splendens", "Calochortus tolmiei", "Calochortus venustus", "Notholirion bulbuliferum", "Cardiocrinum cathayanum", - "Cardiocrinum giganteum", "Cardiocrinum giganteum var. yunnanense", "Medeola virginiana", "Streptopus simplex", - "Tricyrtis formosana", "Tricyrtis macropoda", "Tricyrtis pilosa", "Erythronium albidum", "Erythronium americanum", - "Erythronium grandiflorum", "Erythronium japonicum", "Erythronium oregonum", "Erythronium sibiricum", - "Lilium bakerianum var. rubrum", "Lilium brownii", "Lilium canadense", "Lilium columbianum", "Lilium concolor", - "Lilium concolor var. pulchellum", "Lilium dauricum", "Lilium davidii", "Lilium davidii var. unicolor", "Lilium distichum", - "Lilium duchartrei", "Lilium lankongense", "Lilium longiflorum", "Lilium lophophorum", "Lilium martagon", - "Lilium nanum var. flavidum", "Lilium pardalinum", "Lilium parvum", "Lilium philadelphicum", "Lilium primulinum var. ochraceum", - "Lilium pumilum", "Lilium regale", "Lilium rosthornii", "Lilium souliei", "Lilium speciosum var. gloriosoides", "Lilium taliense", - "Lilium tigrinum", "Amana edulis", "Nomocharis aperta", "Nomocharis pardanthina", "Fritillaria affinis", - "Fritillaria camschatcensis", "Fritillaria imperialis", "Fritillaria maximowiczii", "Fritillaria meleagris", "Fritillaria persica", - "Fritillaria thunbergii", "Fritillaria ussuriensis", "Tulipa gesneriana", "Tulipa iliensis", "Welwitschia mirabilis", - "Stemona japonica", "Stemona mairei", "Stemona tuberosa", "Turpinia arguta", "Euscaphis japonica", "Potamogeton crispus", - "Potamogeton distinctus", "Victoria amazonica", "Victoria cruziana", "Nymphaea", "Nymphaea alba", "Nymphaea nouchali", - "Nymphaea odorata", "Euryale ferox", "Nuphar pumila", "Menyanthes trifoliata", "Nymphoides coreana", "Nymphoides cristata", - "Nymphoides indica", "Nymphoides peltata", "Palhinhaea cernua", "Diphasiastrum digitatum", "Dendrolycopodium obscurum", - "Lycopodiastrum casuarinoides", "Lychnis chalcedonica", "Lychnis fulgens", "Lychnis senno", "Cerastium glomeratum", - "Arenaria smithiana", "Sagina japonica", "Gypsophila oldhamiana", "Gypsophila paniculata", "Dianthus armeria", "Dianthus barbatus", - "Dianthus caryophyllus", "Dianthus chinensis", "Dianthus superbus", "Stellaria alsine", "Stellaria chinensis", "Stellaria media", - "Saponaria officinalis", "Silene armeria", "Silene baccifera", "Silene conoidea", "Silene davidii", "Silene gallica", - "Silene latifolia", "Silene vulgaris", "Myosoton aquaticum", "Agrostemma githago", "Vaccaria hispanica", "Nothoscordum bivalve", - "Boophone disticha", "Eucharis amazonica", "Clivia miniata", "Clivia nobilis", "Clivia × hybrida", "Amaryllis belladonna", - "Crinum amabile", "Crinum asiaticum var. sinicum", "Crinum moorei", "Ipheion uniflorum", "Polianthes tuberosa", - "Cyrtanthus mackenii", "Hippeastrum reticulatum", "Hippeastrum rutilum", "Narcissus bulbocodium", "Narcissus poeticus", - "Narcissus pseudonarcissus", "Narcissus tazetta var. chinensis", "Narcissus triandrus", "Hymenocallis speciosa&littoralis", - "Agapanthus africanus", "Agapanthus praecox", "Lycoris aurea", "Lycoris chinensis", "Lycoris haywardii", "Lycoris incarnata", - "Lycoris longituba", "Lycoris radiata", "Lycoris sprengeri", "Lycoris squamigera", "Lycoris straminea", "Lycoris × rosea", - "Tulbaghia violacea", "Allium carolinianum", "Allium cepa", "Allium chinense", "Allium fistulosum", "Allium giganteum", - "Allium prattii", "Allium sativum", "Allium senescens", "Allium sikkimense", "Allium triquetrum", "Allium tuberosum", - "Allium wallichii", "Zephyranthes candida", "Zephyranthes carinata", "Zephyranthes citrina", "Haemanthus albiflos", - "Haemanthus multiflorus", "Galanthus elwesii", "Leucojum aestivum", "Leucojum vernum", "Eucrosia bicolor", "Histiopteris incisa", - "Pteridium aquilinum", "Lagurus ovatus", "Phyllostachys nigra", "Hordeum jubatum", "Bothriochloa ischaemum", - "Chasmanthium latifolium", "Triticum aestivum", "Poa annua", "Phaenosperma globosa", "Isachne globosa", "Polypogon monspeliensis", - "Oplismenus undulatifolius", "Avena fatua", "Setaria italica var. germanica", "Setaria palmifolia", "Setaria pumila", - "Setaria viridis", "Cynodon dactylon", "Pennisetum alopecuroides", "Pennisetum glaucum", "Pennisetum orientale", - "Pennisetum setaceum 'Rubrum'", "Zea mays", "Saccharum officinarum", "Imperata cylindrica", "Alopecurus aequalis", - "Echinochloa caudata", "Echinochloa crus-galli", "Oryza sativa", "Eleusine indica", "Bambusoideae", "Indocalamus latifolius", - "Bambusa ventricosa", "Miscanthus sinensis 'Gracillimus'", "Miscanthus sinensis 'Zebrinus'", "Arundo donax", "Phragmites australis", - "Microstegium vimineum", "Zizania latifolia", "Cortaderia selloana", "Coix lacryma-jobi", "Phalaris arundinacea", - "Paspalum dilatatum", "Sorghum bicolor", "Sorghum halepense", "Dactylis glomerata", "Panicum virgatum", "Lolium perenne", - "Disporum cantoniense", "Disporum longistylum", "Disporum megalanthum", "Disporum uniflorum", "Disporum viridescens", - "Gloriosa superba", "Sandersonia aurantiaca", "Colchicum autumnale", "Begonia boliviensis", "Begonia circumlobata", - "Begonia cucullata", "Begonia fimbristipula", "Begonia grandis subsp. sinensis", "Begonia leprosa", "Begonia maculata", - "Begonia masoniana", "Begonia palmata", "Begonia soli-mutata", "Begonia × hiemalis", "Ctenanthe setosa", "Thalia dealbata", - "Thalia geniculata", "Maranta leuconeura", "Maranta&Calathea", "Stromanthe sanguinea", "Calathea warscewiczii", "Calathea zebrina", - "Bougainvillea spectabilis&glabra", "Mirabilis jalapa", "Boerhavia diffusa", "Myosotis alpestris", "Ehretia acuminata", - "Ehretia longiflora", "Carmona microphylla", "Heliotropium arborescens", "Heliotropium curassavicum", "Heliotropium indicum", - "Microula sikkimensis", "Bothriospermum chinense", "Bothriospermum zeylanicum", "Onosma hookeri var. longiflorum", - "Mertensia virginica", "Borago officinalis", "Cynoglossum amabile", "Cynoglossum grande", "Cynoglossum lanceolatum", - "Thyrocarpus sampsonii", "Cordia dichotoma", "Cordia subcordata", "Nemophila maculata", "Nemophila menziesii", - "Tournefortia montana", "Tournefortia sibirica", "Stenosolenium saxatile", "Lithospermum incisum", "Lithospermum zollingeri", - "Symphytum officinale", "Echium vulgare", "Echium wildpretii", "Trigonotis peduncularis", "Osmundastrum cinnamomeum", - "Osmunda claytoniana", "Campsis grandiflora", "Campsis radicans", "Kigelia africana", "Catalpa bungei", "Catalpa fargesii", - "Catalpa ovata", "Catalpa speciosa", "Mayodendron igneum", "Spathodea campanulata", "Pyrostegia venusta", - "Markhamia stipulata var. kerrii", "Macfadyena unguis-cati", "Pandorea jasminoides", "Tabebuia impetiginosa", "Tabebuia rosea", - "Radermachera sinica&hainanensis", "Crescentia alata", "Mansoa alliacea", "Jacaranda mimosifolia", "Incarvillea arguta", - "Incarvillea mairei var. multifoliolata", "Incarvillea sinensis", "Clytostoma callistegioides", "Podranea ricasoliana", - "Handroanthus chrysanthus", "Tecoma capensis", "Tecoma stans", "Calophyllum inophyllum", "Calophyllum membranaceum", "Mesua ferrea", - "Bixa orellana", "Bruguiera gymnorhiza", "Kandelia obovata", "Cephalotaxus sinensis", "Torreya grandis 'Merrillii'", - "Taxus baccata", "Taxus wallichiana var. chinensis", "Philadelphus laxiflorus", "Philadelphus pekinensis", - "Philadelphus zhejiangensis", "Dichroa febrifuga", "Deutzia baroniana", "Deutzia crenata", "Deutzia glauca", - "Deutzia glomeruliflora", "Deutzia gracilis", "Deutzia longifolia", "Deutzia ningpoensis", "Deutzia scabra", - "Deutzia scabra var. plena", "Hydrangea", "Hydrangea chinensis", "Hydrangea lingii", "Hydrangea paniculata", - "Hydrangea quercifolia", "Hydrangea strigosa", "Platycrater arguta", "Macleaya cordata", "Chelidonium majus", - "Dicranostigma leptopodum", "Corydalis bungeana", "Corydalis caudata", "Corydalis curviflora", "Corydalis decumbens", - "Corydalis edulis", "Corydalis fangshanensis", "Corydalis flexuosa", "Corydalis hamata", "Corydalis hemidicentra", - "Corydalis incisa", "Corydalis linarioides", "Corydalis melanochlora", "Corydalis mucronata", "Corydalis pachycentra", - "Corydalis pallida", "Corydalis pseudobarbisepala", "Corydalis racemosa", "Corydalis repens", "Corydalis sheareri", - "Corydalis speciosa", "Corydalis turtschaninovii", "Corydalis yanhusuo", "Meconopsis", "Meconopsis balangensis", - "Meconopsis betonicifolia", "Meconopsis chelidoniifolia", "Meconopsis delavayi", "Meconopsis henrici", "Meconopsis horridula", - "Meconopsis impedita", "Meconopsis integrifolia", "Meconopsis lancifolia", "Meconopsis paniculata", "Meconopsis pseudointegrifolia", - "Meconopsis punicea", "Meconopsis quintuplinervia", "Meconopsis racemosa", "Meconopsis simplicifolia", "Meconopsis speciosa", - "Meconopsis sulphurea", "Meconopsis venusta", "Meconopsis wilsonii", "Papaver orientale", "Papaver radicatum var. pseudoradicatum", - "Papaver rhoeas", "Papaver somniferum", "Eschscholzia californica", "Lamprocapnos spectabilis", "Lamprocapnos spectabilis f. alba", - "Hylomecon japonica", "Argemone mexicana", "Sanguinaria canadensis", "Eomecon chionantha", "Dicentra cucullaria", - "Dicentra formosa", "Nageia nagi", "Podocarpus macrophyllus", "Canna", "Canna generalis", "Canna glauca", "Canna indica", - "Canna indica var. flava", "Canna orchioides", "Canna warscewiezii", "Astelia fragrans", "Nephrolepis cordifolia", - "Platycarya strobilacea", "Carya illinoinensis", "Pterocarya stenoptera", "Engelhardia roxburghiana", "Juglans mandshurica", - "Juglans nigra", "Juglans regia", "Cyclocarya paliurus", "Piper aduncum", "Piper hancei", "Piper kadsura", "Piper nigrum", - "Piper sarmentosum", "Peperomia argyreia", "Peperomia caperata", "Peperomia pellucida", "Peperomia polybotrya", - "Peperomia tetraphylla", "Hippophae rhamnoides", "Elaeagnus angustifolia", "Elaeagnus argyi", "Elaeagnus conferta", - "Elaeagnus glabra", "Elaeagnus lanceolata", "Elaeagnus mollis", "Elaeagnus multiflora", "Elaeagnus pungens", - "Elaeagnus Pungens 'Aurea'", "Elaeagnus umbellata", "Paeonia delavayi", "Paeonia lactiflora", "Paeonia obovata", - "Paeonia suffruticosa", "Sesamum indicum", "Uncarina grandidieri", "Musella lasiocarpa", "Musa nana", "Ensete glaucum", - "Stylidium uliginosum", "Cobaea scandens", "Phlox", "Phlox drummondii", "Phlox paniculata", "Phlox subulata", "Ipomopsis aggregata", - "Polemonium caeruleum", "Polemonium chinense", "Butomus umbellatus", "Murraya exotica", "Tetradium austrosinense", - "Tetradium glabrifolium", "Tetradium ruticarpum", "Glycosmis pentaphylla", "Acronychia pedunculata", "Citrus australasica", - "Citrus japonica", "Citrus maxima", "Citrus medica 'Fingered'", "Citrus reticulata", "Citrus reticulata", "Citrus sinensis", - "Citrus trifoliata", "Citrus × limon", "Ptelea trifoliata", "Dictamnus dasycarpus", "Boenninghausenia albiflora", - "Zanthoxylum ailanthoides", "Zanthoxylum bungeanum", "Zanthoxylum nitidum", "Zanthoxylum piperitum", "Zanthoxylum scandens", - "Zanthoxylum simulans", "Skimmia reevesiana", "Melicope pteleifolia", "Toddalia asiatica", "Clausena excavata", "Clausena lansium", - "Gomphrena globosa", "Kochia scoparia", "Cyathula prostrata", "Achyranthes bidentata", "Beta vulgaris", "Salsola tragus", - "Amaranthus caudatus", "Amaranthus hypochondriacus", "Amaranthus spinosus", "Amaranthus tricolor", "Alternanthera bettzickiana", - "Alternanthera philoxeroides", "Spinacia oleracea", "Chenopodium album", "Celosia argentea", "Celosia cristata", "Cycas revoluta", - "Ailanthus altissima", "Brucea javanica", "Hemiboea cavaleriei", "Hemiboea subcapitata", "Didymostigma obtusum", - "Titanotrichum oldhamii", "Lysionotus pauciflorus", "Lysionotus serratus", "Chirita eburnea", "Chirita fimbrisepala", - "Chirita lutea", "Chirita pinnatifida", "Chirita pumila", "Episcia cupreata", "Gyrocheilos chorisepalus", "Sinningia leucotricha", - "Sinningia speciosa", "Gloxinia sylvatica", "Primulina xiziae", "Streptocarpus hybrids", "Streptocarpus saxorum", - "Briggsia chienii", "Rhynchotechum ellipticum", "Didissandra sesquifolia", "Aeschynanthus acuminatus", "Aeschynanthus buxifolius", - "Aeschynanthus sp", "Aeschynanthus speciosus", "Aeschynanthus superbus", "Paraboea sinensis", "Nematanthus wettsteinii", - "Saintpaulia ionantha", "Oreocharis auricula", "Oreocharis benthamii var. reticulata", "Oreocharis maximowiczii", - "Nicandra physalodes", "Cestrum aurantiacum", "Cestrum nocturnum", "Hyoscyamus niger", "Anisodus tanguticus", "Datura inoxia", - "Datura stramonium", "Datura wrightii", "Brugmansia arborea", "Brugmansia aurea", "Brugmansia suaveolens", "Lycium chinense", - "Cyphomandra betacea", "Juanulloa aurantiaca", "Nicotiana alata", "Nicotiana glauca", "Nicotiana tabacum", - "Lycopersicon esculentum", "Petunia × hybrida", "Lycianthes biflora", "Calibrachoa hybrids", "Mandragora caulescens", - "Solanum aculeatissimum", "Solanum capsicoides", "Solanum carolinense", "Solanum dulcamara", "Solanum elaeagnifolium", - "Solanum erianthum", "Solanum jasminoides", "Solanum laciniatum", "Solanum lyratum", "Solanum mammosum", "Solanum melongena", - "Solanum muricatum", "Solanum nigrum&americanum", "Solanum pseudocapsicum", "Solanum pseudocapsicum var. diflorum", - "Solanum rantonnetii", "Solanum rostratum", "Solanum septemlobum", "Solanum texanum", "Solanum torvum", "Solanum tuberosum", - "Solanum virginianum", "Solanum wrightii", "Schizanthus pinnatus", "Capsicum annuum", "Capsicum annuum subsp. cerasiforme", - "Capsicum annuum var. conoides", "Physalis", "Physalis minima", "Physalis philadelphica", "Solandra longiflora", "Solandra maxima", - "Brunfelsia brasiliensis", "Brunfelsia calycina", "Dionaea muscipula", "Drosera burmanni", "Drosera peltata", - "Drosera rotundifolia", "Drosera spatulata", "Psychotria serpens", "Pentas lanceolata", "Coffea", "Pavetta hongkongensis", - "Bouvardia ternifolia", "Morinda citrifolia", "Morinda parvifolia", "Galium aparine", "Galium spurium", "Galium verum", - "Gardenia jasminoides", "Gardenia scabrella", "Adina pilulifera", "Adina rubella", "Coptosapelta diffusa", "Luculia pinceana", - "Diplospora dubia", "Canthium horridum", "Mussaenda 'Alicia'", "Mussaenda erosa", "Mussaenda erythrophylla", "Mussaenda parviflora", - "Mussaenda pubescens", "Mussaenda shikokiana", "Sherardia arvensis", "Serissa japonica", "Serissa japonica 'Variegata'", - "Serissa serissoides", "Neohymenopogon parasiticus", "Lasianthus chinensis", "Houstonia caerulea", "Hedyotis caudatifolia", - "Hedyotis chrysotricha", "Hedyotis diffusa", "Hedyotis hedyotidea", "Hedyotis tenuipes", "Mycetia sinensis", "Coprosma robusta", - "Mitchella repens", "Damnacanthus giganteus", "Ophiorrhiza japonica", "Ophiorrhiza pumila", "Rondeletia leucophylla", - "Rondeletia odorata", "Leptodermis oblonga", "Uncaria hirsuta", "Spermacoce alata", "Hamelia patens", "Cephalanthus occidentalis", - "Cephalanthus tetrandrus", "Paederia foetida", "Ixora chinensis", "Ixora coccinea f. lutea", "Ixora finlaysoniana", - "Ixora paraopaca", "Mappianthus iodoides", "Ribes burejense", "Ribes himalense var. verruculosum", "Ribes nigrum", "Ribes odoratum", - "Ribes reclinatum", "Ribes rubrum", "Ribes rubrum", "Scaevola aemula", "Scaevola taccada", "Goodenia pilosa subsp. chinensis", - "Pilea aquarum", "Pilea cadierei", "Pilea microphylla", "Pilea notata", "Pilea pumila", "Cecropia peltata", "Elatostema cuspidatum", - "Debregeasia orientalis", "Gonostegia hirta", "Oreocnide frutescens", "Nanocnide lobata", "Boehmeria japonica", "Boehmeria nivea", - "Boehmeria tricuspis", "Urtica dioica", "Girardinia diversifolia subsp. suborbiculata", "Pellionia repens", "Pouzolzia zeylanica", - "Calceolaria crenatiflora", "Rhynchospora colorata", "Schoenoplectus tabernaemontani", "Kyllinga brevifolia", "Kyllinga polyphylla", - "Eleocharis dulcis", "Cyperus difformis", "Cyperus glomeratus", "Cyperus involucratus", "Cyperus prolifer", "Cyperus rotundus", - "Trichophorum subcapitatum", "Carex baccans", "Carex scaposa", "Fimbristylis dichotoma", "Illigera celebica", "Illigera rhodantha", - "Nelumbo nucifera", "Brasenia schreberi", "Mycelis muralis", "Solidago canadensis", "Emilia prenanthoidea", "Emilia sonchifolia", - "Tagetes erecta", "Calyptocarpus vialis", "Parasyncalathium souliei", "Mikania micrantha", "Paraprenanthes sororia", - "Praxelis clematidea", "Crepidiastrum lanceolatum", "Crepidiastrum sonchifolium", "Heterotheca subaxillaris", - "Syneilesis aconitifolia", "Ainsliaea fragrans", "Ainsliaea kawakamii", "Gazania rigens", "Smallanthus sonchifolius", - "Senecio analogus", "Senecio cineraria", "Senecio faberi", "Senecio haworthii", "Senecio rowleyanus", "Senecio scandens", - "Senecio serpens", "Senecio vulgaris", "Helianthus annuus", "Helianthus decapetalus", "Helianthus maxillianii", - "Helianthus tuberosus", "Cremanthodium campanulatum", "Helenium amarum", "Helenium autumnale", "Dahlia pinnata", - "Farfugium japonicum", "Gaillardia pulchella&aristata", "Carpesium abrotanoides", "Tragopogon dubius", "Tragopogon porrifolius", - "Tragopogon pratensis", "Wollastonia biflora", "Ixeridium dentatum", "Hieracium aurantiacum", "Dolomiaea souliei", - "Pseudognaphalium hypoleucum", "Inula helenium", "Inula helianthusaquatilis", "Inula japonica", "Argyranthemum frutescens", - "Echinacea purpurea", "Silphium laciniatum", "Silphium perfoliatum", "Nouelia insignis", "Engelmannia peristenia", - "Ligularia sibirica", "Tussilago farfara", "Matricaria chamomilla", "Matricaria discoidea", "Melanoseris atropurpurea", - "Silybum marianum", "Hemisteptia lyrata", "Eupatorium fortunei", "Eupatorium perfoliatum", "Eupatorium serotinum", - "Leucanthemum maximum", "Leucanthemum vulgare", "Rhaponticum chinense", "Rhaponticum uniflorum", "Gerbera jamesonii", - "Leontopodium japonicum", "Leontopodium leontopodioides", "Galinsoga parviflora", "Galinsoga quadriradiata", - "Helminthotheca echioides", "Arctium lappa", "Hypochaeris radicata", "Pericallis hybrida", "Stevia rebaudiana", - "Centaurea solstitialis", "Zinnia elegans", "Cyanus segetum", "Cosmos bipinnatus", "Cosmos sulphureus", "Lapsanastrum apogonoides", - "Ageratina adenophora", "Ageratina altissima", "Aster altaicus", "Aster baccharoides", "Aster hispidus", "Aster indicus", - "Aster likiangensis", "Aster novi-belgii", "Aster pekinensis", "Aster scaber", "Aster trinervius subsp. ageratoides", - "Aster turbinatus", "Carthamus tinctorius", "Eriophyllum confertiflorum", "Eriophyllum staechadifolium", "Thelesperma filifolium", - "Callistephus chinensis", "Symphyotrichum novae-angliae", "Symphyotrichum subulatum", "Tithonia diversifolia", - "Encelia californica", "Blumea megacephala", "Crossostephium chinensis", "Xanthium strumarium", "Sonchus asper", - "Sonchus oleraceus", "Ixeris chinensis", "Glebionis coronaria", "Glebionis segetum", "Ratibida columnifera", "Lactuca indica", - "Lactuca sativa", "Lactuca sativa var. ramosa", "Lactuca serriola", "Lactuca sibirica", "Gynura aurantiaca", "Gynura bicolor", - "Gynura divaricata", "Chrysanthemum multicaule", "Chrysanthemum × morifolium", "Cichorium endivia", "Cichorium intybus", - "Tanacetum vulgare", "Cynara cardunculus", "Cynara scolymus", "Sinosenecio oldhamianus", "Taraxacum mongolicum", - "Taraxacum officinale", "Artemisia argyi", "Artemisia californica", "Artemisia caruifolia", "Artemisia douglasiana", - "Artemisia lactiflora", "Artemisia selengensis", "Achillea millefolium", "Centratherum punctatum", "Echinops gmelinii", - "Cirsium arvense", "Cirsium arvense var. integrifolium", "Cirsium japonicum", "Cirsium leo", "Cirsium souliei", "Cirsium vulgare", - "Ageratum conyzoides", "Ageratum houstonianum", "Myripnois dioica", "Liatris spicata", "Petasites japonicus", - "Xerochrysum bracteatum", "Sphagneticola calendulacea", "Sphagneticola trilobata", "Ambrosia artemisiifolia", "Ambrosia trifida", - "Sigesbeckia orientalis", "Heliopsis helianthoides", "Heliopsis helianthoides var. scabra", "Baccharis halimifolia", - "Baccharis pilularis", "Baccharis salicifolia", "Crassocephalum crepidioides", "Crassocephalum rubens", "Rudbeckia bicolor", - "Rudbeckia fulgida", "Rudbeckia fulgida 'Goldsturm'", "Rudbeckia hirta", "Rudbeckia laciniata", - "Rudbeckia laciniata var. hortensia", "Calendula officinalis", "Synedrella nodiflora", "Acmella paniculata", "Coreopsis basalis", - "Coreopsis lanceolata", "Coreopsis tinctoria", "Coreopsis verticillata", "Vernonia baldwinii", "Vernonia gratiosa", - "Vernonia volkameriifolia", "Parthenium hysterophorus", "Conoclinium coelestinum", "Bellis perennis", "Saussurea involucrata", - "Saussurea medusa", "Saussurea przewalskii", "Saussurea stella", "Saussurea tibetica", "Saussurea velutina", "Carduus crispus", - "Carduus nutans", "Carduus pycnocephalus", "Erigeron annuus", "Erigeron canadensis", "Erigeron glaucus", "Erigeron philadelphicus", - "Erigeron sumatrensis", "Anaphalis margaritacea", "Anaphalis nepalensis", "Anaphalis nepalensis var. monocephala", - "Verbesina virginica", "Osteospermum ecklonis", "Bidens biternata", "Bidens cernua", "Bidens frondosa", "Bidens pilosa", - "Eclipta prostrata", "Brachyscome angustifolia", "Brachyscome iberidifolia", "Euryops pectinatus", "Flaveria bidentis", - "Youngia heterophylla", "Youngia japonica", "Gnaphalium", "Gnaphalium japonicum", "Acorus calamus", "Smilax bona-nox", - "Smilax china", "Smilax davidiana", "Smilax riparia", "Biondia microcentra", "Basella alba", "Anredera cordifolia", - "Cayratia albifolia", "Cayratia japonica", "Yua austro-orientalis", "Parthenocissus laetevirens", "Parthenocissus quinquefolia", - "Parthenocissus tricuspidata", "Tetrastigma hemsleyanum", "Tetrastigma planicaule", "Cissus hexangularis", "Vitis bryoniifolia", - "Vitis flexuosa", "Vitis vinifera", "Ampelopsis aconitifolia", "Ampelopsis delavayana", "Ampelopsis glandulosa", - "Ampelopsis glandulosa var. heterophylla", "Marah fabacea", "Marah macrocarpa", "Luffa aegyptiaca", "Sechium edule", - "Benincasa hispida", "Cucurbita foetidissima", "Cucurbita moschata", "Cucurbita pepo", "Trichosanthes anguina", - "Trichosanthes cucumeroides", "Trichosanthes kirilowii", "Trichosanthes rubriflos", "Diplocyclos palmatus", "Melothria pendula", - "Melothria scabra", "Actinostemma tenerum", "Coccinia grandis", "Gynostemma pentaphyllum", "Momordica charantia", - "Momordica cochinchinensis", "Lagenaria siceraria", "Lagenaria siceraria ‘Hispida’", "Citrullus lanatus", "Thladiantha dubia", - "Thladiantha longifolia", "Thladiantha nudiflora", "Gymnopetalum chinense", "Zehneria japonica", "Cucumis melo", "Cucumis melo", - "Cucumis melo", "Cucumis melo subsp. agrestis", "Cucumis metuliferus", "Cucumis sativus", "Rivina humilis", "Larrea tridentata", - "Tribulus terrestris", "Zygophyllum mucronatum", "Camptotheca acuminata", "Davidia involucrata", "Nyssa sinensis", - "Fallopia multiflora", "Muehlenbeckia complexa", "Rheum alexandrae", "Rheum nobile", "Rheum rhabarbarum", "Oxyria sinensis", - "Coccoloba uvifera", "Antigonon leptopus", "Eriogonum fasciculatum", "Eriogonum latifolium", "Fagopyrum dibotrys", - "Fagopyrum esculentum", "Polygonum aviculare", "Polygonum capitatum", "Polygonum chinense", "Polygonum coriaceum", - "Polygonum japonicum", "Polygonum longisetum", "Polygonum macrophyllum", "Polygonum muricatum", "Polygonum orientale", - "Polygonum perfoliatum", "Polygonum plebeium", "Polygonum pubescens", "Polygonum runcinatum", "Polygonum senticosum", - "Polygonum thunbergii", "Polygonum viscosum", "Persicaria virginiana", "Reynoutria japonica", "Rumex acetosa", "Rumex acetosella", - "Rumex crispus", "Rumex hastatus", "Rumex japonicus", "Rumex obtusifolius", "Antenoron filiforme", - "Antenoron filiforme var. neofiliforme", "Dryas octopetala", "Aruncus sylvester", "Amelanchier canadensis", - "Sanguisorba officinalis", "Potentilla anserina", "Potentilla discolor", "Potentilla fragarioides", "Potentilla freyniana", - "Potentilla fruticosa", "Potentilla glabra", "Potentilla kleiniana", "Potentilla recta", "Potentilla supina", - "Stephanandra chinensis", "Crataegus cuneata", "Crataegus maximowiczii", "Crataegus monogyna", "Crataegus pinnatifida", - "Rubus alceifolius", "Rubus armeniacus", "Rubus buergeri", "Rubus chingii", "Rubus corchorifolius", "Rubus coreanus", - "Rubus crataegifolius", "Rubus fockeanus", "Rubus fruticosus", "Rubus idaeus&hirsutus", "Rubus lambertianus", "Rubus odoratus", - "Rubus pacificus", "Rubus parviflorus", "Rubus parvifolius", "Rubus phoenicolasius", "Rubus pirifolius", "Rubus rosifolius", - "Rubus setchuenensis", "Rubus spectabilis", "Rubus sumatranus", "Rubus swinhoei", "Rubus trianthus", "Rubus ursinus", - "Prinsepia utilis", "Chaenomeles cathayensis", "Chaenomeles sinensis", "Chaenomeles speciosa", "Prunus cerasifera f. atropurpurea", - "Prunus laurocerasus", "Prunus salicina", "Prunus serotina", "Prunus spinosa", "Prunus virginiana", "Armeniaca mume", - "Armeniaca mume var. mume f. alphandii", "Armeniaca mume var. mume f. purpurea", "Armeniaca mume var. mume f. viridicalyx", - "Armeniaca vulgaris", "Eriobotrya japonica", "Adenostoma fasciculatum", "Heteromeles arbutifolia", "Cotoneaster adpressus", - "Cotoneaster horizontalis", "Cotoneaster microphyllus", "Cotoneaster multiflorus", "Amygdalus communis", "Amygdalus persica", - "Amygdalus persica 'Compressa'", "Amygdalus persica 'Juhuatao'", "Amygdalus triloba", "Pyrus", "Pyrus betulifolia", - "Pyrus calleryana", "Pyrus phaeocarpa", "Pyrus sinkiangensis", "Kerria japonica", "Kerria japonica f. pleniflora", - "Cydonia oblonga", "Cerasus campanulata", "Cerasus cerasoides", "Cerasus dielsiana", "Cerasus glandulosa", "Cerasus japonica", - "Cerasus pseudocerasus", "Cerasus serrulata var. lannesiana", "Cerasus tomentosa", "Pyracantha angustifolia", - "Pyracantha fortuneana", "Pyracantha fortuneana 'Harlequin'", "Sorbaria sorbifolia", "Exochorda racemosa", "Rhaphiolepis indica", - "Rhaphiolepis umbellata", "Photinia beauverdiana", "Photinia bodinieri", "Photinia glomerata", "Photinia komarovii", - "Photinia serratifolia", "Photinia × fraseri", "Padus avium", "Padus buergeriana", "Holodiscus discolor", "Neillia sinensis", - "Spiraea alpina", "Spiraea blumei", "Spiraea cantoniensis", "Spiraea fritschiana", "Spiraea japonica", "Spiraea mongolica", - "Spiraea myrtilloides", "Spiraea prunifolia", "Spiraea prunifolia var. simpliciflora", "Spiraea pubescens", "Spiraea thunbergii", - "Spiraea trilobata", "Spiraea × bumalda 'coldfiame'", "Spiraea × bumalda 'Goalden Mound'", "Spiraea × vanhouttei", - "Potaninia mongolica", "Sorbus alnifolia", "Sorbus folgneri", "Sorbus pohuashanensis", "Malus 'American'", "Malus baccata", - "Malus halliana", "Malus hupehensis", "Malus pumila", "Malus × micromalus", "Malus × robusta", "Fragaria orientalis", - "Fragaria vesca", "Fragaria virginiana", "Fragaria × ananassa", "Rosa banksiae", "Rosa banksiae f. lutea", "Rosa bracteata", - "Rosa californica", "Rosa chinensis", "Rosa cymosa", "Rosa davurica", "Rosa henryi", "Rosa laevigata", "Rosa multiflora", - "Rosa multiflora var. carnea", "Rosa multiflora var. cathayensis", "Rosa omeiensis", "Rosa roxburghii", - "Rosa roxburghii f. normalis", "Rosa rugosa", "Rosa rugosa f. albo-plena", "Rosa xanthina", "Rosa xanthina var. normalis", - "Filipendula palmata", "Duchesnea indica", "Geum aleppicum", "Geum canadense", "Geum japonicum var. chinense", - "Physocarpus amurensis", "Spenceria ramalana", "Agrimonia pilosa", "Liquidambar formosana", "Liquidambar styraciflua", - "Altingia chinensis", "Tacca chantrieri", "Tacca plantaginea", "Dioscorea bulbifera", "Dioscorea cirrhosa", - "Dioscorea elephantipes", "Dioscorea japonica", "Dioscorea polystachya", "Ypsilandra thibetica", "Trillium cernuum", - "Trillium chloropetalum", "Trillium cuneatum", "Trillium erectum", "Trillium grandiflorum", "Trillium luteum", "Trillium ovatum", - "Trillium recurvatum", "Trillium undulatum", "Toxicoscordion fremontii", "Chionographis chinensis", "Veratrum californicum", - "Veratrum nigrum", "Veratrum schindleri", "Veratrum viride", "Paris", "Paris luquanensis", "Paris polyphylla", - "Paris polyphylla var. chinensis", "Paris verticillata", "Garcinia cowa", "Garcinia mangostana", "Garcinia multiflora", - "Garcinia oblongifolia", "Garcinia subelliptica", "Garcinia xanthochymus", "Daphniphyllum calycinum", "Daphniphyllum macropodum", - "Mukdenia rossii", "Oresitrophe rupifraga", "Heuchera", "Astilbe chinensis", "Saxifraga egregia", "Saxifraga przewalskii", - "Saxifraga stolonifera", "Tiarella cordifolia", "Tiarella polyphylla", "Balanophora harlandii", "Balanophora laxiflora", - "Calycanthus chinensis", "Calycanthus floridus", "Chimonanthus nitens", "Chimonanthus praecox", "Heliconia latispatha", - "Heliconia metallica", "Heliconia rostrata", "Turnera subulata", "Turnera ulmifolia", "Passiflora alata", "Passiflora amethystina", - "Passiflora caerulea", "Passiflora coccinea", "Passiflora edulis", "Passiflora foetida", "Passiflora incarnata", "Passiflora lutea", - "Passiflora suberosa", "Passiflora yucatanensis", "Eriocaulon buergerianum", "Eriocaulon sexangulare", "Acmispon glaber", - "Amphicarpaea edgeworthii", "Caesalpinia bonduc", "Caesalpinia decapetala", "Caesalpinia minax", "Caesalpinia pulcherrima", - "Caesalpinia pulcherrima 'Flava'", "Caesalpinia sappan", "Lysidice brevicalyx", "Lysidice rhodostegia", "Dendrolobium triangulare", - "Senna alata", "Senna bicapsularis", "Senna occidentalis", "Senna sophera", "Senna spectabilis", "Senna surattensis", - "Delonix regia", "Canavalia gladiata", "Canavalia rosea", "Erythrina corallodendron", "Erythrina crista-galli", - "Erythrina variegata", "Robinia pseudoacacia", "Robinia pseudoacacia f. decaisneana", "Albizia julibrissin", "Albizia kalkora", - "Albizia lebbeck", "Aeschynomene indica", "Mimosa bimucronata", "Mimosa pudica", "Apios carnea", "Apios fortunei", "Glycine max", - "Glycine soja", "Coronilla varia", "Chamaecrista fasciculata", "Chamaecrista mimosoides", "Desmodium heterocarpon", - "Desmodium microphyllum", "Desmodium triflorum", "Lathyrus latifolius", "Lathyrus odoratus", "Fordia cauliflora", - "Lablab purpureus", "Phyllodium pulchellum", "Saraca dives", "Indigofera bungeana", "Indigofera decora", "Indigofera hendecaphylla", - "Indigofera kirilowii", "Cajanus cajan", "Calliandra haematocephala", "Calliandra tergemina var. emarginata", - "Campylotropis macrocarpa", "Campylotropis polyantha", "Castanospermum australe", "Erythrophleum fordii", "Oxytropis aciphylla", - "Oxytropis caerulea", "Oxytropis myriophylla", "Styphnolobium japonicum", "Ammopiptanthus mongolicus", "Sindora glabra", - "Mucuna bennettii", "Mucuna birdwoodiana", "Mucuna lamellata", "Mucuna macrocarpa", "Mucuna sempervirens", - "Adenanthera microsperma", "Prosopis glandulosa", "Uraria crinita", "Uraria picta", "Crotalaria assamica", "Crotalaria pallida", - "Crotalaria sessiliflora", "Crotalaria spectabilis", "Crotalaria trichotoma", "Archidendron clypearia", "Glycyrrhiza uralensis", - "Sesbania cannabina", "Sesbania grandiflora", "Lotus corniculatus", "Gleditsia japonica", "Gleditsia triacanthos", - "Abrus precatorius", "Acacia auriculiformis", "Acacia catechu", "Acacia confusa", "Acacia farnesiana", "Acacia podalyriifolia", - "Peltophorum pterocarpum", "Butea monosperma", "Amorpha fruticosa", "Cercis canadensis", "Cercis chinensis", "Cercis chingii", - "Cercis chuniana", "Cercis glabra", "Wisteria sinensis&villosa", "Ormosia henryi", "Corethrodendron scoparium", - "Bauhinia acuminata", "Bauhinia brachycarpa", "Bauhinia championii", "Bauhinia corymbosa", "Bauhinia didyma", "Bauhinia galpinii", - "Bauhinia glauca", "Bauhinia glauca subsp. tenuiflora", "Bauhinia kockiana", "Bauhinia tomentosa", "Bauhinia touranensis", - "Bauhinia variegata", "Bauhinia variegata var. candida", "Bauhinia × blakeana", "Lupinus arboreus", - "Lupinus micranthus&polyphyllus", "Lupinus texensis", "Strongylodon macrobotrys", "Lespedeza bicolor", "Lespedeza buergeri", - "Lespedeza chinensis", "Lespedeza cuneata", "Lespedeza davidii", "Lespedeza dunnii", "Lespedeza floribunda", "Lespedeza pilosa", - "Lespedeza thunbergii subsp. formosa", "Lespedeza tomentosa", "Lespedeza virgata", "Cassia fistula", "Codoriocalyx motorius", - "Medicago lupulina", "Medicago polymorpha", "Medicago sativa", "Sophora davidii", "Sophora flavescens", "Sphaerophysa salsula", - "Ulex europaeus", "Melilotus albus", "Melilotus indicus", "Melilotus officinalis", "Phaseolus coccineus", "Phaseolus vulgaris", - "Arachis duranensis", "Arachis hypogaea", "Pueraria montana", "Pueraria wallichii", "Bowringia callicarpa", "Clitoria ternatea", - "Cullen corylifolium", "Pachyrhizus erosus", "Vigna radiata", "Vigna umbellata", "Vigna unguiculata", "Vigna vexillata", - "Pisum sativum", "Baptisia australis", "Centrosema pubescens", "Trifolium pratense", "Trifolium repens", "Tamarindus indica", - "Thermopsis barbata", "Thermopsis lanceolata", "Vicia amoena", "Vicia cracca", "Vicia faba", "Vicia sativa", "Vicia sepium", - "Vicia tetrasperma", "Vicia villosa", "Cytisus scoparius", "Leucaena leucocephala", "Caragana jubata", "Caragana rosea", - "Caragana sinica", "Caragana tibetica", "Hylodesmum podocarpum", "Hylodesmum podocarpum subsp. fallax", - "Hylodesmum podocarpum subsp. oxyphyllum", "Chesneya polystichoides", "Tibetia yunnanensis", "Derris alborubra", "Derris fordii", - "Colutea arborescens", "Kummerowia striata", "Callerya dielsiana", "Callerya nitida", "Callerya reticulata", "Callerya speciosa", - "Spartium junceum", "Rhynchosia volubilis", "Dalbergia assamica", "Dalbergia hupeana", "Astragalus sinicus", - "Athyrium filix-femina", "Bacopa diffusa", "Pseudolysimachion longifolium", "Pseudolysimachion spicatum", "Lagotis brevituba", - "Veronica anagallis-aquatica", "Veronica arvensis", "Veronica henryi", "Veronica persica", "Veronica undulata", "Linaria maroccana", - "Linaria vulgaris", "Linaria vulgaris subsp. chinensis", "Digitalis purpurea", "Adenosma glutinosum", "Russelia equisetiformis", - "Veronicastrum axillare", "Otacanthus azureus", "Cymbalaria muralis", "Plantago asiatica", "Plantago depressa", - "Plantago lanceolata", "Plantago major", "Plantago virginica", "Antirrhinum majus", "Penstemon", "Penstemon barbatus", - "Penstemon digitalis", "Collinsia heterophylla", "Hemiphragma heterophyllum", "Angelonia angustifolia", "Chelone glabra", - "Moringa drouhardii", "Moringa oleifera", "Polygala arillata", "Polygala fallax", "Polygala hongkongensis", - "Polygala hongkongensis var. stenophylla", "Polygala japonica", "Polygala latouchei", "Polygala myrtifolia", "Polygala sibirica", - "Polygala tenuifolia", "Salomonia cantoniensis", "Cercidiphyllum japonicum", "Mimulus aurantiacus", "Mimulus guttatus", - "Mimulus szechuanensis", "Lancea tibetica", "Mazus caducifer", "Mazus pumilus", "Oxalis", "Oxalis articulata", "Oxalis barrelieri", - "Oxalis corniculata", "Oxalis corymbosa", "Oxalis griffithii", "Oxalis oregana", "Oxalis palmifrons", "Oxalis pes-caprae", - "Oxalis purpurea", "Oxalis stricta", "Oxalis triangularis 'Urpurea'", "Oxalis violacea", "Averrhoa carambola", - "Oxyspora paniculata", "Blastus cochinchinensis", "Blastus pauciflorus", "Fordiophyton faberi", "Tibouchina semidecandra", - "Tigridiopalma exalata", "Tigridiopalma magnifica", "Sonerila cantonensis", "Memecylon ligustrifolium", "Memecylon octocostatum", - "Medinilla formosana", "Medinilla magnifica", "Bredia fordii", "Bredia quadrangularis", "Melastoma dodecandrum", - "Melastoma malabathricum", "Melastoma malabathricum var. alba", "Melastoma sanguineum", "Osbeckia chinensis", "Osbeckia stellata", - "Phyllagathis cavaleriei", "Hypericum 'Excellent Flair'", "Hypericum androsaemum", "Hypericum faberi", "Hypericum japonicum", - "Hypericum monogynum", "Hypericum patulum", "Hypericum perforatum", "Hypericum sampsonii", "Cratoxylum cochinchinense", - "Phegopteris connectilis", "Sarcandra glabra", "Chloranthus fortunei", "Chloranthus henryi", "Chloranthus japonicus", - "Chloranthus serratus", "Chloranthus spicatus", "Mytilaria laosensis", "Loropetalum chinense", "Loropetalum chinense var. rubrum", - "Loropetalum subcordatum", "Sycopsis sinensis", "Fortunearia sinensis", "Eustigma oblongifolium", "Rhodoleia championii", - "Distylium buxifolium", "Distylium racemosum", "Corylopsis multiflora var. nivea", "Corylopsis sinensis", "Hamamelis mollis", - "Hamamelis virginiana", "Hamamelis × intermedia", "Ochna integerrima", "Ochna serrulata", "Ochna thomasiana", - "Tristellateia australasiae", "Heteropterys glabra", "Thryallis gracilis", "Malpighia glabra", "Hiptage benghalensis", - "Ceratophyllum demersum", "Gelsemium elegans", "Gelsemium sempervirens", "Ancistrocladus tectorius", "Asplenium bulbiferum", - "Asplenium nidus", "Asplenium oblongifolium", "Asplenium platyneuron", "Asplenium trichomanes", "Erythropalum scandens", - "Ginkgo biloba", "", "Byttneria grandifolia", "Triumfetta annua", "Triumfetta cana", "Triumfetta rhomboidea", - "Pentapetes phoenicea", "Anisodontea capensis", "Theobroma cacao", "Ceiba pentandra", "Ceiba speciosa", "Helicteres angustifolia", - "Helicteres hirsuta", "Malvaviscus arboreus", "Malvaviscus arboreus var. mexicanus", "Malvaviscus penduliflorus", "Grewia biloba", - "Grewia biloba var. parviflora", "Grewia occidentalis", "Ambroma augustum", "Bombax ceiba", "Hibiscus acetosella", - "Hibiscus aridicola", "Hibiscus coccineus", "Hibiscus grandiflorus", "Hibiscus grewiifolius", "Hibiscus hamabo", - "Hibiscus moscheutos", "Hibiscus mutabilis", "Hibiscus rosa-sinensis", "Hibiscus sabdariffa", "Hibiscus schizopetalus", - "Hibiscus syriacus", "Hibiscus syriacus var. syriacus f. totus-albus", "Hibiscus tiliaceus", "Hibiscus trionum", - "Firmiana kwangsiensis", "Firmiana simplex", "Reevesia pubescens", "Reevesia thyrsoidea", "Urena lobata", "Urena procumbens", - "Urena procumbens var. microphylla", "Gossypium", "Sidalcea malviflora", "Tilia americana", "Durio zibethinus", - "Diplodiscus trichospermus", "Adansonia digitata", "Pachira glabra", "Corchoropsis crenata", "Microcos paniculata", - "Abelmoschus esculentus", "Abelmoschus manihot", "Abelmoschus sagittifolius", "Pavonia hastata", "Callirhoe involucrata", - "Pterygota alata", "Scaphium wallichii", "Abutilon indicum", "Abutilon megapotamicum", "Abutilon pictum", "Abutilon theophrasti", - "Sterculia lanceolata", "Sterculia monosperma", "Althaea officinalis", "Waltheria indica", "Alcea rosea", - "Malvastrum coromandelianum", "Brachychiton acerifolius", "Brachychiton rupestris", "Heritiera littoralis", "Heritiera parvifolia", - "Malva cathayensis", "Malva pusilla", "Malva verticillata var. crispa", "Dombeya wallichii", "Melochia corchorifolia", - "Kleinhovia hospita", "Sida subcordata", "Corchorus aestuans", "Costus barbatus", "Costus lucanusianus", "Costus woodsonii", - "Stephania cephalantha", "Stephania epigaea&cephalantha", "Stephania longa", "Stephania tetrandra", "Cocculus orbiculatus", - "Diploclisia affinis", "Diploclisia glaucescens", "Menispermum dauricum", "Cyclea racemosa", "Sinomenium acutum", - "Haworthia cooperi var. pilifera", "Haworthia fasciata", "Haworthia truncata", "Dianella ensifolia", "Stypandra glauca", - "Asphodeline lutea", "Kniphofia uvaria", "Geitonoplesium cymosum", "Aloe arborescens", "Aloe ferox", "Aloe mitriformis", - "Aloe vera", "Hemerocallis citrina", "Hemerocallis fulva", "Hemerocallis fulva 'Golden Doll'", "Hemerocallis hybridus", - "Asphodelus fistulosus", "Asphodelus ramosus", "Bulbine bulbosa", "Tricoryne elatior", "Gasteria gracilis var. minima", - "Phormium tenax", "Eichhornia crassipes", "Pontederia cordata", "Pontederia cordata var. alba", "Monochoria korsakowii", - "Monochoria vaginalis", "Sciaphila secundiflora", "Pandanus tectorius", "Schoepfia chinensis", "Helwingia chinensis", - "Helwingia japonica", "Helwingia omeiensis", "Hydnocarpus anthelminthicus", "Hydnocarpus hainanensis", "Typha", - "Typha angustifolia", "Typha latifolia", "Typha orientalis", "Sparganium stoloniferum", "Asarum canadense", "Asarum caudigerum", - "Asarum forbesii", "Asarum heterotropoides", "Aristolochia arborea", "Aristolochia contorta", "Aristolochia debilis", - "Aristolochia elegans", "Aristolochia gentilis", "Aristolochia gibertii", "Aristolochia grandiflora", "Aristolochia griffithii", - "Aristolochia hainanensis", "Aristolochia kwangsiensis", "Aristolochia manshuriensis", "Aristolochia mollissima", - "Aristolochia ringens", "Aristolochia tagala", "Aristolochia tubiflora", "Aristolochia westlandii", "Coriaria nepalensis", - "Mitrasacme pygmaea", "Gardneria multiflora", "Strychnos angustiflora", "Duranta erecta", "Duranta erecta 'Alba'", - "Glandularia bipinnatifida", "Glandularia tenera", "Glandularia × hybrida", "Petrea volubilis", "Phyla canescens", - "Phyla nodiflora", "Lantana camara", "Lantana fucata", "Lantana montevidensis", "Verbena bonariensis", "Verbena brasiliensis", - "Verbena halei", "Verbena hastata", "Verbena officinalis", "Verbena stricta", "Portulaca gilliesii", "Portulaca grandiflora", - "Portulaca molokiniensis", "Portulaca oleracea", "Portulaca pilosa", "Portulaca umbraticola", "", "", "Polystichum acrostichoides", - "Polystichum munitum", "Polystichum vestitum", "Gladiolus communis", "Gladiolus dalenii", "Gladiolus gandavensis", - "Gladiolus imbricatus", "Belamcanda chinensis", "Neomarica gracilis", "Sisyrinchium albidum", "Sisyrinchium angustifolium", - "Sisyrinchium bellum", "Sisyrinchium campestre", "Sisyrinchium micranthum", "Sisyrinchium montanum", "sisyrinchium rosulatum", - "Alophia drummondii", "Olsynium douglasii", "Romulea columnae", "Romulea rosea", "Herbertia lahue", "Crocus biflorus", - "Crocus nudiflorus", "Crocus sativus", "Crocus tommasinianus", "Crocus vernus", "Dietes bicolor", "Nemastylis geminiflora", - "Tigridia pavonia", "Ixia viridiflora", "Trimezia martinicensis", "Crocosmia × crocosmiiflora", "Freesia refracta", - "Sparaxis tricolor", "Iris bulleyana", "Iris chrysographes", "Iris confusa", "Iris cristata", "Iris douglasiana", "Iris ensata", - "Iris foetidissima", "Iris fulva 'Louisiana Hybrids'", "Iris germanica", "Iris hartwegii", "Iris japonica", "Iris lactea", - "Iris lutescens", "Iris macrosiphon", "Iris missouriensis", "Iris pseudacorus", "Iris pumila", "Iris ruthenica", "Iris sanguinea", - "Iris setosa", "Iris sibirica", "Iris speculatrix", "Iris tectorum", "Iris tenax", "Iris verna", "Iris versicolor", - "Iris virginica", "Tinantia anomala", "Tinantia erecta", "Pollia japonica", "Murdannia loriformis", "Murdannia nudiflora", - "Murdannia triquetra", "Amischotolype hispida", "Tradescantia cerinthoides 'Nanouk'", "Tradescantia fluminensis", - "Tradescantia ohiensis", "Tradescantia pallida", "Tradescantia sillamontana", "Tradescantia spathacea", "Tradescantia virginiana", - "Tradescantia zanonia", "Tradescantia zebrina", "Floscopa scandens", "Cyanotis arachnoidea", "Commelina benghalensis", - "Commelina communis", "Commelina diffusa", "Commelina erecta", "Strelitzia nicolai", "Strelitzia reginae", "Ephedra aspera", - "Ephedra californica", "Ephedra distachya", "Ephedra trifurca", "Ephedra viridis", "Pachysandra terminalis", - "Sarcococca hookeriana", "Sarcococca ruscifolia", "Buxus harlandii", "Buxus sinica", "Itea omeiensis", "Berchemia floribunda", - "Berchemia lineata", "Berchemia sinica", "Ziziphus jujuba", "Ziziphus mauritiana", "Hovenia acerba", "Ceanothus", - "Ventilago leiocarpa", "Frangula californica", "Sageretia thea", "Paliurus hemsleyanus", "Paliurus ramosissimus", - "Rhamnus cathartica", "Rhamnus crenata", "Rhamnus davurica", "Rhamnus utilis", "Gentianella azurea", "Latouchea fokienensis", - "Tripterospermum chinense", "Tripterospermum nienkui", "Comastoma pulmonarium", "Megacodon stylophorus", "Gentianopsis barbata", - "Cotylanthera paucisquama", "Eustoma grandiflorum", "Fagraea ceilanica", "Fagraea ceilanica 'Variegata'", "Swertia bimaculata", - "Swertia decora", "Swertia hickinii", "Swertia pseudochinensis", "Centaurium pulchellum var. altaicum", "Canscora lucidissima", - "Sabatia campestris", "Halenia elliptica", "Exacum affine", "Gentiana arethusae var. delicatula", "Gentiana aristata", - "Gentiana dahurica", "Gentiana davidii", "Gentiana lawrencei var. farreri", "Gentiana loureiroi", "Gentiana panthaica", - "Gentiana pseudoaquatica", "Gentiana pudica", "Gentiana rubicunda", "Gentiana squarrosa", "Gentiana straminea", "Gentiana striata", - "Gentiana tatsienensis", "Gentiana urnula", "Gentiana veitchiorum", "Gentiana zollingeri", "Hopea chinensis", "Hopea hainanensis", - "Vatica mangachapoi", "Marsilea quadrifolia" - }; - }; -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_PLANTID_H diff --git a/lite/tnn/cv/tnn_resnet.cpp b/lite/tnn/cv/tnn_resnet.cpp deleted file mode 100644 index 5070e91b..00000000 --- a/lite/tnn/cv/tnn_resnet.cpp +++ /dev/null @@ -1,94 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "tnn_resnet.h" -#include "lite/utils.h" - -using tnncv::TNNResNet; - -TNNResNet::TNNResNet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNResNet::transform(const cv::Mat &mat_rs) -{ - // push into input_mat (1,3,224,224) - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNResNet::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch. - tnn::MatConvertParam cvt_param; - std::shared_ptr logits_mat; // (1,1000) - status = instance->GetOutputMat(logits_mat, cvt_param, "logits", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto logits_dims = logits_mat->GetDims(); - const unsigned int num_classes = logits_dims.at(1); // 1000 - const float *logits = (float *) logits_mat->GetData(); - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/tnn/cv/tnn_resnet.h b/lite/tnn/cv/tnn_resnet.h deleted file mode 100644 index 471fa62a..00000000 --- a/lite/tnn/cv/tnn_resnet.h +++ /dev/null @@ -1,414 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_RESNET_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_RESNET_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNResNet : public BasicTNNHandler - { - public: - explicit TNNResNet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNResNet() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - std::vector bias_vals = {-0.485f * 255.f * (1.0f / 0.229f) * (1.0 / 255.f), - -0.456f * 255.f * (1.0f / 0.224f) * (1.0 / 255.f), - -0.406f * 255.f * (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - - }; -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_RESNET_H diff --git a/lite/tnn/cv/tnn_resnext.cpp b/lite/tnn/cv/tnn_resnext.cpp deleted file mode 100644 index bb976798..00000000 --- a/lite/tnn/cv/tnn_resnext.cpp +++ /dev/null @@ -1,94 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "tnn_resnext.h" -#include "lite/utils.h" - -using tnncv::TNNResNeXt; - -TNNResNeXt::TNNResNeXt(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNResNeXt::transform(const cv::Mat &mat_rs) -{ - // push into input_mat (1,3,224,224) - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNResNeXt::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch. - tnn::MatConvertParam cvt_param; - std::shared_ptr logits_mat; // (1,1000) - status = instance->GetOutputMat(logits_mat, cvt_param, "logits", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto logits_dims = logits_mat->GetDims(); - const unsigned int num_classes = logits_dims.at(1); // 1000 - const float *logits = (float *) logits_mat->GetData(); - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/tnn/cv/tnn_resnext.h b/lite/tnn/cv/tnn_resnext.h deleted file mode 100644 index 69bc826a..00000000 --- a/lite/tnn/cv/tnn_resnext.h +++ /dev/null @@ -1,414 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_RESNEXT_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_RESNEXT_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNResNeXt : public BasicTNNHandler - { - public: - explicit TNNResNeXt(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNResNeXt() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - std::vector bias_vals = {-0.485f * 255.f * (1.0f / 0.229f) * (1.0 / 255.f), - -0.456f * 255.f * (1.0f / 0.224f) * (1.0 / 255.f), - -0.406f * 255.f * (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - - }; -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_RESNEXT_H diff --git a/lite/tnn/cv/tnn_retinaface.cpp b/lite/tnn/cv/tnn_retinaface.cpp deleted file mode 100644 index 358f8478..00000000 --- a/lite/tnn/cv/tnn_retinaface.cpp +++ /dev/null @@ -1,245 +0,0 @@ -// -// Created by DefTruth on 2021/11/20. -// - -#include "tnn_retinaface.h" -#include "lite/utils.h" - -using tnncv::TNNRetinaFace; - -TNNRetinaFace::TNNRetinaFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNRetinaFace::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNRetinaFace::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(bbox_collection, instance, score_threshold, img_height, img_width); - // 5. hard|blend nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void TNNRetinaFace::generate_anchors(const int target_height, const int target_width, - std::vector &anchors) -{ - std::vector> feature_maps; - for (auto step: steps) - { - feature_maps.push_back( - { - (int) std::ceil((float) target_height / (float) step), - (int) std::ceil((float) target_width / (float) step) - } // ceil - ); - } - - anchors.clear(); - const int num_feature_map = feature_maps.size(); - - for (int k = 0; k < num_feature_map; ++k) - { - auto f_map = feature_maps.at(k); // e.g [640//8,640//8] - auto tmp_min_sizes = min_sizes.at(k); // e.g [8,16] - int f_h = f_map.at(0); - int f_w = f_map.at(1); - - for (int i = 0; i < f_h; ++i) - { - for (int j = 0; j < f_w; ++j) - { - for (auto min_size: tmp_min_sizes) - { - float s_kx = (float) min_size / (float) target_width; // e.g 16/w - float s_ky = (float) min_size / (float) target_height; // e.g 16/h - // (x + 0.5) * step / w normalized loc mapping to input width - // (y + 0.5) * step / h normalized loc mapping to input height - float cx = ((float) j + 0.5f) * (float) steps.at(k) / (float) target_width; - float cy = ((float) i + 0.5f) * (float) steps.at(k) / (float) target_height; - - anchors.push_back(RetinaAnchor{cx, cy, s_kx, s_ky}); // without clip - } - } - } - } -} - -void TNNRetinaFace::generate_bboxes(std::vector &bbox_collection, - std::shared_ptr &_instance, - float score_threshold, float img_height, - float img_width) -{ - std::shared_ptr bboxes; // (1,n,4) - std::shared_ptr probs; // (1,n,2) - tnn::MatConvertParam cvt_param; - tnn::Status status_bboxes; - tnn::Status status_probs; - - status_bboxes = _instance->GetOutputMat(bboxes, cvt_param, "bbox", output_device_type); - status_probs = _instance->GetOutputMat(probs, cvt_param, "conf", output_device_type); - - if (status_bboxes != tnn::TNN_OK || status_probs != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status_bboxes.description().c_str() << ": " - << status_probs.description().c_str() << "\n"; -#endif - return; - } - auto bbox_dims = bboxes->GetDims(); - const unsigned int bbox_num = bbox_dims.at(1); // n = ? - - std::vector anchors; - this->generate_anchors(input_height, input_width, anchors); - - const unsigned int num_anchors = anchors.size(); - if (num_anchors != bbox_num) - throw std::runtime_error("mismatch num_anchors != bbox_num"); - - const float *bboxes_ptr = (float *) bboxes->GetData(); - const float *probs_ptr = (float *) probs->GetData(); - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - float conf = probs_ptr[2 * i + 1]; - if (conf < score_threshold) continue; // filter first. - - float prior_cx = anchors.at(i).cx; - float prior_cy = anchors.at(i).cy; - float prior_s_kx = anchors.at(i).s_kx; - float prior_s_ky = anchors.at(i).s_ky; - - float dx = bboxes_ptr[4 * i + 0]; - float dy = bboxes_ptr[4 * i + 1]; - float dw = bboxes_ptr[4 * i + 2]; - float dh = bboxes_ptr[4 * i + 3]; - // ref: https://github.com/biubug6/Pytorch_Retinaface/blob/master/utils/box_utils.py - float cx = prior_cx + dx * variance[0] * prior_s_kx; - float cy = prior_cy + dy * variance[0] * prior_s_ky; - float w = prior_s_kx * std::exp(dw * variance[1]); - float h = prior_s_ky * std::exp(dh * variance[1]); // norm coor (0.,1.) - - types::Boxf box; - box.x1 = (cx - w / 2.f) * img_width; - box.y1 = (cy - h / 2.f) * img_height; - box.x2 = (cx + w / 2.f) * img_width; - box.y2 = (cy + h / 2.f) * img_height; - box.score = conf; - box.label = 1; - box.label_text = "face"; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITETNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void TNNRetinaFace::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lite/tnn/cv/tnn_retinaface.h b/lite/tnn/cv/tnn_retinaface.h deleted file mode 100644 index 57b6132b..00000000 --- a/lite/tnn/cv/tnn_retinaface.h +++ /dev/null @@ -1,75 +0,0 @@ -// -// Created by DefTruth on 2021/11/20. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_RETINAFACE_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_RETINAFACE_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNRetinaFace : public BasicTNNHandler - { - public: - explicit TNNRetinaFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNRetinaFace() override = default; - - private: - // nested classes - struct RetinaAnchor - { - float cx; - float cy; - float s_kx; - float s_ky; - }; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f, 1.f, 1.f}; - std::vector bias_vals = { - -104.f * 1.0f, - -117.f * 1.0f, - -123.f * 1.0f - }; // bgr order - const float variance[2] = {0.1f, 0.2f}; - std::vector steps = {8, 16, 32}; - std::vector> min_sizes = { - {16, 32}, - {64, 128}, - {256, 512} - }; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void transform(const cv::Mat &mat_rs) override; // - - void generate_anchors(const int target_height, - const int target_width, - std::vector &anchors); - - void generate_bboxes(std::vector &bbox_collection, - std::shared_ptr &_instance, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.7f, float iou_threshold = 0.3f, - unsigned int topk = 300, unsigned int nms_type = 0); - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_RETINAFACE_H diff --git a/lite/tnn/cv/tnn_rexnet_emotion7.cpp b/lite/tnn/cv/tnn_rexnet_emotion7.cpp deleted file mode 100644 index 5aa38ea9..00000000 --- a/lite/tnn/cv/tnn_rexnet_emotion7.cpp +++ /dev/null @@ -1,86 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#include "tnn_rexnet_emotion7.h" -#include "lite/utils.h" - -using tnncv::TNNReXNetEmotion7; - -TNNReXNetEmotion7::TNNReXNetEmotion7(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNReXNetEmotion7::transform(const cv::Mat &mat_rs) -{ - // push into input_mat (1,3,224,224) - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNReXNetEmotion7::detect(const cv::Mat &mat, types::Emotions &emotions) -{ - if (mat.empty()) return; - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch. - tnn::MatConvertParam cvt_param; - std::shared_ptr emotion_logits; // (1,7) - status = instance->GetOutputMat(emotion_logits, cvt_param, "logits", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto emotion_dims = emotion_logits->GetDims(); - const unsigned int num_emotions = emotion_dims.at(1); // 7 - - unsigned int pred_label = 0; - const float *pred_logits_ptr = (float *) emotion_logits->GetData(); - - auto softmax_probs = lite::utils::math::softmax(pred_logits_ptr, num_emotions, pred_label); - emotions.label = pred_label; - emotions.score = softmax_probs[pred_label]; - emotions.text = emotion_texts[pred_label]; - emotions.flag = true; -} diff --git a/lite/tnn/cv/tnn_rexnet_emotion7.h b/lite/tnn/cv/tnn_rexnet_emotion7.h deleted file mode 100644 index bc5aeb21..00000000 --- a/lite/tnn/cv/tnn_rexnet_emotion7.h +++ /dev/null @@ -1,40 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_REXNET_EMOTION7_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_REXNET_EMOTION7_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNReXNetEmotion7 : public BasicTNNHandler - { - public: - explicit TNNReXNetEmotion7(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNReXNetEmotion7() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f / (255.f * 0.229f), - 1.f / (255.f * 0.224f), - 1.f / (255.f * 0.225f)}; - std::vector bias_vals = {-255.f * 0.485f * 1.f / (255.f * 0.229f), - -255.f * 0.456f * 1.f / (255.f * 0.224f), - -255.f * 0.406f * 1.f / (255.f * 0.225f)}; - const char *emotion_texts[7] = { - "angry", "disgust", "fear", "happiness", "neutral", "sadness", "surprise" - }; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::Emotions &emotions); - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_REXNET_EMOTION7_H diff --git a/lite/tnn/cv/tnn_rvm.cpp b/lite/tnn/cv/tnn_rvm.cpp deleted file mode 100644 index aa12fda3..00000000 --- a/lite/tnn/cv/tnn_rvm.cpp +++ /dev/null @@ -1,492 +0,0 @@ -// -// Created by DefTruth on 2021/10/18. -// - -#include "tnn_rvm.h" -#include "lite/utils.h" - - -using tnncv::TNNRobustVideoMatting; - -TNNRobustVideoMatting::TNNRobustVideoMatting( - const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads -) : proto_path(_proto_path.data()), - model_path(_model_path.data()), - log_id(_proto_path.data()), - num_threads(_num_threads) -{ - initialize_instance(); - initialize_context(); -} - -TNNRobustVideoMatting::~TNNRobustVideoMatting() -{ - net = nullptr; - src_mat = nullptr; - r1i_mat = nullptr; - r2i_mat = nullptr; - r3i_mat = nullptr; - r4i_mat = nullptr; - instance = nullptr; -} - -void TNNRobustVideoMatting::initialize_instance() -{ - std::string proto_content_buffer, model_content_buffer; - proto_content_buffer = BasicTNNHandler::content_buffer_from(proto_path); - model_content_buffer = BasicTNNHandler::content_buffer_from(model_path); - - tnn::ModelConfig model_config; - model_config.model_type = tnn::MODEL_TYPE_TNN; - model_config.params = {proto_content_buffer, model_content_buffer}; - - // 1. init TNN net - tnn::Status status; - net = std::make_shared(); - status = net->Init(model_config); - if (status != tnn::TNN_OK || !net) - { -#ifdef LITETNN_DEBUG - std::cout << "net->Init failed!\n"; -#endif - return; - } - // 2. init device type, change this default setting - // for better performance. such as CUDA/OPENCL/... -#ifdef __ANDROID__ - network_device_type = tnn::DEVICE_ARM; // CPU,GPU - input_device_type = tnn::DEVICE_ARM; // CPU only - output_device_type = tnn::DEVICE_ARM; -#else - network_device_type = tnn::DEVICE_X86; // CPU,GPU - input_device_type = tnn::DEVICE_X86; // CPU only - output_device_type = tnn::DEVICE_X86; -#endif - // 3. init instance - tnn::NetworkConfig network_config; - network_config.library_path = {""}; - network_config.device_type = network_device_type; - - instance = net->CreateInst(network_config, status); - if (status != tnn::TNN_OK || !instance) - { -#ifdef LITETNN_DEBUG - std::cout << "CreateInst failed!" << status.description().c_str() << "\n"; -#endif - return; - } - // 4. setting up num_threads - instance->SetCpuNumThreads((int) num_threads); - // 5. init input information. - for (auto &name: input_names) - input_shapes[name] = BasicTNNHandler::get_input_shape(instance, name); - auto src_shape = input_shapes.at("src"); - if (src_shape.size() != 4) - { -#ifdef LITETNN_DEBUG - throw std::runtime_error("Found src_shape.size()!=4, but " - "src input only support 4 dims." - "Such as NCHW, NHWC ..."); -#else - return; -#endif - } - input_mat_type = BasicTNNHandler::get_input_mat_type(instance, "src"); - input_data_format = BasicTNNHandler::get_input_data_format(instance, "src"); - if (input_data_format == tnn::DATA_FORMAT_NCHW) - { - input_height = src_shape.at(2); - input_width = src_shape.at(3); - } // NHWC - else if (input_data_format == tnn::DATA_FORMAT_NHWC) - { - input_height = src_shape.at(1); - input_width = src_shape.at(2); - } // unsupport - else - { -#ifdef LITETNN_DEBUG - std::cout << "src input only support NCHW and NHWC " - "input_data_format, but found others.\n"; -#endif - return; - } - src_size = 1 * 3 * input_height * input_width; - // 6. init output information, debug only. - for (auto &name: output_names) - output_shapes[name] = BasicTNNHandler::get_output_shape(instance, name); -#ifdef LITETNN_DEBUG - this->print_debug_string(); -#endif -} - -int TNNRobustVideoMatting::value_size_of(tnn::DimsVector &shape) -{ - if (shape.empty()) return 0; - int _size = 1; - for (auto &s: shape) _size *= s; - return _size; -} - -void TNNRobustVideoMatting::print_debug_string() -{ - std::cout << "LITETNN_DEBUG LogId: " << log_id << "\n"; - std::cout << "=============== Input-Dims ==============\n"; - for (auto &in: input_shapes) - BasicTNNHandler::print_name_shape(in.first, in.second); - std::string data_format_string = - (input_data_format == tnn::DATA_FORMAT_NCHW) ? "NCHW" : "NHWC"; - std::cout << "Input Data Format: " << data_format_string << "\n"; - std::cout << "=============== Output-Dims ==============\n"; - for (auto &out: output_shapes) - BasicTNNHandler::print_name_shape(out.first, out.second); - std::cout << "========================================\n"; -} - -void TNNRobustVideoMatting::initialize_context() -{ - r1i_mat = std::make_shared( - input_device_type, - tnn::NCHW_FLOAT, - input_shapes.at("r1i") - ); - r2i_mat = std::make_shared( - input_device_type, - tnn::NCHW_FLOAT, - input_shapes.at("r2i") - ); - r3i_mat = std::make_shared( - input_device_type, - tnn::NCHW_FLOAT, - input_shapes.at("r3i") - ); - r4i_mat = std::make_shared( - input_device_type, - tnn::NCHW_FLOAT, - input_shapes.at("r4i") - ); - r1i_size = this->value_size_of(input_shapes.at("r1i")); - r2i_size = this->value_size_of(input_shapes.at("r2i")); - r3i_size = this->value_size_of(input_shapes.at("r3i")); - r4i_size = this->value_size_of(input_shapes.at("r4i")); - // init 0. - std::fill_n((float *) r1i_mat->GetData(), r1i_size, 0.f); - std::fill_n((float *) r2i_mat->GetData(), r2i_size, 0.f); - std::fill_n((float *) r3i_mat->GetData(), r3i_size, 0.f); - std::fill_n((float *) r4i_mat->GetData(), r4i_size, 0.f); - - context_is_initialized = true; -} - -void TNNRobustVideoMatting::transform(const cv::Mat &mat_rs) -{ - // cv::Mat canvas; - // cv::resize(mat, canvas, cv::Size(input_width, input_height)); - // cv::cvtColor(canvas, canvas, cv::COLOR_BGR2RGB); - // reference: https://github.com/DefTruth/lite.ai.toolkit/issues/240 - // push into src_mat - src_mat = std::make_shared( - input_device_type, - tnn::N8UC3, - input_shapes.at("src"), - (void *) mat_rs.data - ); - if (!src_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNRobustVideoMatting::detect(const cv::Mat &mat, types::MattingContent &content, bool video_mode, - bool remove_noise, bool minimum_post_process) -{ - if (mat.empty()) return; - int img_h = mat.rows; - int img_w = mat.cols; - if (!context_is_initialized) return; - - // 1. make input tensor - cv::Mat mat_rs; - // resize mat outside 'transform' to prevent memory overflow - // reference: https://github.com/DefTruth/lite.ai.toolkit/issues/240 - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam src_cvt_param, ctx_cvt_param; - src_cvt_param.scale = scale_vals; - src_cvt_param.bias = bias_vals; - - tnn::Status status_src, status_r1i, status_r2i, status_r3i, status_r4i; - status_src = instance->SetInputMat(src_mat, src_cvt_param, "src"); - status_r1i = instance->SetInputMat(r1i_mat, ctx_cvt_param, "r1i"); - status_r2i = instance->SetInputMat(r2i_mat, ctx_cvt_param, "r2i"); - status_r3i = instance->SetInputMat(r3i_mat, ctx_cvt_param, "r3i"); - status_r4i = instance->SetInputMat(r4i_mat, ctx_cvt_param, "r4i"); - if (status_src != tnn::TNN_OK || status_r1i != tnn::TNN_OK || - status_r2i != tnn::TNN_OK || status_r3i != tnn::TNN_OK || - status_r4i != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status_src.description().c_str() << ": " - << status_r1i.description().c_str() << ": " - << status_r2i.description().c_str() << ": " - << status_r3i.description().c_str() << ": " - << status_r4i.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - auto status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - // 4. generate matting - this->generate_matting(instance, content, img_h, img_w, remove_noise, minimum_post_process); - // 5. update context (needed for video matting) - if (video_mode) - { - context_is_update = false; // init state. - this->update_context(instance); - } - -} - -void TNNRobustVideoMatting::detect_video( - const std::string &video_path, const std::string &output_path, - std::vector &contents, bool save_contents, - unsigned int writer_fps, bool remove_noise, bool minimum_post_process, - const cv::Mat &background) -{ -// 0. init video capture - cv::VideoCapture video_capture(video_path); - const unsigned int width = video_capture.get(cv::CAP_PROP_FRAME_WIDTH); - const unsigned int height = video_capture.get(cv::CAP_PROP_FRAME_HEIGHT); - const unsigned int frame_count = video_capture.get(cv::CAP_PROP_FRAME_COUNT); - if (!video_capture.isOpened()) - { - std::cout << "Can not open video: " << video_path << "\n"; - return; - } - // 1. init video writer - cv::VideoWriter video_writer(output_path, cv::VideoWriter::fourcc('m', 'p', '4', 'v'), - writer_fps, cv::Size(width, height)); - if (!video_writer.isOpened()) - { - std::cout << "Can not open writer: " << output_path << "\n"; - return; - } - - // 2. matting loop - cv::Mat mat; - unsigned int i = 0; - while (video_capture.read(mat)) - { - i += 1; - types::MattingContent content; - this->detect(mat, content, true, remove_noise, minimum_post_process); // video_mode true - // 3. save contents and writing out. - if (content.flag) - { -// if (save_contents) contents.push_back(content); -// if (!content.merge_mat.empty()) video_writer.write(content.merge_mat); - - if (save_contents) contents.push_back(content); - // 3.1 do nothing if set minimum_post_process as true - if (background.empty()) - { - if (!content.merge_mat.empty() && !minimum_post_process) - video_writer.write(content.merge_mat); - else if (!content.fgr_mat.empty()) - video_writer.write(content.fgr_mat); - } // - else - { - cv::Mat out_mat; - // 3.2 merge user custom background - if (!content.pha_mat.empty()) - { - if (!content.fgr_mat.empty()) - lite::utils::swap_background(content.fgr_mat, content.pha_mat, - background, out_mat, false); - else - lite::utils::swap_background(mat, content.pha_mat, - background, out_mat, false); - } - if (!out_mat.empty()) video_writer.write(out_mat); - - } - - } - // 4. check context states. - if (!context_is_update) break; -#ifdef LITETNN_DEBUG - std::cout << i << "/" << frame_count << " done!" << "\n"; -#endif - } - - // 5. release - video_capture.release(); - video_writer.release(); -} - -void TNNRobustVideoMatting::generate_matting(std::shared_ptr &_instance, - types::MattingContent &content, - int img_h, int img_w, - bool remove_noise, - bool minimum_post_process) -{ - std::shared_ptr fgr_mat; - std::shared_ptr pha_mat; - tnn::MatConvertParam cvt_param; - tnn::Status status_fgr, status_pha; - - status_fgr = _instance->GetOutputMat(fgr_mat, cvt_param, "fgr", output_device_type); - status_pha = _instance->GetOutputMat(pha_mat, cvt_param, "pha", output_device_type); - - if (status_fgr != tnn::TNN_OK || status_pha != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status_fgr.description().c_str() << ": " - << status_pha.description().c_str() << "\n"; -#endif - return; - } - - float *fgr_ptr = (float *) fgr_mat->GetData(); - float *pha_ptr = (float *) pha_mat->GetData(); - const unsigned int channel_step = input_height * input_width; - - // fast assign & channel transpose(CHW->HWC). - cv::Mat rmat(input_height, input_width, CV_32FC1, fgr_ptr); - cv::Mat gmat(input_height, input_width, CV_32FC1, fgr_ptr + channel_step); - cv::Mat bmat(input_height, input_width, CV_32FC1, fgr_ptr + 2 * channel_step); - cv::Mat pmat(input_height, input_width, CV_32FC1, pha_ptr); // ref only, zero-copy. - if (remove_noise) lite::utils::remove_small_connected_area(pmat, 0.05f); - - rmat *= 255.f; - bmat *= 255.f; - gmat *= 255.f; - std::vector fgr_channel_mats; - fgr_channel_mats.push_back(bmat); - fgr_channel_mats.push_back(gmat); - fgr_channel_mats.push_back(rmat); - - // need clone to allocate a new continuous memory. - content.pha_mat = pmat.clone(); // allocated - cv::merge(fgr_channel_mats, content.fgr_mat); - content.fgr_mat.convertTo(content.fgr_mat, CV_8UC3); - - if (!minimum_post_process) - { - cv::Mat rest = 1.f - pmat; - cv::Mat mbmat = bmat.mul(pmat) + rest * 153.f; - cv::Mat mgmat = gmat.mul(pmat) + rest * 255.f; - cv::Mat mrmat = rmat.mul(pmat) + rest * 120.f; - std::vector merge_channel_mats; - merge_channel_mats.push_back(mbmat); - merge_channel_mats.push_back(mgmat); - merge_channel_mats.push_back(mrmat); - cv::merge(merge_channel_mats, content.merge_mat); - content.merge_mat.convertTo(content.merge_mat, CV_8UC3); - } - - if (img_w != input_width || img_h != input_height) - { - cv::resize(content.pha_mat, content.pha_mat, cv::Size(img_w, img_h)); - cv::resize(content.fgr_mat, content.fgr_mat, cv::Size(img_w, img_h)); - if (!minimum_post_process) - cv::resize(content.merge_mat, content.merge_mat, cv::Size(img_w, img_h)); - } - - content.flag = true; -} - -void TNNRobustVideoMatting::update_context(std::shared_ptr &_instance) -{ - std::shared_ptr r1o_mat; - std::shared_ptr r2o_mat; - std::shared_ptr r3o_mat; - std::shared_ptr r4o_mat; - tnn::MatConvertParam cvt_param; - tnn::Status status_r1o; - tnn::Status status_r2o; - tnn::Status status_r3o; - tnn::Status status_r4o; - - status_r1o = _instance->GetOutputMat(r1o_mat, cvt_param, "r1o", output_device_type); - status_r2o = _instance->GetOutputMat(r2o_mat, cvt_param, "r2o", output_device_type); - status_r3o = _instance->GetOutputMat(r3o_mat, cvt_param, "r3o", output_device_type); - status_r4o = _instance->GetOutputMat(r4o_mat, cvt_param, "r4o", output_device_type); - - if (status_r1o != tnn::TNN_OK || status_r2o != tnn::TNN_OK || - status_r3o != tnn::TNN_OK || status_r4o != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat context failed!:" - << status_r1o.description().c_str() << ": " - << status_r2o.description().c_str() << ": " - << status_r3o.description().c_str() << ": " - << status_r4o.description().c_str() << "\n"; -#endif - return; - } - void *command_queue = nullptr; - auto status_cmd = _instance->GetCommandQueue(&command_queue); - if (status_cmd != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetCommandQueue failed!:" - << status_cmd.description().c_str() << "\n"; -#endif - return; - } - - tnn::MatUtils::Copy(*r1o_mat, *r1i_mat, command_queue); - tnn::MatUtils::Copy(*r2o_mat, *r2i_mat, command_queue); - tnn::MatUtils::Copy(*r3o_mat, *r3i_mat, command_queue); - tnn::MatUtils::Copy(*r4o_mat, *r4i_mat, command_queue); - - context_is_update = true; -} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lite/tnn/cv/tnn_rvm.h b/lite/tnn/cv/tnn_rvm.h deleted file mode 100644 index fd73da21..00000000 --- a/lite/tnn/cv/tnn_rvm.h +++ /dev/null @@ -1,145 +0,0 @@ -// -// Created by DefTruth on 2021/10/18. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_RVM_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_RVM_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNRobustVideoMatting - { - public: - explicit TNNRobustVideoMatting(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); - - ~TNNRobustVideoMatting(); - - private: - const char *log_id = nullptr; - const char *proto_path = nullptr; - const char *model_path = nullptr; - // Note, tnn:: actually is TNN_NS::, I prefer the first one. - std::shared_ptr net; - std::shared_ptr instance; - - private: - std::vector scale_vals = {1.f / 255.f, 1.f / 255.f, 1.f / 255.f}; - std::vector bias_vals = {0.f, 0.f, 0.f}; // RGB - // hardcode input node names, hint only. - // downsample_ratio has been freeze while onnx exported - // and, the input size of each input has been freeze, also. - std::vector input_names = { - "src", - "r1i", - "r2i", - "r3i", - "r4i" - }; - // hardcode output node names, hint only. - std::vector output_names = { - "fgr", - "pha", - "r1o", - "r2o", - "r3o", - "r4o" - }; - bool context_is_update = false; - bool context_is_initialized = false; - - private: - const unsigned int num_threads; // initialize at runtime. - // multi inputs, rxi will be update inner video matting process. - std::shared_ptr src_mat; - std::shared_ptr r1i_mat; - std::shared_ptr r2i_mat; - std::shared_ptr r3i_mat; - std::shared_ptr r4i_mat; - // input size , initialize at runtime. - int input_height; - int input_width; - tnn::DataFormat input_data_format; // e.g DATA_FORMAT_NHWC - tnn::MatType input_mat_type; // e.g NCHW_FLOAT - tnn::DeviceType input_device_type; // only CPU, namely ARM or X86 - tnn::DeviceType output_device_type; // only CPU, namely ARM or X86 - tnn::DeviceType network_device_type; // e.g DEVICE_X86 DEVICE_NAIVE DEVICE_ARM - std::map input_shapes; - std::map output_shapes; - unsigned int src_size; - unsigned int r1i_size; - unsigned int r2i_size; - unsigned int r3i_size; - unsigned int r4i_size; - - // un-copyable - protected: - TNNRobustVideoMatting(const TNNRobustVideoMatting &) = delete; // - TNNRobustVideoMatting(TNNRobustVideoMatting &&) = delete; // - TNNRobustVideoMatting &operator=(const TNNRobustVideoMatting &) = delete; // - TNNRobustVideoMatting &operator=(TNNRobustVideoMatting &&) = delete; // - - private: - void print_debug_string(); // debug information - - private: - void transform(const cv::Mat &mat_rs); // - - void initialize_instance(); // init net & instance - - void initialize_context(); - - int value_size_of(tnn::DimsVector &shape); - - void generate_matting(std::shared_ptr &_instance, - types::MattingContent &content, - int img_h, int img_w, - bool remove_noise = false, - bool minimum_post_process = false); - - void update_context(std::shared_ptr &_instance); - - public: - /** - * Image Matting Using RVM(https://github.com/PeterL1n/RobustVideoMatting) - * @param mat: cv::Mat BGR HWC - * @param content: types::MattingContent to catch the detected results. - * @param video_mode: false by default. - * See https://github.com/PeterL1n/RobustVideoMatting/blob/master/documentation/inference_zh_Hans.md - * @param remove_noise: remove small connected area or not - * @param minimum_post_process: if True, will run matting with minimum post process - * in order to speed up the matting processes. - */ - void detect(const cv::Mat &mat, types::MattingContent &content, bool video_mode = false, - bool remove_noise = false, bool minimum_post_process = false); - - /** - * Video Matting Using RVM(https://github.com/PeterL1n/RobustVideoMatting) - * @param video_path: eg. xxx/xxx/input.mp4 - * @param output_path: eg. xxx/xxx/output.mp4 - * @param contents: vector of MattingContent to catch the detected results. - * @param save_contents: false by default, whether to save MattingContent. - * See https://github.com/PeterL1n/RobustVideoMatting/blob/master/documentation/inference_zh_Hans.md - * @param writer_fps: FPS for VideoWriter, 20 by default. - * @param remove_noise: remove small connected area or not - * @param minimum_post_process: if True, will run matting with minimum post process - * in order to speed up the matting processes. - * @param background: user's custom background setting, will return with this target - * background if background Mat is not empty instead of green background. - */ - void detect_video(const std::string &video_path, - const std::string &output_path, - std::vector &contents, - bool save_contents = false, - unsigned int writer_fps = 20, - bool remove_noise = false, - bool minimum_post_process = false, - const cv::Mat &background = cv::Mat()); - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_RVM_H diff --git a/lite/tnn/cv/tnn_scrfd.cpp b/lite/tnn/cv/tnn_scrfd.cpp deleted file mode 100644 index 1fead55d..00000000 --- a/lite/tnn/cv/tnn_scrfd.cpp +++ /dev/null @@ -1,482 +0,0 @@ -// -// Created by DefTruth on 2021/12/30. -// - -#include "tnn_scrfd.h" - -using tnncv::TNNSCRFD; - -TNNSCRFD::TNNSCRFD(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ - initial_context(); -} - -void TNNSCRFD::initial_context() -{ - if (num_outputs == 6) - { - fmc = 3; - feat_stride_fpn = {8, 16, 32}; - num_anchors = 2; - use_kps = false; - } // kps - else if (num_outputs == 9) - { - fmc = 3; - feat_stride_fpn = {8, 16, 32}; - num_anchors = 2; - use_kps = true; - } - -} - -void TNNSCRFD::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - SCRFDScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(0, 0, 0)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.ratio = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.flag = true; -} - -void TNNSCRFD::transform(const cv::Mat &mat_rs) -{ - // push into input_mat, RGB - // be carefully, no deepcopy inside this tnn::Mat constructor, - // so, we can not pass a local cv::Mat to this constructor. - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNSCRFD::detect(const cv::Mat &mat, std::vector &detected_boxes_kps, - float score_threshold, float iou_threshold, unsigned int topk) -{ - if (mat.empty()) return; - auto img_height = static_cast(mat.rows); - auto img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - SCRFDScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input mat - cv::Mat mat_rs_; - cv::cvtColor(mat_rs, mat_rs_, cv::COLOR_BGR2RGB); - this->transform(mat_rs_); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 3. rescale & exclude. - std::vector bbox_kps_collection; - this->generate_bboxes_kps(scale_params, bbox_kps_collection, instance, - score_threshold, img_height, img_width); - // 4. hard nms with topk. - this->nms_bboxes_kps(bbox_kps_collection, detected_boxes_kps, iou_threshold, topk); -} - -void TNNSCRFD::generate_points(const int target_height, const int target_width) -{ - if (center_points_is_update) return; - // 8, 16, 32 - for (auto stride : feat_stride_fpn) - { - unsigned int num_grid_w = target_width / stride; - unsigned int num_grid_h = target_height / stride; - // y - for (unsigned int i = 0; i < num_grid_h; ++i) - { - // x - for (unsigned int j = 0; j < num_grid_w; ++j) - { - // num_anchors, col major - for (unsigned int k = 0; k < num_anchors; ++k) - { - SCRFDPoint point; - point.cx = (float) j; - point.cy = (float) i; - point.stride = (float) stride; - center_points[stride].push_back(point); - } - - } - } - } - - center_points_is_update = true; -} - -void TNNSCRFD::generate_bboxes_kps(const SCRFDScaleParams &scale_params, - std::vector &bbox_kps_collection, - std::shared_ptr &_instance, - float score_threshold, float img_height, - float img_width) -{ - tnn::MatConvertParam cvt_param; - std::shared_ptr score_8, score_16, score_32, bbox_8, bbox_16, bbox_32; - tnn::Status status_score_8, status_score_16, status_score_32, status_bbox_8, status_bbox_16, status_bbox_32; - status_score_8 = _instance->GetOutputMat(score_8, cvt_param, "score_8", output_device_type); // e.g [1,12800,1] - status_score_16 = _instance->GetOutputMat(score_16, cvt_param, "score_16", output_device_type); // e.g [1,3200,1] - status_score_32 = _instance->GetOutputMat(score_32, cvt_param, "score_32", output_device_type); // e.g [1,800,1] - status_bbox_8 = _instance->GetOutputMat(bbox_8, cvt_param, "bbox_8", output_device_type); // e.g [1,12800,4] - status_bbox_16 = _instance->GetOutputMat(bbox_16, cvt_param, "bbox_16", output_device_type); // e.g [1,3200,4] - status_bbox_32 = _instance->GetOutputMat(bbox_32, cvt_param, "bbox_32", output_device_type); // e.g [1,800,4] - this->generate_points(input_height, input_width); - - if (status_score_8 != tnn::TNN_OK || status_score_16 != tnn::TNN_OK || status_score_32 != tnn::TNN_OK || - status_bbox_8 != tnn::TNN_OK || status_bbox_16 != tnn::TNN_OK || status_bbox_32 != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" << status_score_8.description().c_str() << "\n"; - std::cout << "instance->GetOutputMat failed!:" << status_score_16.description().c_str() << "\n"; - std::cout << "instance->GetOutputMat failed!:" << status_score_32.description().c_str() << "\n"; - std::cout << "instance->GetOutputMat failed!:" << status_bbox_8.description().c_str() << "\n"; - std::cout << "instance->GetOutputMat failed!:" << status_bbox_16.description().c_str() << "\n"; - std::cout << "instance->GetOutputMat failed!:" << status_bbox_32.description().c_str() << "\n"; -#endif - return; - } - - bbox_kps_collection.clear(); - - if (use_kps) - { - std::shared_ptr kps_8, kps_16, kps_32; - tnn::Status status_kps_8, status_kps_16, status_kps_32; - status_kps_8 = _instance->GetOutputMat(kps_8, cvt_param, "kps_8", output_device_type); // e.g [1,12800,10] - status_kps_16 = _instance->GetOutputMat(kps_16, cvt_param, "kps_16", output_device_type); // e.g [1,3200,10] - status_kps_32 = _instance->GetOutputMat(kps_32, cvt_param, "kps_32", output_device_type); // e.g [1,800,10] - if (status_kps_8 != tnn::TNN_OK || status_kps_16 != tnn::TNN_OK || status_kps_32 != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" << status_kps_8.description().c_str() << "\n"; - std::cout << "instance->GetOutputMat failed!:" << status_kps_16.description().c_str() << "\n"; - std::cout << "instance->GetOutputMat failed!:" << status_kps_32.description().c_str() << "\n"; -#endif - return; - } - - // level 8 & 16 & 32 with kps - this->generate_bboxes_kps_single_stride(scale_params, score_8, bbox_8, kps_8, 8, score_threshold, - img_height, img_width, bbox_kps_collection); - this->generate_bboxes_kps_single_stride(scale_params, score_16, bbox_16, kps_16, 16, score_threshold, - img_height, img_width, bbox_kps_collection); - this->generate_bboxes_kps_single_stride(scale_params, score_32, bbox_32, kps_32, 32, score_threshold, - img_height, img_width, bbox_kps_collection); - } // no kps - else - { - // level 8 & 16 & 32 - this->generate_bboxes_single_stride(scale_params, score_8, bbox_8, 8, score_threshold, - img_height, img_width, bbox_kps_collection); - this->generate_bboxes_single_stride(scale_params, score_16, bbox_16, 16, score_threshold, - img_height, img_width, bbox_kps_collection); - this->generate_bboxes_single_stride(scale_params, score_32, bbox_32, 32, score_threshold, - img_height, img_width, bbox_kps_collection); - } -#if LITETNN_DEBUG - std::cout << "generate_bboxes_kps num: " << bbox_kps_collection.size() << "\n"; -#endif -} - - -void TNNSCRFD::generate_bboxes_single_stride( - const SCRFDScaleParams &scale_params, std::shared_ptr &score_pred, - std::shared_ptr &bbox_pred, unsigned int stride, float score_threshold, - float img_height, float img_width, std::vector &bbox_kps_collection) -{ - unsigned int nms_pre_ = (stride / 8) * nms_pre; // 1 * 1000,2*1000,... - nms_pre_ = nms_pre_ >= nms_pre ? nms_pre_ : nms_pre; - - auto stride_dims = score_pred->GetDims(); - const unsigned int num_points = stride_dims.at(1); // 12800 - const float *score_ptr = (float *) score_pred->GetData(); // [1,12800,1] - const float *bbox_ptr = (float *) bbox_pred->GetData(); // [1,12800,4] - - float ratio = scale_params.ratio; - int dw = scale_params.dw; - int dh = scale_params.dh; - - unsigned int count = 0; - auto &stride_points = center_points[stride]; - - for (unsigned int i = 0; i < num_points; ++i) - { - const float cls_conf = score_ptr[i]; - if (cls_conf < score_threshold) continue; // filter - auto &point = stride_points.at(i); - const float cx = point.cx; // cx - const float cy = point.cy; // cy - const float s = point.stride; // stride - - // bbox - const float *offsets = bbox_ptr + i * 4; - float l = offsets[0]; // left - float t = offsets[1]; // top - float r = offsets[2]; // right - float b = offsets[3]; // bottom - - types::BoxfWithLandmarks box_kps; - float x1 = ((cx - l) * s - (float) dw) / ratio; // cx - l x1 - float y1 = ((cy - t) * s - (float) dh) / ratio; // cy - t y1 - float x2 = ((cx + r) * s - (float) dw) / ratio; // cx + r x2 - float y2 = ((cy + b) * s - (float) dh) / ratio; // cy + b y2 - box_kps.box.x1 = std::max(0.f, x1); - box_kps.box.y1 = std::max(0.f, y1); - box_kps.box.x2 = std::min(img_width - 1.f, x2); - box_kps.box.y2 = std::min(img_height - 1.f, y2); - box_kps.box.score = cls_conf; - box_kps.box.label = 1; - box_kps.box.label_text = "face"; - box_kps.box.flag = true; - box_kps.flag = true; - - bbox_kps_collection.push_back(box_kps); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - - if (bbox_kps_collection.size() > nms_pre_) - { - std::sort( - bbox_kps_collection.begin(), bbox_kps_collection.end(), - [](const types::BoxfWithLandmarks &a, const types::BoxfWithLandmarks &b) - { return a.box.score > b.box.score; } - ); // sort inplace - // trunc - bbox_kps_collection.resize(nms_pre_); - } - -} - -void TNNSCRFD::generate_bboxes_kps_single_stride( - const SCRFDScaleParams &scale_params, std::shared_ptr &score_pred, - std::shared_ptr &bbox_pred, std::shared_ptr &kps_pred, - unsigned int stride, float score_threshold, float img_height, float img_width, - std::vector &bbox_kps_collection) -{ - unsigned int nms_pre_ = (stride / 8) * nms_pre; // 1 * 1000,2*1000,... - nms_pre_ = nms_pre_ >= nms_pre ? nms_pre_ : nms_pre; - - auto stride_dims = score_pred->GetDims(); - const unsigned int num_points = stride_dims.at(1); // 12800 - const float *score_ptr = (float *) score_pred->GetData(); // [1,12800,1] - const float *bbox_ptr = (float *) bbox_pred->GetData(); // [1,12800,4] - const float *kps_ptr = (float *) kps_pred->GetData(); // [1,12800,10] - - float ratio = scale_params.ratio; - int dw = scale_params.dw; - int dh = scale_params.dh; - - unsigned int count = 0; - auto &stride_points = center_points[stride]; - - for (unsigned int i = 0; i < num_points; ++i) - { - const float cls_conf = score_ptr[i]; - if (cls_conf < score_threshold) continue; // filter - auto &point = stride_points.at(i); - const float cx = point.cx; // cx - const float cy = point.cy; // cy - const float s = point.stride; // stride - - // bbox - const float *offsets = bbox_ptr + i * 4; - float l = offsets[0]; // left - float t = offsets[1]; // top - float r = offsets[2]; // right - float b = offsets[3]; // bottom - - types::BoxfWithLandmarks box_kps; - float x1 = ((cx - l) * s - (float) dw) / ratio; // cx - l x1 - float y1 = ((cy - t) * s - (float) dh) / ratio; // cy - t y1 - float x2 = ((cx + r) * s - (float) dw) / ratio; // cx + r x2 - float y2 = ((cy + b) * s - (float) dh) / ratio; // cy + b y2 - box_kps.box.x1 = std::max(0.f, x1); - box_kps.box.y1 = std::max(0.f, y1); - box_kps.box.x2 = std::min(img_width - 1.f, x2); - box_kps.box.y2 = std::min(img_height - 1.f, y2); - box_kps.box.score = cls_conf; - box_kps.box.label = 1; - box_kps.box.label_text = "face"; - box_kps.box.flag = true; - - // landmarks - const float *kps_offsets = kps_ptr + i * 10; - for (unsigned int j = 0; j < 10; j += 2) - { - cv::Point2f kps; - float kps_l = kps_offsets[j]; - float kps_t = kps_offsets[j + 1]; - float kps_x = ((cx + kps_l) * s - (float) dw) / ratio; // cx - l x - float kps_y = ((cy + kps_t) * s - (float) dh) / ratio; // cy - t y - kps.x = std::min(std::max(0.f, kps_x), img_width - 1.f); - kps.y = std::min(std::max(0.f, kps_y), img_height - 1.f); - box_kps.landmarks.points.push_back(kps); - } - box_kps.landmarks.flag = true; - box_kps.flag = true; - - bbox_kps_collection.push_back(box_kps); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - - if (bbox_kps_collection.size() > nms_pre_) - { - std::sort( - bbox_kps_collection.begin(), bbox_kps_collection.end(), - [](const types::BoxfWithLandmarks &a, const types::BoxfWithLandmarks &b) - { return a.box.score > b.box.score; } - ); // sort inplace - // trunc - bbox_kps_collection.resize(nms_pre_); - } -} - -void TNNSCRFD::nms_bboxes_kps(std::vector &input, - std::vector &output, - float iou_threshold, unsigned int topk) -{ - if (input.empty()) return; - std::sort( - input.begin(), input.end(), - [](const types::BoxfWithLandmarks &a, const types::BoxfWithLandmarks &b) - { return a.box.score > b.box.score; } - ); - const unsigned int box_num = input.size(); - std::vector merged(box_num, 0); - - unsigned int count = 0; - for (unsigned int i = 0; i < box_num; ++i) - { - if (merged[i]) continue; - std::vector buf; - - buf.push_back(input[i]); - merged[i] = 1; - - for (unsigned int j = i + 1; j < box_num; ++j) - { - if (merged[j]) continue; - - float iou = static_cast(input[i].box.iou_of(input[j].box)); - - if (iou > iou_threshold) - { - merged[j] = 1; - buf.push_back(input[j]); - } - - } - output.push_back(buf[0]); - - // keep top k - count += 1; - if (count >= topk) - break; - } -} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lite/tnn/cv/tnn_scrfd.h b/lite/tnn/cv/tnn_scrfd.h deleted file mode 100644 index c0f1485c..00000000 --- a/lite/tnn/cv/tnn_scrfd.h +++ /dev/null @@ -1,161 +0,0 @@ -// -// Created by DefTruth on 2021/12/30. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_SCRFD_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_SCRFD_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNSCRFD : public BasicTNNHandler - { - public: - explicit TNNSCRFD(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNSCRFD() override = default; - - private: - // nested classes - typedef struct - { - float cx; - float cy; - float stride; - } SCRFDPoint; - typedef struct - { - float ratio; - int dw; - int dh; - bool flag; - } SCRFDScaleParams; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f / 128.f, 1.f / 128.f, 1.f / 128.f}; // RGB - std::vector bias_vals = {-127.5f / 128.f, -127.5f / 128.f, -127.5f / 128.f}; - unsigned int fmc = 3; // feature map count - bool use_kps = false; - unsigned int num_anchors = 2; - std::vector feat_stride_fpn = {8, 16, 32}; // steps, may [8, 16, 32, 64, 128] - // if num_anchors>1, then stack points in col major -> (height*num_anchor*width,2) - // anchor_centers = np.stack([anchor_centers]*self._num_anchors, axis=1).reshape( (-1,2) ) - std::unordered_map> center_points; - bool center_points_is_update = false; - static constexpr const unsigned int nms_pre = 1000; - static constexpr const unsigned int max_nms = 30000; - - private: - void transform(const cv::Mat &mat_rs) override; // without resize - - // initial steps and num_anchors - // https://github.com/deepinsight/insightface/blob/master/detection/scrfd/tools/scrfd.py - void initial_context(); - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - SCRFDScaleParams &scale_params); - - // generate once. - void generate_points(const int target_height, const int target_width); - - void generate_bboxes_single_stride(const SCRFDScaleParams &scale_params, - std::shared_ptr &score_pred, - std::shared_ptr &bbox_pred, - unsigned int stride, - float score_threshold, - float img_height, - float img_width, - std::vector &bbox_kps_collection); - - void generate_bboxes_kps_single_stride(const SCRFDScaleParams &scale_params, - std::shared_ptr &score_pred, - std::shared_ptr &bbox_pred, - std::shared_ptr &kps_pred, - unsigned int stride, - float score_threshold, - float img_height, - float img_width, - std::vector &bbox_kps_collection); - - void generate_bboxes_kps(const SCRFDScaleParams &scale_params, - std::vector &bbox_kps_collection, - std::shared_ptr &_instance, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms_bboxes_kps(std::vector &input, - std::vector &output, - float iou_threshold, unsigned int topk); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes_kps, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 400); - - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_SCRFD_H - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lite/tnn/cv/tnn_shufflenetv2.cpp b/lite/tnn/cv/tnn_shufflenetv2.cpp deleted file mode 100644 index 8c610ded..00000000 --- a/lite/tnn/cv/tnn_shufflenetv2.cpp +++ /dev/null @@ -1,94 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "tnn_shufflenetv2.h" -#include "lite/utils.h" - -using tnncv::TNNShuffleNetV2; - -TNNShuffleNetV2::TNNShuffleNetV2(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNShuffleNetV2::transform(const cv::Mat &mat_rs) -{ - // push into input_mat (1,3,224,224) - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNShuffleNetV2::detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k) -{ - if (mat.empty()) return; - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch. - tnn::MatConvertParam cvt_param; - std::shared_ptr logits_mat; // (1,1000) - status = instance->GetOutputMat(logits_mat, cvt_param, "output", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto logits_dims = logits_mat->GetDims(); - const unsigned int num_classes = logits_dims.at(1); // 1000 - const float *logits = (float *) logits_mat->GetData(); - - unsigned int max_id; - std::vector scores = lite::utils::math::softmax(logits, num_classes, max_id); - std::vector sorted_indices = lite::utils::math::argsort(scores); - if (top_k > num_classes) top_k = num_classes; - - content.scores.clear(); - content.labels.clear(); - content.texts.clear(); - for (unsigned int i = 0; i < top_k; ++i) - { - content.labels.push_back(sorted_indices[i]); - content.scores.push_back(scores[sorted_indices[i]]); - content.texts.push_back(class_names[sorted_indices[i]]); - } - content.flag = true; -} \ No newline at end of file diff --git a/lite/tnn/cv/tnn_shufflenetv2.h b/lite/tnn/cv/tnn_shufflenetv2.h deleted file mode 100644 index 02b79b86..00000000 --- a/lite/tnn/cv/tnn_shufflenetv2.h +++ /dev/null @@ -1,414 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_SHUFFLENETV2_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_SHUFFLENETV2_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNShuffleNetV2 : public BasicTNNHandler - { - public: - explicit TNNShuffleNetV2(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNShuffleNetV2() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {(1.0f / 0.229f) * (1.0 / 255.f), - (1.0f / 0.224f) * (1.0 / 255.f), - (1.0f / 0.225f) * (1.0 / 255.f)}; - std::vector bias_vals = {-0.485f * 255.f * (1.0f / 0.229f) * (1.0 / 255.f), - -0.456f * 255.f * (1.0f / 0.224f) * (1.0 / 255.f), - -0.406f * 255.f * (1.0f / 0.225f) * (1.0 / 255.f)}; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::ImageNetContent &content, unsigned int top_k = 5); - - private: - const char *class_names[1000] = { - "tench, Tinca tinca", "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", "stingray", "cock", - "hen", "ostrich, Struthio camelus", "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", "bulbul", "jay", - "magpie", "chickadee", "water ouzel, dipper", - "kite", "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", "great grey owl, great gray owl, Strix nebulosa", "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", "eft", "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", "terrapin", "box turtle, box tortoise", - "banded gecko", "common iguana, iguana, Iguana iguana", "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", "agama", "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", "Gila monster, Heloderma suspectum", "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", "American alligator, Alligator mississipiensis", - "triceratops", "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", "hognose snake, puff adder, sand viper", - "green snake, grass snake", "king snake, kingsnake", "garter snake, grass snake", - "water snake", "vine snake", "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", "green mamba", "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", "harvestman, daddy longlegs, Phalangium opilio", "scorpion", - "black and gold garden spider, Argiope aurantia", "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", "black widow, Latrodectus mactans", - "tarantula", "wolf spider, hunting spider", "tick", - "centipede", "black grouse", "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", "prairie chicken, prairie grouse, prairie fowl", - "peacock", "quail", "partridge", - "African grey, African gray, Psittacus erithacus", "macaw", "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", "coucal", "bee eater", - "hornbill", "hummingbird", "jacamar", - "toucan", "drake", "red-breasted merganser, Mergus serrator", - "goose", "black swan, Cygnus atratus", "tusker", - "echidna, spiny anteater, anteater", "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", "jellyfish", "sea anemone, anemone", - "brain coral", "flatworm, platyhelminth", "nematode, nematode worm, roundworm", - "conch", "snail", "slug", - "sea slug, nudibranch", "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", "hermit crab", "isopod", - "white stork, Ciconia ciconia", "black stork, Ciconia nigra", "spoonbill", - "flamingo", "little blue heron, Egretta caerulea", "American egret, great white heron, Egretta albus", - "bittern", "crane", "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", "ruddy turnstone, Arenaria interpres", "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", "dowitcher", "oystercatcher, oyster catcher", - "pelican", "king penguin, Aptenodytes patagonica", "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", "sea lion", "Chihuahua", - "Japanese spaniel", "Maltese dog, Maltese terrier, Maltese", "Pekinese, Pekingese, Peke", - "Shih-Tzu", "Blenheim spaniel", "papillon", - "toy terrier", "Rhodesian ridgeback", "Afghan hound, Afghan", - "basset, basset hound", "beagle", "bloodhound, sleuthhound", - "bluetick", "black-and-tan coonhound", "Walker hound, Walker foxhound", - "English foxhound", "redbone", "borzoi, Russian wolfhound", - "Irish wolfhound", "Italian greyhound", "whippet", - "Ibizan hound, Ibizan Podenco", "Norwegian elkhound, elkhound", - "otterhound, otter hound", "Saluki, gazelle hound", "Scottish deerhound, deerhound", - "Weimaraner", "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", "Border terrier", "Kerry blue terrier", - "Irish terrier", "Norfolk terrier", "Norwich terrier", - "Yorkshire terrier", "wire-haired fox terrier", "Lakeland terrier", - "Sealyham terrier, Sealyham", "Airedale, Airedale terrier", "cairn, cairn terrier", - "Australian terrier", "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", "miniature schnauzer", "giant schnauzer", - "standard schnauzer", "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", "silky terrier, Sydney silky", - "soft-coated wheaten terrier", "West Highland white terrier", "Lhasa, Lhasa apso", - "flat-coated retriever", "curly-coated retriever", "golden retriever", - "Labrador retriever", "Chesapeake Bay retriever", "German short-haired pointer", - "vizsla, Hungarian pointer", "English setter", "Irish setter, red setter", - "Gordon setter", "Brittany spaniel", "clumber, clumber spaniel", - "English springer, English springer spaniel", "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", "Sussex spaniel", - "Irish water spaniel", "kuvasz", "schipperke", - "groenendael", "malinois", "briard", - "kelpie", "komondor", "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", "collie", "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", "Rottweiler", "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", "miniature pinscher", "Greater Swiss Mountain dog", - "Bernese mountain dog", "Appenzeller", "EntleBucher", - "boxer", "bull mastiff", "Tibetan mastiff", - "French bulldog", "Great Dane", "Saint Bernard, St Bernard", - "Eskimo dog, husky", "malamute, malemute, Alaskan malamute", "Siberian husky", - "dalmatian, coach dog, carriage dog", "affenpinscher, monkey pinscher, monkey dog", - "basenji", "pug, pug-dog", "Leonberg", - "Newfoundland, Newfoundland dog", "Great Pyrenees", "Samoyed, Samoyede", - "Pomeranian", "chow, chow chow", "keeshond", - "Brabancon griffon", "Pembroke, Pembroke Welsh corgi", "Cardigan, Cardigan Welsh corgi", - "toy poodle", "miniature poodle", "standard poodle", - "Mexican hairless", "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", "red fox, Vulpes vulpes", "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", "tiger cat", "Persian cat", - "Siamese cat, Siamese", "Egyptian cat", "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", "leopard, Panthera pardus", "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", "cheetah, chetah, Acinonyx jubatus", "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", "mongoose", "meerkat, mierkat", - "tiger beetle", "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", "dung beetle", "rhinoceros beetle", - "weevil", "fly", "bee", - "ant, emmet, pismire", "grasshopper, hopper", "cricket", - "walking stick, walkingstick, stick insect", "cockroach, roach", - "mantis, mantid", "cicada, cicala", "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", "admiral", "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", "sulphur butterfly, sulfur butterfly", "lycaenid, lycaenid butterfly", - "starfish, sea star", "sea urchin", "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", "hare", "Angora, Angora rabbit", - "hamster", "porcupine, hedgehog", "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", "beaver", "guinea pig, Cavia cobaya", - "sorrel", "zebra", "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", "warthog", "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", "ram, tup", "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", "hartebeest", "impala, Aepyceros melampus", - "gazelle", "Arabian camel, dromedary, Camelus dromedarius", "llama", - "weasel", "mink", "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", "otter", "skunk, polecat, wood pussy", - "badger", "armadillo", "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", "gibbon, Hylobates lar", "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", "patas, hussar monkey, Erythrocebus patas", - "baboon", "macaque", "langur", - "colobus, colobus monkey", "proboscis monkey, Nasalis larvatus", - "marmoset", "capuchin, ringtail, Cebus capucinus", "howler monkey, howler", - "titi, titi monkey", "spider monkey, Ateles geoffroyi", "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", "eel", "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", "anemone fish", "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", "lionfish", - "puffer, pufferfish, blowfish, globefish", "abacus", "abaya", - "academic gown, academic robe, judge's robe", "accordion, piano accordion, squeeze box", - "acoustic guitar", "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", "airship, dirigible", "altar", - "ambulance", "amphibian, amphibious vehicle", "analog clock", - "apiary, bee house", "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", "balance beam, beam", "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", "Band Aid", "banjo", - "bannister, banister, balustrade, balusters, handrail", "barbell", - "barber chair", "barbershop", "barn", - "barometer", "barrel, cask", "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", "basketball", "bassinet", - "bassoon", "bathing cap, swimming cap", "bath towel", - "bathtub, bathing tub, bath, tub", "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", "beaker", "bearskin, busby, shako", - "beer bottle", "beer glass", "bell cote, bell cot", - "bib", "bicycle-built-for-two, tandem bicycle, tandem", "bikini, two-piece", - "binder, ring-binder", "binoculars, field glasses, opera glasses", - "birdhouse", "boathouse", "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", "bonnet, poke bonnet", "bookcase", - "bookshop, bookstore, bookstall", "bottlecap", "bow", - "bow tie, bow-tie, bowtie", "brass, memorial tablet, plaque", "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", "breastplate, aegis, egis", - "broom", "bucket, pail", "buckle", - "bulletproof vest", "bullet train, bullet", "butcher shop, meat market", - "cab, hack, taxi, taxicab", "caldron, cauldron", "candle, taper, wax light", - "cannon", "canoe", "can opener, tin opener", - "cardigan", "car mirror", "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", "carton", "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", "cassette player", "castle", - "catamaran", "CD player", "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", "chainlink fence", "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", "chest", "chiffonier, commode", - "chime, bell, gong", "china cabinet, china closet", "Christmas stocking", - "church, church building", "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", "cliff dwelling", "cloak", - "clog, geta, patten, sabot", "cocktail shaker", "coffee mug", - "coffeepot", "coil, spiral, volute, whorl, helix", "combination lock", - "computer keyboard, keypad", "confectionery, confectionary, candy store", - "container ship, containership, container vessel", "convertible", - "corkscrew, bottle screw", "cornet, horn, trumpet, trump", "cowboy boot", - "cowboy hat, ten-gallon hat", "cradle", "crane", - "crash helmet", "crate", "crib, cot", - "Crock Pot", "croquet ball", "crutch", - "cuirass", "dam, dike, dyke", "desk", - "desktop computer", "dial telephone, dial phone", "diaper, nappy, napkin", - "digital clock", "digital watch", "dining table, board", - "dishrag, dishcloth", "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", "dock, dockage, docking facility", "dogsled, dog sled, dog sleigh", - "dome", "doormat, welcome mat", "drilling platform, offshore rig", - "drum, membranophone, tympan", "drumstick", "dumbbell", - "Dutch oven", "electric fan, blower", "electric guitar", - "electric locomotive", "entertainment center", "envelope", - "espresso maker", "face powder", "feather boa, boa", - "file, file cabinet, filing cabinet", "fireboat", "fire engine, fire truck", - "fire screen, fireguard", "flagpole, flagstaff", "flute, transverse flute", - "folding chair", "football helmet", "forklift", - "fountain", "fountain pen", "four-poster", - "freight car", "French horn, horn", "frying pan, frypan, skillet", - "fur coat", "garbage truck, dustcart", "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", "goblet", - "go-kart", "golf ball", "golfcart, golf cart", - "gondola", "gong, tam-tam", "gown", - "grand piano, grand", "greenhouse, nursery, glasshouse", "grille, radiator grille", - "grocery store, grocery, food market, market", "guillotine", "hair slide", - "hair spray", "half track", "hammer", - "hamper", "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", "harmonica, mouth organ, harp, mouth harp", - "harp", "harvester, reaper", "hatchet", - "holster", "home theater, home theatre", "honeycomb", - "hook, claw", "hoopskirt, crinoline", "horizontal bar, high bar", - "horse cart, horse-cart", "hourglass", "iPod", - "iron, smoothing iron", "jack-o'-lantern", "jean, blue jean, denim", - "jeep, landrover", "jersey, T-shirt, tee shirt", "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", "joystick", "kimono", - "knee pad", "knot", "lab coat, laboratory coat", - "ladle", "lampshade, lamp shade", "laptop, laptop computer", - "lawn mower, mower", "lens cap, lens cover", "letter opener, paper knife, paperknife", - "library", "lifeboat", "lighter, light, igniter, ignitor", - "limousine, limo", "liner, ocean liner", "lipstick, lip rouge", - "Loafer", "lotion", "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", "lumbermill, sawmill", "magnetic compass", - "mailbag, postbag", "mailbox, letter box", "maillot", - "maillot, tank suit", "manhole cover", "maraca", - "marimba, xylophone", "mask", "matchstick", - "maypole", "maze, labyrinth", "measuring cup", - "medicine chest, medicine cabinet", "megalith, megalithic structure", - "microphone, mike", "microwave, microwave oven", "military uniform", - "milk can", "minibus", "miniskirt, mini", - "minivan", "missile", "mitten", - "mixing bowl", "mobile home, manufactured home", "Model T", - "modem", "monastery", "monitor", - "moped", "mortar", "mortarboard", - "mosque", "mosquito net", "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", "mountain tent", - "mouse, computer mouse", "mousetrap", "moving van", - "muzzle", "nail", "neck brace", - "necklace", "nipple", "notebook, notebook computer", - "obelisk", "oboe, hautboy, hautbois", "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", "oil filter", "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", "overskirt", - "oxcart", "oxygen mask", "packet", - "paddle, boat paddle", "paddlewheel, paddle wheel", "padlock", - "paintbrush", "pajama, pyjama, pj's, jammies", "palace", - "panpipe, pandean pipe, syrinx", "paper towel", "parachute, chute", - "parallel bars, bars", "park bench", "parking meter", - "passenger car, coach, carriage", "patio, terrace", "pay-phone, pay-station", - "pedestal, plinth, footstall", "pencil box, pencil case", "pencil sharpener", - "perfume, essence", "Petri dish", "photocopier", - "pick, plectrum, plectron", "pickelhaube", "picket fence, paling", - "pickup, pickup truck", "pier", "piggy bank, penny bank", - "pill bottle", "pillow", "ping-pong ball", - "pinwheel", "pirate, pirate ship", "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", "planetarium", "plastic bag", - "plate rack", "plow, plough", "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", "pole", "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", "pool table, billiard table, snooker table", "pop bottle, soda bottle", - "pot, flowerpot", "potter's wheel", "power drill", - "prayer rug, prayer mat", "printer", "prison, prison house", - "projectile, missile", "projector", "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", "purse", "quill, quill pen", - "quilt, comforter, comfort, puff", "racer, race car, racing car", - "racket, racquet", "radiator", "radio, wireless", - "radio telescope, radio reflector", "rain barrel", "recreational vehicle, RV, R.V.", - "reel", "reflex camera", "refrigerator, icebox", - "remote control, remote", "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", "rifle", "rocking chair, rocker", - "rotisserie", "rubber eraser, rubber, pencil eraser", "rugby ball", - "rule, ruler", "running shoe", "safe", - "safety pin", "saltshaker, salt shaker", "sandal", - "sarong", "sax, saxophone", "scabbard", - "scale, weighing machine", "school bus", "schooner", - "scoreboard", "screen, CRT screen", "screw", - "screwdriver", "seat belt, seatbelt", "sewing machine", - "shield, buckler", "shoe shop, shoe-shop, shoe store", "shoji", - "shopping basket", "shopping cart", "shovel", - "shower cap", "shower curtain", "ski", - "ski mask", "sleeping bag", "slide rule, slipstick", - "sliding door", "slot, one-armed bandit", "snorkel", - "snowmobile", "snowplow, snowplough", "soap dispenser", - "soccer ball", "sock", "solar dish, solar collector, solar furnace", - "sombrero", "soup bowl", "space bar", - "space heater", "space shuttle", "spatula", - "speedboat", "spider web, spider's web", "spindle", - "sports car, sport car", "spotlight, spot", "stage", - "steam locomotive", "steel arch bridge", "steel drum", - "stethoscope", "stole", "stone wall", - "stopwatch, stop watch", "stove", "strainer", - "streetcar, tram, tramcar, trolley, trolley car", "stretcher", "studio couch, day bed", - "stupa, tope", "submarine, pigboat, sub, U-boat", "suit, suit of clothes", - "sundial", "sunglass", "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", "suspension bridge", "swab, swob, mop", - "sweatshirt", "swimming trunks, bathing trunks", "swing", - "switch, electric switch, electrical switch", "syringe", "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", "teapot", "teddy, teddy bear", - "television, television system", "tennis ball", "thatch, thatched roof", - "theater curtain, theatre curtain", "thimble", "thresher, thrasher, threshing machine", - "throne", "tile roof", "toaster", - "tobacco shop, tobacconist shop, tobacconist", "toilet seat", "torch", - "totem pole", "tow truck, tow car, wrecker", "toyshop", - "tractor", "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", "trench coat", "tricycle, trike, velocipede", - "trimaran", "tripod", "triumphal arch", - "trolleybus, trolley coach, trackless trolley", "trombone", "tub, vat", - "turnstile", "typewriter keyboard", "umbrella", - "unicycle, monocycle", "upright, upright piano", "vacuum, vacuum cleaner", - "vase", "vault", "velvet", - "vending machine", "vestment", "viaduct", - "violin, fiddle", "volleyball", "waffle iron", - "wall clock", "wallet, billfold, notecase, pocketbook", "wardrobe, closet, press", - "warplane, military plane", "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", "water bottle", "water jug", - "water tower", "whiskey jug", "whistle", - "wig", "window screen", "window shade", - "Windsor tie", "wine bottle", "wing", - "wok", "wooden spoon", "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", "wreck", - "yawl", "yurt", "web site, website, internet site, site", - "comic book", "crossword puzzle, crossword", "street sign", - "traffic light, traffic signal, stoplight", "book jacket, dust cover, dust jacket, dust wrapper", - "menu", "plate", "guacamole", - "consomme", "hot pot, hotpot", "trifle", - "ice cream, icecream", "ice lolly, lolly, lollipop, popsicle", "French loaf", - "bagel, beigel", "pretzel", "cheeseburger", - "hotdog, hot dog, red hot", "mashed potato", "head cabbage", - "broccoli", "cauliflower", "zucchini, courgette", - "spaghetti squash", "acorn squash", "butternut squash", - "cucumber, cuke", "artichoke, globe artichoke", "bell pepper", - "cardoon", "mushroom", "Granny Smith", - "strawberry", "orange", "lemon", - "fig", "pineapple, ananas", "banana", - "jackfruit, jak, jack", "custard apple", "pomegranate", - "hay", "carbonara", "chocolate sauce, chocolate syrup", - "dough", "meat loaf, meatloaf", "pizza, pizza pie", - "potpie", "burrito", "red wine", - "espresso", "cup", "eggnog", - "alp", "bubble", "cliff, drop, drop-off", - "coral reef", "geyser", "lakeside, lakeshore", - "promontory, headland, head, foreland", "sandbar, sand bar", "seashore, coast, seacoast, sea-coast", - "valley, vale", "volcano", "ballplayer, baseball player", - "groom, bridegroom", "scuba diver", "rapeseed", - "daisy", "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", "acorn", "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", "coral fungus", "agaric", - "gyromitra", "stinkhorn, carrion fungus", "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", "ear, spike, capitulum", "toilet tissue, toilet paper, bathroom tissue" - }; - - }; -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_SHUFFLENETV2_H diff --git a/lite/tnn/cv/tnn_sphere_face.cpp b/lite/tnn/cv/tnn_sphere_face.cpp deleted file mode 100644 index 6e9f2b80..00000000 --- a/lite/tnn/cv/tnn_sphere_face.cpp +++ /dev/null @@ -1,86 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "tnn_sphere_face.h" - -using tnncv::TNNSphereFace; - -TNNSphereFace::TNNSphereFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNSphereFace::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNSphereFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 4. fetch output mat - std::shared_ptr embedding_mat; - tnn::MatConvertParam embed_cvt_param; // default - - status = instance->GetOutputMat(embedding_mat, embed_cvt_param, "embedding", output_device_type); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - auto embedding_dims = embedding_mat->GetDims(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = (float *) embedding_mat->GetData(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} - diff --git a/lite/tnn/cv/tnn_sphere_face.h b/lite/tnn/cv/tnn_sphere_face.h deleted file mode 100644 index c3d0c857..00000000 --- a/lite/tnn/cv/tnn_sphere_face.h +++ /dev/null @@ -1,34 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_SPHERE_FACE_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_SPHERE_FACE_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNSphereFace : public BasicTNNHandler - { - public: - explicit TNNSphereFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNSphereFace() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f / 128.0f, 1.f / 128.0f, 1.f / 128.0f}; - std::vector bias_vals = {-127.5f / 128.0f, -127.5f / 128.0f, -127.5f / 128.0f}; - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_SPHERE_FACE_H diff --git a/lite/tnn/cv/tnn_ssrnet.cpp b/lite/tnn/cv/tnn_ssrnet.cpp deleted file mode 100644 index b9fc9aad..00000000 --- a/lite/tnn/cv/tnn_ssrnet.cpp +++ /dev/null @@ -1,108 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#include "tnn_ssrnet.h" - -using tnncv::TNNSSRNet; - -TNNSSRNet::TNNSSRNet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNSSRNet::transform(const cv::Mat &mat_rs) -{ - // push into input_mat (1,3,64,64) - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNSSRNet::detect(const cv::Mat &mat, types::Age &age) -{ - if (mat.empty()) return; - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - // 4. fetch. - tnn::MatConvertParam cvt_param; - std::shared_ptr age_mat; // (1,1) - status = instance->GetOutputMat(age_mat, cvt_param, "age", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - const float *age_ptr = (float *) age_mat->GetData(); - const float pred_age = age_ptr[0]; - - const unsigned int interval_min = static_cast(pred_age - 2.f > 0.f ? pred_age - 2.f : 0.f); - const unsigned int interval_max = static_cast(pred_age + 3.f < 100.f ? pred_age + 3.f : 100.f); - - age.age = pred_age; - age.age_interval[0] = interval_min; - age.age_interval[1] = interval_max; - age.interval_prob = 1.0f; - age.flag = true; -} - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lite/tnn/cv/tnn_ssrnet.h b/lite/tnn/cv/tnn_ssrnet.h deleted file mode 100644 index 220d7d88..00000000 --- a/lite/tnn/cv/tnn_ssrnet.h +++ /dev/null @@ -1,36 +0,0 @@ -// -// Created by DefTruth on 2021/11/27. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_SSRNET_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_SSRNET_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNSSRNet : public BasicTNNHandler - { - public: - explicit TNNSSRNet(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNSSRNet() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {(1.0f / 0.229f) * (1.0f / 255.0f), - (1.0f / 0.224f) * (1.0f / 255.0f), - (1.0f / 0.225f) * (1.0f / 255.0f)}; - std::vector bias_vals = {-0.485f * (1.0f / 0.229f), - -0.456f * (1.0f / 0.229f), - -0.406f * (1.0f / 0.229f)}; - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::Age &age); - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_SSRNET_H diff --git a/lite/tnn/cv/tnn_subpixel_cnn.cpp b/lite/tnn/cv/tnn_subpixel_cnn.cpp deleted file mode 100644 index edbe07a0..00000000 --- a/lite/tnn/cv/tnn_subpixel_cnn.cpp +++ /dev/null @@ -1,110 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#include "tnn_subpixel_cnn.h" - -using tnncv::TNNSubPixelCNN; - -TNNSubPixelCNN::TNNSubPixelCNN(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNSubPixelCNN::transform(const cv::Mat &mat_y) -{ - input_mat = std::make_shared(input_device_type, tnn::NCHW_FLOAT, - input_shape, (void *) mat_y.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNSubPixelCNN::detect(const cv::Mat &mat, types::SuperResolutionContent &super_resolution_content) -{ - if (mat.empty()) return; - cv::Mat mat_copy = mat.clone(); - cv::resize(mat_copy, mat_copy, cv::Size(input_width, input_height)); // (224,224,3) - cv::Mat mat_ycrcb, mat_y, mat_cr, mat_cb; - cv::cvtColor(mat_copy, mat_ycrcb, cv::COLOR_BGR2YCrCb); - - // 0. split - std::vector split_mats; - cv::split(mat_ycrcb, split_mats); - mat_y = split_mats.at(0); // (224,224,1) uchar CV_8UC1 - mat_cr = split_mats.at(1); - mat_cb = split_mats.at(2); - - // 1. make input tensor - cv::Mat mat_y_; // assume that input mat is Y of YCrCb - mat_y.convertTo(mat_y_, CV_32FC1, 1.0f / 255.0f, 0.f); // (224,224,1) range (0.,1.0) - this->transform(mat_y_); // (1,1,224,224) - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch - tnn::MatConvertParam cvt_param; - std::shared_ptr pred_mat; // (1,1,672,672) - status = instance->GetOutputMat(pred_mat, cvt_param, "output", output_device_type); - - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - - auto pred_dims = pred_mat->GetDims(); // (1,2,256,256) - const unsigned int rows = pred_dims.at(2); // H 256 - const unsigned int cols = pred_dims.at(3); // W 256 - - float *pred_ptr = (float *) pred_mat->GetData(); - - mat_y = cv::Mat(rows, cols, CV_32FC1, pred_ptr); // release & create - - mat_y *= 255.0f; - - mat_y.convertTo(mat_y, CV_8UC1); - - cv::resize(mat_cr, mat_cr, cv::Size(cols, rows)); - cv::resize(mat_cb, mat_cb, cv::Size(cols, rows)); - - std::vector out_mats; - out_mats.push_back(mat_y); - out_mats.push_back(mat_cr); - out_mats.push_back(mat_cb); - - // 3. merge - cv::merge(out_mats, super_resolution_content.mat); - if (super_resolution_content.mat.empty()) - { - super_resolution_content.flag = false; - return; - } - cv::cvtColor(super_resolution_content.mat, super_resolution_content.mat, cv::COLOR_YCrCb2BGR); - super_resolution_content.flag = true; -} \ No newline at end of file diff --git a/lite/tnn/cv/tnn_subpixel_cnn.h b/lite/tnn/cv/tnn_subpixel_cnn.h deleted file mode 100644 index 483d8a51..00000000 --- a/lite/tnn/cv/tnn_subpixel_cnn.h +++ /dev/null @@ -1,28 +0,0 @@ -// -// Created by DefTruth on 2021/11/29. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_SUBPIXEL_CNN_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_SUBPIXEL_CNN_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNSubPixelCNN : public BasicTNNHandler - { - public: - explicit TNNSubPixelCNN(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNSubPixelCNN() override = default; - - private: - void transform(const cv::Mat &mat_y) override; // - - public: - void detect(const cv::Mat &mat, types::SuperResolutionContent &super_resolution_content); - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_SUBPIXEL_CNN_H diff --git a/lite/tnn/cv/tnn_tencent_cifp_face.cpp b/lite/tnn/cv/tnn_tencent_cifp_face.cpp deleted file mode 100644 index 81d00c41..00000000 --- a/lite/tnn/cv/tnn_tencent_cifp_face.cpp +++ /dev/null @@ -1,86 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "tnn_tencent_cifp_face.h" - -using tnncv::TNNTencentCifpFace; - -TNNTencentCifpFace::TNNTencentCifpFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNTencentCifpFace::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNTencentCifpFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 4. fetch output mat - std::shared_ptr embedding_mat; - tnn::MatConvertParam embed_cvt_param; // default - - status = instance->GetOutputMat(embedding_mat, embed_cvt_param, "embedding", output_device_type); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - auto embedding_dims = embedding_mat->GetDims(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = (float *) embedding_mat->GetData(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} diff --git a/lite/tnn/cv/tnn_tencent_cifp_face.h b/lite/tnn/cv/tnn_tencent_cifp_face.h deleted file mode 100644 index dead9a31..00000000 --- a/lite/tnn/cv/tnn_tencent_cifp_face.h +++ /dev/null @@ -1,34 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_TENCENT_CIFP_FACE_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_TENCENT_CIFP_FACE_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNTencentCifpFace : public BasicTNNHandler - { - public: - explicit TNNTencentCifpFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNTencentCifpFace() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - std::vector bias_vals = {-1.f, -1.f, -1.f}; // RGB - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_TENCENT_CIFP_FACE_H diff --git a/lite/tnn/cv/tnn_tencent_curricular_face.cpp b/lite/tnn/cv/tnn_tencent_curricular_face.cpp deleted file mode 100644 index 03fc61f8..00000000 --- a/lite/tnn/cv/tnn_tencent_curricular_face.cpp +++ /dev/null @@ -1,87 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#include "tnn_tencent_curricular_face.h" - -using tnncv::TNNTencentCurricularFace; - -TNNTencentCurricularFace::TNNTencentCurricularFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNTencentCurricularFace::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNTencentCurricularFace::detect(const cv::Mat &mat, types::FaceContent &face_content) -{ - if (mat.empty()) return; - // 1. make input tensor - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 4. fetch output mat - std::shared_ptr embedding_mat; - tnn::MatConvertParam embed_cvt_param; // default - - status = instance->GetOutputMat(embedding_mat, embed_cvt_param, "embedding", output_device_type); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - auto embedding_dims = embedding_mat->GetDims(); // (1,512) - const unsigned int hidden_dim = embedding_dims.at(1); - const float *embedding_values = (float *) embedding_mat->GetData(); - - std::vector embedding_norm(embedding_values, embedding_values + hidden_dim); - cv::normalize(embedding_norm, embedding_norm); // l2 normalize - face_content.embedding.assign(embedding_norm.begin(), embedding_norm.end()); - face_content.dim = hidden_dim; - face_content.flag = true; -} - diff --git a/lite/tnn/cv/tnn_tencent_curricular_face.h b/lite/tnn/cv/tnn_tencent_curricular_face.h deleted file mode 100644 index 145d9728..00000000 --- a/lite/tnn/cv/tnn_tencent_curricular_face.h +++ /dev/null @@ -1,34 +0,0 @@ -// -// Created by DefTruth on 2021/11/14. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_TENCENT_CURRICULAR_FACE_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_TENCENT_CURRICULAR_FACE_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNTencentCurricularFace : public BasicTNNHandler - { - public: - explicit TNNTencentCurricularFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNTencentCurricularFace() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f / 127.5f, 1.f / 127.5f, 1.f / 127.5f}; - std::vector bias_vals = {-1.f, -1.f, -1.f}; // RGB - - private: - void transform(const cv::Mat &mat_rs) override; // - - public: - void detect(const cv::Mat &mat, types::FaceContent &face_content); - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_TENCENT_CURRICULAR_FACE_H diff --git a/lite/tnn/cv/tnn_ultraface.cpp b/lite/tnn/cv/tnn_ultraface.cpp deleted file mode 100644 index f8ecb8e1..00000000 --- a/lite/tnn/cv/tnn_ultraface.cpp +++ /dev/null @@ -1,174 +0,0 @@ -// -// Created by DefTruth on 2021/11/20. -// - -#include "tnn_ultraface.h" -#include "lite/utils.h" - -using tnncv::TNNUltraFace; - -TNNUltraFace::TNNUltraFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNUltraFace::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNUltraFace::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // 1. make input mat - cv::Mat mat_rs; - cv::resize(mat, mat_rs, cv::Size(input_width, input_height)); - cv::cvtColor(mat_rs, mat_rs, cv::COLOR_BGR2RGB); - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status.description().c_str() << "\n"; -#endif - return; - } - // 4. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(bbox_collection, instance, score_threshold, img_height, img_width); - // 5. hard|blend nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void TNNUltraFace::generate_bboxes(std::vector &bbox_collection, - std::shared_ptr &_instance, - float score_threshold, float img_height, - float img_width) -{ - std::shared_ptr boxes; // (1,n,4) - std::shared_ptr scores; // (1,n,2) - tnn::MatConvertParam cvt_param; - tnn::Status status_boxes; - tnn::Status status_scores; - - status_boxes = _instance->GetOutputMat(boxes, cvt_param, "boxes", output_device_type); - status_scores = _instance->GetOutputMat(scores, cvt_param, "scores", output_device_type); - - if (status_boxes != tnn::TNN_OK || status_scores != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << status_boxes.description().c_str() << ": " - << status_scores.description().c_str() << "\n"; -#endif - return; - } - - auto scores_dims = scores->GetDims(); - const unsigned int num_anchors = scores_dims.at(1); // n = 17640 (640x480) - const float *scores_ptr = (float *) scores->GetData(); - const float *boxes_ptr = (float *) boxes->GetData(); - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - float confidence = scores_ptr[2 * i + 1]; - if (confidence < score_threshold) continue; - types::Boxf box; - box.x1 = boxes_ptr[4 * i + 0] * img_width; - box.y1 = boxes_ptr[4 * i + 1] * img_height; - box.x2 = boxes_ptr[4 * i + 2] * img_width; - box.y2 = boxes_ptr[4 * i + 3] * img_height; - box.score = confidence; - box.label_text = "face"; - box.label = 1; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITETNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void TNNUltraFace::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lite/tnn/cv/tnn_ultraface.h b/lite/tnn/cv/tnn_ultraface.h deleted file mode 100644 index 284dc66e..00000000 --- a/lite/tnn/cv/tnn_ultraface.h +++ /dev/null @@ -1,53 +0,0 @@ -// -// Created by DefTruth on 2021/11/20. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_ULTRAFACE_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_ULTRAFACE_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNUltraFace : public BasicTNNHandler - { - public: - explicit TNNUltraFace(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNUltraFace() override = default; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.0f / 128.0f, 1.0f / 128.0f, 1.0f / 128.0f}; - std::vector bias_vals = { - -127.0f * (1.0f / 128.0f), - -127.0f * (1.0f / 128.0f), - -127.0f * (1.0f / 128.0f) - }; // RGB - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void transform(const cv::Mat &mat_rs) override; // - - void generate_bboxes(std::vector &bbox_collection, - std::shared_ptr &_instance, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.7f, float iou_threshold = 0.3f, - unsigned int topk = 300, unsigned int nms_type = 0); - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_ULTRAFACE_H diff --git a/lite/tnn/cv/tnn_yolo5face.cpp b/lite/tnn/cv/tnn_yolo5face.cpp deleted file mode 100644 index 024580db..00000000 --- a/lite/tnn/cv/tnn_yolo5face.cpp +++ /dev/null @@ -1,242 +0,0 @@ -// -// Created by DefTruth on 2022/1/16. -// - -#include "tnn_yolo5face.h" - -using tnncv::TNNYOLO5Face; - -TNNYOLO5Face::TNNYOLO5Face(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNYOLO5Face::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YOLO5FaceScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(0, 0, 0)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.ratio = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.flag = true; -} - -void TNNYOLO5Face::transform(const cv::Mat &mat_rs) -{ - // push into input_mat, RGB - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNYOLO5Face::detect(const cv::Mat &mat, std::vector &detected_boxes_kps, - float score_threshold, float iou_threshold, unsigned int topk) -{ - if (mat.empty()) return; - auto img_height = static_cast(mat.rows); - auto img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - YOLO5FaceScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input mat - cv::Mat mat_rs_; - cv::cvtColor(mat_rs, mat_rs_, cv::COLOR_BGR2RGB); - this->transform(mat_rs_); - // this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 3. rescale & exclude. - std::vector bbox_kps_collection; - this->generate_bboxes_kps(scale_params, bbox_kps_collection, instance, - score_threshold, img_height, img_width); - // 4. hard nms with topk. - this->nms_bboxes_kps(bbox_kps_collection, detected_boxes_kps, iou_threshold, topk); -} - -void TNNYOLO5Face::generate_bboxes_kps(const YOLO5FaceScaleParams &scale_params, - std::vector &bbox_kps_collection, - std::shared_ptr &_instance, - float score_threshold, float img_height, - float img_width) -{ - tnn::MatConvertParam cvt_param; - std::shared_ptr output; - tnn::Status status; - - status = _instance->GetOutputMat(output, cvt_param, "output", output_device_type); // [1,N,16] - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" << status.description().c_str() << "\n"; -#endif - return; - } - - auto output_dims = output->GetDims(); - const unsigned int num_anchors = output_dims.at(1); // n = ? - const float *output_ptr = (float *) output->GetData(); - - float r_ = scale_params.ratio; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - bbox_kps_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *row_ptr = output_ptr + i * 16; - float obj_conf = row_ptr[4]; - if (obj_conf < score_threshold) continue; // filter first. - float cls_conf = row_ptr[15]; - if (cls_conf < score_threshold) continue; // face score. - - // bounding box - const float *offsets = row_ptr; - float cx = offsets[0]; - float cy = offsets[1]; - float w = offsets[2]; - float h = offsets[3]; - - types::BoxfWithLandmarks box_kps; - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - box_kps.box.x1 = std::max(0.f, x1); - box_kps.box.y1 = std::max(0.f, y1); - box_kps.box.x2 = std::min(img_width - 1.f, x2); - box_kps.box.y2 = std::min(img_height - 1.f, y2); - box_kps.box.score = cls_conf; - box_kps.box.label = 1; - box_kps.box.label_text = "face"; - box_kps.box.flag = true; - - // landmarks - const float *kps_offsets = row_ptr + 5; - for (unsigned int j = 0; j < 10; j += 2) - { - cv::Point2f kps; - float kps_x = (kps_offsets[j] - (float) dw_) / r_; - float kps_y = (kps_offsets[j + 1] - (float) dh_) / r_; - kps.x = std::min(std::max(0.f, kps_x), img_width - 1.f); - kps.y = std::min(std::max(0.f, kps_y), img_height - 1.f); - box_kps.landmarks.points.push_back(kps); - } - box_kps.landmarks.flag = true; - box_kps.flag = true; - - bbox_kps_collection.push_back(box_kps); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } - -#if LITETNN_DEBUG - std::cout << "generate_bboxes_kps num: " << bbox_kps_collection.size() << "\n"; -#endif -} - -void TNNYOLO5Face::nms_bboxes_kps(std::vector &input, - std::vector &output, - float iou_threshold, unsigned int topk) -{ - if (input.empty()) return; - std::sort( - input.begin(), input.end(), - [](const types::BoxfWithLandmarks &a, const types::BoxfWithLandmarks &b) - { return a.box.score > b.box.score; } - ); - const unsigned int box_num = input.size(); - std::vector merged(box_num, 0); - - unsigned int count = 0; - for (unsigned int i = 0; i < box_num; ++i) - { - if (merged[i]) continue; - std::vector buf; - - buf.push_back(input[i]); - merged[i] = 1; - - for (unsigned int j = i + 1; j < box_num; ++j) - { - if (merged[j]) continue; - - float iou = static_cast(input[i].box.iou_of(input[j].box)); - - if (iou > iou_threshold) - { - merged[j] = 1; - buf.push_back(input[j]); - } - - } - output.push_back(buf[0]); - - // keep top k - count += 1; - if (count >= topk) - break; - } -} \ No newline at end of file diff --git a/lite/tnn/cv/tnn_yolo5face.h b/lite/tnn/cv/tnn_yolo5face.h deleted file mode 100644 index 503967b6..00000000 --- a/lite/tnn/cv/tnn_yolo5face.h +++ /dev/null @@ -1,63 +0,0 @@ -// -// Created by DefTruth on 2022/1/16. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_YOLO5FACE_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_YOLO5FACE_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNYOLO5Face : public BasicTNNHandler - { - public: - explicit TNNYOLO5Face(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); - - ~TNNYOLO5Face() override = default; - - private: - // nested classes - typedef struct - { - float ratio; - int dw; - int dh; - bool flag; - } YOLO5FaceScaleParams; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f / 255.f, 1.f / 255.f, 1.f / 255.f}; - std::vector bias_vals = {0.f, 0.f, 0.f}; // RGB - static constexpr const unsigned int max_nms = 30000; - - private: - void transform(const cv::Mat &mat_rs) override; // without resize - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YOLO5FaceScaleParams &scale_params); - - void generate_bboxes_kps(const YOLO5FaceScaleParams &scale_params, - std::vector &bbox_kps_collection, - std::shared_ptr &_instance, - float score_threshold, float img_height, - float img_width); // rescale & exclude - - void nms_bboxes_kps(std::vector &input, - std::vector &output, - float iou_threshold, unsigned int topk); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes_kps, - float score_threshold = 0.3f, float iou_threshold = 0.45f, - unsigned int topk = 400); - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_YOLO5FACE_H diff --git a/lite/tnn/cv/tnn_yolop.cpp b/lite/tnn/cv/tnn_yolop.cpp deleted file mode 100644 index 34697927..00000000 --- a/lite/tnn/cv/tnn_yolop.cpp +++ /dev/null @@ -1,300 +0,0 @@ -// -// Created by DefTruth on 2021/10/18. -// - -#include "tnn_yolop.h" -#include "lite/utils.h" - -using tnncv::TNNYOLOP; - -TNNYOLOP::TNNYOLOP(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNYOLOP::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YOLOPScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat = mat.clone(); - cv::resize(new_unpad_mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void TNNYOLOP::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNYOLOP::detect(const cv::Mat &mat, - std::vector &detected_boxes, - types::SegmentContent &da_seg_content, - types::SegmentContent &ll_seg_content, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - float img_height = static_cast(mat.rows); - float img_width = static_cast(mat.cols); - - // resize & unscale - cv::Mat mat_rs; - YOLOPScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - if ((!scale_params.flag) || mat_rs.empty()) return; - // 1. make input mat - cv::Mat mat_rs_; - cv::cvtColor(mat_rs, mat_rs_, cv::COLOR_BGR2RGB); - this->transform(mat_rs_); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - // 4. rescale & fetch da|ll seg. - std::vector bbox_collection; - this->generate_bboxes_da_ll(scale_params, instance, bbox_collection, - da_seg_content, ll_seg_content, score_threshold, - img_height, img_width); - // 5. hard|blend nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void TNNYOLOP::generate_bboxes_da_ll(const YOLOPScaleParams &scale_params, - std::shared_ptr &_instance, - std::vector &bbox_collection, - types::SegmentContent &da_seg_content, - types::SegmentContent &ll_seg_content, - float score_threshold, float img_height, - float img_width) -{ - std::shared_ptr det_out_mat; - std::shared_ptr da_seg_out_mat; - std::shared_ptr ll_seg_out_mat; - tnn::MatConvertParam cvt_param; - tnn::Status status_det_out; - tnn::Status status_da_seg_out; - tnn::Status status_ll_seg_out; - - // (1,n,6=5+1=cxcy+cwch+obj_conf+cls_conf) (1,2,640,640) (1,2,640,640) - status_det_out = _instance->GetOutputMat(det_out_mat, cvt_param, "det_out", output_device_type); - status_da_seg_out = _instance->GetOutputMat(da_seg_out_mat, cvt_param, "drive_area_seg", output_device_type); - status_ll_seg_out = _instance->GetOutputMat(ll_seg_out_mat, cvt_param, "lane_line_seg", output_device_type); - - if (status_det_out != tnn::TNN_OK || status_da_seg_out != tnn::TNN_OK - || status_ll_seg_out != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status_det_out.description().c_str() << ": " - << status_ll_seg_out.description().c_str() << ": " - << status_da_seg_out.description().c_str() << "\n"; -#endif - return; - } - - auto det_dims = det_out_mat->GetDims(); - const unsigned int num_anchors = det_dims.at(1); // n = ? - - float r = scale_params.r; - int dw = scale_params.dw; - int dh = scale_params.dh; - int new_unpad_w = scale_params.new_unpad_w; - int new_unpad_h = scale_params.new_unpad_h; - - // generate bounding boxes. - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *offset_obj_cls_ptr = (float *) det_out_mat->GetData() + (i * 6); - float obj_conf = offset_obj_cls_ptr[4]; - if (obj_conf < score_threshold) continue; // filter first. - - unsigned int label = 1; // 1 class only - float cls_conf = offset_obj_cls_ptr[5]; - float conf = obj_conf * cls_conf; // cls_conf (0.,1.) - if (conf < score_threshold) continue; // filter - - float cx = offset_obj_cls_ptr[0]; - float cy = offset_obj_cls_ptr[1]; - float w = offset_obj_cls_ptr[2]; - float h = offset_obj_cls_ptr[3]; - float x1 = ((cx - w / 2.f) - (float) dw) / r; - float y1 = ((cy - h / 2.f) - (float) dh) / r; - float x2 = ((cx + w / 2.f) - (float) dw) / r; - float y2 = ((cy + h / 2.f) - (float) dh) / r; - - types::Boxf box; - // de-padding & rescaling - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(x2, (float) img_width); - box.y2 = std::min(y2, (float) img_height); - box.score = conf; - box.label = label; - box.label_text = "traffic car"; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITETNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif - - // generate da && ll seg. - da_seg_content.names_map.clear(); - da_seg_content.class_mat = cv::Mat(new_unpad_h, new_unpad_w, CV_8UC1, cv::Scalar(0)); - da_seg_content.color_mat = cv::Mat(new_unpad_h, new_unpad_w, CV_8UC3, cv::Scalar(0, 0, 0)); - ll_seg_content.names_map.clear(); - ll_seg_content.class_mat = cv::Mat(new_unpad_h, new_unpad_w, CV_8UC1, cv::Scalar(0)); - ll_seg_content.color_mat = cv::Mat(new_unpad_h, new_unpad_w, CV_8UC3, cv::Scalar(0, 0, 0)); - - const unsigned int channel_step = input_height * input_width; - const float *da_seg_bg_ptr = (float *) da_seg_out_mat->GetData(); // background - const float *da_seg_fg_ptr = (float *) da_seg_out_mat->GetData() + channel_step; // foreground - const float *ll_seg_bg_ptr = (float *) ll_seg_out_mat->GetData(); // background - const float *ll_seg_fg_ptr = (float *) ll_seg_out_mat->GetData() + channel_step; // foreground - - for (int i = dh; i < dh + new_unpad_h; ++i) - { - // row ptr. - uchar *da_p_class = da_seg_content.class_mat.ptr(i - dh); - uchar *ll_p_class = ll_seg_content.class_mat.ptr(i - dh); - cv::Vec3b *da_p_color = da_seg_content.color_mat.ptr(i - dh); - cv::Vec3b *ll_p_color = ll_seg_content.color_mat.ptr(i - dh); - - for (int j = dw; j < dw + new_unpad_w; ++j) - { - // argmax - float da_bg_prob = da_seg_bg_ptr[i * input_height + j]; - float da_fg_prob = da_seg_fg_ptr[i * input_height + j]; - float ll_bg_prob = ll_seg_bg_ptr[i * input_height + j]; - float ll_fg_prob = ll_seg_fg_ptr[i * input_height + j]; - unsigned int da_label = da_bg_prob < da_fg_prob ? 1 : 0; - unsigned int ll_label = ll_bg_prob < ll_fg_prob ? 1 : 0; - - if (da_label == 1) - { - // assign label for pixel(i,j) - da_p_class[j - dw] = 1 * 255; // 255 indicate drivable area, for post resize - // assign color for detected class at pixel(i,j). - da_p_color[j - dw][0] = 0; - da_p_color[j - dw][1] = 255; // green - da_p_color[j - dw][2] = 0; - // assign names map - da_seg_content.names_map[255] = "drivable area"; - } - - if (ll_label == 1) - { - // assign label for pixel(i,j) - ll_p_class[j - dw] = 1 * 255; // 255 indicate lane line, for post resize - // assign color for detected class at pixel(i,j). - ll_p_color[j - dw][0] = 0; - ll_p_color[j - dw][1] = 0; - ll_p_color[j - dw][2] = 255; // red - // assign names map - ll_seg_content.names_map[255] = "lane line"; - } - - } - } - // resize to original size. - const unsigned int img_h = static_cast(img_height); - const unsigned int img_w = static_cast(img_width); - // da_seg_mask 255 or 0 - cv::resize(da_seg_content.class_mat, da_seg_content.class_mat, - cv::Size(img_w, img_h), cv::INTER_LINEAR); - cv::resize(da_seg_content.color_mat, da_seg_content.color_mat, - cv::Size(img_w, img_h), cv::INTER_LINEAR); - // ll_seg_mask 255 or 0 - cv::resize(ll_seg_content.class_mat, ll_seg_content.class_mat, - cv::Size(img_w, img_h), cv::INTER_LINEAR); - cv::resize(ll_seg_content.color_mat, ll_seg_content.color_mat, - cv::Size(img_w, img_h), cv::INTER_LINEAR); - - da_seg_content.flag = true; - ll_seg_content.flag = true; -} - -void TNNYOLOP::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - - - - - - - - - - diff --git a/lite/tnn/cv/tnn_yolop.h b/lite/tnn/cv/tnn_yolop.h deleted file mode 100644 index 3ab7a2a8..00000000 --- a/lite/tnn/cv/tnn_yolop.h +++ /dev/null @@ -1,75 +0,0 @@ -// -// Created by DefTruth on 2021/10/18. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_YOLOP_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_YOLOP_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNYOLOP : public BasicTNNHandler - { - public: - explicit TNNYOLOP(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNYOLOP() override = default; - - private: - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } YOLOPScaleParams; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {0.0171247f, 0.0175070f, 0.0174291f}; // RGB - std::vector bias_vals = {-123.675f * 0.0171247f, -116.28f * 0.0175070f, -103.53f * 0.0174291f}; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void transform(const cv::Mat &mat_rs) override; // without resize - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YOLOPScaleParams &scale_params); - - void generate_bboxes_da_ll(const YOLOPScaleParams &scale_params, - std::shared_ptr &_instance, - std::vector &bbox_collection, - types::SegmentContent &da_seg_content, - types::SegmentContent &ll_seg_content, - float score_threshold, float img_height, - float img_width); // det,da_seg,ll_seg - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, - std::vector &detected_boxes, - types::SegmentContent &da_seg_content, - types::SegmentContent &ll_seg_content, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - - }; - -} - - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_YOLOP_H diff --git a/lite/tnn/cv/tnn_yolor.cpp b/lite/tnn/cv/tnn_yolor.cpp deleted file mode 100644 index 11636f54..00000000 --- a/lite/tnn/cv/tnn_yolor.cpp +++ /dev/null @@ -1,212 +0,0 @@ -// -// Created by DefTruth on 2021/11/7. -// - -#include "tnn_yolor.h" -#include "lite/utils.h" - -using tnncv::TNNYoloR; - -TNNYoloR::TNNYoloR(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNYoloR::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YoloRScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void TNNYoloR::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNYoloR::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - // resize & unscale - cv::Mat mat_rs; - YoloRScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - cv::Mat mat_rs_; - cv::cvtColor(mat_rs, mat_rs_, cv::COLOR_BGR2RGB); - this->transform(mat_rs_); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - // 5. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, instance, score_threshold, img_height, img_width); - // 6. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void TNNYoloR::generate_bboxes(const YoloRScaleParams &scale_params, - std::vector &bbox_collection, - std::shared_ptr &_instance, - float score_threshold, int img_height, - int img_width) -{ - // 4. fetch output mat - std::shared_ptr pred_mat; - tnn::MatConvertParam pred_cvt_param; // default - tnn::Status status; - - // (1,n,85=5+80=cxcy+cwch+obj_conf+cls_conf) - status = _instance->GetOutputMat(pred_mat, pred_cvt_param, "output", output_device_type); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - auto pred_dims = pred_mat->GetDims(); - const unsigned int num_anchors = pred_dims.at(1); // n = ? - const unsigned int num_classes = pred_dims.at(2) - 5; // 80 - - float r_ = scale_params.r; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *offset_obj_cls_ptr = - (float *) pred_mat->GetData() + (i * (num_classes + 5)); // row ptr - float obj_conf = offset_obj_cls_ptr[4]; - if (obj_conf < score_threshold) continue; // filter first. - - float cls_conf = offset_obj_cls_ptr[5]; - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = offset_obj_cls_ptr[j + 5]; - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - - float conf = obj_conf * cls_conf; // cls_conf (0.,1.) - if (conf < score_threshold) continue; // filter - - float cx = offset_obj_cls_ptr[0]; - float cy = offset_obj_cls_ptr[1]; - float w = offset_obj_cls_ptr[2]; - float h = offset_obj_cls_ptr[3]; - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - - types::Boxf box; - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(x2, (float) img_width - 1.f); - box.y2 = std::min(y2, (float) img_height - 1.f); - box.score = conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITETNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void TNNYoloR::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - - - - diff --git a/lite/tnn/cv/tnn_yolor.h b/lite/tnn/cv/tnn_yolor.h deleted file mode 100644 index 4867e4a7..00000000 --- a/lite/tnn/cv/tnn_yolor.h +++ /dev/null @@ -1,80 +0,0 @@ -// -// Created by DefTruth on 2021/11/7. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_YOLOR_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_YOLOR_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNYoloR : public BasicTNNHandler - { - public: - explicit TNNYoloR(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNYoloR() override = default; - - private: - // nested classes - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } YoloRScaleParams; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.0 / 255.f, 1.0 / 255.f, 1.0 / 255.f}; // RGB - std::vector bias_vals = {0.f, 0.f, 0.f}; - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void transform(const cv::Mat &mat_rs) override; // without resize - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YoloRScaleParams &scale_params); - - void generate_bboxes(const YoloRScaleParams &scale_params, - std::vector &bbox_collection, - std::shared_ptr &_instance, - float score_threshold, int img_height, - int img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_YOLOR_H diff --git a/lite/tnn/cv/tnn_yolov5.cpp b/lite/tnn/cv/tnn_yolov5.cpp deleted file mode 100644 index 6049ac8f..00000000 --- a/lite/tnn/cv/tnn_yolov5.cpp +++ /dev/null @@ -1,231 +0,0 @@ -// -// Created by DefTruth on 2021/11/6. -// - -#include "tnn_yolov5.h" -#include "lite/utils.h" - -using tnncv::TNNYoloV5; - -TNNYoloV5::TNNYoloV5(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNYoloV5::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YoloV5ScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void TNNYoloV5::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNYoloV5::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - // resize & unscale - cv::Mat mat_rs; - YoloV5ScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - cv::Mat mat_rs_; - cv::cvtColor(mat_rs, mat_rs_, cv::COLOR_BGR2RGB); - this->transform(mat_rs_); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - // 5. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, instance, score_threshold, img_height, img_width); - // 6. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void TNNYoloV5::generate_bboxes(const YoloV5ScaleParams &scale_params, - std::vector &bbox_collection, - std::shared_ptr &_instance, - float score_threshold, int img_height, - int img_width) -{ - // 4. fetch output mat - std::shared_ptr pred_mat; - tnn::MatConvertParam pred_cvt_param; // default - tnn::Status status; - - // (1,n,85=5+80=cxcy+cwch+obj_conf+cls_conf) - status = _instance->GetOutputMat(pred_mat, pred_cvt_param, "pred", output_device_type); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - auto pred_dims = pred_mat->GetDims(); - const unsigned int num_anchors = pred_dims.at(1); // n = ? - const unsigned int num_classes = pred_dims.at(2) - 5; // 80 - - float r_ = scale_params.r; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *offset_obj_cls_ptr = - (float *) pred_mat->GetData() + (i * (num_classes + 5)); // row ptr - float obj_conf = offset_obj_cls_ptr[4]; - if (obj_conf < score_threshold) continue; // filter first. - - float cls_conf = offset_obj_cls_ptr[5]; - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = offset_obj_cls_ptr[j + 5]; - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - - float conf = obj_conf * cls_conf; // cls_conf (0.,1.) - if (conf < score_threshold) continue; // filter - - float cx = offset_obj_cls_ptr[0]; - float cy = offset_obj_cls_ptr[1]; - float w = offset_obj_cls_ptr[2]; - float h = offset_obj_cls_ptr[3]; - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - - types::Boxf box; - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(x2, (float) img_width - 1.f); - box.y2 = std::min(y2, (float) img_height - 1.f); - box.score = conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITETNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void TNNYoloV5::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lite/tnn/cv/tnn_yolov5.h b/lite/tnn/cv/tnn_yolov5.h deleted file mode 100644 index de68c35f..00000000 --- a/lite/tnn/cv/tnn_yolov5.h +++ /dev/null @@ -1,79 +0,0 @@ -// -// Created by DefTruth on 2021/11/6. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_YOLOV5_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_YOLOV5_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNYoloV5 : public BasicTNNHandler - { - public: - explicit TNNYoloV5(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNYoloV5() override = default; - - private: - // nested classes - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } YoloV5ScaleParams; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.0 / 255.f, 1.0 / 255.f, 1.0 / 255.f}; // RGB - std::vector bias_vals = {0.f, 0.f, 0.f}; - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void transform(const cv::Mat &mat_rs) override; // without resize - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YoloV5ScaleParams &scale_params); - - void generate_bboxes(const YoloV5ScaleParams &scale_params, - std::vector &bbox_collection, - std::shared_ptr &_instance, - float score_threshold, int img_height, - int img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - - }; -} -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_YOLOV5_H diff --git a/lite/tnn/cv/tnn_yolov5_v6.0.cpp b/lite/tnn/cv/tnn_yolov5_v6.0.cpp deleted file mode 100644 index fdc33fd9..00000000 --- a/lite/tnn/cv/tnn_yolov5_v6.0.cpp +++ /dev/null @@ -1,233 +0,0 @@ -// -// Created by DefTruth on 2021/11/10. -// - -#include "tnn_yolov5_v6.0.h" -#include "lite/utils.h" - -using tnncv::TNNYoloV5_V_6_0; - -TNNYoloV5_V_6_0::TNNYoloV5_V_6_0(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNYoloV5_V_6_0::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YoloV5ScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void TNNYoloV5_V_6_0::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - // be carefully, no deepcopy inside this tnn::Mat constructor, - // so, we can not pass a local cv::Mat to this constructor. - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNYoloV5_V_6_0::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - // resize & unscale - cv::Mat mat_rs; - YoloV5ScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - cv::Mat mat_rs_; - cv::cvtColor(mat_rs, mat_rs_, cv::COLOR_BGR2RGB); - this->transform(mat_rs_); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - // 5. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, instance, score_threshold, img_height, img_width); - // 6. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void TNNYoloV5_V_6_0::generate_bboxes(const YoloV5ScaleParams &scale_params, - std::vector &bbox_collection, - std::shared_ptr &_instance, - float score_threshold, int img_height, - int img_width) -{ - // 4. fetch output mat - std::shared_ptr pred_mat; - tnn::MatConvertParam pred_cvt_param; // default - tnn::Status status; - - // (1,n,85=5+80=cxcy+cwch+obj_conf+cls_conf) - status = _instance->GetOutputMat(pred_mat, pred_cvt_param, "output", output_device_type); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - auto pred_dims = pred_mat->GetDims(); - const unsigned int num_anchors = pred_dims.at(1); // n = ? - const unsigned int num_classes = pred_dims.at(2) - 5; // 80 - - float r_ = scale_params.r; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *offset_obj_cls_ptr = - (float *) pred_mat->GetData() + (i * (num_classes + 5)); // row ptr - float obj_conf = offset_obj_cls_ptr[4]; - if (obj_conf < score_threshold) continue; // filter first. - - float cls_conf = offset_obj_cls_ptr[5]; - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = offset_obj_cls_ptr[j + 5]; - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - - float conf = obj_conf * cls_conf; // cls_conf (0.,1.) - if (conf < score_threshold) continue; // filter - - float cx = offset_obj_cls_ptr[0]; - float cy = offset_obj_cls_ptr[1]; - float w = offset_obj_cls_ptr[2]; - float h = offset_obj_cls_ptr[3]; - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - - types::Boxf box; - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(x2, (float) img_width - 1.f); - box.y2 = std::min(y2, (float) img_height - 1.f); - box.score = conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITETNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void TNNYoloV5_V_6_0::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lite/tnn/cv/tnn_yolov5_v6.0.h b/lite/tnn/cv/tnn_yolov5_v6.0.h deleted file mode 100644 index 68a0e5ff..00000000 --- a/lite/tnn/cv/tnn_yolov5_v6.0.h +++ /dev/null @@ -1,80 +0,0 @@ -// -// Created by DefTruth on 2021/11/10. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_YOLOV5_V6_0_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_YOLOV5_V6_0_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNYoloV5_V_6_0 : public BasicTNNHandler - { - public: - explicit TNNYoloV5_V_6_0(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNYoloV5_V_6_0() override = default; - - private: - // nested classes - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } YoloV5ScaleParams; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.0 / 255.f, 1.0 / 255.f, 1.0 / 255.f}; // RGB - std::vector bias_vals = {0.f, 0.f, 0.f}; - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void transform(const cv::Mat &mat_rs) override; // without resize - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YoloV5ScaleParams &scale_params); - - void generate_bboxes(const YoloV5ScaleParams &scale_params, - std::vector &bbox_collection, - std::shared_ptr &_instance, - float score_threshold, int img_height, - int img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_YOLOV5_V6_0_H diff --git a/lite/tnn/cv/tnn_yolov6.cpp b/lite/tnn/cv/tnn_yolov6.cpp deleted file mode 100644 index d03a013b..00000000 --- a/lite/tnn/cv/tnn_yolov6.cpp +++ /dev/null @@ -1,212 +0,0 @@ -// -// Created by DefTruth on 2022/6/25. -// - -#include "tnn_yolov6.h" -#include "lite/utils.h" - -using tnncv::TNNYOLOv6; - -TNNYOLOv6::TNNYOLOv6(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -// letterbox -void TNNYOLOv6::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YOLOv6ScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void TNNYOLOv6::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - // be carefully, no deepcopy inside this tnn::Mat constructor, - // so, we can not pass a local cv::Mat to this constructor. - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNYOLOv6::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - // resize & unscale - cv::Mat mat_rs; - YOLOv6ScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - cv::Mat mat_rs_; - cv::cvtColor(mat_rs, mat_rs_, cv::COLOR_BGR2RGB); - this->transform(mat_rs_); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - // 5. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, instance, score_threshold, img_height, img_width); - // 6. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void TNNYOLOv6::generate_bboxes(const YOLOv6ScaleParams &scale_params, - std::vector &bbox_collection, - std::shared_ptr &_instance, - float score_threshold, int img_height, - int img_width) -{ - // 4. fetch output mat - std::shared_ptr pred_mat; - tnn::MatConvertParam pred_cvt_param; // default - tnn::Status status; - - // (1,n,85=5+80=cxcy+cwch+obj_conf+cls_conf) - status = _instance->GetOutputMat(pred_mat, pred_cvt_param, "outputs", output_device_type); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - auto pred_dims = pred_mat->GetDims(); - const unsigned int num_anchors = pred_dims.at(1); // n = ? - const unsigned int num_classes = pred_dims.at(2) - 5; // 80 - - float r_ = scale_params.r; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *offset_obj_cls_ptr = - (float *) pred_mat->GetData() + (i * (num_classes + 5)); // row ptr - float obj_conf = offset_obj_cls_ptr[4]; - if (obj_conf < score_threshold) continue; // filter first. - - float cls_conf = offset_obj_cls_ptr[5]; - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = offset_obj_cls_ptr[j + 5]; - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - - float conf = obj_conf * cls_conf; // cls_conf (0.,1.) - if (conf < score_threshold) continue; // filter - - float cx = offset_obj_cls_ptr[0]; - float cy = offset_obj_cls_ptr[1]; - float w = offset_obj_cls_ptr[2]; - float h = offset_obj_cls_ptr[3]; - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - - types::Boxf box; - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(x2, (float) img_width - 1.f); - box.y2 = std::min(y2, (float) img_height - 1.f); - box.score = conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITETNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void TNNYOLOv6::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - diff --git a/lite/tnn/cv/tnn_yolov6.h b/lite/tnn/cv/tnn_yolov6.h deleted file mode 100644 index 09b9580a..00000000 --- a/lite/tnn/cv/tnn_yolov6.h +++ /dev/null @@ -1,80 +0,0 @@ -// -// Created by DefTruth on 2022/6/25. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_YOLOV6_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_YOLOV6_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNYOLOv6 : public BasicTNNHandler - { - public: - explicit TNNYOLOv6(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNYOLOv6() override = default; - - private: - // nested classes - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } YOLOv6ScaleParams; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.0 / 255.f, 1.0 / 255.f, 1.0 / 255.f}; // RGB - std::vector bias_vals = {0.f, 0.f, 0.f}; - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void transform(const cv::Mat &mat_rs) override; // without resize - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YOLOv6ScaleParams &scale_params); - - void generate_bboxes(const YOLOv6ScaleParams &scale_params, - std::vector &bbox_collection, - std::shared_ptr &_instance, - float score_threshold, int img_height, - int img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - - }; -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_YOLOV6_H diff --git a/lite/tnn/cv/tnn_yolox.cpp b/lite/tnn/cv/tnn_yolox.cpp deleted file mode 100644 index ea0bd916..00000000 --- a/lite/tnn/cv/tnn_yolox.cpp +++ /dev/null @@ -1,267 +0,0 @@ -// -// Created by DefTruth on 2021/10/17. -// - -#include "tnn_yolox.h" -#include "lite/utils.h" - -using tnncv::TNNYoloX; - -TNNYoloX::TNNYoloX(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNYoloX::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - // be carefully, no deepcopy inside this tnn::Mat constructor, - // so, we can not pass a local cv::Mat to this constructor. - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNYoloX::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YoloXScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat; - // cv::Mat new_unpad_mat = mat.clone(); // may not need clone. - cv::resize(mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void TNNYoloX::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - // resize & unscale - cv::Mat mat_rs; - YoloXScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - cv::Mat mat_rs_; - cv::cvtColor(mat_rs, mat_rs_, cv::COLOR_BGR2RGB); - this->transform(mat_rs_); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch output mat - std::shared_ptr pred_mat; - tnn::MatConvertParam pred_cvt_param; // default - - status = instance->GetOutputMat(pred_mat, pred_cvt_param, "outputs", output_device_type); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - // 5. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, pred_mat, score_threshold, img_height, img_width); - // 6. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void TNNYoloX::generate_anchors(const int target_height, - const int target_width, - std::vector &strides, - std::vector &anchors) -{ - for (auto stride: strides) - { - int num_grid_w = target_width / stride; - int num_grid_h = target_height / stride; - for (int g1 = 0; g1 < num_grid_h; ++g1) - { - for (int g0 = 0; g0 < num_grid_w; ++g0) - { -#ifdef LITE_WIN32 - YoloXAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchors.push_back(anchor); -#else - anchors.push_back((YoloXAnchor) {g0, g1, stride}); -#endif - } - } - } -} - -void TNNYoloX::generate_bboxes(const YoloXScaleParams &scale_params, - std::vector &bbox_collection, - const std::shared_ptr &pred_mat, - float score_threshold, int img_height, - int img_width) -{ - auto pred_dims = pred_mat->GetDims(); - const unsigned int num_anchors = pred_dims.at(1); // n = ? - const unsigned int num_classes = pred_dims.at(2) - 5; - - std::vector anchors; - std::vector strides = {8, 16, 32}; // might have stride=64 - this->generate_anchors(input_height, input_width, strides, anchors); - - float r_ = scale_params.r; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *offset_obj_cls_ptr = - (float *) pred_mat->GetData() + (i * (num_classes + 5)); - float obj_conf = offset_obj_cls_ptr[4]; - if (obj_conf < score_threshold) continue; // filter first. - - float cls_conf = offset_obj_cls_ptr[5]; - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = offset_obj_cls_ptr[j + 5]; - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - - float conf = obj_conf * cls_conf; // cls_conf (0.,1.) - if (conf < score_threshold) continue; // filter - - const int grid0 = anchors.at(i).grid0; - const int grid1 = anchors.at(i).grid1; - const int stride = anchors.at(i).stride; - - float dx = offset_obj_cls_ptr[0]; - float dy = offset_obj_cls_ptr[1]; - float dw = offset_obj_cls_ptr[2]; - float dh = offset_obj_cls_ptr[3]; - - float cx = (dx + (float) grid0) * (float) stride; - float cy = (dy + (float) grid1) * (float) stride; - float w = std::exp(dw) * (float) stride; - float h = std::exp(dh) * (float) stride; - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - - types::Boxf box; - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(x2, (float) img_width - 1.f); - box.y2 = std::min(y2, (float) img_height - 1.f); - box.score = conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITETNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void TNNYoloX::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - - - - - - - - - - - - - - - - - - - - diff --git a/lite/tnn/cv/tnn_yolox.h b/lite/tnn/cv/tnn_yolox.h deleted file mode 100644 index 5010c418..00000000 --- a/lite/tnn/cv/tnn_yolox.h +++ /dev/null @@ -1,93 +0,0 @@ -// -// Created by DefTruth on 2021/10/17. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_YOLOX_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_YOLOX_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNYoloX : public BasicTNNHandler - { - public: - explicit TNNYoloX(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNYoloX() override = default; - - private: - // nested classes - typedef struct GridAndStride - { - int grid0; - int grid1; - int stride; - } YoloXAnchor; - - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } YoloXScaleParams; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {0.0171247f, 0.0175070f, 0.0174291f}; - std::vector bias_vals = {-123.675f * 0.0171247f, -116.28f * 0.0175070f,-103.53f * 0.0174291f}; // RGB - - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void transform(const cv::Mat &mat_rs) override; // - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YoloXScaleParams &scale_params); - - void generate_anchors(const int target_height, - const int target_width, - std::vector &strides, - std::vector &anchors); - - void generate_bboxes(const YoloXScaleParams &scale_params, - std::vector &bbox_collection, - const std::shared_ptr &pred_mat, - float score_threshold, int img_height, - int img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - - }; - -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_YOLOX_H diff --git a/lite/tnn/cv/tnn_yolox_v0.1.1.cpp b/lite/tnn/cv/tnn_yolox_v0.1.1.cpp deleted file mode 100644 index 26b8886f..00000000 --- a/lite/tnn/cv/tnn_yolox_v0.1.1.cpp +++ /dev/null @@ -1,264 +0,0 @@ -// -// Created by DefTruth on 2021/11/6. -// - -#include "tnn_yolox_v0.1.1.h" -#include "lite/utils.h" - -using tnncv::TNNYoloX_V_0_1_1; - -TNNYoloX_V_0_1_1::TNNYoloX_V_0_1_1(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads) : - BasicTNNHandler(_proto_path, _model_path, _num_threads) -{ -} - -void TNNYoloX_V_0_1_1::transform(const cv::Mat &mat_rs) -{ - // push into input_mat - // be carefully, no deepcopy inside this tnn::Mat constructor, - // so, we can not pass a local cv::Mat to this constructor. - input_mat = std::make_shared(input_device_type, tnn::N8UC3, - input_shape, (void *) mat_rs.data); - if (!input_mat->GetData()) - { -#ifdef LITETNN_DEBUG - std::cout << "input_mat == nullptr! transform failed\n"; -#endif - } -} - -void TNNYoloX_V_0_1_1::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, - int target_height, int target_width, - YoloXScaleParams &scale_params) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - - mat_rs = cv::Mat(target_height, target_width, CV_8UC3, - cv::Scalar(114, 114, 114)); - // scale ratio (new / old) new_shape(h,w) - float w_r = (float) target_width / (float) img_width; - float h_r = (float) target_height / (float) img_height; - float r = std::min(w_r, h_r); - // compute padding - int new_unpad_w = static_cast((float) img_width * r); // floor - int new_unpad_h = static_cast((float) img_height * r); // floor - int pad_w = target_width - new_unpad_w; // >=0 - int pad_h = target_height - new_unpad_h; // >=0 - - int dw = pad_w / 2; - int dh = pad_h / 2; - - // resize with unscaling - cv::Mat new_unpad_mat = mat.clone(); - cv::resize(new_unpad_mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h)); - new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h))); - - // record scale params. - scale_params.r = r; - scale_params.dw = dw; - scale_params.dh = dh; - scale_params.new_unpad_w = new_unpad_w; - scale_params.new_unpad_h = new_unpad_h; - scale_params.flag = true; -} - -void TNNYoloX_V_0_1_1::detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold, float iou_threshold, - unsigned int topk, unsigned int nms_type) -{ - if (mat.empty()) return; - int img_height = static_cast(mat.rows); - int img_width = static_cast(mat.cols); - // resize & unscale - cv::Mat mat_rs; - YoloXScaleParams scale_params; - this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params); - - // 1. make input tensor - this->transform(mat_rs); - // 2. set input_mat - tnn::MatConvertParam input_cvt_param; - input_cvt_param.scale = scale_vals; - input_cvt_param.bias = bias_vals; - - tnn::Status status; - status = instance->SetInputMat(input_mat, input_cvt_param); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->SetInputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - - // 3. forward - status = instance->Forward(); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->Forward failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - // 4. fetch output mat - std::shared_ptr pred_mat; - tnn::MatConvertParam pred_cvt_param; // default - - status = instance->GetOutputMat(pred_mat, pred_cvt_param, "output", output_device_type); - if (status != tnn::TNN_OK) - { -#ifdef LITETNN_DEBUG - std::cout << "instance->GetOutputMat failed!:" - << status.description().c_str() << "\n"; -#endif - return; - } - // 5. rescale & exclude. - std::vector bbox_collection; - this->generate_bboxes(scale_params, bbox_collection, pred_mat, score_threshold, img_height, img_width); - // 6. hard|blend|offset nms with topk. - this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type); -} - -void TNNYoloX_V_0_1_1::generate_anchors(const int target_height, - const int target_width, - std::vector &strides, - std::vector &anchors) -{ - for (auto stride : strides) - { - int num_grid_w = target_width / stride; - int num_grid_h = target_height / stride; - for (int g1 = 0; g1 < num_grid_h; ++g1) - { - for (int g0 = 0; g0 < num_grid_w; ++g0) - { -#ifdef LITE_WIN32 - YoloXAnchor anchor; - anchor.grid0 = g0; - anchor.grid1 = g1; - anchor.stride = stride; - anchors.push_back(anchor); -#else - anchors.push_back((YoloXAnchor) {g0, g1, stride}); -#endif - } - } - } -} - -void TNNYoloX_V_0_1_1::generate_bboxes(const YoloXScaleParams &scale_params, - std::vector &bbox_collection, - const std::shared_ptr &pred_mat, - float score_threshold, int img_height, - int img_width) -{ - auto pred_dims = pred_mat->GetDims(); - const unsigned int num_anchors = pred_dims.at(1); // n = ? - const unsigned int num_classes = pred_dims.at(2) - 5; - - std::vector anchors; - std::vector strides = {8, 16, 32}; // might have stride=64 - this->generate_anchors(input_height, input_width, strides, anchors); - - float r_ = scale_params.r; - int dw_ = scale_params.dw; - int dh_ = scale_params.dh; - - bbox_collection.clear(); - unsigned int count = 0; - for (unsigned int i = 0; i < num_anchors; ++i) - { - const float *offset_obj_cls_ptr = - (float *) pred_mat->GetData() + (i * (num_classes + 5)); - float obj_conf = offset_obj_cls_ptr[4]; - if (obj_conf < score_threshold) continue; // filter first. - - float cls_conf = offset_obj_cls_ptr[5]; - unsigned int label = 0; - for (unsigned int j = 0; j < num_classes; ++j) - { - float tmp_conf = offset_obj_cls_ptr[j + 5]; - if (tmp_conf > cls_conf) - { - cls_conf = tmp_conf; - label = j; - } - } // argmax - - float conf = obj_conf * cls_conf; // cls_conf (0.,1.) - if (conf < score_threshold) continue; // filter - - const int grid0 = anchors.at(i).grid0; - const int grid1 = anchors.at(i).grid1; - const int stride = anchors.at(i).stride; - - float dx = offset_obj_cls_ptr[0]; - float dy = offset_obj_cls_ptr[1]; - float dw = offset_obj_cls_ptr[2]; - float dh = offset_obj_cls_ptr[3]; - - float cx = (dx + (float) grid0) * (float) stride; - float cy = (dy + (float) grid1) * (float) stride; - float w = std::exp(dw) * (float) stride; - float h = std::exp(dh) * (float) stride; - float x1 = ((cx - w / 2.f) - (float) dw_) / r_; - float y1 = ((cy - h / 2.f) - (float) dh_) / r_; - float x2 = ((cx + w / 2.f) - (float) dw_) / r_; - float y2 = ((cy + h / 2.f) - (float) dh_) / r_; - - types::Boxf box; - box.x1 = std::max(0.f, x1); - box.y1 = std::max(0.f, y1); - box.x2 = std::min(x2, (float) img_width); - box.y2 = std::min(y2, (float) img_height); - box.score = conf; - box.label = label; - box.label_text = class_names[label]; - box.flag = true; - bbox_collection.push_back(box); - - count += 1; // limit boxes for nms. - if (count > max_nms) - break; - } -#if LITETNN_DEBUG - std::cout << "detected num_anchors: " << num_anchors << "\n"; - std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n"; -#endif -} - -void TNNYoloX_V_0_1_1::nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, - unsigned int nms_type) -{ - if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk); - else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk); - else lite::utils::hard_nms(input, output, iou_threshold, topk); -} - - - - - - - - - - - - - - - - - - - - diff --git a/lite/tnn/cv/tnn_yolox_v0.1.1.h b/lite/tnn/cv/tnn_yolox_v0.1.1.h deleted file mode 100644 index 579dde56..00000000 --- a/lite/tnn/cv/tnn_yolox_v0.1.1.h +++ /dev/null @@ -1,93 +0,0 @@ -// -// Created by DefTruth on 2021/11/6. -// - -#ifndef LITE_AI_TOOLKIT_TNN_CV_TNN_YOLOX_V0_1_1_H -#define LITE_AI_TOOLKIT_TNN_CV_TNN_YOLOX_V0_1_1_H - -#include "lite/tnn/core/tnn_core.h" - -namespace tnncv -{ - class LITE_EXPORTS TNNYoloX_V_0_1_1 : public BasicTNNHandler - { - public: - explicit TNNYoloX_V_0_1_1(const std::string &_proto_path, - const std::string &_model_path, - unsigned int _num_threads = 1); // - ~TNNYoloX_V_0_1_1() override = default; - - private: - // nested classes - typedef struct GridAndStride - { - int grid0; - int grid1; - int stride; - } YoloXAnchor; - - typedef struct - { - float r; - int dw; - int dh; - int new_unpad_w; - int new_unpad_h; - bool flag; - } YoloXScaleParams; - - private: - // In TNN: x*scale + bias - std::vector scale_vals = {1.f, 1.f, 1.f}; - std::vector bias_vals = {0.f, 0.f, 0.f}; // RGB - - const char *class_names[80] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", - "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush" - }; - enum NMS - { - HARD = 0, BLEND = 1, OFFSET = 2 - }; - static constexpr const unsigned int max_nms = 30000; - - private: - void transform(const cv::Mat &mat_rs) override; // - - void resize_unscale(const cv::Mat &mat, - cv::Mat &mat_rs, - int target_height, - int target_width, - YoloXScaleParams &scale_params); - - void generate_anchors(const int target_height, - const int target_width, - std::vector &strides, - std::vector &anchors); - - void generate_bboxes(const YoloXScaleParams &scale_params, - std::vector &bbox_collection, - const std::shared_ptr &pred_mat, - float score_threshold, int img_height, - int img_width); // rescale & exclude - - void nms(std::vector &input, std::vector &output, - float iou_threshold, unsigned int topk, unsigned int nms_type); - - public: - void detect(const cv::Mat &mat, std::vector &detected_boxes, - float score_threshold = 0.25f, float iou_threshold = 0.45f, - unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET); - - }; - -} - -#endif //LITE_AI_TOOLKIT_TNN_CV_TNN_YOLOX_V0_1_1_H From fadba054dbb5bdf4e663a6d17041062795879f71 Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Sun, 31 May 2026 16:57:24 +0800 Subject: [PATCH 08/30] docs(readme): reframe around extreme-GPU-inference + FaceFusion flagship Rewrite the README from the upstream "300+ models, 5 backends" catalog into a focused flagship narrative: FaceFusion face-swap on TensorRT, with the GPU-optimization benchmark as the hero. 1152 -> 183 lines. - lead with the benchmark (GFPGAN 4.4x) and the FaceFusion pipeline - TensorRT-first build + verified flagship/YOLOv5 GPU code samples - drop MNN/NCNN/TNN columns, the per-model code dump, Mixed-with-MNN + docker-hub sections - 100+ ORT CV models demoted to a one-line pointer to docs/hub - note the legacy multi-backend build is frozen at tag v0.2-all-backends - keep all xlite-dev URLs / assets / citation unchanged Co-Authored-By: Claude Opus 4.8 --- README.md | 1153 +++++------------------------------------------------ 1 file changed, 94 insertions(+), 1059 deletions(-) diff --git a/README.md b/README.md index 592e0c9b..678b2491 100644 --- a/README.md +++ b/README.md @@ -1,101 +1,35 @@ -
- - - +
![lite-ai-toolkit](https://github.com/user-attachments/assets/11568474-57e3-4ef7-96c0-d2ce7028bb5f)
- - - + + + -
+ -🛠**Lite.Ai.ToolKit**: A lite C++ toolkit of 100+ Awesome AI models, such as [Object Detection](#lite.ai.toolkit-object-detection), [Face Detection](#lite.ai.toolkit-face-detection), [Face Recognition](#lite.ai.toolkit-face-recognition), [Segmentation](#lite.ai.toolkit-segmentation), [Matting](#lite.ai.toolkit-matting), etc. See [Model Zoo](#lite.ai.toolkit-Model-Zoo) and [ONNX Hub](https://github.com/xlite-dev/lite.ai.toolkit/tree/main/docs/hub/lite.ai.toolkit.hub.onnx.md), [MNN Hub](https://github.com/xlite-dev/lite.ai.toolkit/tree/main/docs/hub/lite.ai.toolkit.hub.mnn.md), [TNN Hub](https://github.com/xlite-dev/lite.ai.toolkit/tree/main/docs/hub/lite.ai.toolkit.hub.tnn.md), [NCNN Hub](https://github.com/xlite-dev/lite.ai.toolkit/tree/main/docs/hub/lite.ai.toolkit.hub.ncnn.md). Welcome to 🌟👆🏻star this repo to support me, many thanks ~ 🎉🎉 +🛠 **Lite.Ai.ToolKit** is a C++ toolkit for **extreme GPU inference**. The flagship is an end-to-end +**FaceFusion face-swap pipeline** (detect → landmark → recognize → swap → restore) running entirely on +**TensorRT**, with the CPU pre/post-processing glue rewritten as **hand-fused CUDA kernels**. The goal is +not breadth — it is to make one real pipeline as fast as a single GPU can make it, and to show the work +honestly with a reproducible benchmark harness. Welcome to 🌟 star this repo to support us ~ 🎉🎉 -
- - - - - - - - - -
+> **Heads up (>= 0.3):** the active line targets **TensorRT only**. ONNXRuntime is kept as the numerical +> reference + the host for the test suite. The legacy multi-backend build (MNN / NCNN / TNN, 300+ thin +> model wrappers) is frozen on tag **[`v0.2-all-backends`](https://github.com/xlite-dev/lite.ai.toolkit/tree/main)** — check it out if you need those backends. ## 📖 News 🔥🔥
-- Now, [lite.ai.toolkit](https://github.com/xlite-dev/lite.ai.toolkit) ![](https://img.shields.io/github/stars/xlite-dev/lite.ai.toolkit.svg?style=social) is mainly maintained by 🎉[@wangzijian1010](https://github.com/wangzijian1010). Many thanks ~ 🎉🎉 - -## Citations 🎉🎉 -```BibTeX -@misc{lite.ai.toolkit@2021, - title={lite.ai.toolkit: A lite C++ toolkit of 100+ Awesome AI models.}, - url={https://github.com/xlite-dev/lite.ai.toolkit}, - note={Open-source software available at https://github.com/xlite-dev/lite.ai.toolkit}, - author={xlite-dev, wangzijian1010 etc}, - year={2021} -} -``` - -## Features 👏👋 - -* **Simply and User friendly.** Simply and Consistent syntax like **lite::cv::Type::Class**, see [examples](#lite.ai.toolkit-Examples-for-Lite.AI.ToolKit). -* **Minimum Dependencies.** Only **OpenCV** and **ONNXRuntime** are required by default, see [build](#lite.ai.toolkit-Build-Lite.AI.ToolKit). -* **Many Models Supported.** **[300+](#lite.ai.toolkit-Supported-Models-Matrix)** C++ implementations and **[500+](https://github.com/xlite-dev/lite.ai.toolkit/tree/main/docs/hub/lite.ai.toolkit.hub.onnx.md)** weights 👉 **[Supported-Matrix](#lite.ai.toolkit-Supported-Models-Matrix)**. - -## Build 👇👇 -Download prebuilt lite.ai.toolkit library from [tag/v0.2.0](https://github.com/xlite-dev/lite.ai.toolkit/releases/tag/v0.2.0), or just build it from source: -```shell -git clone --depth=1 https://github.com/xlite-dev/lite.ai.toolkit.git # latest -cd lite.ai.toolkit && sh ./build.sh # >= 0.2.0, support Linux only, tested on Ubuntu 20.04.6 LTS -``` - -## Quick Start 🌟🌟 -
- -#### Example0: Object Detection using [YOLOv5](https://github.com/ultralytics/yolov5). Download model from Model-Zoo[2](#lite.ai.toolkit-2). -```c++ -#include "lite/lite.h" - -int main(int argc, char *argv[]) { - std::string onnx_path = "yolov5s.onnx"; - std::string test_img_path = "test_yolov5.jpg"; - std::string save_img_path = "test_results.jpg"; - - auto *yolov5 = new lite::cv::detection::YoloV5(onnx_path); - std::vector detected_boxes; - cv::Mat img_bgr = cv::imread(test_img_path); - yolov5->detect(img_bgr, detected_boxes); - - lite::utils::draw_boxes_inplace(img_bgr, detected_boxes); - cv::imwrite(save_img_path, img_bgr); - delete yolov5; - return 0; -} -``` -You can download the prebuilt lite.ai.tooklit library and test resources from [tag/v0.2.0](https://github.com/xlite-dev/lite.ai.toolkit/releases/tag/v0.2.0). -```bash -export LITE_AI_TAG_URL=https://github.com/xlite-dev/lite.ai.toolkit/releases/download/v0.2.0 -wget ${LITE_AI_TAG_URL}/lite-ort1.17.1+ocv4.9.0+ffmpeg4.2.2-linux-x86_64.tgz -wget ${LITE_AI_TAG_URL}/yolov5s.onnx && wget ${LITE_AI_TAG_URL}/test_yolov5.jpg -``` -#### 🎉🎉[TensorRT](https://github.com/NVIDIA/TensorRT): Boost inference performance with NVIDIA GPU via TensorRT. -Run `bash ./build.sh tensorrt` to build lite.ai.toolkit with TensorRT support, and then test yolov5 with the codes below. NOTE: lite.ai.toolkit need TensorRT 10.x (or later) and CUDA 12.x (or later). Please check [build.sh](./build.sh), [tensorrt-linux-x86_64-install.zh.md](./docs/tensorrt/tensorrt-linux-x86_64.zh.md), [test_lite_yolov5.cpp](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolov5.cpp) and [NVIDIA/TensorRT](https://github.com/NVIDIA/TensorRT) for more details. -```c++ -// trtexec --onnx=yolov5s.onnx --saveEngine=yolov5s.engine -auto *yolov5 = new lite::trt::cv::detection::YOLOV5(engine_path); -``` +- **GPU-inference optimization in progress** — the FaceFusion face-restoration stage (GFPGAN 1.4) was + taken from **78.2 ms → 17.7 ms (4.4×, 12.8 → 56.6 FPS)** on an RTX 4090 by moving paste-back and + preprocessing into fused CUDA kernels. See [Benchmark](#benchmark) below. The rest of the pipeline + (detect / landmark / swap, FP16) is being optimized stage by stage. +- [lite.ai.toolkit](https://github.com/xlite-dev/lite.ai.toolkit) is mainly maintained by 🎉[@wangzijian1010](https://github.com/wangzijian1010). ## ⚡ Benchmark 🔥
@@ -125,1016 +59,117 @@ paste-back is numerically equivalent to the CPU path (max |diff| = 2/255). The s -## Quick Setup 👀 - -To quickly setup `lite.ai.toolkit`, you can follow the `CMakeLists.txt` listed as belows. 👇👀 - -```cmake -set(lite.ai.toolkit_DIR YOUR-PATH-TO-LITE-INSTALL) -find_package(lite.ai.toolkit REQUIRED PATHS ${lite.ai.toolkit_DIR}) -add_executable(lite_yolov5 test_lite_yolov5.cpp) -target_link_libraries(lite_yolov5 ${lite.ai.toolkit_LIBS}) -``` - -## Mixed with MNN or ONNXRuntime 👇👇 -The goal of lite.ai.toolkit is not to abstract on top of MNN and ONNXRuntime. So, you can use lite.ai.toolkit mixed with MNN(`-DENABLE_MNN=ON, default OFF`) or ONNXRuntime(`-DENABLE_ONNXRUNTIME=ON, default ON`). The lite.ai.toolkit installation package contains complete MNN and ONNXRuntime. The workflow may looks like: -```C++ -#include "lite/lite.h" -// 0. use yolov5 from lite.ai.toolkit to detect objs. -auto *yolov5 = new lite::cv::detection::YoloV5(onnx_path); -// 1. use OnnxRuntime or MNN to implement your own classfier. -interpreter = std::shared_ptr(MNN::Interpreter::createFromFile(mnn_path)); -// or: session = new Ort::Session(ort_env, onnx_path, session_options); -classfier = interpreter->createSession(schedule_config); -// 2. then, classify the detected objs use your own classfier ... -``` -The included headers of MNN and ONNXRuntime can be found at [mnn_config.h](./lite/mnn/core/mnn_config.h) and [ort_config.h](./lite/ort/core/ort_config.h). - -
- 🔑️ Check the detailed Quick Start！Click here! - -### Download resources - -You can download the prebuilt lite.ai.tooklit library and test resources from [tag/v0.2.0](https://github.com/xlite-dev/lite.ai.toolkit/releases/tag/v0.2.0). -```bash -export LITE_AI_TAG_URL=https://github.com/xlite-dev/lite.ai.toolkit/releases/download/v0.2.0 -wget ${LITE_AI_TAG_URL}/lite-ort1.17.1+ocv4.9.0+ffmpeg4.2.2-linux-x86_64.tgz -wget ${LITE_AI_TAG_URL}/yolov5s.onnx && wget ${LITE_AI_TAG_URL}/test_yolov5.jpg -tar -zxvf lite-ort1.17.1+ocv4.9.0+ffmpeg4.2.2-linux-x86_64.tgz -``` -### Write test code - -write YOLOv5 example codes and name it `test_lite_yolov5.cpp`: -```c++ -#include "lite/lite.h" - -int main(int argc, char *argv[]) { - std::string onnx_path = "yolov5s.onnx"; - std::string test_img_path = "test_yolov5.jpg"; - std::string save_img_path = "test_results.jpg"; - - auto *yolov5 = new lite::cv::detection::YoloV5(onnx_path); - std::vector detected_boxes; - cv::Mat img_bgr = cv::imread(test_img_path); - yolov5->detect(img_bgr, detected_boxes); - - lite::utils::draw_boxes_inplace(img_bgr, detected_boxes); - cv::imwrite(save_img_path, img_bgr); - delete yolov5; - return 0; -} -``` - -### Setup CMakeLists.txt -```cmake -cmake_minimum_required(VERSION 3.10) -project(lite_yolov5) -set(CMAKE_CXX_STANDARD 17) - -set(lite.ai.toolkit_DIR YOUR-PATH-TO-LITE-INSTALL) -find_package(lite.ai.toolkit REQUIRED PATHS ${lite.ai.toolkit_DIR}) -if (lite.ai.toolkit_Found) - message(STATUS "lite.ai.toolkit_INCLUDE_DIRS: ${lite.ai.toolkit_INCLUDE_DIRS}") - message(STATUS " lite.ai.toolkit_LIBS: ${lite.ai.toolkit_LIBS}") - message(STATUS " lite.ai.toolkit_LIBS_DIRS: ${lite.ai.toolkit_LIBS_DIRS}") -endif() -add_executable(lite_yolov5 test_lite_yolov5.cpp) -target_link_libraries(lite_yolov5 ${lite.ai.toolkit_LIBS}) -``` -### Build example - -```bash -mkdir build && cd build && cmake .. && make -j1 -``` -Then, export the lib paths to `LD_LIBRARY_PATH` which listed by `lite.ai.toolkit_LIBS_DIRS`. -```bash -export LD_LIBRARY_PATH=YOUR-PATH-TO-LITE-INSTALL/lib:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=YOUR-PATH-TO-LITE-INSTALL/third_party/opencv/lib:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=YOUR-PATH-TO-LITE-INSTALL/third_party/onnxruntime/lib:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=YOUR-PATH-TO-LITE-INSTALL/third_party/MNN/lib:$LD_LIBRARY_PATH # if -DENABLE_MNN=ON -``` - -### Run binary: -```bash -cp ../yolov5s.onnx ../test_yolov.jpg . -./lite_yolov5 -``` -The output logs: -```bash -LITEORT_DEBUG LogId: ../examples/hub/onnx/cv/yolov5s.onnx -=============== Input-Dims ============== -Name: images -Dims: 1 -Dims: 3 -Dims: 640 -Dims: 640 -=============== Output-Dims ============== -Output: 0 Name: pred Dim: 0 :1 -Output: 0 Name: pred Dim: 1 :25200 -Output: 0 Name: pred Dim: 2 :85 -Output: 1 Name: output2 Dim: 0 :1 -...... -Output: 3 Name: output4 Dim: 1 :3 -Output: 3 Name: output4 Dim: 2 :20 -Output: 3 Name: output4 Dim: 3 :20 -Output: 3 Name: output4 Dim: 4 :85 -======================================== -detected num_anchors: 25200 -generate_bboxes num: 48 -``` -
- -
- - - -## Supported Models Matrix - -* / = not supported now. -* ✅ = known work and official supported now. -* ✔️ = known work, but unofficial supported now. -* ❔ = in my plan, but not coming soon, maybe a few months later. - -### NVIDIA GPU Inference: TensorRT - -|Class|Class|Class|Class|Class| System | Engine | -|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -|✅[YOLOv5](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolov5.cpp)|✅[YOLOv6](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolov6.cpp)|✅[YOLOv8](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolov8.cpp)|✅[YOLOv8Face](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolov8face.cpp)|✅[YOLOv5Face](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolo5face.cpp)| Linux | TensorRT | -|✅[YOLOX](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolox.cpp)|✅[YOLOv5BlazeFace](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolov5_blazeface.cpp) |✅[StableDiffusion](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/sd/test_lite_sd_pipeline.cpp)| ✅[FaceFusion](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_facefusion_pipeline_gpu.cpp) | / | Linux | TensorRT | - - -### CPU Inference: ONNXRuntime, MNN, NCNN and TNN -| Class | Size | Type | Demo | ONNXRuntime | MNN | NCNN | TNN | Linux | MacOS | Windows | Android | -|:-----------------------------------------------------------------------------------------------------------------:|:-----:|:----------------:|:----------------------------------------------------------------------------------------------------------------------:|:-----------:|:---:|:----:|:---:|:-----:|:-----:|:-------:|:-------:| -| [YoloV5](https://github.com/ultralytics/yolov5) | 28M | *detection* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolov5.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [YoloV3](https://github.com/onnx/models/blob/master/vision/object_detection_segmentation/yolov3) | 236M | *detection* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolov3.cpp) | ✅ | / | / | / | ✅ | ✔️ | ✔️ | / | -| [TinyYoloV3](https://github.com/onnx/models/blob/master/vision/object_detection_segmentation/tiny-yolov3) | 33M | *detection* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_tiny_yolov3.cpp) | ✅ | / | / | / | ✅ | ✔️ | ✔️ | / | -| [YoloV4](https://github.com/argusswift/YOLOv4-pytorch) | 176M | *detection* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolov4.cpp) | ✅ | / | / | / | ✅ | ✔️ | ✔️ | / | -| [SSD](https://github.com/onnx/models/blob/master/vision/object_detection_segmentation/ssd) | 76M | *detection* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_ssd.cpp) | ✅ | / | / | / | ✅ | ✔️ | ✔️ | / | -| [SSDMobileNetV1](https://github.com/onnx/models/blob/master/vision/object_detection_segmentation/ssd-mobilenetv1) | 27M | *detection* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_ssd_mobilenetv1.cpp) | ✅ | / | / | / | ✅ | ✔️ | ✔️ | / | -| [YoloX](https://github.com/Megvii-BaseDetection/YOLOX) | 3.5M | *detection* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolox.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [TinyYoloV4VOC](https://github.com/bubbliiiing/yolov4-tiny-pytorch) | 22M | *detection* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_tiny_yolov4_voc.cpp) | ✅ | / | / | / | ✅ | ✔️ | ✔️ | / | -| [TinyYoloV4COCO](https://github.com/bubbliiiing/yolov4-tiny-pytorch) | 22M | *detection* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_tiny_yolov4_coco.cpp) | ✅ | / | / | / | ✅ | ✔️ | ✔️ | / | -| [YoloR](https://github.com/WongKinYiu/yolor) | 39M | *detection* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolor.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [ScaledYoloV4](https://github.com/WongKinYiu/ScaledYOLOv4) | 270M | *detection* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_scaled_yolov4.cpp) | ✅ | / | / | / | ✅ | ✔️ | ✔️ | / | -| [EfficientDet](https://github.com/zylo117/Yet-Another-EfficientDet-Pytorch) | 15M | *detection* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_efficientdet.cpp) | ✅ | / | / | / | ✅ | ✔️ | ✔️ | / | -| [EfficientDetD7](https://github.com/zylo117/Yet-Another-EfficientDet-Pytorch) | 220M | *detection* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_efficientdet_d7.cpp) | ✅ | / | / | / | ✅ | ✔️ | ✔️ | / | -| [EfficientDetD8](https://github.com/zylo117/Yet-Another-EfficientDet-Pytorch) | 322M | *detection* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_efficientdet_d8.cpp) | ✅ | / | / | / | ✅ | ✔️ | ✔️ | / | -| [YOLOP](https://github.com/hustvl/YOLOP) | 30M | *detection* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolop.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [NanoDet](https://github.com/RangiLyu/nanodet) | 1.1M | *detection* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_nanodet.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [NanoDetPlus](https://github.com/RangiLyu/nanodet) | 4.5M | *detection* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_nanodet_plus.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [NanoDetEffi...](https://github.com/RangiLyu/nanodet) | 12M | *detection* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_nanodet_efficientnet_lite.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [YoloX_V_0_1_1](https://github.com/Megvii-BaseDetection/YOLOX) | 3.5M | *detection* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolox_v0.1.1.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [YoloV5_V_6_0](https://github.com/ultralytics/yolov5) | 7.5M | *detection* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolov5_v6.0.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [GlintArcFace](https://github.com/deepinsight/insightface/tree/master/recognition/arcface_torch) | 92M | *faceid* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_glint_arcface.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [GlintCosFace](https://github.com/deepinsight/insightface/tree/master/recognition/arcface_torch) | 92M | *faceid* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_glint_cosface.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | / | -| [GlintPartialFC](https://github.com/deepinsight/insightface/tree/master/recognition/partial_fc) | 170M | *faceid* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_glint_partial_fc.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | / | -| [FaceNet](https://github.com/timesler/facenet-pytorch) | 89M | *faceid* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_facenet.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | / | -| [FocalArcFace](https://github.com/ZhaoJ9014/face.evoLVe.PyTorch) | 166M | *faceid* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_focal_arcface.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | / | -| [FocalAsiaArcFace](https://github.com/ZhaoJ9014/face.evoLVe.PyTorch) | 166M | *faceid* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_focal_asia_arcface.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | / | -| [TencentCurricularFace](https://github.com/Tencent/TFace/tree/master/tasks/distfc) | 249M | *faceid* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_tencent_curricular_face.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | / | -| [TencentCifpFace](https://github.com/Tencent/TFace/tree/master/tasks/cifp) | 130M | *faceid* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_tencent_cifp_face.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | / | -| [CenterLossFace](https://github.com/louis-she/center-loss.pytorch) | 280M | *faceid* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_center_loss_face.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | / | -| [SphereFace](https://github.com/clcarwin/sphereface_pytorch) | 80M | *faceid* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_sphere_face.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | / | -| [PoseRobustFace](https://github.com/penincillin/DREAM) | 92M | *faceid* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_pose_robust_face.cpp) | ✅ | / | / | / | ✅ | ✔️ | ✔️ | / | -| [NaivePoseRobustFace](https://github.com/penincillin/DREAM) | 43M | *faceid* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_naive_pose_robust_face.cpp) | ✅ | / | / | / | ✅ | ✔️ | ✔️ | / | -| [MobileFaceNet](https://github.com/Xiaoccer/MobileFaceNet_Pytorch) | 3.8M | *faceid* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_mobile_facenet.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [CavaGhostArcFace](https://github.com/cavalleria/cavaface.pytorch) | 15M | *faceid* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_cava_ghost_arcface.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [CavaCombinedFace](https://github.com/cavalleria/cavaface.pytorch) | 250M | *faceid* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_cava_combined_face.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | / | -| [MobileSEFocalFace](https://github.com/grib0ed0v/face_recognition.pytorch) | 4.5M | *faceid* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_mobilese_focal_face.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [RobustVideoMatting](https://github.com/PeterL1n/RobustVideoMatting) | 14M | *matting* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_rvm.cpp) | ✅ | ✅ | / | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [MGMatting](https://github.com/yucornetto/MGMatting) | 113M | *matting* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_mg_matting.cpp) | ✅ | ✅ | / | ✅ | ✅ | ✔️ | ✔️ | / | -| [MODNet](https://github.com/ZHKKKe/MODNet) | 24M | *matting* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_modnet.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | / | -| [MODNetDyn](https://github.com/ZHKKKe/MODNet) | 24M | *matting* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_modnet_dyn.cpp) | ✅ | / | / | / | ✅ | ✔️ | ✔️ | / | -| [BackgroundMattingV2](https://github.com/PeterL1n/BackgroundMattingV2) | 20M | *matting* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_backgroundmattingv2.cpp) | ✅ | ✅ | / | ✅ | ✅ | ✔️ | ✔️ | / | -| [BackgroundMattingV2Dyn](https://github.com/PeterL1n/BackgroundMattingV2) | 20M | *matting* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_backgroundmattingv2_dyn.cpp) | ✅ | / | / | / | ✅ | ✔️ | ✔️ | / | -| [UltraFace](https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB) | 1.1M | *face::detect* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_ultraface.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [RetinaFace](https://github.com/biubug6/Pytorch_Retinaface) | 1.6M | *face::detect* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_retinaface.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [FaceBoxes](https://github.com/zisianw/FaceBoxes.PyTorch) | 3.8M | *face::detect* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_faceboxes.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [FaceBoxesV2](https://github.com/jhb86253817/FaceBoxesV2) | 3.8M | *face::detect* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_faceboxesv2.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [SCRFD](https://github.com/deepinsight/insightface/blob/master/detection/scrfd/) | 2.5M | *face::detect* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_scrfd.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [YOLO5Face](https://github.com/deepcam-cn/yolov5-face) | 4.8M | *face::detect* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolo5face.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [PFLD](https://github.com/Hsintao/pfld_106_face_landmarks) | 1.0M | *face::align* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_pfld.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [PFLD98](https://github.com/polarisZhao/PFLD-pytorch) | 4.8M | *face::align* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_pfld98.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [MobileNetV268](https://github.com/cunjian/pytorch_face_landmark) | 9.4M | *face::align* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_mobilenetv2_68.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [MobileNetV2SE68](https://github.com/cunjian/pytorch_face_landmark) | 11M | *face::align* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_mobilenetv2_se_68.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [PFLD68](https://github.com/cunjian/pytorch_face_landmark) | 2.8M | *face::align* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_pfld68.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [FaceLandmark1000](https://github.com/Single430/FaceLandmark1000) | 2.0M | *face::align* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_face_landmarks_1000.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [PIPNet98](https://github.com/jhb86253817/PIPNet) | 44.0M | *face::align* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_pipnet98.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [PIPNet68](https://github.com/jhb86253817/PIPNet) | 44.0M | *face::align* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_pipnet68.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [PIPNet29](https://github.com/jhb86253817/PIPNet) | 44.0M | *face::align* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_pipnet29.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [PIPNet19](https://github.com/jhb86253817/PIPNet) | 44.0M | *face::align* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_pipnet19.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [FSANet](https://github.com/omasaht/headpose-fsanet-pytorch) | 1.2M | *face::pose* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_fsanet.cpp) | ✅ | ✅ | / | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [AgeGoogleNet](https://github.com/onnx/models/tree/master/vision/body_analysis/age_gender) | 23M | *face::attr* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_age_googlenet.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [GenderGoogleNet](https://github.com/onnx/models/tree/master/vision/body_analysis/age_gender) | 23M | *face::attr* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_gender_googlenet.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [EmotionFerPlus](https://github.com/onnx/models/blob/master/vision/body_analysis/emotion_ferplus) | 33M | *face::attr* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_emotion_ferplus.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [VGG16Age](https://github.com/onnx/models/tree/master/vision/body_analysis/age_gender) | 514M | *face::attr* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_vgg16_age.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | / | -| [VGG16Gender](https://github.com/onnx/models/tree/master/vision/body_analysis/age_gender) | 512M | *face::attr* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_vgg16_gender.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | / | -| [SSRNet](https://github.com/oukohou/SSR_Net_Pytorch) | 190K | *face::attr* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_ssrnet.cpp) | ✅ | ✅ | / | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [EfficientEmotion7](https://github.com/HSE-asavchenko/face-emotion-recognition) | 15M | *face::attr* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_efficient_emotion7.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [EfficientEmotion8](https://github.com/HSE-asavchenko/face-emotion-recognition) | 15M | *face::attr* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_efficient_emotion8.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [MobileEmotion7](https://github.com/HSE-asavchenko/face-emotion-recognition) | 13M | *face::attr* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_mobile_emotion7.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [ReXNetEmotion7](https://github.com/HSE-asavchenko/face-emotion-recognition) | 30M | *face::attr* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_rexnet_emotion7.cpp) | ✅ | ✅ | / | ✅ | ✅ | ✔️ | ✔️ | / | -| [EfficientNetLite4](https://github.com/onnx/models/blob/master/vision/classification/efficientnet-lite4) | 49M | *classification* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_efficientnet_lite4.cpp) | ✅ | ✅ | / | ✅ | ✅ | ✔️ | ✔️ | / | -| [ShuffleNetV2](https://github.com/onnx/models/blob/master/vision/classification/shufflenet) | 8.7M | *classification* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_shufflenetv2.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [DenseNet121](https://pytorch.org/hub/pytorch_vision_densenet/) | 30.7M | *classification* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_densenet.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | / | -| [GhostNet](https://pytorch.org/hub/pytorch_vision_ghostnet/) | 20M | *classification* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_ghostnet.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [HdrDNet](https://pytorch.org/hub/pytorch_vision_hardnet//) | 13M | *classification* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_hardnet.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [IBNNet](https://pytorch.org/hub/pytorch_vision_ibnnet/) | 97M | *classification* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_ibnnet.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | / | -| [MobileNetV2](https://pytorch.org/hub/pytorch_vision_mobilenet_v2/) | 13M | *classification* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_mobilenetv2.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [ResNet](https://pytorch.org/hub/pytorch_vision_resnet/) | 44M | *classification* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_resnet.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | / | -| [ResNeXt](https://pytorch.org/hub/pytorch_vision_resnext/) | 95M | *classification* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_resnext.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | / | -| [DeepLabV3ResNet101](https://pytorch.org/hub/pytorch_vision_deeplabv3_resnet101/) | 232M | *segmentation* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_deeplabv3_resnet101.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | / | -| [FCNResNet101](https://pytorch.org/hub/pytorch_vision_fcn_resnet101/) | 207M | *segmentation* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_fcn_resnet101.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | / | -| [FastStyleTransfer](https://github.com/onnx/models/blob/master/vision/style_transfer/fast_neural_style) | 6.4M | *style* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_fast_style_transfer.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [Colorizer](https://github.com/richzhang/colorization) | 123M | *colorization* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_colorizer.cpp) | ✅ | ✅ | / | ✅ | ✅ | ✔️ | ✔️ | / | -| [SubPixelCNN](https://github.com/niazwazir/SUB_PIXEL_CNN) | 234K | *resolution* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_subpixel_cnn.cpp) | ✅ | ✅ | / | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [SubPixelCNN](https://github.com/niazwazir/SUB_PIXEL_CNN) | 234K | *resolution* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_subpixel_cnn.cpp) | ✅ | ✅ | / | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [InsectDet](https://github.com/quarrying/quarrying-insect-id) | 27M | *detection* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_insectdet.cpp) | ✅ | ✅ | / | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [InsectID](https://github.com/quarrying/quarrying-insect-id) | 22M | *classification* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_insectid.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ✔️ | ❔ | -| [PlantID](https://github.com/quarrying/quarrying-plant-id) | 30M | *classification* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_plantid.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ✔️ | ❔ | -| [YOLOv5BlazeFace](https://github.com/deepcam-cn/yolov5-face) | 3.4M | *face::detect* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolov5_blazeface.cpp) | ✅ | ✅ | / | / | ✅ | ✔️ | ✔️ | ❔ | -| [YoloV5_V_6_1](https://github.com/ultralytics/yolov5/releases/tag/v6.1) | 7.5M | *detection* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolov5_v6.1.cpp) | ✅ | ✅ | / | / | ✅ | ✔️ | ✔️ | ❔ | -| [HeadSeg](https://github.com/minivision-ai/photo2cartoon) | 31M | *segmentation* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_head_seg.cpp) | ✅ | ✅ | / | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [FemalePhoto2Cartoon](https://github.com/minivision-ai/photo2cartoon) | 15M | *style* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_female_photo2cartoon.cpp) | ✅ | ✅ | / | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [FastPortraitSeg](https://github.com/YexingWan/Fast-Portrait-Segmentation) | 400k | *segmentation* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_fast_portrait_seg.cpp) | ✅ | ✅ | / | / | ✅ | ✔️ | ✔️ | ❔ | -| [PortraitSegSINet](https://github.com/clovaai/ext_portrait_segmentation) | 380k | *segmentation* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_portrait_seg_sinet.cpp) | ✅ | ✅ | / | / | ✅ | ✔️ | ✔️ | ❔ | -| [PortraitSegExtremeC3Net](https://github.com/clovaai/ext_portrait_segmentation) | 180k | *segmentation* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_portrait_seg_extremec3net.cpp) | ✅ | ✅ | / | / | ✅ | ✔️ | ✔️ | ❔ | -| [FaceHairSeg](https://github.com/kampta/face-seg) | 18M | *segmentation* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_face_hair_seg.cpp) | ✅ | ✅ | / | / | ✅ | ✔️ | ✔️ | ❔ | -| [HairSeg](https://github.com/akirasosa/mobile-semantic-segmentation) | 18M | *segmentation* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_hair_seg.cpp) | ✅ | ✅ | / | / | ✅ | ✔️ | ✔️ | ❔ | -| [MobileHumanMatting](https://github.com/lizhengwei1992/mobile_phone_human_matting) | 3M | *matting* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_mobile_human_matting.cpp) | ✅ | ✅ | / | / | ✅ | ✔️ | ✔️ | ❔ | -| [MobileHairSeg](https://github.com/wonbeomjang/mobile-hair-segmentation-pytorch) | 14M | *segmentation* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_mobile_hair_seg.cpp) | ✅ | ✅ | / | / | ✅ | ✔️ | ✔️ | ❔ | -| [YOLOv6](https://github.com/meituan/YOLOv6) | 17M | *detection* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolov6.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [FaceParsingBiSeNet](https://github.com/zllrunning/face-parsing.PyTorch) | 50M | *segmentation* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_face_parsing_bisenet.cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✔️ | ✔️ | ❔ | -| [FaceParsingBiSeNetDyn](https://github.com/zllrunning/face-parsing.PyTorch) | 50M | *segmentation* | [demo](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_face_parsing_bisenet_dyn.cpp) | ✅ | / | / | / | / | ✔️ | ✔️ | ❔ | - - - -
- -
- 🔑️ Model Zoo！Click here! - -## Model Zoo. +## Features 👏👋 -
+- **GPU-first.** The whole FaceFusion pipeline runs on TensorRT; the pre/post-processing that usually + lingers on the CPU (warp / color-convert / normalize / layout / paste-back / NMS) is implemented as + **fused CUDA kernels** under [`lite/trt/kernel/`](https://github.com/xlite-dev/lite.ai.toolkit/tree/main/lite/trt/kernel), with reused device buffers and pinned + async copies. +- **Measured, not claimed.** A header-only profiler ([`lite/bench/profiler.h`](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/lite/bench/profiler.h)) gives CPU-chrono + GPU-cudaEvent timings (p50 / p99 / FPS / CSV). Every optimization ships with a before/after `lite_*_bench` binary. +- **Multi-threaded TRT path.** `_mt` pipelines (e.g. `trt_face_restoration_mt`) run a thread pool with one + `IExecutionContext` + `cudaStream_t` + buffer set per thread and an async task queue. +- **Consistent C++ API.** Same `lite::trt::cv::Type::Class` syntax across models, e.g. `lite::trt::cv::detection::YOLOV5`. -**Lite.Ai.ToolKit** contains almost **[100+](https://github.com/xlite-dev/lite.ai.toolkit/tree/main/docs/hub/lite.ai.toolkit.hub.onnx.md)** AI models with **[500+](https://github.com/xlite-dev/lite.ai.toolkit/tree/main/docs/hub/lite.ai.toolkit.hub.onnx.md)** frozen pretrained files now. Most of the files are converted by myself. You can use it through **lite::cv::Type::Class** syntax, such as **[lite::cv::detection::YoloV5](#lite.ai.toolkit-object-detection)**. More details can be found at [Examples for Lite.Ai.ToolKit](#lite.ai.toolkit-Examples-for-Lite.AI.ToolKit). Note, for Google Drive, I can not upload all the *.onnx files because of the storage limitation (15G). +## Build 👇👇 -| File | Baidu Drive | Google Drive | Docker Hub | Hub (Docs) | -|:----:|:-------------------------------------------------------------------------:|:----------------------------------------------------------------------------------------------------:|:----------------------------------------------------------------------------------------------------------------------------:|:------------------------------------------------------------------------------------------------------:| -| ONNX | [Baidu Drive](https://pan.baidu.com/s/1elUGcx7CZkkjEoYhTMwTRQ) code: 8gin | [Google Drive](https://drive.google.com/drive/folders/1p6uBcxGeyS1exc-T61vL8YRhwjYL4iD2?usp=sharing) | [ONNX Docker v0.1.22.01.08 (28G), v0.1.22.02.02 (400M)](https://hub.docker.com/r/qyjdefdocker/lite.ai.toolkit-onnx-hub/tags) | [ONNX Hub](https://github.com/xlite-dev/lite.ai.toolkit/tree/main/docs/hub/lite.ai.toolkit.hub.onnx.md) | -| MNN | [Baidu Drive](https://pan.baidu.com/s/1KyO-bCYUv6qPq2M8BH_Okg) code: 9v63 | ❔ | [MNN Docker v0.1.22.01.08 (11G), v0.1.22.02.02 (213M)](https://hub.docker.com/r/qyjdefdocker/lite.ai.toolkit-mnn-hub/tags) | [MNN Hub](https://github.com/xlite-dev/lite.ai.toolkit/tree/main/docs/hub/lite.ai.toolkit.hub.mnn.md) | -| NCNN | [Baidu Drive](https://pan.baidu.com/s/1hlnqyNsFbMseGFWscgVhgQ) code: sc7f | ❔ | [NCNN Docker v0.1.22.01.08 (9G), v0.1.22.02.02 (197M)](https://hub.docker.com/r/qyjdefdocker/lite.ai.toolkit-ncnn-hub/tags) | [NCNN Hub](https://github.com/xlite-dev/lite.ai.toolkit/tree/main/docs/hub/lite.ai.toolkit.hub.ncnn.md) | -| TNN | [Baidu Drive](https://pan.baidu.com/s/1lvM2YKyUbEc5HKVtqITpcw) code: 6o6k | ❔ | [TNN Docker v0.1.22.01.08 (11G), v0.1.22.02.02 (217M)](https://hub.docker.com/r/qyjdefdocker/lite.ai.toolkit-tnn-hub/tags) | [TNN Hub](https://github.com/xlite-dev/lite.ai.toolkit/tree/main/docs/hub/lite.ai.toolkit.hub.tnn.md) | +TensorRT is the maintained backend. It needs **TensorRT 10.x** and **CUDA 12.x** (Linux only). The first +build downloads third-party libs into `third_party/` automatically. ```shell - docker pull qyjdefdocker/lite.ai.toolkit-onnx-hub:v0.1.22.01.08 # (28G) - docker pull qyjdefdocker/lite.ai.toolkit-mnn-hub:v0.1.22.01.08 # (11G) - docker pull qyjdefdocker/lite.ai.toolkit-ncnn-hub:v0.1.22.01.08 # (9G) - docker pull qyjdefdocker/lite.ai.toolkit-tnn-hub:v0.1.22.01.08 # (11G) - docker pull qyjdefdocker/lite.ai.toolkit-onnx-hub:v0.1.22.02.02 # (400M) + YOLO5Face - docker pull qyjdefdocker/lite.ai.toolkit-mnn-hub:v0.1.22.02.02 # (213M) + YOLO5Face - docker pull qyjdefdocker/lite.ai.toolkit-ncnn-hub:v0.1.22.02.02 # (197M) + YOLO5Face - docker pull qyjdefdocker/lite.ai.toolkit-tnn-hub:v0.1.22.02.02 # (217M) + YOLO5Face -``` - -### 🔑️ How to download Model Zoo from Docker Hub? - -* Firstly, pull the image from docker hub. - ```shell - docker pull qyjdefdocker/lite.ai.toolkit-mnn-hub:v0.1.22.01.08 # (11G) - docker pull qyjdefdocker/lite.ai.toolkit-ncnn-hub:v0.1.22.01.08 # (9G) - docker pull qyjdefdocker/lite.ai.toolkit-tnn-hub:v0.1.22.01.08 # (11G) - docker pull qyjdefdocker/lite.ai.toolkit-onnx-hub:v0.1.22.01.08 # (28G) - ``` -* Secondly, run the container with local `share` dir using `docker run -idt xxx`. A minimum example will show you as follows. - * make a `share` dir in your local device. - ```shell - mkdir share # any name is ok. - ``` - * write `run_mnn_docker_hub.sh` script like: - ```shell - #!/bin/bash - PORT1=6072 - PORT2=6084 - SERVICE_DIR=/Users/xxx/Desktop/your-path-to/share - CONRAINER_DIR=/home/hub/share - CONRAINER_NAME=mnn_docker_hub_d - - docker run -idt -p ${PORT2}:${PORT1} -v ${SERVICE_DIR}:${CONRAINER_DIR} --shm-size=16gb --name ${CONRAINER_NAME} qyjdefdocker/lite.ai.toolkit-mnn-hub:v0.1.22.01.08 - - ``` -* Finally, copy the model weights from `/home/hub/mnn/cv` to your local `share` dir. - ```shell - # activate mnn docker. - sh ./run_mnn_docker_hub.sh - docker exec -it mnn_docker_hub_d /bin/bash - # copy the models to the share dir. - cd /home/hub - cp -rf mnn/cv share/ - ``` - - -### Model Hubs -The pretrained and converted ONNX files provide by lite.ai.toolkit are listed as follows. Also, see [Model Zoo](#lite.ai.toolkit-Model-Zoo) and [ONNX Hub](https://github.com/xlite-dev/lite.ai.toolkit/tree/main/docs/hub/lite.ai.toolkit.hub.onnx.md), [MNN Hub](https://github.com/xlite-dev/lite.ai.toolkit/tree/main/docs/hub/lite.ai.toolkit.hub.mnn.md), [TNN Hub](https://github.com/xlite-dev/lite.ai.toolkit/tree/main/docs/hub/lite.ai.toolkit.hub.tnn.md), [NCNN Hub](https://github.com/xlite-dev/lite.ai.toolkit/tree/main/docs/hub/lite.ai.toolkit.hub.ncnn.md) for more details. - -
- - -
- -
- 🔑️ More Examples！Click here! - -## 🔑️ More Examples. - -More examples can be found at [examples](https://github.com/xlite-dev/lite.ai.toolkit/tree/main/examples/lite/cv). - -
- -#### Example0: Object Detection using [YOLOv5](https://github.com/ultralytics/yolov5). Download model from Model-Zoo[2](#lite.ai.toolkit-2). -```c++ -#include "lite/lite.h" - -static void test_default() -{ - std::string onnx_path = "../../../examples/hub/onnx/cv/yolov5s.onnx"; - std::string test_img_path = "../../../examples/lite/resources/test_lite_yolov5_1.jpg"; - std::string save_img_path = "../../../examples/logs/test_lite_yolov5_1.jpg"; - - auto *yolov5 = new lite::cv::detection::YoloV5(onnx_path); - std::vector detected_boxes; - cv::Mat img_bgr = cv::imread(test_img_path); - yolov5->detect(img_bgr, detected_boxes); - - lite::utils::draw_boxes_inplace(img_bgr, detected_boxes); - cv::imwrite(save_img_path, img_bgr); - - delete yolov5; -} -``` - -The output is: -
- - -
- -Or you can use Newest 🔥🔥 ! YOLO series's detector [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) or [YoloR](https://github.com/WongKinYiu/yolor). They got the similar results. - -More classes for general object detection (80 classes, COCO). -```c++ -auto *detector = new lite::cv::detection::YoloX(onnx_path); // Newest YOLO detector !!! 2021-07 -auto *detector = new lite::cv::detection::YoloV4(onnx_path); -auto *detector = new lite::cv::detection::YoloV3(onnx_path); -auto *detector = new lite::cv::detection::TinyYoloV3(onnx_path); -auto *detector = new lite::cv::detection::SSD(onnx_path); -auto *detector = new lite::cv::detection::YoloV5(onnx_path); -auto *detector = new lite::cv::detection::YoloR(onnx_path); // Newest YOLO detector !!! 2021-05 -auto *detector = new lite::cv::detection::TinyYoloV4VOC(onnx_path); -auto *detector = new lite::cv::detection::TinyYoloV4COCO(onnx_path); -auto *detector = new lite::cv::detection::ScaledYoloV4(onnx_path); -auto *detector = new lite::cv::detection::EfficientDet(onnx_path); -auto *detector = new lite::cv::detection::EfficientDetD7(onnx_path); -auto *detector = new lite::cv::detection::EfficientDetD8(onnx_path); -auto *detector = new lite::cv::detection::YOLOP(onnx_path); -auto *detector = new lite::cv::detection::NanoDet(onnx_path); // Super fast and tiny! -auto *detector = new lite::cv::detection::NanoDetPlus(onnx_path); // Super fast and tiny! 2021/12/25 -auto *detector = new lite::cv::detection::NanoDetEfficientNetLite(onnx_path); // Super fast and tiny! -auto *detector = new lite::cv::detection::YoloV5_V_6_0(onnx_path); -auto *detector = new lite::cv::detection::YoloV5_V_6_1(onnx_path); -auto *detector = new lite::cv::detection::YoloX_V_0_1_1(onnx_path); // Newest YOLO detector !!! 2021-07 -auto *detector = new lite::cv::detection::YOLOv6(onnx_path); // Newest 2022 YOLO detector !!! -``` - - -**** - -
- -#### Example1: Video Matting using [RobustVideoMatting2021🔥🔥🔥](https://github.com/PeterL1n/RobustVideoMatting). Download model from Model-Zoo[2](#lite.ai.toolkit-2). - -```c++ -#include "lite/lite.h" - -static void test_default() -{ - std::string onnx_path = "../../../examples/hub/onnx/cv/rvm_mobilenetv3_fp32.onnx"; - std::string video_path = "../../../examples/lite/resources/test_lite_rvm_0.mp4"; - std::string output_path = "../../../examples/logs/test_lite_rvm_0.mp4"; - std::string background_path = "../../../examples/lite/resources/test_lite_matting_bgr.jpg"; - - auto *rvm = new lite::cv::matting::RobustVideoMatting(onnx_path, 16); // 16 threads - std::vector contents; - - // 1. video matting. - cv::Mat background = cv::imread(background_path); - rvm->detect_video(video_path, output_path, contents, false, 0.4f, - 20, true, true, background); - - delete rvm; -} +git clone --depth=1 https://github.com/xlite-dev/lite.ai.toolkit.git +cd lite.ai.toolkit +bash ./build.sh tensorrt # GPU / TensorRT backend +# bash ./build.sh # ONNXRuntime backend (CPU reference + 100+ CV models, builds the tests) ``` -The output is: -
- - - - -
- - - - -
+See [tensorrt-linux-x86_64.zh.md](./docs/tensorrt/tensorrt-linux-x86_64.zh.md) for the TensorRT/CUDA setup. -More classes for matting (image matting, video matting, trimap/mask-free, trimap/mask-based) -```c++ -auto *matting = new lite::cv::matting::RobustVideoMatting:(onnx_path); // WACV 2022. -auto *matting = new lite::cv::matting::MGMatting(onnx_path); // CVPR 2021 -auto *matting = new lite::cv::matting::MODNet(onnx_path); // AAAI 2022 -auto *matting = new lite::cv::matting::MODNetDyn(onnx_path); // AAAI 2022 Dynamic Shape Inference. -auto *matting = new lite::cv::matting::BackgroundMattingV2(onnx_path); // CVPR 2020 -auto *matting = new lite::cv::matting::BackgroundMattingV2Dyn(onnx_path); // CVPR 2020 Dynamic Shape Inference. -auto *matting = new lite::cv::matting::MobileHumanMatting(onnx_path); // 3Mb only !!! -``` - - -**** - -
- -#### Example2: 1000 Facial Landmarks Detection using [FaceLandmarks1000](https://github.com/Single430/FaceLandmark1000). Download model from Model-Zoo[2](#lite.ai.toolkit-2). -```c++ -#include "lite/lite.h" - -static void test_default() -{ - std::string onnx_path = "../../../examples/hub/onnx/cv/FaceLandmark1000.onnx"; - std::string test_img_path = "../../../examples/lite/resources/test_lite_face_landmarks_0.png"; - std::string save_img_path = "../../../examples/logs/test_lite_face_landmarks_1000.jpg"; - - auto *face_landmarks_1000 = new lite::cv::face::align::FaceLandmark1000(onnx_path); - - lite::types::Landmarks landmarks; - cv::Mat img_bgr = cv::imread(test_img_path); - face_landmarks_1000->detect(img_bgr, landmarks); - lite::utils::draw_landmarks_inplace(img_bgr, landmarks); - cv::imwrite(save_img_path, img_bgr); - - delete face_landmarks_1000; -} -``` -The output is: -
- - - -
- -More classes for face alignment (68 points, 98 points, 106 points, 1000 points) -```c++ -auto *align = new lite::cv::face::align::PFLD(onnx_path); // 106 landmarks, 1.0Mb only! -auto *align = new lite::cv::face::align::PFLD98(onnx_path); // 98 landmarks, 4.8Mb only! -auto *align = new lite::cv::face::align::PFLD68(onnx_path); // 68 landmarks, 2.8Mb only! -auto *align = new lite::cv::face::align::MobileNetV268(onnx_path); // 68 landmarks, 9.4Mb only! -auto *align = new lite::cv::face::align::MobileNetV2SE68(onnx_path); // 68 landmarks, 11Mb only! -auto *align = new lite::cv::face::align::FaceLandmark1000(onnx_path); // 1000 landmarks, 2.0Mb only! -auto *align = new lite::cv::face::align::PIPNet98(onnx_path); // 98 landmarks, CVPR2021! -auto *align = new lite::cv::face::align::PIPNet68(onnx_path); // 68 landmarks, CVPR2021! -auto *align = new lite::cv::face::align::PIPNet29(onnx_path); // 29 landmarks, CVPR2021! -auto *align = new lite::cv::face::align::PIPNet19(onnx_path); // 19 landmarks, CVPR2021! -``` - - -**** - -
- -#### Example3: Colorization using [colorization](https://github.com/richzhang/colorization). Download model from Model-Zoo[2](#lite.ai.toolkit-2). -```c++ -#include "lite/lite.h" - -static void test_default() -{ - std::string onnx_path = "../../../examples/hub/onnx/cv/eccv16-colorizer.onnx"; - std::string test_img_path = "../../../examples/lite/resources/test_lite_colorizer_1.jpg"; - std::string save_img_path = "../../../examples/logs/test_lite_eccv16_colorizer_1.jpg"; - - auto *colorizer = new lite::cv::colorization::Colorizer(onnx_path); - - cv::Mat img_bgr = cv::imread(test_img_path); - lite::types::ColorizeContent colorize_content; - colorizer->detect(img_bgr, colorize_content); - - if (colorize_content.flag) cv::imwrite(save_img_path, colorize_content.mat); - delete colorizer; -} -``` -The output is: - -
- - - -
- - - -
- -More classes for colorization (gray to rgb) -```c++ -auto *colorizer = new lite::cv::colorization::Colorizer(onnx_path); -``` - -**** - -
- -#### Example4: Face Recognition using [ArcFace](https://github.com/deepinsight/insightface/tree/master/recognition/arcface_torch). Download model from Model-Zoo[2](#lite.ai.toolkit-2). - -```c++ -#include "lite/lite.h" - -static void test_default() -{ - std::string onnx_path = "../../../examples/hub/onnx/cv/ms1mv3_arcface_r100.onnx"; - std::string test_img_path0 = "../../../examples/lite/resources/test_lite_faceid_0.png"; - std::string test_img_path1 = "../../../examples/lite/resources/test_lite_faceid_1.png"; - std::string test_img_path2 = "../../../examples/lite/resources/test_lite_faceid_2.png"; - - auto *glint_arcface = new lite::cv::faceid::GlintArcFace(onnx_path); - - lite::types::FaceContent face_content0, face_content1, face_content2; - cv::Mat img_bgr0 = cv::imread(test_img_path0); - cv::Mat img_bgr1 = cv::imread(test_img_path1); - cv::Mat img_bgr2 = cv::imread(test_img_path2); - glint_arcface->detect(img_bgr0, face_content0); - glint_arcface->detect(img_bgr1, face_content1); - glint_arcface->detect(img_bgr2, face_content2); - - if (face_content0.flag && face_content1.flag && face_content2.flag) - { - float sim01 = lite::utils::math::cosine_similarity( - face_content0.embedding, face_content1.embedding); - float sim02 = lite::utils::math::cosine_similarity( - face_content0.embedding, face_content2.embedding); - std::cout << "Detected Sim01: " << sim << " Sim02: " << sim02 << std::endl; - } - - delete glint_arcface; -} -``` - -The output is: -
- - - -
- -> Detected Sim01: 0.721159 Sim02: -0.0626267 - -More classes for face recognition (face id vector extract) -```c++ -auto *recognition = new lite::cv::faceid::GlintCosFace(onnx_path); // DeepGlint(insightface) -auto *recognition = new lite::cv::faceid::GlintArcFace(onnx_path); // DeepGlint(insightface) -auto *recognition = new lite::cv::faceid::GlintPartialFC(onnx_path); // DeepGlint(insightface) -auto *recognition = new lite::cv::faceid::FaceNet(onnx_path); -auto *recognition = new lite::cv::faceid::FocalArcFace(onnx_path); -auto *recognition = new lite::cv::faceid::FocalAsiaArcFace(onnx_path); -auto *recognition = new lite::cv::faceid::TencentCurricularFace(onnx_path); // Tencent(TFace) -auto *recognition = new lite::cv::faceid::TencentCifpFace(onnx_path); // Tencent(TFace) -auto *recognition = new lite::cv::faceid::CenterLossFace(onnx_path); -auto *recognition = new lite::cv::faceid::SphereFace(onnx_path); -auto *recognition = new lite::cv::faceid::PoseRobustFace(onnx_path); -auto *recognition = new lite::cv::faceid::NaivePoseRobustFace(onnx_path); -auto *recognition = new lite::cv::faceid::MobileFaceNet(onnx_path); // 3.8Mb only ! -auto *recognition = new lite::cv::faceid::CavaGhostArcFace(onnx_path); -auto *recognition = new lite::cv::faceid::CavaCombinedFace(onnx_path); -auto *recognition = new lite::cv::faceid::MobileSEFocalFace(onnx_path); // 4.5Mb only ! -``` - -**** - -
- -#### Example5: Face Detection using [SCRFD 2021](https://github.com/deepinsight/insightface/blob/master/detection/scrfd/). Download model from Model-Zoo[2](#lite.ai.toolkit-2). -```c++ -#include "lite/lite.h" - -static void test_default() -{ - std::string onnx_path = "../../../examples/hub/onnx/cv/scrfd_2.5g_bnkps_shape640x640.onnx"; - std::string test_img_path = "../../../examples/lite/resources/test_lite_face_detector.jpg"; - std::string save_img_path = "../../../examples/logs/test_lite_scrfd.jpg"; - - auto *scrfd = new lite::cv::face::detect::SCRFD(onnx_path); - - std::vector detected_boxes; - cv::Mat img_bgr = cv::imread(test_img_path); - scrfd->detect(img_bgr, detected_boxes); - - lite::utils::draw_boxes_with_landmarks_inplace(img_bgr, detected_boxes); - cv::imwrite(save_img_path, img_bgr); - - delete scrfd; -} -``` -The output is: -
- - - -
- -More classes for face detection (super fast face detection) -```c++ -auto *detector = new lite::face::detect::UltraFace(onnx_path); // 1.1Mb only ! -auto *detector = new lite::face::detect::FaceBoxes(onnx_path); // 3.8Mb only ! -auto *detector = new lite::face::detect::FaceBoxesv2(onnx_path); // 4.0Mb only ! -auto *detector = new lite::face::detect::RetinaFace(onnx_path); // 1.6Mb only ! CVPR2020 -auto *detector = new lite::face::detect::SCRFD(onnx_path); // 2.5Mb only ! CVPR2021, Super fast and accurate!! -auto *detector = new lite::face::detect::YOLO5Face(onnx_path); // 2021, Super fast and accurate!! -auto *detector = new lite::face::detect::YOLOv5BlazeFace(onnx_path); // 2021, Super fast and accurate!! -``` - -**** +## Quick Start 🌟🌟 +
-
+#### Flagship: FaceFusion face-swap pipeline on the GPU +End-to-end source→target face swap, fully on TensorRT. See [`test_lite_facefusion_pipeline.cpp`](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_facefusion_pipeline.cpp) for the full example (engine paths + I/O). -#### Example6: Object Segmentation using [DeepLabV3ResNet101](https://pytorch.org/hub/pytorch_vision_deeplabv3_resnet101/). Download model from Model-Zoo[2](#lite.ai.toolkit-2). ```c++ #include "lite/lite.h" - -static void test_default() -{ - std::string onnx_path = "../../../examples/hub/onnx/cv/deeplabv3_resnet101_coco.onnx"; - std::string test_img_path = "../../../examples/lite/resources/test_lite_deeplabv3_resnet101.png"; - std::string save_img_path = "../../../examples/logs/test_lite_deeplabv3_resnet101.jpg"; - - auto *deeplabv3_resnet101 = new lite::cv::segmentation::DeepLabV3ResNet101(onnx_path, 16); // 16 threads - - lite::types::SegmentContent content; - cv::Mat img_bgr = cv::imread(test_img_path); - deeplabv3_resnet101->detect(img_bgr, content); - - if (content.flag) - { - cv::Mat out_img; - cv::addWeighted(img_bgr, 0.2, content.color_mat, 0.8, 0., out_img); - cv::imwrite(save_img_path, out_img); - if (!content.names_map.empty()) - { - for (auto it = content.names_map.begin(); it != content.names_map.end(); ++it) - { - std::cout << it->first << " Name: " << it->second << std::endl; - } - } - } - delete deeplabv3_resnet101; -} -``` - -The output is: -
- - -
- -More classes for object segmentation (general objects segmentation) -```c++ -auto *segment = new lite::cv::segmentation::FCNResNet101(onnx_path); -auto *segment = new lite::cv::segmentation::DeepLabV3ResNet101(onnx_path); +// build the 5 engines once, e.g. trtexec --onnx=gfpgan_1.4.onnx --saveEngine=gfpgan_1.4_fp32.engine +auto pipeline = lite::trt::cv::face::swap::FaceFusionPipeLine( + face_detect_engine, // yoloface_8n + face_landmarks_68_engine, // 2dfan4 + face_recognizer_engine, // arcface_w600k_r50 + face_swap_engine, // inswapper_128 + face_restoration_engine); // gfpgan_1.4 +// swap face #0 of the source onto face #0 of the target, then write the result +pipeline.detect(source_image_path, 0, target_image_path, 0, save_image_path); ``` -**** - -
- -#### Example7: Age Estimation using [SSRNet](https://github.com/oukohou/SSR_Net_Pytorch) . Download model from Model-Zoo[2](#lite.ai.toolkit-2). +#### Single model on the GPU (YOLOv5) ```c++ #include "lite/lite.h" - -static void test_default() -{ - std::string onnx_path = "../../../examples/hub/onnx/cv/ssrnet.onnx"; - std::string test_img_path = "../../../examples/lite/resources/test_lite_ssrnet.jpg"; - std::string save_img_path = "../../../examples/logs/test_lite_ssrnet.jpg"; - - auto *ssrnet = new lite::cv::face::attr::SSRNet(onnx_path); - - lite::types::Age age; - cv::Mat img_bgr = cv::imread(test_img_path); - ssrnet->detect(img_bgr, age); - lite::utils::draw_age_inplace(img_bgr, age); - cv::imwrite(save_img_path, img_bgr); - - delete ssrnet; -} -``` -The output is: -
- - - -
- -More classes for face attributes analysis (age, gender, emotion) -```c++ -auto *attribute = new lite::cv::face::attr::AgeGoogleNet(onnx_path); -auto *attribute = new lite::cv::face::attr::GenderGoogleNet(onnx_path); -auto *attribute = new lite::cv::face::attr::EmotionFerPlus(onnx_path); -auto *attribute = new lite::cv::face::attr::VGG16Age(onnx_path); -auto *attribute = new lite::cv::face::attr::VGG16Gender(onnx_path); -auto *attribute = new lite::cv::face::attr::EfficientEmotion7(onnx_path); // 7 emotions, 15Mb only! -auto *attribute = new lite::cv::face::attr::EfficientEmotion8(onnx_path); // 8 emotions, 15Mb only! -auto *attribute = new lite::cv::face::attr::MobileEmotion7(onnx_path); // 7 emotions, 13Mb only! -auto *attribute = new lite::cv::face::attr::ReXNetEmotion7(onnx_path); // 7 emotions -auto *attribute = new lite::cv::face::attr::SSRNet(onnx_path); // age estimation, 190kb only!!! -``` - -**** - -
- -#### Example8: 1000 Classes Classification using [DenseNet](https://pytorch.org/hub/pytorch_vision_densenet/). Download model from Model-Zoo[2](#lite.ai.toolkit-2). -```c++ -#include "lite/lite.h" - -static void test_default() -{ - std::string onnx_path = "../../../examples/hub/onnx/cv/densenet121.onnx"; - std::string test_img_path = "../../../examples/lite/resources/test_lite_densenet.jpg"; - - auto *densenet = new lite::cv::classification::DenseNet(onnx_path); - - lite::types::ImageNetContent content; - cv::Mat img_bgr = cv::imread(test_img_path); - densenet->detect(img_bgr, content); - if (content.flag) - { - const unsigned int top_k = content.scores.size(); - if (top_k > 0) - { - for (unsigned int i = 0; i < top_k; ++i) - std::cout << i + 1 - << ": " << content.labels.at(i) - << ": " << content.texts.at(i) - << ": " << content.scores.at(i) - << std::endl; - } - } - delete densenet; -} -``` - -The output is: -
- - -
- -More classes for image classification (1000 classes) -```c++ -auto *classifier = new lite::cv::classification::EfficientNetLite4(onnx_path); -auto *classifier = new lite::cv::classification::ShuffleNetV2(onnx_path); // 8.7Mb only! -auto *classifier = new lite::cv::classification::GhostNet(onnx_path); -auto *classifier = new lite::cv::classification::HdrDNet(onnx_path); -auto *classifier = new lite::cv::classification::IBNNet(onnx_path); -auto *classifier = new lite::cv::classification::MobileNetV2(onnx_path); // 13Mb only! -auto *classifier = new lite::cv::classification::ResNet(onnx_path); -auto *classifier = new lite::cv::classification::ResNeXt(onnx_path); -``` - -**** - -
- -#### Example9: Head Pose Estimation using [FSANet](https://github.com/omasaht/headpose-fsanet-pytorch). Download model from Model-Zoo[2](#lite.ai.toolkit-2). - -```c++ -#include "lite/lite.h" - -static void test_default() -{ - std::string onnx_path = "../../../examples/hub/onnx/cv/fsanet-var.onnx"; - std::string test_img_path = "../../../examples/lite/resources/test_lite_fsanet.jpg"; - std::string save_img_path = "../../../examples/logs/test_lite_fsanet.jpg"; - - auto *fsanet = new lite::cv::face::pose::FSANet(onnx_path); - cv::Mat img_bgr = cv::imread(test_img_path); - lite::types::EulerAngles euler_angles; - fsanet->detect(img_bgr, euler_angles); - - if (euler_angles.flag) - { - lite::utils::draw_axis_inplace(img_bgr, euler_angles); - cv::imwrite(save_img_path, img_bgr); - std::cout << "yaw:" << euler_angles.yaw << " pitch:" << euler_angles.pitch << " row:" << euler_angles.roll << std::endl; - } - delete fsanet; -} -``` - -The output is: -
- - - -
- -More classes for head pose estimation (euler angle, yaw, pitch, roll) -```c++ -auto *pose = new lite::cv::face::pose::FSANet(onnx_path); // 1.2Mb only! +// trtexec --onnx=yolov5s.onnx --saveEngine=yolov5s.engine +auto *yolov5 = new lite::trt::cv::detection::YOLOV5(engine_path); +std::vector boxes; +cv::Mat img = cv::imread(test_img_path); +yolov5->detect(img, boxes); +lite::utils::draw_boxes_inplace(img, boxes); +cv::imwrite(save_img_path, img); +delete yolov5; ``` -**** - -
- -#### Example10: Style Transfer using [FastStyleTransfer](https://github.com/onnx/models/tree/master/vision/style_transfer/fast_neural_style). Download model from Model-Zoo[2](#lite.ai.toolkit-2). -```c++ -#include "lite/lite.h" - -static void test_default() -{ - std::string onnx_path = "../../../examples/hub/onnx/cv/style-candy-8.onnx"; - std::string test_img_path = "../../../examples/lite/resources/test_lite_fast_style_transfer.jpg"; - std::string save_img_path = "../../../examples/logs/test_lite_fast_style_transfer_candy.jpg"; - - auto *fast_style_transfer = new lite::cv::style::FastStyleTransfer(onnx_path); - - lite::types::StyleContent style_content; - cv::Mat img_bgr = cv::imread(test_img_path); - fast_style_transfer->detect(img_bgr, style_content); - - if (style_content.flag) cv::imwrite(save_img_path, style_content.mat); - delete fast_style_transfer; -} -``` -The output is: +## Quick Setup 👀 -
- - - -
- - - -
+To use the installed library from your own project, point `find_package` at the install dir: -More classes for style transfer (neural style transfer, others) -```c++ -auto *transfer = new lite::cv::style::FastStyleTransfer(onnx_path); // 6.4Mb only +```cmake +set(lite.ai.toolkit_DIR YOUR-PATH-TO-LITE-INSTALL) +find_package(lite.ai.toolkit REQUIRED PATHS ${lite.ai.toolkit_DIR}) +add_executable(lite_yolov5 test_lite_yolov5.cpp) +target_link_libraries(lite_yolov5 ${lite.ai.toolkit_LIBS}) ``` -**** - -#### Example11: Human Head Segmentation using [HeadSeg](https://github.com/minivision-ai/photo2cartoon). Download model from Model-Zoo[2](#lite.ai.toolkit-2). -```c++ -#include "lite/lite.h" - -static void test_default() -{ - std::string onnx_path = "../../../examples/hub/onnx/cv/minivision_head_seg.onnx"; - std::string test_img_path = "../../../examples/lite/resources/test_lite_head_seg.png"; - std::string save_img_path = "../../../examples/logs/test_lite_head_seg.jpg"; +## Supported Models (TensorRT) 🚀 +
- auto *head_seg = new lite::cv::segmentation::HeadSeg(onnx_path, 4); // 4 threads +|Class|Class|Class|Class|Class| System | Engine | +|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +|✅[YOLOv5](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolov5.cpp)|✅[YOLOv6](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolov6.cpp)|✅[YOLOv8](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolov8.cpp)|✅[YOLOv8Face](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolov8face.cpp)|✅[YOLOv5Face](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolo5face.cpp)| Linux | TensorRT | +|✅[YOLOX](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolox.cpp)|✅[YOLOv5BlazeFace](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolov5_blazeface.cpp)|✅[StableDiffusion](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/sd/test_lite_sd_pipeline.cpp)|✅[FaceFusion](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_facefusion_pipeline.cpp)| / | Linux | TensorRT | - lite::types::HeadSegContent content; - cv::Mat img_bgr = cv::imread(test_img_path); - head_seg->detect(img_bgr, content); - if (content.flag) cv::imwrite(save_img_path, content.mask * 255.f); +> Also includes **100+ CPU / ONNXRuntime CV models** (detection, face recognition, segmentation, matting, +> classification, …) behind the same `lite::cv::Type::Class` API. They are not the focus of the active +> line but remain available — see the [ONNX Hub](https://github.com/xlite-dev/lite.ai.toolkit/tree/main/docs/hub/lite.ai.toolkit.hub.onnx.md) for the full catalog and weights, or tag [`v0.2-all-backends`](https://github.com/xlite-dev/lite.ai.toolkit/tree/main) for the legacy multi-backend matrix. - delete head_seg; -} -``` -The output is: +## Architecture 🧩 -
- - - - -
- -More classes for human segmentation (head, portrait, hair, others) -```c++ -auto *segment = new lite::cv::segmentation::HeadSeg(onnx_path); // 31Mb -auto *segment = new lite::cv::segmentation::FastPortraitSeg(onnx_path); // <= 400Kb !!! -auto *segment = new lite::cv::segmentation::PortraitSegSINet(onnx_path); // <= 380Kb !!! -auto *segment = new lite::cv::segmentation::PortraitSegExtremeC3Net(onnx_path); // <= 180Kb !!! Extreme Tiny !!! -auto *segment = new lite::cv::segmentation::FaceHairSeg(onnx_path); // 18M -auto *segment = new lite::cv::segmentation::HairSeg(onnx_path); // 18M -auto *segment = new lite::cv::segmentation::MobileHairSeg(onnx_path); // 14M ``` - -**** - -#### Example12: Photo transfer to Cartoon [Photo2Cartoon](https://github.com/minivision-ai/photo2cartoon). Download model from Model-Zoo[2](#lite.ai.toolkit-2). -```c++ -#include "lite/lite.h" - -static void test_default() -{ - std::string head_seg_onnx_path = "../../../examples/hub/onnx/cv/minivision_head_seg.onnx"; - std::string cartoon_onnx_path = "../../../examples/hub/onnx/cv/minivision_female_photo2cartoon.onnx"; - std::string test_img_path = "../../../examples/lite/resources/test_lite_female_photo2cartoon.jpg"; - std::string save_mask_path = "../../../examples/logs/test_lite_female_photo2cartoon_seg.jpg"; - std::string save_cartoon_path = "../../../examples/logs/test_lite_female_photo2cartoon_cartoon.jpg"; - - auto *head_seg = new lite::cv::segmentation::HeadSeg(head_seg_onnx_path, 4); // 4 threads - auto *female_photo2cartoon = new lite::cv::style::FemalePhoto2Cartoon(cartoon_onnx_path, 4); // 4 threads - - lite::types::HeadSegContent head_seg_content; - cv::Mat img_bgr = cv::imread(test_img_path); - head_seg->detect(img_bgr, head_seg_content); - - if (head_seg_content.flag && !head_seg_content.mask.empty()) - { - cv::imwrite(save_mask_path, head_seg_content.mask * 255.f); - // Female Photo2Cartoon Style Transfer - lite::types::FemalePhoto2CartoonContent female_cartoon_content; - female_photo2cartoon->detect(img_bgr, head_seg_content.mask, female_cartoon_content); - - if (female_cartoon_content.flag && !female_cartoon_content.cartoon.empty()) - cv::imwrite(save_cartoon_path, female_cartoon_content.cartoon); - } - - delete head_seg; - delete female_photo2cartoon; -} -``` -The output is: - -
- - - - -
- -More classes for photo style transfer. -```c++ -auto *transfer = new lite::cv::style::FemalePhoto2Cartoon(onnx_path); +lite/ +├── trt/ # TensorRT backend — the maintained high-performance path +│ ├── core/ # trt_handler base (engine load, buffers, streams) +│ ├── cv/ # one .h/.cpp per model + the facefusion pipeline (+ _mt variants) +│ ├── kernel/ # hand-written fused CUDA kernels (.cu/.cuh) + host-side managers +│ └── sd/ # Stable Diffusion components (clip / unet / vae / scheduler) +├── ort/ # ONNXRuntime backend — numerical reference + test host (100+ CV models) +├── bench/ # header-only profiler (CPU chrono + GPU cudaEvent, p50/p99/FPS/CSV) +└── lite.h # single public include ``` -**** +`lite::cv` is a compile-time namespace alias resolved in [`lite/models.h`](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/lite/models.h). Pin a backend explicitly with `lite::trt::cv::...` (GPU) or `lite::onnxruntime::cv::...` (CPU reference). -#### Example13: Face Parsing using [FaceParsing](https://github.com/zllrunning/face-parsing.PyTorch). Download model from Model-Zoo[2](#lite.ai.toolkit-2). -```c++ -#include "lite/lite.h" - -static void test_default() -{ - std::string onnx_path = "../../../examples/hub/onnx/cv/face_parsing_512x512.onnx"; - std::string test_img_path = "../../../examples/lite/resources/test_lite_face_parsing.png"; - std::string save_img_path = "../../../examples/logs/test_lite_face_parsing_bisenet.jpg"; - - auto *face_parsing_bisenet = new lite::cv::segmentation::FaceParsingBiSeNet(onnx_path, 8); // 8 threads - - lite::types::FaceParsingContent content; - cv::Mat img_bgr = cv::imread(test_img_path); - face_parsing_bisenet->detect(img_bgr, content); - - if (content.flag && !content.merge.empty()) - cv::imwrite(save_img_path, content.merge); - - delete face_parsing_bisenet; +## Citations 🎉🎉 +```BibTeX +@misc{lite.ai.toolkit@2021, + title={lite.ai.toolkit: A lite C++ toolkit of 100+ Awesome AI models.}, + url={https://github.com/xlite-dev/lite.ai.toolkit}, + note={Open-source software available at https://github.com/xlite-dev/lite.ai.toolkit}, + author={xlite-dev, wangzijian1010 etc}, + year={2021} } -``` -The output is: - -
- - - - -
- -More classes for face parsing (hair, eyes, nose, mouth, others) -```c++ -auto *segment = new lite::cv::segmentation::FaceParsingBiSeNet(onnx_path); // 50Mb -auto *segment = new lite::cv::segmentation::FaceParsingBiSeNetDyn(onnx_path); // Dynamic Shape Inference. ``` -
## ©️License GNU General Public License v3.0 ## 🎉Contribute -Please consider ⭐ this repo if you like it, as it is the simplest way to support me. +Please consider ⭐ this repo if you like it, as it is the simplest way to support us. - + From 24f8efe03d04a2e791124b4bff6340ecd0987318 Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Sun, 31 May 2026 17:04:11 +0800 Subject: [PATCH 09/30] feat(facefusion): out-of-box CLI runner + quickstart for the flagship pipeline Minimal-viable path to actually *use* the flagship without editing source: - lite_facefusion_cli: argv-driven runner (engine_dir + source/target/output), guarded by ENABLE_TENSORRT, using the verified FaceFusionPipeLine API - scripts/build_facefusion_engines.sh: one trtexec per model -> 5 engines (GFPGAN kept FP32 on purpose) - docs/facefusion_quickstart.md: build -> get ONNX -> build engines -> run - README: out-of-box run block + quickstart link - remove dead test_lite_facefusion_pipeline_gpu.cpp (referenced a missing header, never registered in CMake) Not yet build-verified (no GPU host); pending remote 4090 via scripts/remote.sh. Co-Authored-By: Claude Opus 4.8 --- README.md | 12 +++- build_facefusion_engines.sh | 40 +++++++++++ docs/facefusion_quickstart.md | 62 ++++++++++++++++ examples/lite/CMakeLists.txt | 1 + examples/lite/cv/test_lite_facefusion_cli.cpp | 72 +++++++++++++++++++ .../cv/test_lite_facefusion_pipeline_gpu.cpp | 25 ------- 6 files changed, 186 insertions(+), 26 deletions(-) create mode 100755 build_facefusion_engines.sh create mode 100644 docs/facefusion_quickstart.md create mode 100644 examples/lite/cv/test_lite_facefusion_cli.cpp delete mode 100644 examples/lite/cv/test_lite_facefusion_pipeline_gpu.cpp diff --git a/README.md b/README.md index 678b2491..7bd4a4c1 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,17 @@ See [tensorrt-linux-x86_64.zh.md](./docs/tensorrt/tensorrt-linux-x86_64.zh.md) f
#### Flagship: FaceFusion face-swap pipeline on the GPU -End-to-end source→target face swap, fully on TensorRT. See [`test_lite_facefusion_pipeline.cpp`](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_facefusion_pipeline.cpp) for the full example (engine paths + I/O). +End-to-end source→target face swap, fully on TensorRT. **Out of the box**, build with +`bash ./build.sh tensorrt` and run the CLI on your own images — no source editing: + +```bash +# build the 5 engines once, then run: +bash ./build_facefusion_engines.sh +./build/install/bin/lite_facefusion_cli source.jpg target.jpg output.jpg +``` + +Full walkthrough: **[docs/facefusion_quickstart.md](./docs/facefusion_quickstart.md)**. The C++ API +(see [`test_lite_facefusion_pipeline.cpp`](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_facefusion_pipeline.cpp)): ```c++ #include "lite/lite.h" diff --git a/build_facefusion_engines.sh b/build_facefusion_engines.sh new file mode 100755 index 00000000..2d1dab4c --- /dev/null +++ b/build_facefusion_engines.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Build the 5 TensorRT engines the FaceFusion pipeline needs, from their ONNX files. +# +# Usage: +# bash ./build_facefusion_engines.sh +# +# directory holding the 5 ONNX models (see docs/facefusion_quickstart.md) +# where the .engine files are written (create if missing) +# +# Requires `trtexec` on PATH (ships with TensorRT 10.x). Override with TRTEXEC=... +set -euo pipefail + +ONNX_DIR="${1:?usage: $0 }" +ENGINE_DIR="${2:?usage: $0 }" +TRTEXEC="${TRTEXEC:-trtexec}" + +mkdir -p "$ENGINE_DIR" + +# onnx_basename engine_basename extra_flags +# GFPGAN stays FP32 on purpose: FP16 makes its StyleGAN modulated convs blow up +# (grey-block artifacts). The other four are fine in FP16. +build() { + local onnx="$ONNX_DIR/$1" engine="$ENGINE_DIR/$2"; shift 2 + if [[ ! -f "$onnx" ]]; then + echo "[build_facefusion_engines] MISSING onnx: $onnx" >&2; exit 1 + fi + if [[ -f "$engine" ]]; then + echo "[build_facefusion_engines] skip (exists): $engine"; return + fi + echo "[build_facefusion_engines] $onnx -> $engine ($*)" + "$TRTEXEC" --onnx="$onnx" --saveEngine="$engine" "$@" +} + +build yoloface_8n.onnx yoloface_8n_fp16.engine --fp16 +build 2dfan4.onnx 2dfan4_fp16.engine --fp16 +build arcface_w600k_r50.onnx arcface_w600k_r50_fp16.engine --fp16 +build inswapper_128.onnx inswapper_128_fp16.engine --fp16 +build gfpgan_1.4.onnx gfpgan_1.4_fp32.engine + +echo "[build_facefusion_engines] done -> $ENGINE_DIR" diff --git a/docs/facefusion_quickstart.md b/docs/facefusion_quickstart.md new file mode 100644 index 00000000..f76ce521 --- /dev/null +++ b/docs/facefusion_quickstart.md @@ -0,0 +1,62 @@ +# FaceFusion Pipeline — Quickstart (TensorRT) + +Run the flagship end-to-end face-swap pipeline (detect → 68 landmarks → recognize → +swap → restore) on your own images, on an NVIDIA GPU. Linux only, **TensorRT 10.x + +CUDA 12.x**. + +## 1. Build + +```bash +git clone --depth=1 https://github.com/xlite-dev/lite.ai.toolkit.git +cd lite.ai.toolkit +bash ./build.sh tensorrt +``` + +Binaries land in `build/install/bin/` (the CLI runner is `lite_facefusion_cli`). + +## 2. Get the 5 ONNX models + +The pipeline uses these 5 models (the standard FaceFusion / InsightFace assets): + +| Stage | ONNX file | +|--|--| +| face detect | `yoloface_8n.onnx` | +| 68 landmarks | `2dfan4.onnx` | +| face recognize | `arcface_w600k_r50.onnx` | +| face swap | `inswapper_128.onnx` | +| face restore | `gfpgan_1.4.onnx` | + +Put all 5 in one directory, e.g. `~/ff_onnx/`. + +## 3. Build the TensorRT engines + +```bash +bash ./build_facefusion_engines.sh ~/ff_onnx ~/ff_engines +``` + +This runs `trtexec` once per model and writes the 5 `.engine` files into `~/ff_engines/`. +GFPGAN is kept FP32 on purpose (FP16 produces grey-block artifacts on its StyleGAN +modulated convs); the other four are FP16. Engines are GPU/TensorRT-version specific — +rebuild them if you change GPU or TensorRT version. + +## 4. Run + +```bash +./build/install/bin/lite_facefusion_cli \ + ~/ff_engines \ + source.jpg \ # face to take + target.jpg \ # image to paste it onto + output.jpg # result + +# optionally pick which detected face to use on each side (default 0 0): +# ... output.jpg +``` + +That's it — `output.jpg` is the swapped + restored result. + +## Performance + +The face-restoration stage is GPU-fused (paste-back + preprocess moved into CUDA +kernels): **78.2 ms → 17.7 ms (4.4×)** on an RTX 4090. See the +[Benchmark](../README.md#benchmark) section. Other stages are being optimized stage +by stage. diff --git a/examples/lite/CMakeLists.txt b/examples/lite/CMakeLists.txt index ae0a1dbf..fa798024 100644 --- a/examples/lite/CMakeLists.txt +++ b/examples/lite/CMakeLists.txt @@ -110,6 +110,7 @@ add_lite_executable(lite_face_swap cv) add_lite_executable(lite_face_restoration cv) add_lite_executable(lite_face_restoration_bench cv) add_lite_executable(lite_facefusion_pipeline cv) +add_lite_executable(lite_facefusion_cli cv) add_lite_executable(lite_yolov8 cv) add_lite_executable(lite_yolov11 cv) add_lite_executable(lite_sd_pipeline sd) diff --git a/examples/lite/cv/test_lite_facefusion_cli.cpp b/examples/lite/cv/test_lite_facefusion_cli.cpp new file mode 100644 index 00000000..b48acf0b --- /dev/null +++ b/examples/lite/cv/test_lite_facefusion_cli.cpp @@ -0,0 +1,72 @@ +// +// FaceFusion face-swap pipeline — out-of-box CLI runner (TensorRT). +// +// Unlike the other examples, this one takes every path from argv so you can run +// the flagship pipeline on your own images without editing/recompiling source. +// +// lite_facefusion_cli [src_face_idx] [tgt_face_idx] +// +// `engine_dir` must contain the 5 TensorRT engines (build them once from ONNX with +// ./build_facefusion_engines.sh). See docs/facefusion_quickstart.md. +// +#include "lite/lite.h" +#include +#include + +// Default engine filenames expected inside . +static const char *kFaceDetectEngine = "yoloface_8n_fp16.engine"; +static const char *kFaceLandmarksEngine = "2dfan4_fp16.engine"; +static const char *kFaceRecognizerEngine = "arcface_w600k_r50_fp16.engine"; +static const char *kFaceSwapEngine = "inswapper_128_fp16.engine"; +static const char *kFaceRestoreEngine = "gfpgan_1.4_fp32.engine"; + +static void usage(const char *prog) +{ + std::cout + << "Usage: " << prog + << " [src_face_idx=0] [tgt_face_idx=0]\n\n" + << " engine_dir directory holding the 5 TensorRT engines:\n" + << " " << kFaceDetectEngine << ", " << kFaceLandmarksEngine << ",\n" + << " " << kFaceRecognizerEngine << ", " << kFaceSwapEngine << ", " + << kFaceRestoreEngine << "\n" + << " source_img image whose face is taken\n" + << " target_img image whose face is replaced\n" + << " output_img where to write the swapped result\n"; +} + +int main(int argc, char *argv[]) +{ +#ifdef ENABLE_TENSORRT + if (argc < 5) + { + usage(argv[0]); + return 1; + } + const std::string engine_dir = argv[1]; + const std::string source_img = argv[2]; + const std::string target_img = argv[3]; + const std::string output_img = argv[4]; + const int src_idx = (argc > 5) ? std::stoi(argv[5]) : 0; + const int tgt_idx = (argc > 6) ? std::stoi(argv[6]) : 0; + + const std::string sep = + (engine_dir.empty() || engine_dir.back() == '/') ? "" : "/"; + auto engine = [&](const char *name) { return engine_dir + sep + name; }; + + auto pipeline = lite::trt::cv::face::swap::FaceFusionPipeLine( + engine(kFaceDetectEngine), + engine(kFaceLandmarksEngine), + engine(kFaceRecognizerEngine), + engine(kFaceSwapEngine), + engine(kFaceRestoreEngine)); + + pipeline.detect(source_img, src_idx, target_img, tgt_idx, output_img); + std::cout << "[FaceFusion] wrote: " << output_img << std::endl; + return 0; +#else + (void) argc; + (void) argv; + std::cerr << "This binary needs the TensorRT backend. Rebuild with: bash ./build.sh tensorrt\n"; + return 1; +#endif +} diff --git a/examples/lite/cv/test_lite_facefusion_pipeline_gpu.cpp b/examples/lite/cv/test_lite_facefusion_pipeline_gpu.cpp deleted file mode 100644 index 8a11aca5..00000000 --- a/examples/lite/cv/test_lite_facefusion_pipeline_gpu.cpp +++ /dev/null @@ -1,25 +0,0 @@ -// -// Created by wangzijian on 3/5/25. -// -#include "lite/trt/cv/trt_facefusion_pipeline_gpu.h" - -void test_default(){ - std::string face_swap_onnx_path = "/home/lite.ai.toolkit/examples/hub/trt/inswapper_128_fp16.engine"; - std::string face_detect_onnx_path = "/home/lite.ai.toolkit/examples/hub/trt/yoloface_8n_fp16.engine"; - std::string face_landmarks_68 = "/home/lite.ai.toolkit/examples/hub/trt/2dfan4_fp16.engine"; - std::string face_recognizer_onnx_path = "/home/lite.ai.toolkit/examples/hub/trt/arcface_w600k_r50_fp16.engine"; - std::string face_restoration_onnx_path = "/home/lite.ai.toolkit/examples/hub/trt/gfpgan_1.4_fp32.engine"; - std::vector model_list{face_swap_onnx_path,face_detect_onnx_path,face_landmarks_68, - face_recognizer_onnx_path,face_restoration_onnx_path}; - - trt_facefusion_pipeline_gpu test(model_list); - cv::Mat test1 = cv::imread("/home/lite.ai.toolkit/1.jpg"); - cv::Mat test2; - - test.detect(test1,test2,1,2); -} - - -int main(){ - test_default(); -} \ No newline at end of file From 0014e40977b649a7be2834e8d35ed0c0b07f7875 Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Mon, 1 Jun 2026 00:27:32 +0800 Subject: [PATCH 10/30] fix(facefusion): fail fast with clear errors instead of segfaulting The pipeline had ~5 latent crashes on bad input. Replace them with explicit, message-carrying exceptions (a prerequisite for shipping to users / wrapping in a binding): - constructor: check all 5 engine files exist before deserializing - detect: check source/target images actually loaded (imread != empty) - detect: check a face was found and the face index is in range, on both source and target, instead of out-of-bounds vector access - drop a dead, unconditional out-of-bounds access (target_test_bounding_box) No API or happy-path behavior change. Not build-verified (no GPU host). Co-Authored-By: Claude Opus 4.8 --- lite/trt/cv/trt_facefusion_pipeline.cpp | 44 ++++++++++++++++++++----- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/lite/trt/cv/trt_facefusion_pipeline.cpp b/lite/trt/cv/trt_facefusion_pipeline.cpp index 83d2e4ce..77c82c5b 100644 --- a/lite/trt/cv/trt_facefusion_pipeline.cpp +++ b/lite/trt/cv/trt_facefusion_pipeline.cpp @@ -3,13 +3,30 @@ // #include "trt_facefusion_pipeline.h" +#include +#include +#include using trtcv::TRTFaceFusionPipeLine; +namespace { +// Fail fast with a clear message instead of crashing deep inside TensorRT/OpenCV. +inline void require_file(const std::string &path, const char *what) { + if (!std::filesystem::exists(path)) + throw std::runtime_error(std::string("[FaceFusion] ") + what + " not found: " + path); +} +} + TRTFaceFusionPipeLine::TRTFaceFusionPipeLine(const std::string &face_detect_engine_path, const std::string &face_landmarks_68_engine_path, const std::string &face_recognizer_engine_path, const std::string &face_swap_engine_path, const std::string &face_restoration_engine_path) { + require_file(face_detect_engine_path, "face-detect engine"); + require_file(face_landmarks_68_engine_path, "face-landmarks engine"); + require_file(face_recognizer_engine_path, "face-recognizer engine"); + require_file(face_swap_engine_path, "face-swap engine"); + require_file(face_restoration_engine_path, "face-restoration engine"); + face_detect = std::make_unique(face_detect_engine_path,1); face_landmarks = std::make_unique(face_landmarks_68_engine_path,1); face_recognizer = std::make_unique(face_recognizer_engine_path,1); @@ -28,6 +45,8 @@ void TRTFaceFusionPipeLine::detect(const std::string &source_image, int src_inde // 最终也就是 image -> embeding std::vector detected_boxes; cv::Mat img_bgr = cv::imread(source_image); + if (img_bgr.empty()) + throw std::runtime_error("[FaceFusion] cannot read source image: " + source_image); auto img_bgr_src = img_bgr.clone(); face_detect->detect(img_bgr,detected_boxes,0.25f,0.45f); @@ -40,16 +59,15 @@ void TRTFaceFusionPipeLine::detect(const std::string &source_image, int src_inde } } - + if (src_final_boxes.empty()) + throw std::runtime_error("[FaceFusion] no face detected in source image: " + source_image); + if (src_index < 0 || src_index >= static_cast(src_final_boxes.size())) + throw std::runtime_error("[FaceFusion] source face index " + std::to_string(src_index) + + " out of range (" + std::to_string(src_final_boxes.size()) + " face(s) detected)"); std::vector face_landmark_5of68; - - if (src_final_boxes.size()==1) - { - face_landmarks->detect(img_bgr, src_final_boxes[0],face_landmark_5of68); - }else{ - face_landmarks->detect(img_bgr, src_final_boxes[src_index],face_landmark_5of68); - } + int src_pick = (src_final_boxes.size() == 1) ? 0 : src_index; + face_landmarks->detect(img_bgr, src_final_boxes[src_pick], face_landmark_5of68); // 这里准备使用多线程来进行操作 因为这里的操作和下面target的操作是独立的 // 这段代码仅仅是为了测试多线程的效果 @@ -70,6 +88,8 @@ void TRTFaceFusionPipeLine::detect(const std::string &source_image, int src_inde // 最终也就是 image -> landmarks std::vector target_detected_boxes; cv::Mat target_img_bgr = cv::imread(target_image); + if (target_img_bgr.empty()) + throw std::runtime_error("[FaceFusion] cannot read target image: " + target_image); auto target_img_bgr_src = target_img_bgr.clone(); face_detect->detect(target_img_bgr, target_detected_boxes,0.25f,0.45f); @@ -81,7 +101,13 @@ void TRTFaceFusionPipeLine::detect(const std::string &source_image, int src_inde target_final_boxes.emplace_back(current_box); } } - auto target_test_bounding_box = target_final_boxes[target_index]; + + if (target_final_boxes.empty()) + throw std::runtime_error("[FaceFusion] no face detected in target image: " + target_image); + if (target_index < 0 || target_index >= static_cast(target_final_boxes.size())) + throw std::runtime_error("[FaceFusion] target face index " + std::to_string(target_index) + + " out of range (" + std::to_string(target_final_boxes.size()) + " face(s) detected)"); + std::vector target_face_landmark_5of68; // face68Landmarks->detect_async(target_img_bgr_src, target_test_bounding_box, target_face_landmark_5of68); // face68Landmarks->wait_for_completion(); From e590a7b6260d9978e7bbc5c41e09c3234dc3bcea Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Fri, 5 Jun 2026 22:22:09 +0800 Subject: [PATCH 11/30] feat(facefusion): per-stage pipeline profiling + whole-pipeline benchmark Add an opt-in Profiler* to TRTFaceFusionPipeLine::detect() (LITE_CPU_SCOPE_OPT, zero-overhead when null) breaking the pipeline into imread / detect / landmark (x2) / recognizer / swap / restoration, plus lite_facefusion_pipeline_bench to drive it. Also tidies the dead commented-out multithread lines in detect(). First whole-pipeline measurement (4090, fp32, 1024x768) reframes the bottleneck: no single dominant stage -- restoration 27% / detect 24% / swap 19% / imread 15% (disk I/O) / landmark 12%. Surfaced a CUDA OOM (per-iteration memory growth, likely the swap paste-back) to fix next. Co-Authored-By: Claude Opus 4.8 --- examples/lite/CMakeLists.txt | 1 + .../test_lite_facefusion_pipeline_bench.cpp | 70 +++++++++++++ lite/trt/cv/trt_facefusion_pipeline.cpp | 99 +++++++------------ lite/trt/cv/trt_facefusion_pipeline.h | 4 +- 4 files changed, 112 insertions(+), 62 deletions(-) create mode 100644 examples/lite/cv/test_lite_facefusion_pipeline_bench.cpp diff --git a/examples/lite/CMakeLists.txt b/examples/lite/CMakeLists.txt index fa798024..d32c09ef 100644 --- a/examples/lite/CMakeLists.txt +++ b/examples/lite/CMakeLists.txt @@ -111,6 +111,7 @@ add_lite_executable(lite_face_restoration cv) add_lite_executable(lite_face_restoration_bench cv) add_lite_executable(lite_facefusion_pipeline cv) add_lite_executable(lite_facefusion_cli cv) +add_lite_executable(lite_facefusion_pipeline_bench cv) add_lite_executable(lite_yolov8 cv) add_lite_executable(lite_yolov11 cv) add_lite_executable(lite_sd_pipeline sd) diff --git a/examples/lite/cv/test_lite_facefusion_pipeline_bench.cpp b/examples/lite/cv/test_lite_facefusion_pipeline_bench.cpp new file mode 100644 index 00000000..6e0f657b --- /dev/null +++ b/examples/lite/cv/test_lite_facefusion_pipeline_bench.cpp @@ -0,0 +1,70 @@ +// +// Whole-pipeline benchmark for the FaceFusion face-swap pipeline. +// lite_facefusion_pipeline_bench \ +// \ +// [iters=30] [warmup=5] [csv] +// +// Profiles the pipeline into per-stage times (imread / detect / landmark — each x2 for +// source+target — / recognizer / swap / restoration) so we can see where the end-to-end +// time actually goes BEFORE optimizing anything. Each stage returns a host-visible result, +// so CPU-side timing is accurate. +// +// NOTE: the "restoration" stage here includes the final imwrite (the pipeline writes the +// result to disk), so it reads a bit higher than the compute-only restoration bench. +// +#include "lite/lite.h" +#include "lite/bench/profiler.h" +#include +#include +#include + +#ifdef ENABLE_TENSORRT +int main(int argc, char *argv[]) { + if (argc < 8) { + std::cout << "Usage: " << argv[0] + << " " + " [iters=30] [warmup=5] [csv]\n"; + return 1; + } + const std::string detect_engine = argv[1]; + const std::string landmark_engine = argv[2]; + const std::string recognizer_engine = argv[3]; + const std::string swap_engine = argv[4]; + const std::string restoration_engine = argv[5]; + const std::string source_img = argv[6]; + const std::string target_img = argv[7]; + const int iters = argc > 8 ? std::atoi(argv[8]) : 30; + const int warmup = argc > 9 ? std::atoi(argv[9]) : 5; + const std::string csv_path = argc > 10 ? argv[10] : "bench_facefusion_pipeline.csv"; + + const std::string out_path = "/tmp/bench_facefusion_out.jpg"; + + lite::trt::cv::face::swap::FaceFusionPipeLine pipeline( + detect_engine, landmark_engine, recognizer_engine, swap_engine, restoration_engine); + + std::cout << "[bench] source=" << source_img << " target=" << target_img + << "\n[bench] warmup=" << warmup << " iters=" << iters << std::endl; + + // Warmup (lazy engine/context init, cudnn autotune) — excluded from stats. + for (int i = 0; i < warmup; ++i) + pipeline.detect(source_img, 0, target_img, 0, out_path); + + lite::bench::Profiler prof; + for (int i = 0; i < iters; ++i) { + lite::bench::CpuTimer t; + t.start(); + pipeline.detect(source_img, 0, target_img, 0, out_path, &prof); + prof.tick(t.stop_ms()); + } + + prof.report("FaceFusion pipeline (per-stage, end-to-end)"); + prof.to_csv(csv_path); + std::cout << "[bench] sample result: " << out_path << std::endl; + return 0; +} +#else +int main() { + std::cerr << "This benchmark requires ENABLE_TENSORRT=ON.\n"; + return 0; +} +#endif diff --git a/lite/trt/cv/trt_facefusion_pipeline.cpp b/lite/trt/cv/trt_facefusion_pipeline.cpp index 77c82c5b..fe8c57ea 100644 --- a/lite/trt/cv/trt_facefusion_pipeline.cpp +++ b/lite/trt/cv/trt_facefusion_pipeline.cpp @@ -32,32 +32,31 @@ TRTFaceFusionPipeLine::TRTFaceFusionPipeLine(const std::string &face_detect_engi face_recognizer = std::make_unique(face_recognizer_engine_path,1); face_swap = std::make_unique(face_swap_engine_path,1); face_restoration = std::make_unique(face_restoration_engine_path,1); - - } - - -void TRTFaceFusionPipeLine::detect(const std::string &source_image, int src_index, const std::string &target_image, - int target_index, const std::string &save_image) { - // source 的全部流程 - // image -> detect -> landmarks -> recognizer -> embeding - // 最终也就是 image -> embeding - std::vector detected_boxes; - cv::Mat img_bgr = cv::imread(source_image); +// Per-stage timing is opt-in: pass a Profiler to break the pipeline down into +// imread / detect / landmark (x2, source+target) / recognizer / swap / restoration. +// When prof == nullptr the LITE_CPU_SCOPE_OPT scopes are zero-overhead. Each stage +// returns a host-visible result (boxes / landmarks / Mat), so it synchronizes and +// CPU-side wall-clock timing is accurate. +void TRTFaceFusionPipeLine::detect(const std::string &source_image, int src_index, + const std::string &target_image, int target_index, + const std::string &save_image, + lite::bench::Profiler *prof) { + // ---- source: image -> detect -> landmarks -> recognizer -> embedding ---- + cv::Mat img_bgr; + { LITE_CPU_SCOPE_OPT(prof, "imread_src"); img_bgr = cv::imread(source_image); } if (img_bgr.empty()) throw std::runtime_error("[FaceFusion] cannot read source image: " + source_image); auto img_bgr_src = img_bgr.clone(); - face_detect->detect(img_bgr,detected_boxes,0.25f,0.45f); + + std::vector detected_boxes; + { LITE_CPU_SCOPE_OPT(prof, "detect_src"); + face_detect->detect(img_bgr, detected_boxes, 0.25f, 0.45f); } std::vector src_final_boxes; for (auto current_box : detected_boxes) - { - if (current_box.score != 0) - { - src_final_boxes.emplace_back(current_box); - } - } + if (current_box.score != 0) src_final_boxes.emplace_back(current_box); if (src_final_boxes.empty()) throw std::runtime_error("[FaceFusion] no face detected in source image: " + source_image); @@ -67,40 +66,26 @@ void TRTFaceFusionPipeLine::detect(const std::string &source_image, int src_inde std::vector face_landmark_5of68; int src_pick = (src_final_boxes.size() == 1) ? 0 : src_index; - face_landmarks->detect(img_bgr, src_final_boxes[src_pick], face_landmark_5of68); - - // 这里准备使用多线程来进行操作 因为这里的操作和下面target的操作是独立的 - // 这段代码仅仅是为了测试多线程的效果 - // 到时候需要更改 -// std::string engine_path = "/home/lite.ai.toolkit/examples/hub/trt/2dfan4_fp16.engine"; -// trt_face_68landmarks_mt *face68Landmarks = new trt_face_68landmarks_mt(engine_path,2); -// face68Landmarks->detect_async(img_bgr, test_bounding_box, face_landmark_5of68); -// face68Landmarks->wait_for_completion(); + { LITE_CPU_SCOPE_OPT(prof, "landmark_src"); + face_landmarks->detect(img_bgr, src_final_boxes[src_pick], face_landmark_5of68); } - - -// face_landmarks->detect(img_bgr, test_bounding_box, face_landmark_5of68); std::vector source_image_embeding; - face_recognizer->detect(img_bgr_src,face_landmark_5of68,source_image_embeding); + { LITE_CPU_SCOPE_OPT(prof, "recognizer"); + face_recognizer->detect(img_bgr_src, face_landmark_5of68, source_image_embeding); } - // target 的全部流程 - // image -> detect -> landmarks - // 最终也就是 image -> landmarks - std::vector target_detected_boxes; - cv::Mat target_img_bgr = cv::imread(target_image); + // ---- target: image -> detect -> landmarks ---- + cv::Mat target_img_bgr; + { LITE_CPU_SCOPE_OPT(prof, "imread_tgt"); target_img_bgr = cv::imread(target_image); } if (target_img_bgr.empty()) throw std::runtime_error("[FaceFusion] cannot read target image: " + target_image); - auto target_img_bgr_src = target_img_bgr.clone(); - face_detect->detect(target_img_bgr, target_detected_boxes,0.25f,0.45f); + + std::vector target_detected_boxes; + { LITE_CPU_SCOPE_OPT(prof, "detect_tgt"); + face_detect->detect(target_img_bgr, target_detected_boxes, 0.25f, 0.45f); } std::vector target_final_boxes; for (auto current_box : target_detected_boxes) - { - if (current_box.score != 0) - { - target_final_boxes.emplace_back(current_box); - } - } + if (current_box.score != 0) target_final_boxes.emplace_back(current_box); if (target_final_boxes.empty()) throw std::runtime_error("[FaceFusion] no face detected in target image: " + target_image); @@ -109,22 +94,14 @@ void TRTFaceFusionPipeLine::detect(const std::string &source_image, int src_inde " out of range (" + std::to_string(target_final_boxes.size()) + " face(s) detected)"); std::vector target_face_landmark_5of68; -// face68Landmarks->detect_async(target_img_bgr_src, target_test_bounding_box, target_face_landmark_5of68); -// face68Landmarks->wait_for_completion(); -// face68Landmarks->shutdown(); - -// face_landmarks->detect(target_img_bgr, target_test_bounding_box,target_face_landmark_5of68); - - if (target_final_boxes.size()==1) - { - face_landmarks->detect(target_img_bgr, target_final_boxes[0],target_face_landmark_5of68); - }else{ - face_landmarks->detect(target_img_bgr, target_final_boxes[target_index],target_face_landmark_5of68); - } - // 公共部分 + int tgt_pick = (target_final_boxes.size() == 1) ? 0 : target_index; + { LITE_CPU_SCOPE_OPT(prof, "landmark_tgt"); + face_landmarks->detect(target_img_bgr, target_final_boxes[tgt_pick], target_face_landmark_5of68); } + + // ---- swap + restore ---- cv::Mat face_swap_image; - face_swap->detect(target_img_bgr,source_image_embeding,target_face_landmark_5of68,face_swap_image); - face_restoration->detect(face_swap_image,target_face_landmark_5of68,save_image); + { LITE_CPU_SCOPE_OPT(prof, "swap"); + face_swap->detect(target_img_bgr, source_image_embeding, target_face_landmark_5of68, face_swap_image); } + { LITE_CPU_SCOPE_OPT(prof, "restoration"); + face_restoration->detect(face_swap_image, target_face_landmark_5of68, save_image); } } - - diff --git a/lite/trt/cv/trt_facefusion_pipeline.h b/lite/trt/cv/trt_facefusion_pipeline.h index 5ba2dd3c..2d8715b3 100644 --- a/lite/trt/cv/trt_facefusion_pipeline.h +++ b/lite/trt/cv/trt_facefusion_pipeline.h @@ -13,6 +13,7 @@ #include "lite/trt/cv/trt_face_68landmarks.h" #include "lite/trt/cv/trt_face_68landmarks_mt.h" #include "lite/trt/cv/trt_yolofacev8_mt.h" +#include "lite/bench/profiler.h" namespace trtcv{ class TRTFaceFusionPipeLine{ @@ -35,7 +36,8 @@ namespace trtcv{ std::unique_ptr face_landmarks_mt; public: - void detect(const std::string &source_image,int src_index,const std::string &target_image,int target_index,const std::string &save_image); + void detect(const std::string &source_image,int src_index,const std::string &target_image,int target_index,const std::string &save_image, + lite::bench::Profiler *prof = nullptr); }; } From eca2d233205af73fbafbd3c58f9ca9ffb69f7408 Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Fri, 5 Jun 2026 22:44:17 +0800 Subject: [PATCH 12/30] fix(trt): stop unbounded GPU/host memory growth in the facefusion pipeline Two bugs surfaced by the whole-pipeline benchmark (CUDA OOM after ~13 frames on a 24GB 4090, GPU memory growing geometrically ~4x per frame): 1. nms_cuda_manager: perform_nms() had 'if (true) init(max(num_boxes, max_boxes_num*2))', re-allocating the NMS device buffers at DOUBLE the size on every call (x4 per frame since detect runs twice) -> geometric growth -> OOM. Now 'init(num_boxes)', a no-op once capacity fits. Side benefit: detect ~20% faster (no per-frame reallocs) and stable timing. 2. trt_face_swap: crop_list (a member) was emplace_back'd every call and never cleared -> unbounded host growth. Only crop_list[0] is used and the mask is constant, so clear() first. Verified: 50 frames, GPU memory flat at 2636 MiB (was OOM at ~13). Co-Authored-By: Claude Opus 4.8 --- .../lite/cv/test_lite_facefusion_pipeline_bench.cpp | 8 ++++++++ lite/trt/cv/trt_face_swap.cpp | 4 ++++ lite/trt/kernel/nms_cuda_manager.cpp | 10 ++++++---- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/examples/lite/cv/test_lite_facefusion_pipeline_bench.cpp b/examples/lite/cv/test_lite_facefusion_pipeline_bench.cpp index 6e0f657b..a5a6e8a5 100644 --- a/examples/lite/cv/test_lite_facefusion_pipeline_bench.cpp +++ b/examples/lite/cv/test_lite_facefusion_pipeline_bench.cpp @@ -51,6 +51,14 @@ int main(int argc, char *argv[]) { lite::bench::Profiler prof; for (int i = 0; i < iters; ++i) { + // Sanity: GPU memory should stay flat across iterations (no leak / no + // per-call buffer growth). Printed sparsely to avoid flooding the output. + if (i == 0 || i == iters - 1 || i % 10 == 0) { + size_t freeB = 0, totalB = 0; + cudaMemGetInfo(&freeB, &totalB); + std::cout << "[mem] iter " << i << " used=" << (totalB - freeB) / (1024 * 1024) + << " MiB" << std::endl; + } lite::bench::CpuTimer t; t.start(); pipeline.detect(source_img, 0, target_img, 0, out_path, &prof); diff --git a/lite/trt/cv/trt_face_swap.cpp b/lite/trt/cv/trt_face_swap.cpp index f007e34a..b845d2f6 100644 --- a/lite/trt/cv/trt_face_swap.cpp +++ b/lite/trt/cv/trt_face_swap.cpp @@ -12,6 +12,10 @@ void TRTFaceFusionFaceSwap::preprocess(cv::Mat &target_face, std::vector std::tie(preprocessed_mat, affine_martix) = face_utils::warp_face_by_face_landmark_5(target_face,target_landmark_5,face_utils::ARCFACE_128_V2); std::vector crop_size= {128.0,128.0}; + // crop_list is a member; it was emplace_back'd every call and never cleared, + // growing unbounded. Only crop_list[0] is used and the mask is identical each + // call, so keep exactly the current one. + crop_list.clear(); crop_list.emplace_back(face_utils::create_static_box_mask(crop_size)); cv::cvtColor(preprocessed_mat,preprocessed_mat,cv::COLOR_BGR2RGB); diff --git a/lite/trt/kernel/nms_cuda_manager.cpp b/lite/trt/kernel/nms_cuda_manager.cpp index 30c5cc90..f5e43584 100644 --- a/lite/trt/kernel/nms_cuda_manager.cpp +++ b/lite/trt/kernel/nms_cuda_manager.cpp @@ -77,11 +77,13 @@ std::vector NMSCudaManager::perform_nms( throw std::invalid_argument("Box and confidence sizes must match"); } - // 初始化或调整资源大小 + // Grow the device buffers only when the current capacity is too small; + // init() is a no-op when num_boxes already fits. (Previously this was + // `if (true) init(max(num_boxes, max_boxes_num * 2))`, which RE-ALLOCATED + // the buffers at double the size on EVERY call -> geometric GPU memory + // growth (x4 per pipeline frame, since detect runs twice) -> CUDA OOM.) const int num_boxes = boxes.size(); - if (true ) { - init(fmax(num_boxes, max_boxes_num * 2)); - } + init(num_boxes); // 准备数据 std::vector box_data(num_boxes * 5); From 159eac6d4c26e59d0c9b5ba019b125caa3167f8c Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Fri, 5 Jun 2026 22:56:51 +0800 Subject: [PATCH 13/30] feat(facefusion): in-memory Mat-in/Mat-out pipeline API + compute-only benchmark Add 'cv::Mat detect(src, tgt) -> Mat' that does NO disk I/O (uses restoration's restore() instead of writing to a path); the file-path detect() becomes a thin wrapper (imread -> core -> imwrite). This is both the production API (video/server feed frames, not paths) and what lets the benchmark measure compute only. The pipeline bench now decodes the two images once, outside the loop, and times the Mat-based detect. Effect (4090, fp32, 50 frames): TOTAL 90.5 -> 63.8 ms (the ~27ms removed was pure imread x2 + imwrite); restoration 27.4 -> 15.1 ms (it had been inflated by a per-frame full-frame imwrite). Real compute bottleneck: detect 28% / swap 26% / restoration 24% / landmark 17% / recognizer 3%. Co-Authored-By: Claude Opus 4.8 --- .../test_lite_facefusion_pipeline_bench.cpp | 25 +++++-- lite/trt/cv/trt_facefusion_pipeline.cpp | 66 ++++++++++++------- lite/trt/cv/trt_facefusion_pipeline.h | 8 +++ 3 files changed, 70 insertions(+), 29 deletions(-) diff --git a/examples/lite/cv/test_lite_facefusion_pipeline_bench.cpp b/examples/lite/cv/test_lite_facefusion_pipeline_bench.cpp index a5a6e8a5..5a6496b2 100644 --- a/examples/lite/cv/test_lite_facefusion_pipeline_bench.cpp +++ b/examples/lite/cv/test_lite_facefusion_pipeline_bench.cpp @@ -42,14 +42,26 @@ int main(int argc, char *argv[]) { lite::trt::cv::face::swap::FaceFusionPipeLine pipeline( detect_engine, landmark_engine, recognizer_engine, swap_engine, restoration_engine); + // Decode the two images ONCE, outside the timed loop. Real pipelines (video / + // server) decode at the edge, not per frame; keeping imread/imwrite out of the + // loop is what makes this a *compute-only* benchmark (the file-path detect() + // overload would re-read both images and write the result every iteration). + cv::Mat src = cv::imread(source_img); + cv::Mat tgt = cv::imread(target_img); + if (src.empty() || tgt.empty()) { + std::cerr << "[bench] cannot read source/target image" << std::endl; + return 1; + } std::cout << "[bench] source=" << source_img << " target=" << target_img - << "\n[bench] warmup=" << warmup << " iters=" << iters << std::endl; + << "\n[bench] warmup=" << warmup << " iters=" << iters + << " (compute-only: imread/imwrite excluded)" << std::endl; // Warmup (lazy engine/context init, cudnn autotune) — excluded from stats. for (int i = 0; i < warmup; ++i) - pipeline.detect(source_img, 0, target_img, 0, out_path); + pipeline.detect(src, 0, tgt, 0); lite::bench::Profiler prof; + cv::Mat out; for (int i = 0; i < iters; ++i) { // Sanity: GPU memory should stay flat across iterations (no leak / no // per-call buffer growth). Printed sparsely to avoid flooding the output. @@ -61,13 +73,16 @@ int main(int argc, char *argv[]) { } lite::bench::CpuTimer t; t.start(); - pipeline.detect(source_img, 0, target_img, 0, out_path, &prof); + out = pipeline.detect(src, 0, tgt, 0, &prof); prof.tick(t.stop_ms()); } - prof.report("FaceFusion pipeline (per-stage, end-to-end)"); + prof.report("FaceFusion pipeline (per-stage, compute-only)"); prof.to_csv(csv_path); - std::cout << "[bench] sample result: " << out_path << std::endl; + if (!out.empty()) { + cv::imwrite(out_path, out); // save one result (outside the timed loop) for visual check + std::cout << "[bench] sample result: " << out_path << std::endl; + } return 0; } #else diff --git a/lite/trt/cv/trt_facefusion_pipeline.cpp b/lite/trt/cv/trt_facefusion_pipeline.cpp index fe8c57ea..01cc0d80 100644 --- a/lite/trt/cv/trt_facefusion_pipeline.cpp +++ b/lite/trt/cv/trt_facefusion_pipeline.cpp @@ -34,21 +34,20 @@ TRTFaceFusionPipeLine::TRTFaceFusionPipeLine(const std::string &face_detect_engi face_restoration = std::make_unique(face_restoration_engine_path,1); } -// Per-stage timing is opt-in: pass a Profiler to break the pipeline down into -// imread / detect / landmark (x2, source+target) / recognizer / swap / restoration. -// When prof == nullptr the LITE_CPU_SCOPE_OPT scopes are zero-overhead. Each stage -// returns a host-visible result (boxes / landmarks / Mat), so it synchronizes and -// CPU-side wall-clock timing is accurate. -void TRTFaceFusionPipeLine::detect(const std::string &source_image, int src_index, - const std::string &target_image, int target_index, - const std::string &save_image, - lite::bench::Profiler *prof) { - // ---- source: image -> detect -> landmarks -> recognizer -> embedding ---- - cv::Mat img_bgr; - { LITE_CPU_SCOPE_OPT(prof, "imread_src"); img_bgr = cv::imread(source_image); } - if (img_bgr.empty()) - throw std::runtime_error("[FaceFusion] cannot read source image: " + source_image); - auto img_bgr_src = img_bgr.clone(); +// Compute-only core: in-memory images in, restored frame out. NO disk I/O. +// Per-stage timing is opt-in via prof (LITE_CPU_SCOPE_OPT is zero-overhead when null); +// each stage returns a host-visible result, so CPU-side wall-clock timing is accurate. +cv::Mat TRTFaceFusionPipeLine::detect(const cv::Mat &source_image, int src_index, + const cv::Mat &target_image, int target_index, + lite::bench::Profiler *prof) { + if (source_image.empty()) + throw std::runtime_error("[FaceFusion] source image is empty"); + if (target_image.empty()) + throw std::runtime_error("[FaceFusion] target image is empty"); + + // ---- source: detect -> landmarks -> recognizer -> embedding ---- + cv::Mat img_bgr = source_image.clone(); // sub-models take a non-const cv::Mat& + cv::Mat img_bgr_src = img_bgr.clone(); std::vector detected_boxes; { LITE_CPU_SCOPE_OPT(prof, "detect_src"); @@ -59,7 +58,7 @@ void TRTFaceFusionPipeLine::detect(const std::string &source_image, int src_inde if (current_box.score != 0) src_final_boxes.emplace_back(current_box); if (src_final_boxes.empty()) - throw std::runtime_error("[FaceFusion] no face detected in source image: " + source_image); + throw std::runtime_error("[FaceFusion] no face detected in source image"); if (src_index < 0 || src_index >= static_cast(src_final_boxes.size())) throw std::runtime_error("[FaceFusion] source face index " + std::to_string(src_index) + " out of range (" + std::to_string(src_final_boxes.size()) + " face(s) detected)"); @@ -73,11 +72,8 @@ void TRTFaceFusionPipeLine::detect(const std::string &source_image, int src_inde { LITE_CPU_SCOPE_OPT(prof, "recognizer"); face_recognizer->detect(img_bgr_src, face_landmark_5of68, source_image_embeding); } - // ---- target: image -> detect -> landmarks ---- - cv::Mat target_img_bgr; - { LITE_CPU_SCOPE_OPT(prof, "imread_tgt"); target_img_bgr = cv::imread(target_image); } - if (target_img_bgr.empty()) - throw std::runtime_error("[FaceFusion] cannot read target image: " + target_image); + // ---- target: detect -> landmarks ---- + cv::Mat target_img_bgr = target_image.clone(); std::vector target_detected_boxes; { LITE_CPU_SCOPE_OPT(prof, "detect_tgt"); @@ -88,7 +84,7 @@ void TRTFaceFusionPipeLine::detect(const std::string &source_image, int src_inde if (current_box.score != 0) target_final_boxes.emplace_back(current_box); if (target_final_boxes.empty()) - throw std::runtime_error("[FaceFusion] no face detected in target image: " + target_image); + throw std::runtime_error("[FaceFusion] no face detected in target image"); if (target_index < 0 || target_index >= static_cast(target_final_boxes.size())) throw std::runtime_error("[FaceFusion] target face index " + std::to_string(target_index) + " out of range (" + std::to_string(target_final_boxes.size()) + " face(s) detected)"); @@ -98,10 +94,32 @@ void TRTFaceFusionPipeLine::detect(const std::string &source_image, int src_inde { LITE_CPU_SCOPE_OPT(prof, "landmark_tgt"); face_landmarks->detect(target_img_bgr, target_final_boxes[tgt_pick], target_face_landmark_5of68); } - // ---- swap + restore ---- + // ---- swap + restore (restore() returns the frame; no disk write) ---- cv::Mat face_swap_image; { LITE_CPU_SCOPE_OPT(prof, "swap"); face_swap->detect(target_img_bgr, source_image_embeding, target_face_landmark_5of68, face_swap_image); } + + cv::Mat result; { LITE_CPU_SCOPE_OPT(prof, "restoration"); - face_restoration->detect(face_swap_image, target_face_landmark_5of68, save_image); } + result = face_restoration->restore(face_swap_image, target_face_landmark_5of68, nullptr); } + return result; +} + +// Convenience wrapper: file paths in, result written to disk. Thin layer over the +// in-memory core; imread/imwrite are timed separately when a Profiler is passed. +void TRTFaceFusionPipeLine::detect(const std::string &source_image, int src_index, + const std::string &target_image, int target_index, + const std::string &save_image, + lite::bench::Profiler *prof) { + cv::Mat src, tgt; + { LITE_CPU_SCOPE_OPT(prof, "imread_src"); src = cv::imread(source_image); } + if (src.empty()) + throw std::runtime_error("[FaceFusion] cannot read source image: " + source_image); + { LITE_CPU_SCOPE_OPT(prof, "imread_tgt"); tgt = cv::imread(target_image); } + if (tgt.empty()) + throw std::runtime_error("[FaceFusion] cannot read target image: " + target_image); + + cv::Mat out = detect(src, src_index, tgt, target_index, prof); + + { LITE_CPU_SCOPE_OPT(prof, "imwrite"); cv::imwrite(save_image, out); } } diff --git a/lite/trt/cv/trt_facefusion_pipeline.h b/lite/trt/cv/trt_facefusion_pipeline.h index 2d8715b3..6ca4c349 100644 --- a/lite/trt/cv/trt_facefusion_pipeline.h +++ b/lite/trt/cv/trt_facefusion_pipeline.h @@ -36,6 +36,14 @@ namespace trtcv{ std::unique_ptr face_landmarks_mt; public: + // Compute-only, in-memory: decoded images in -> restored frame out, no disk I/O. + // This is the path benchmarks and real (video / server) use should call. + cv::Mat detect(const cv::Mat &source_image, int src_index, + const cv::Mat &target_image, int target_index, + lite::bench::Profiler *prof = nullptr); + + // Convenience wrapper: file paths in, result written to disk. Thin layer over the + // in-memory core above (imread / imwrite are timed separately when benchmarking). void detect(const std::string &source_image,int src_index,const std::string &target_image,int target_index,const std::string &save_image, lite::bench::Profiler *prof = nullptr); From 83c7acc34da26a076828f789ce58d1e24fe53928 Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Fri, 5 Jun 2026 23:03:08 +0800 Subject: [PATCH 14/30] perf(trt): GPU-fuse the face-swap paste-back (swap 16.9 -> 9.4ms) The face-swap stage still used the old CPU launch_paste_back (two full-frame warpAffine + per-frame cudaMalloc/sync), while restoration had already moved to the GPU-fused PasteBackGPU. Give swap a PasteBackGPU member and reuse the same kernel (inverse-mapping sample + blend, reused device buffers, pinned/async), numerically equivalent to the CPU path (max|diff|=2/255). 4090, fp32, compute-only, 50 frames: swap 16.9 -> 9.4 ms; pipeline TOTAL 63.8 -> 56.2 ms (15.7 -> 17.8 FPS). Output verified visually unchanged. Co-Authored-By: Claude Opus 4.8 --- lite/trt/cv/trt_face_swap.cpp | 6 ++++-- lite/trt/cv/trt_face_swap.h | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lite/trt/cv/trt_face_swap.cpp b/lite/trt/cv/trt_face_swap.cpp index b845d2f6..5620a2da 100644 --- a/lite/trt/cv/trt_face_swap.cpp +++ b/lite/trt/cv/trt_face_swap.cpp @@ -122,8 +122,10 @@ void TRTFaceFusionFaceSwap::detect(cv::Mat &target_image, std::vector sou // 计算pasteback时间 auto start_pasteback = std::chrono::high_resolution_clock::now(); -// cv::Mat dst_image = face_utils::paste_back(ori_image,mat,crop_list[0],affine_martix); - cv::Mat dst_image = launch_paste_back(ori_image,mat,crop_list[0],affine_martix); + // GPU-fused paste-back (reused device buffers, no per-frame cudaMalloc), reusing the + // exact kernel restoration uses. Numerically equivalent to launch_paste_back. +// cv::Mat dst_image = launch_paste_back(ori_image,mat,crop_list[0],affine_martix); + cv::Mat dst_image = paste_back_gpu_.paste_back(ori_image, mat, crop_list[0], affine_martix, stream); auto end_pasteback = std::chrono::high_resolution_clock::now(); std::chrono::duration diff_pasteback = end_pasteback-start_pasteback; std::cout << "Face_Swap pasteback Time: " << diff_pasteback.count() * 1000 << " ms\n"; diff --git a/lite/trt/cv/trt_face_swap.h b/lite/trt/cv/trt_face_swap.h index 9b4ad08e..087f83f9 100644 --- a/lite/trt/cv/trt_face_swap.h +++ b/lite/trt/cv/trt_face_swap.h @@ -23,6 +23,7 @@ namespace trtcv{ private: std::vector crop_list; cv::Mat affine_martix; + PasteBackGPU paste_back_gpu_; // GPU-fused paste-back, reused device buffers (same as restoration) public: void detect(cv::Mat &target_image,std::vector source_face_embeding,std::vector target_landmark_5, cv::Mat &face_swap_image); From 76037939b513cf4b9622538c8bb632539bc4de85 Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Sun, 7 Jun 2026 10:15:35 +0800 Subject: [PATCH 15/30] perf(trt): GPU-fuse the face-detect (yoloface) preprocess (detect 9.6 -> 4.2ms) Profiling the detect stage showed it was glue-bound, not infer-bound: of ~9.6ms/call, preprocess was 4.9ms (CPU normalize: resize + copyMakeBorder + split + 3x convertTo + merge + create_tensor) while TRT inference was only 1.4ms -- so FP16 would barely help it. Moved normalize + BGR HWC->CHW into a fused CUDA kernel (YoloFacePreprocessGPU, reused device/pinned buffers) writing straight into the inference input buffer; only resize+letterbox stays on CPU. Also dropped a wasted full-output D2H copy (generate_box reads buffers[1] directly) and the now-dead normalize() method. 4090, fp32, compute-only, 50 frames: detect 9.6 -> 4.2 ms/call (preprocess 4.9 -> 0.8); pipeline TOTAL 56.2 -> 46.2 ms (17.8 -> 21.6 FPS). Output verified visually unchanged. Co-Authored-By: Claude Opus 4.8 --- lite/trt/cv/trt_yolofacev8.cpp | 69 +++++-------------- lite/trt/cv/trt_yolofacev8.h | 4 +- lite/trt/kernel/yoloface_preprocess.cu | 23 +++++++ lite/trt/kernel/yoloface_preprocess.cuh | 10 +++ .../kernel/yoloface_preprocess_manager.cpp | 37 ++++++++++ lite/trt/kernel/yoloface_preprocess_manager.h | 30 ++++++++ 6 files changed, 121 insertions(+), 52 deletions(-) create mode 100644 lite/trt/kernel/yoloface_preprocess.cu create mode 100644 lite/trt/kernel/yoloface_preprocess.cuh create mode 100644 lite/trt/kernel/yoloface_preprocess_manager.cpp create mode 100644 lite/trt/kernel/yoloface_preprocess_manager.h diff --git a/lite/trt/cv/trt_yolofacev8.cpp b/lite/trt/cv/trt_yolofacev8.cpp index ca3381ca..d6c93974 100644 --- a/lite/trt/cv/trt_yolofacev8.cpp +++ b/lite/trt/cv/trt_yolofacev8.cpp @@ -63,41 +63,6 @@ std::vector TRTYoloFaceV8::nms(std::vector boxes, std::v return keep_inds; } -cv::Mat TRTYoloFaceV8::normalize(cv::Mat srcimg) { - const int height = srcimg.rows; - const int width = srcimg.cols; - cv::Mat temp_image = srcimg.clone(); - int input_height = input_node_dims[2]; - int input_width = input_node_dims[3]; - - if (height > input_height || width > input_width) - { - const float scale = std::min((float)input_height / height, (float)input_width / width); - cv::Size new_size = cv::Size(int(width * scale), int(height * scale)); - cv::resize(srcimg, temp_image, new_size); - } - - ratio_height = (float)height / temp_image.rows; - ratio_width = (float)width / temp_image.cols; - - cv::Mat input_img; - cv::copyMakeBorder(temp_image, input_img, 0, input_height - temp_image.rows, - 0, input_width - temp_image.cols, cv::BORDER_CONSTANT, 0); - - std::vector bgrChannels(3); - cv::split(input_img, bgrChannels); - for (int c = 0; c < 3; c++) - { - bgrChannels[c].convertTo(bgrChannels[c], CV_32FC1, 1 / 128.0, -127.5 / 128.0); - } - cv::Mat normalized_image; - cv::merge(bgrChannels,normalized_image); - return normalized_image; - -} - - - void TRTYoloFaceV8::generate_box(float *trt_outputs, std::vector &boxes, float conf_threshold, float iou_threshold) { @@ -157,30 +122,34 @@ void TRTYoloFaceV8::detect(const cv::Mat &mat, std::vector &b } - // 1.normalized the input - cv::Mat normalized_image = normalize(mat); + // 1. letterbox: resize (keep aspect) + pad to the network input size, BGR uint8. Sets ratio_*. + const int input_height = input_node_dims[2]; + const int input_width = input_node_dims[3]; + cv::Mat temp_image = mat; + if (mat.rows > input_height || mat.cols > input_width) { + const float s = std::min((float)input_height / mat.rows, (float)input_width / mat.cols); + cv::resize(mat, temp_image, cv::Size(int(mat.cols * s), int(mat.rows * s))); + } + ratio_height = (float)mat.rows / temp_image.rows; + ratio_width = (float)mat.cols / temp_image.cols; + cv::Mat input_img; + cv::copyMakeBorder(temp_image, input_img, 0, input_height - temp_image.rows, + 0, input_width - temp_image.cols, cv::BORDER_CONSTANT, 0); - // 2.trans to input vector - std::vector input; - trtcv::utils::transform::create_tensor(normalized_image,input,input_node_dims,trtcv::utils::transform::CHW); + // 2. GPU-fused normalize + BGR HWC->CHW straight into the inference input buffer + // (replaces CPU split / 3x convertTo / merge / create_tensor + the separate float H2D). + preprocess_gpu_.run(input_img, static_cast(buffers[0]), stream); // 3. infer - cudaMemcpyAsync(buffers[0], input.data(), input_node_dims[0] * input_node_dims[1] * input_node_dims[2] * input_node_dims[3] * sizeof(float), - cudaMemcpyHostToDevice, stream); bool status = trt_context->enqueueV3(stream); - - if (!status){ std::cerr << "Failed to infer by TensorRT." << std::endl; return; } + cudaStreamSynchronize(stream); // ensure the inference output (buffers[1]) is ready - std::vector output(output_node_dims[0][0] * output_node_dims[0][1] * output_node_dims[0][2]); - - cudaMemcpyAsync(output.data(), buffers[1], output_node_dims[0][0] * output_node_dims[0][1] * output_node_dims[0][2] * sizeof(float), - cudaMemcpyDeviceToHost, stream); - // 4. generate box - generate_box(output.data(),boxes,0.45f,0.5f); + // 4. generate box (reads buffers[1] directly; the trt_outputs param is unused) + generate_box(nullptr, boxes, 0.45f, 0.5f); } diff --git a/lite/trt/cv/trt_yolofacev8.h b/lite/trt/cv/trt_yolofacev8.h index b028c618..2ac2315c 100644 --- a/lite/trt/cv/trt_yolofacev8.h +++ b/lite/trt/cv/trt_yolofacev8.h @@ -9,6 +9,7 @@ #include "lite/trt/kernel/nms_cuda_manager.h" #include "lite/trt/kernel/generate_bbox_cuda_manager.h" #include "lite/trt/kernel/bgr2rgb.cuh" +#include "lite/trt/kernel/yoloface_preprocess_manager.h" namespace trtcv{ class LITE_EXPORTS TRTYoloFaceV8 : public BasicTRTHandler{ @@ -35,6 +36,7 @@ namespace trtcv{ float ratio_width ; float ratio_height; + YoloFacePreprocessGPU preprocess_gpu_; // GPU-fused normalize + BGR HWC->CHW private: // transform func @@ -43,8 +45,6 @@ namespace trtcv{ std::vector nms(std::vector boxes, std::vector confidences, const float nms_thresh); - cv::Mat normalize(cv::Mat srcImg); - void generate_box(float* trt_outputs, std::vector& boxes,float conf_threshold, float iou_threshold); public: void detect(const cv::Mat &mat,std::vector &boxes, diff --git a/lite/trt/kernel/yoloface_preprocess.cu b/lite/trt/kernel/yoloface_preprocess.cu new file mode 100644 index 00000000..32ac3b2f --- /dev/null +++ b/lite/trt/kernel/yoloface_preprocess.cu @@ -0,0 +1,23 @@ +#include "yoloface_preprocess.cuh" + +// One thread per pixel of the letterboxed BGR uint8 image. Writes planar BGR float (CHW), +// normalized v*(1/128) - 127.5/128. Channel order is preserved (plane0=B, plane1=G, +// plane2=R), exactly matching the CPU path (cv::split -> per-channel convertTo -> CHW). +__global__ void yoloface_preprocess_kernel(const unsigned char* img, float* out, int H, int W) { + int x = blockIdx.x * blockDim.x + threadIdx.x; + int y = blockIdx.y * blockDim.y + threadIdx.y; + if (x >= W || y >= H) return; + + int i = (y * W + x) * 3; + float b = static_cast(img[i + 0]); + float g = static_cast(img[i + 1]); + float r = static_cast(img[i + 2]); + + const float scale = 1.f / 128.f; + const float shift = -127.5f / 128.f; + int plane = H * W; + int off = y * W + x; + out[0 * plane + off] = b * scale + shift; + out[1 * plane + off] = g * scale + shift; + out[2 * plane + off] = r * scale + shift; +} diff --git a/lite/trt/kernel/yoloface_preprocess.cuh b/lite/trt/kernel/yoloface_preprocess.cuh new file mode 100644 index 00000000..7a7b7cb9 --- /dev/null +++ b/lite/trt/kernel/yoloface_preprocess.cuh @@ -0,0 +1,10 @@ +#ifndef YOLOFACE_PREPROCESS_CUH +#define YOLOFACE_PREPROCESS_CUH + +#include + +// One thread per pixel of the letterboxed BGR uint8 image. Writes planar BGR float (CHW), +// normalized v*(1/128) - 127.5/128, matching the CPU normalize() in trt_yolofacev8.cpp. +__global__ void yoloface_preprocess_kernel(const unsigned char* img, float* out, int H, int W); + +#endif // YOLOFACE_PREPROCESS_CUH diff --git a/lite/trt/kernel/yoloface_preprocess_manager.cpp b/lite/trt/kernel/yoloface_preprocess_manager.cpp new file mode 100644 index 00000000..e46b3a05 --- /dev/null +++ b/lite/trt/kernel/yoloface_preprocess_manager.cpp @@ -0,0 +1,37 @@ +#include "yoloface_preprocess_manager.h" +#include +#include + +YoloFacePreprocessGPU::~YoloFacePreprocessGPU() { + if (d_img_) cudaFree(d_img_); + if (h_pinned_) cudaFreeHost(h_pinned_); +} + +void YoloFacePreprocessGPU::ensure_capacity(size_t bytes) { + if (bytes > cap_) { + if (d_img_) cudaFree(d_img_); + if (h_pinned_) cudaFreeHost(h_pinned_); + cudaMalloc(&d_img_, bytes); + cudaMallocHost(&h_pinned_, bytes); + cap_ = bytes; + } +} + +void YoloFacePreprocessGPU::run(const cv::Mat& letterboxed_bgr_u8, float* d_out, cudaStream_t stream) { + cv::Mat c = letterboxed_bgr_u8; + if (c.type() != CV_8UC3) c.convertTo(c, CV_8UC3); + if (!c.isContinuous()) c = c.clone(); + + const int H = c.rows, W = c.cols; + const size_t bytes = static_cast(H) * W * 3; + ensure_capacity(bytes); + + std::memcpy(h_pinned_, c.data, bytes); + cudaMemcpyAsync(d_img_, h_pinned_, bytes, cudaMemcpyHostToDevice, stream); + + dim3 block(16, 16); + dim3 grid((W + block.x - 1) / block.x, (H + block.y - 1) / block.y); + yoloface_preprocess_kernel<<>>(d_img_, d_out, H, W); + + cudaStreamSynchronize(stream); +} diff --git a/lite/trt/kernel/yoloface_preprocess_manager.h b/lite/trt/kernel/yoloface_preprocess_manager.h new file mode 100644 index 00000000..e16191a1 --- /dev/null +++ b/lite/trt/kernel/yoloface_preprocess_manager.h @@ -0,0 +1,30 @@ +#ifndef YOLOFACE_PREPROCESS_MANAGER_H +#define YOLOFACE_PREPROCESS_MANAGER_H + +#include "yoloface_preprocess.cuh" +#include + +// Fuses normalize (v/128 - 127.5/128) + BGR HWC->CHW into one kernel and writes the tensor +// straight into the device inference input buffer — replacing the CPU split / 3x convertTo / +// merge / create_tensor + the separate float H2D. Reuses device + pinned staging buffers. +class YoloFacePreprocessGPU { +public: + YoloFacePreprocessGPU() = default; + ~YoloFacePreprocessGPU(); + + YoloFacePreprocessGPU(const YoloFacePreprocessGPU&) = delete; + YoloFacePreprocessGPU& operator=(const YoloFacePreprocessGPU&) = delete; + + // letterboxed_bgr_u8: CV_8UC3, already resized + padded to the network input size. + // d_out: device float CHW buffer (the inference input). + void run(const cv::Mat& letterboxed_bgr_u8, float* d_out, cudaStream_t stream = nullptr); + +private: + void ensure_capacity(size_t bytes); + + unsigned char* d_img_ = nullptr; + unsigned char* h_pinned_ = nullptr; + size_t cap_ = 0; +}; + +#endif // YOLOFACE_PREPROCESS_MANAGER_H From 15aa2ca1bd5142da05a71d152d9b825b19b47a25 Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Sun, 7 Jun 2026 10:57:37 +0800 Subject: [PATCH 16/30] tools: mixed-precision GFPGAN FP16 engine builder for TRT 10.1 Cracks the previously-blocked gfpgan FP16-on-TRT10.1 problem: naive --fp16 grey-blocks (StyleGAN modulated-conv demodulation overflows in FP16); the TRT11 'Cast strong-typing' route crashes on TRT10.1 (matchTypeSpec). This route works: weak FP16 + OBEY_PRECISION_CONSTRAINTS + per-layer FP32 pin on style_conv*/to_rgb* float layers via the builder API (TRT10.1 python wheel). Verified on 4090/TRT10.1: restoration infer 10.8 -> 8.0 ms, output numerically clean (no grey blocks); facefusion pipeline 36.6 -> 33.1 ms (27.4 -> 30.3 FPS). Co-Authored-By: Claude Opus 4.8 --- build_gfpgan_fp16_engine.py | 84 +++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 build_gfpgan_fp16_engine.py diff --git a/build_gfpgan_fp16_engine.py b/build_gfpgan_fp16_engine.py new file mode 100644 index 00000000..f8bc65ba --- /dev/null +++ b/build_gfpgan_fp16_engine.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# Build a *mixed-precision* TensorRT engine for GFPGAN that the lite.ai.toolkit C++ +# (TensorRT 10.1) can load — fast FP16 everywhere EXCEPT the StyleGAN modulated convs, +# which are kept in FP32. +# +# Why this exists: +# A naive `trtexec --fp16` GFPGAN engine produces grey-block artifacts: the StyleGAN +# "modulated conv" demodulation (sum-of-squares -> rsqrt) overflows/underflows in FP16. +# The clean fix is to keep just those layers (style_conv* / to_rgb*) in FP32 and run the +# rest in FP16. On TensorRT 10.1 the "strong typing via Cast nodes in the ONNX" route +# crashes (matchTypeSpec); the route that works is weak FP16 + OBEY_PRECISION_CONSTRAINTS +# with per-layer FP32 precision set through the builder API (this script). +# +# Result on RTX 4090 / TRT 10.1: restoration infer 10.8 -> 8.0 ms, output numerically clean +# (no grey blocks); facefusion pipeline 36.6 -> 33.1 ms (27 -> 30 FPS). +# +# Requirements: the TensorRT 10.1 *python* wheel (ships in the TRT tarball under python/), +# e.g. python -m venv env && env/bin/pip install /usr/local/tensorrt/python/tensorrt-10.1.0-cp312-*.whl +# +# Usage: +# LD_LIBRARY_PATH=/usr/local/tensorrt/lib:/usr/local/cuda/lib64 \ +# python build_gfpgan_fp16_engine.py +# +import sys +import os +import tensorrt as trt + +# Substring match on layer names; these are the StyleGAN modulated convs that must stay FP32. +KEEP_FP32 = ("style_conv", "to_rgb") +FLOAT_TYPES = (trt.float32, trt.float16) + + +def main(): + if len(sys.argv) < 3: + print(__doc__) + sys.exit(1) + onnx_path, engine_path = sys.argv[1], sys.argv[2] + + log = trt.Logger(trt.Logger.WARNING) + builder = trt.Builder(log) + network = builder.create_network(0) + parser = trt.OnnxParser(network, log) + with open(onnx_path, "rb") as f: + if not parser.parse(f.read()): + for i in range(parser.num_errors): + print(parser.get_error(i)) + sys.exit(1) + + cfg = builder.create_builder_config() + cfg.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, 4 << 30) + cfg.set_flag(trt.BuilderFlag.FP16) + cfg.set_flag(trt.BuilderFlag.OBEY_PRECISION_CONSTRAINTS) + + def all_float(layer): + return layer.num_outputs > 0 and all( + layer.get_output(j).dtype in FLOAT_TYPES for j in range(layer.num_outputs) + ) + + pinned = 0 + for i in range(network.num_layers): + layer = network.get_layer(i) + if not any(k in layer.name for k in KEEP_FP32): + continue + # Only float compute layers; skip Int64/shape Constants (can't be FP32-typed). + if layer.type == trt.LayerType.CONSTANT or not all_float(layer): + continue + layer.precision = trt.float32 + for j in range(layer.num_outputs): + layer.set_output_type(j, trt.float32) + pinned += 1 + print(f"network layers={network.num_layers} pinned to fp32={pinned}", flush=True) + + print("building serialized engine (this is slow on the first build)...", flush=True) + serialized = builder.build_serialized_network(network, cfg) + if serialized is None: + print("BUILD FAILED") + sys.exit(1) + with open(engine_path, "wb") as f: + f.write(serialized) + print(f"OK wrote {engine_path} ({os.path.getsize(engine_path) / 1e6:.1f} MB)") + + +if __name__ == "__main__": + main() From 8acca60a8adffb001b43945a632db9b5b89af34d Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Sun, 7 Jun 2026 11:11:33 +0800 Subject: [PATCH 17/30] fix(trt): isolate ROI in yoloface letterbox padding (BORDER_ISOLATED) The detect-preprocess refactor (7603793) dropped the old srcimg.clone(); when detect() is given a ROI/submatrix and no resize happens, copyMakeBorder(BORDER_CONSTANT) pulls parent-image pixels (outside the ROI) into the pad region instead of zeros, corrupting the detector input. Add BORDER_ISOLATED to restore the clone()-based behavior. (Latent: the facefusion pipeline passes a clone, so it was not active, but detect() is a public API.) Co-Authored-By: Claude Opus 4.8 --- lite/trt/cv/trt_yolofacev8.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lite/trt/cv/trt_yolofacev8.cpp b/lite/trt/cv/trt_yolofacev8.cpp index d6c93974..b0906901 100644 --- a/lite/trt/cv/trt_yolofacev8.cpp +++ b/lite/trt/cv/trt_yolofacev8.cpp @@ -133,8 +133,13 @@ void TRTYoloFaceV8::detect(const cv::Mat &mat, std::vector &b ratio_height = (float)mat.rows / temp_image.rows; ratio_width = (float)mat.cols / temp_image.cols; cv::Mat input_img; + // BORDER_ISOLATED: when `mat` is a ROI/submatrix of a larger image and no resize + // happened (temp_image == mat), plain copyMakeBorder would pull the parent image's + // pixels (outside the ROI) into the pad region instead of the constant. Isolating the + // ROI restores the old clone()-based behavior. cv::copyMakeBorder(temp_image, input_img, 0, input_height - temp_image.rows, - 0, input_width - temp_image.cols, cv::BORDER_CONSTANT, 0); + 0, input_width - temp_image.cols, + cv::BORDER_CONSTANT | cv::BORDER_ISOLATED, 0); // 2. GPU-fused normalize + BGR HWC->CHW straight into the inference input buffer // (replaces CPU split / 3x convertTo / merge / create_tensor + the separate float H2D). From 1090755aba229dede584c4f9484a8b6654cfcf75 Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Sun, 7 Jun 2026 11:26:09 +0800 Subject: [PATCH 18/30] perf/cleanup(trt): hoist per-frame constants out of the face-swap hot path - trt_face_swap: load the model_matrix .npy ONCE in the constructor (was load_npy() from disk every frame) and cache the static 128 box mask (was rebuilt every frame). Swap result byte-identical. - remove hot-path std::cout spam from trt_face_swap (Face_Swap timing prints + 'done!') and trt_face_recognizer ('done!') -- they pollute stdout and add per-frame latency/jitter. Correctness/hygiene win (no per-frame disk syscall, clean output) rather than a big speedup -- the swap stage's real cost is the CPU CHW->HWC transpose + paste-back (addressed later by the device-path work). Output verified unchanged on a real source/target swap. Co-Authored-By: Claude Opus 4.8 --- lite/trt/cv/trt_face_recognizer.cpp | 5 -- lite/trt/cv/trt_face_swap.cpp | 72 ++++------------------------- lite/trt/cv/trt_face_swap.h | 10 +++- 3 files changed, 16 insertions(+), 71 deletions(-) diff --git a/lite/trt/cv/trt_face_recognizer.cpp b/lite/trt/cv/trt_face_recognizer.cpp index f6f0bf82..b324a2a9 100644 --- a/lite/trt/cv/trt_face_recognizer.cpp +++ b/lite/trt/cv/trt_face_recognizer.cpp @@ -66,9 +66,4 @@ void TRTFaceFusionFaceRecognizer::detect(cv::Mat &input_mat, std::vector std::tie(preprocessed_mat, affine_martix) = face_utils::warp_face_by_face_landmark_5(target_face,target_landmark_5,face_utils::ARCFACE_128_V2); - std::vector crop_size= {128.0,128.0}; - // crop_list is a member; it was emplace_back'd every call and never cleared, - // growing unbounded. Only crop_list[0] is used and the mask is identical each - // call, so keep exactly the current one. - crop_list.clear(); - crop_list.emplace_back(face_utils::create_static_box_mask(crop_size)); - cv::cvtColor(preprocessed_mat,preprocessed_mat,cv::COLOR_BGR2RGB); preprocessed_mat.convertTo(preprocessed_mat,CV_32FC3,1.0 / 255.f); preprocessed_mat.convertTo(preprocessed_mat,CV_32FC3,1.0 / 1.f,0); - // 使用 CMake 传递的 SOURCE_PATH 宏 - std::string model_matrix_path = std::string(SOURCE_PATH) + "/examples/lite/resources/model_matrix.npy"; - std::vector model_martix = face_utils::load_npy(model_matrix_path); - - processed_source_embeding= face_utils::dot_product(source_image_embeding,model_martix,512); + // model_matrix_ and box_mask_ are loaded/built once in the constructor (they are + // constant); they used to be load_npy'd from disk and rebuilt every frame. + processed_source_embeding = face_utils::dot_product(source_image_embeding, model_matrix_, 512); face_utils::normalize(processed_source_embeding); - - std::cout<<"done!"< sou cv::Mat ori_image = target_image.clone(); std::vector source_embeding_input; cv::Mat model_input_mat; - // 预处理时间 - auto start_preprocess = std::chrono::high_resolution_clock::now(); preprocess(target_image,source_face_embeding,target_landmark_5,source_embeding_input,model_input_mat); - auto end_preprocess = std::chrono::high_resolution_clock::now(); - std::chrono::duration diff_preprocess = end_preprocess-start_preprocess; - std::cout << "Face_Swap preprocess Time: " << diff_preprocess.count() * 1000 << " ms\n"; std::vector input_vector; trtcv::utils::transform::create_tensor(model_input_mat,input_vector,input_node_dims,trtcv::utils::transform::CHW); - // 这个是 source 的输入下面写一个 embeding 的输入 + // input 0 = preprocessed target face, input 1 = source embedding cudaMemcpyAsync(buffers[0],input_vector.data(),1 * 3 * 128 * 128 *sizeof(float ), cudaMemcpyHostToDevice,stream); cudaMemcpyAsync(buffers[1],source_embeding_input.data(),512 * sizeof(float), cudaMemcpyHostToDevice,stream); - - // 推理之前先同步一下 cudaStreamSynchronize(stream); - // 这里是推理 bool status = trt_context->enqueueV3(stream); if (!status) { std::cerr << "Failed to enqueue TensorRT model." << std::endl; return; } - auto start = std::chrono::high_resolution_clock::now(); -// 将输出拷贝出来 std::vector output_vector(3 * 128 * 128); cudaMemcpyAsync(output_vector.data(),buffers[2],1 * 3 * 128 * 128 * sizeof(float),cudaMemcpyDeviceToHost,stream); cudaStreamSynchronize(stream); @@ -72,22 +50,9 @@ void TRTFaceFusionFaceSwap::detect(cv::Mat &target_image, std::vector sou std::vector output_swap_image(1 * 3 * 128 * 128); output_swap_image.assign(output_vector.begin(),output_vector.end()); - - + // CHW float -> HWC uint8-range (denormalize); paste-back is done on the GPU below. std::vector transposed(3 * 128 * 128); - int channels = 3; - int height = 128; - int width = 128; -// launch_face_swap_postprocess( -// static_cast(buffers[2]), -// channels, -// height, -// width, -// transposed.data() -// ); - - // 写一个测试时间的代码 - + const int channels = 3, height = 128, width = 128; #pragma omp parallel for collapse(3) for (int c = 0; c < channels; ++c) { for (int h = 0; h < height; ++h) { @@ -98,36 +63,15 @@ void TRTFaceFusionFaceSwap::detect(cv::Mat &target_image, std::vector sou } } } - -// for (int c = 0; c < channels; ++c) { -// for (int h = 0; h < height; ++h) { -// for (int w = 0; w < width; ++w) { -// int src_idx = c * (height * width) + h * width + w; // CHW -// int dst_idx = h * (width * channels) + w * channels + c; // HWC -// transposed[dst_idx] = output_swap_image[src_idx]; -// } -// } -// } - for (auto& val : transposed) { val = std::round(val * 255.0); } - auto end = std::chrono::high_resolution_clock::now(); - std::chrono::duration diff = end-start; - std::cout << "Face_Swap postprocess Time: " << diff.count() * 1000 << " ms\n"; - cv::Mat mat(height, width, CV_32FC3, transposed.data()); cv::cvtColor(mat, mat, cv::COLOR_RGB2BGR); - // 计算pasteback时间 - auto start_pasteback = std::chrono::high_resolution_clock::now(); // GPU-fused paste-back (reused device buffers, no per-frame cudaMalloc), reusing the // exact kernel restoration uses. Numerically equivalent to launch_paste_back. -// cv::Mat dst_image = launch_paste_back(ori_image,mat,crop_list[0],affine_martix); - cv::Mat dst_image = paste_back_gpu_.paste_back(ori_image, mat, crop_list[0], affine_martix, stream); - auto end_pasteback = std::chrono::high_resolution_clock::now(); - std::chrono::duration diff_pasteback = end_pasteback-start_pasteback; - std::cout << "Face_Swap pasteback Time: " << diff_pasteback.count() * 1000 << " ms\n"; + cv::Mat dst_image = paste_back_gpu_.paste_back(ori_image, mat, box_mask_, affine_martix, stream); face_swap_image = dst_image; -} \ No newline at end of file +} diff --git a/lite/trt/cv/trt_face_swap.h b/lite/trt/cv/trt_face_swap.h index 087f83f9..bbd8c780 100644 --- a/lite/trt/cv/trt_face_swap.h +++ b/lite/trt/cv/trt_face_swap.h @@ -15,14 +15,20 @@ namespace trtcv{ class LITE_EXPORTS TRTFaceFusionFaceSwap : BasicTRTHandler{ public: explicit TRTFaceFusionFaceSwap(const std::string& _trt_model_path,unsigned int _num_threads = 1): - BasicTRTHandler(_trt_model_path,_num_threads){}; + BasicTRTHandler(_trt_model_path,_num_threads){ + // Constant inputs — load/build once here (used to be done every frame in preprocess: + // a load_npy() disk read and a create_static_box_mask() rebuild). + model_matrix_ = face_utils::load_npy(std::string(SOURCE_PATH) + "/examples/lite/resources/model_matrix.npy"); + box_mask_ = face_utils::create_static_box_mask(std::vector{128.0f, 128.0f}); + }; private: void preprocess(cv::Mat &target_face,std::vector source_image_embeding,std::vector target_landmark_5, std::vector &processed_source_embeding,cv::Mat &preprocessed_mat); private: - std::vector crop_list; cv::Mat affine_martix; + std::vector model_matrix_; // loaded once in ctor (was load_npy every frame) + cv::Mat box_mask_; // cached static 128 box mask (was rebuilt every frame) PasteBackGPU paste_back_gpu_; // GPU-fused paste-back, reused device buffers (same as restoration) public: void detect(cv::Mat &target_image,std::vector source_face_embeding,std::vector target_landmark_5, From 5885953ace66540a4e22b3db463a6f0e1a342deb Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Sun, 7 Jun 2026 11:37:46 +0800 Subject: [PATCH 19/30] feat(facefusion): split pipeline into prepare_source() + process() (source caching) The face-swap source is fixed across a video / session, so re-running detect+landmark+recognize on it every frame is wasted work. Split the pipeline: prepare_source(src,idx) caches the source embedding once; process(tgt,idx)->Mat runs per frame (detect+landmark target, swap with cached source, restore). detect(src,tgt) kept as a one-shot convenience. The bench now mirrors real video use (prepare source once, time per-frame process). 4090, fp16 + mixed-gfpgan, compute-only: per-frame 38 -> 27.9 ms (26 -> 36 FPS) -- the ~10ms source branch (detect_src+landmark_src+recognizer) is off the per-frame path. Output verified unchanged. Co-Authored-By: Claude Opus 4.8 --- .../test_lite_facefusion_pipeline_bench.cpp | 13 ++++-- lite/trt/cv/trt_facefusion_pipeline.cpp | 40 +++++++++++++------ lite/trt/cv/trt_facefusion_pipeline.h | 18 ++++++++- 3 files changed, 52 insertions(+), 19 deletions(-) diff --git a/examples/lite/cv/test_lite_facefusion_pipeline_bench.cpp b/examples/lite/cv/test_lite_facefusion_pipeline_bench.cpp index 5a6496b2..98b07bcd 100644 --- a/examples/lite/cv/test_lite_facefusion_pipeline_bench.cpp +++ b/examples/lite/cv/test_lite_facefusion_pipeline_bench.cpp @@ -54,11 +54,16 @@ int main(int argc, char *argv[]) { } std::cout << "[bench] source=" << source_img << " target=" << target_img << "\n[bench] warmup=" << warmup << " iters=" << iters - << " (compute-only: imread/imwrite excluded)" << std::endl; + << " (compute-only, video-style: prepare_source once + per-frame process)" << std::endl; + + // Video / server use case: the SOURCE face is fixed, so prepare it ONCE and then time only + // the per-frame process(target). This is what the source-embedding cache buys — the loop no + // longer re-runs detect_src / landmark_src / recognizer every frame. + pipeline.prepare_source(src, 0); // Warmup (lazy engine/context init, cudnn autotune) — excluded from stats. for (int i = 0; i < warmup; ++i) - pipeline.detect(src, 0, tgt, 0); + pipeline.process(tgt, 0); lite::bench::Profiler prof; cv::Mat out; @@ -73,11 +78,11 @@ int main(int argc, char *argv[]) { } lite::bench::CpuTimer t; t.start(); - out = pipeline.detect(src, 0, tgt, 0, &prof); + out = pipeline.process(tgt, 0, &prof); prof.tick(t.stop_ms()); } - prof.report("FaceFusion pipeline (per-stage, compute-only)"); + prof.report("FaceFusion pipeline (per-frame, source cached)"); prof.to_csv(csv_path); if (!out.empty()) { cv::imwrite(out_path, out); // save one result (outside the timed loop) for visual check diff --git a/lite/trt/cv/trt_facefusion_pipeline.cpp b/lite/trt/cv/trt_facefusion_pipeline.cpp index 01cc0d80..901ea0fe 100644 --- a/lite/trt/cv/trt_facefusion_pipeline.cpp +++ b/lite/trt/cv/trt_facefusion_pipeline.cpp @@ -34,18 +34,14 @@ TRTFaceFusionPipeLine::TRTFaceFusionPipeLine(const std::string &face_detect_engi face_restoration = std::make_unique(face_restoration_engine_path,1); } -// Compute-only core: in-memory images in, restored frame out. NO disk I/O. -// Per-stage timing is opt-in via prof (LITE_CPU_SCOPE_OPT is zero-overhead when null); -// each stage returns a host-visible result, so CPU-side wall-clock timing is accurate. -cv::Mat TRTFaceFusionPipeLine::detect(const cv::Mat &source_image, int src_index, - const cv::Mat &target_image, int target_index, - lite::bench::Profiler *prof) { +// Run the SOURCE branch once (detect -> landmark -> recognize) and cache its embedding, +// so per-frame process() only has to handle the target. NO disk I/O. Per-stage timing is +// opt-in via prof (LITE_CPU_SCOPE_OPT is zero-overhead when null). +void TRTFaceFusionPipeLine::prepare_source(const cv::Mat &source_image, int src_index, + lite::bench::Profiler *prof) { if (source_image.empty()) throw std::runtime_error("[FaceFusion] source image is empty"); - if (target_image.empty()) - throw std::runtime_error("[FaceFusion] target image is empty"); - // ---- source: detect -> landmarks -> recognizer -> embedding ---- cv::Mat img_bgr = source_image.clone(); // sub-models take a non-const cv::Mat& cv::Mat img_bgr_src = img_bgr.clone(); @@ -68,11 +64,20 @@ cv::Mat TRTFaceFusionPipeLine::detect(const cv::Mat &source_image, int src_index { LITE_CPU_SCOPE_OPT(prof, "landmark_src"); face_landmarks->detect(img_bgr, src_final_boxes[src_pick], face_landmark_5of68); } - std::vector source_image_embeding; { LITE_CPU_SCOPE_OPT(prof, "recognizer"); - face_recognizer->detect(img_bgr_src, face_landmark_5of68, source_image_embeding); } + face_recognizer->detect(img_bgr_src, face_landmark_5of68, source_embedding_); } + source_ready_ = true; +} + +// Per target frame: detect + landmark on the target, swap the cached source face, restore. +// NO disk I/O. Requires a prior prepare_source(). +cv::Mat TRTFaceFusionPipeLine::process(const cv::Mat &target_image, int target_index, + lite::bench::Profiler *prof) { + if (!source_ready_) + throw std::runtime_error("[FaceFusion] process() called before prepare_source()"); + if (target_image.empty()) + throw std::runtime_error("[FaceFusion] target image is empty"); - // ---- target: detect -> landmarks ---- cv::Mat target_img_bgr = target_image.clone(); std::vector target_detected_boxes; @@ -97,7 +102,7 @@ cv::Mat TRTFaceFusionPipeLine::detect(const cv::Mat &source_image, int src_index // ---- swap + restore (restore() returns the frame; no disk write) ---- cv::Mat face_swap_image; { LITE_CPU_SCOPE_OPT(prof, "swap"); - face_swap->detect(target_img_bgr, source_image_embeding, target_face_landmark_5of68, face_swap_image); } + face_swap->detect(target_img_bgr, source_embedding_, target_face_landmark_5of68, face_swap_image); } cv::Mat result; { LITE_CPU_SCOPE_OPT(prof, "restoration"); @@ -105,6 +110,15 @@ cv::Mat TRTFaceFusionPipeLine::detect(const cv::Mat &source_image, int src_index return result; } +// Convenience one-shot: prepare the source then process the target (recomputes the source +// embedding on every call — for video, call prepare_source() once and process() per frame). +cv::Mat TRTFaceFusionPipeLine::detect(const cv::Mat &source_image, int src_index, + const cv::Mat &target_image, int target_index, + lite::bench::Profiler *prof) { + prepare_source(source_image, src_index, prof); + return process(target_image, target_index, prof); +} + // Convenience wrapper: file paths in, result written to disk. Thin layer over the // in-memory core; imread/imwrite are timed separately when a Profiler is passed. void TRTFaceFusionPipeLine::detect(const std::string &source_image, int src_index, diff --git a/lite/trt/cv/trt_facefusion_pipeline.h b/lite/trt/cv/trt_facefusion_pipeline.h index 6ca4c349..b30227ed 100644 --- a/lite/trt/cv/trt_facefusion_pipeline.h +++ b/lite/trt/cv/trt_facefusion_pipeline.h @@ -35,9 +35,23 @@ namespace trtcv{ std::unique_ptr face_detect_mt; std::unique_ptr face_landmarks_mt; + std::vector source_embedding_; // cached by prepare_source() + bool source_ready_ = false; + public: - // Compute-only, in-memory: decoded images in -> restored frame out, no disk I/O. - // This is the path benchmarks and real (video / server) use should call. + // ---- Split API (the right shape for video / server: the SOURCE face is usually + // fixed, so its embedding is computed ONCE and reused across many target frames). ---- + + // Run detect + landmark + recognize on the source image once; cache its embedding. + void prepare_source(const cv::Mat &source_image, int src_index, + lite::bench::Profiler *prof = nullptr); + + // Per target frame: swap the cached source face onto the target and restore. No disk + // I/O. Requires a prior prepare_source(). + cv::Mat process(const cv::Mat &target_image, int target_index, + lite::bench::Profiler *prof = nullptr); + + // Convenience one-shot: prepare_source() + process() (recomputes source every call). cv::Mat detect(const cv::Mat &source_image, int src_index, const cv::Mat &target_image, int target_index, lite::bench::Profiler *prof = nullptr); From d535d8ab0b37a7d1c6e43014480a4fd5e4bde08a Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Sun, 7 Jun 2026 15:14:43 +0800 Subject: [PATCH 20/30] perf(trt): fold restoration postprocess RGB->BGR + uint8->float into the GPU kernel The restoration transpose kernel emitted uint8 RGB, forcing a CPU uint8->float copy + cv::cvtColor(RGB2BGR) + a clone() before paste-back. Make the kernel write HWC BGR float[0,255] directly; the cv::Mat now aliases that buffer (no clone). Removes ~1.3ms of CPU glue. 4090, mixed-gfpgan, restoration bench: postprocess 3.33 -> 2.02 ms (transpose+dl 1.45->0.57, cvtColor folded away), restoration 12.6 -> 11.2 ms. Output verified color-correct. Co-Authored-By: Claude Opus 4.8 --- lite/trt/cv/trt_face_restoration.cpp | 25 +++++++++++-------- .../kernel/face_restoration_postprocess.cu | 19 +++++++------- .../kernel/face_restoration_postprocess.cuh | 4 +-- .../face_restoration_postprocess_manager.cpp | 17 +++---------- .../face_restoration_postprocess_manager.h | 2 +- 5 files changed, 32 insertions(+), 35 deletions(-) diff --git a/lite/trt/cv/trt_face_restoration.cpp b/lite/trt/cv/trt_face_restoration.cpp index 214190d8..e7c6f707 100644 --- a/lite/trt/cv/trt_face_restoration.cpp +++ b/lite/trt/cv/trt_face_restoration.cpp @@ -58,15 +58,17 @@ cv::Mat TRTFaceFusionFaceRestoration::restore(cv::Mat &face_swap_image, cv::Mat dst_image; { LITE_CPU_SCOPE_OPT(prof, "postprocess"); - std::vector transposed_data(1 * 3 * 512 * 512); - launch_face_restoration_postprocess( - static_cast(buffers[1]), transposed_data.data(), 3, 512, 512); - std::vector transposed_data_float(transposed_data.begin(), transposed_data.end()); - cudaStreamSynchronize(stream); - - int height = 512, width = 512; - cv::Mat mat(height, width, CV_32FC3, transposed_data_float.data()); - cv::cvtColor(mat, mat, cv::COLOR_RGB2BGR); + const int height = 512, width = 512; + std::vector transposed_data(1 * 3 * 512 * 512); + { + // GPU kernel writes HWC, BGR, float[0,255] straight into transposed_data, + // folding the old CPU uint8->float conversion + cv::cvtColor(RGB2BGR). + LITE_CPU_SCOPE_OPT(prof, " transpose+dl"); + launch_face_restoration_postprocess( + static_cast(buffers[1]), transposed_data.data(), 3, 512, 512); + } + // aliases transposed_data (alive for the rest of this scope, i.e. through paste_back) + cv::Mat mat(height, width, CV_32FC3, transposed_data.data()); cv::Mat paste_frame; { @@ -75,7 +77,10 @@ cv::Mat TRTFaceFusionFaceRestoration::restore(cv::Mat &face_swap_image, LITE_CPU_SCOPE_OPT(prof, " paste_back"); paste_frame = paste_back_gpu_.paste_back(ori_image, mat, box_mask, affine_matrix, stream); } - dst_image = face_utils::blend_frame(ori_image, paste_frame); + { + LITE_CPU_SCOPE_OPT(prof, " blend"); + dst_image = face_utils::blend_frame(ori_image, paste_frame); + } } return dst_image; diff --git a/lite/trt/kernel/face_restoration_postprocess.cu b/lite/trt/kernel/face_restoration_postprocess.cu index 8dacd0d0..646a1d6b 100644 --- a/lite/trt/kernel/face_restoration_postprocess.cu +++ b/lite/trt/kernel/face_restoration_postprocess.cu @@ -18,8 +18,8 @@ __device__ unsigned char float_to_uint8_simple(float x) { // 主kernel函数 __global__ void face_restoration_postprocess( - float* input_buffer, // 输入数据（TRT输出，CHW格式） - unsigned char* output_final, // 最终输出（HWC格式，uint8） + float* input_buffer, // 输入数据（TRT输出，CHW格式，RGB） + float* output_final, // output: HWC, BGR, float in [0,255] int channel, int height, int width @@ -28,17 +28,18 @@ __global__ void face_restoration_postprocess( int total_size = channel * height * width; if (idx >= total_size) return; - // 第一步：范围处理 + // clamp + (x+1)/2 -> [0,1] float processed = process_range_single(input_buffer[idx]); - // 第二步：计算CHW中的位置 + // CHW position int c = idx / (height * width); int h = (idx % (height * width)) / width; int w = idx % width; - // 第三步：计算HWC位置并转换 - int hwc_idx = get_hwc_index(c, h, w, channel, width); - - // 第四步：转换为uint8并写入输出 - output_final[hwc_idx] = float_to_uint8_simple(processed); + // Write directly as HWC, BGR, float in [0,255]: this folds the old CPU uint8->float + // conversion AND the cv::cvtColor(RGB2BGR) into the kernel. Model channels are RGB + // (0,1,2) -> BGR positions (2,1,0). + int out_c = channel - 1 - c; + int hwc_idx = get_hwc_index(out_c, h, w, channel, width); + output_final[hwc_idx] = processed * 255.f; } diff --git a/lite/trt/kernel/face_restoration_postprocess.cuh b/lite/trt/kernel/face_restoration_postprocess.cuh index adc276e1..b047e91c 100644 --- a/lite/trt/kernel/face_restoration_postprocess.cuh +++ b/lite/trt/kernel/face_restoration_postprocess.cuh @@ -1,7 +1,7 @@ #include "cuda_runtime.h" extern "C" __global__ void face_restoration_postprocess( - float* input_buffer, // 输入数据（TRT输出，CHW格式） - unsigned char* output_final, // 最终输出（HWC格式，uint8） + float* input_buffer, // 输入数据（TRT输出，CHW格式，RGB） + float* output_final, // output: HWC, BGR, float [0,255] int channel, int height, int width diff --git a/lite/trt/kernel/face_restoration_postprocess_manager.cpp b/lite/trt/kernel/face_restoration_postprocess_manager.cpp index f1ace60b..d21e37e3 100644 --- a/lite/trt/kernel/face_restoration_postprocess_manager.cpp +++ b/lite/trt/kernel/face_restoration_postprocess_manager.cpp @@ -5,23 +5,18 @@ #include "face_restoration_postprocess_manager.h" void launch_face_restoration_postprocess( float* trt_outputs, - unsigned char* output_final, + float* output_final, // HWC, BGR, float [0,255] int channel, int height, int width ){ - // 设计grid和block的尺寸 block直接设置为256的最大值 int block_size = 256; int vec_num = channel * height * width; int grid_size = ( vec_num + block_size - 1) / block_size; - // GPU上的内存空间 - unsigned char* d_output_final; - int* d_output_count; - // 在GPU上分配输出的空间 - cudaMalloc(&d_output_final,vec_num * sizeof(unsigned char )); + float* d_output_final; + cudaMalloc(&d_output_final, vec_num * sizeof(float)); - // 启动内核 face_restoration_postprocess<<>>( trt_outputs, d_output_final, @@ -35,11 +30,7 @@ void launch_face_restoration_postprocess( printf("CUDA error: %s\n", cudaGetErrorString(error)); } - // 将生成的数据复制出来 - cudaMemcpy(output_final,d_output_final,vec_num * sizeof(unsigned char ), + cudaMemcpy(output_final, d_output_final, vec_num * sizeof(float), cudaMemcpyDeviceToHost); - - // 释放cuda上的内存 cudaFree(d_output_final); - } \ No newline at end of file diff --git a/lite/trt/kernel/face_restoration_postprocess_manager.h b/lite/trt/kernel/face_restoration_postprocess_manager.h index 57db9e11..00834e90 100644 --- a/lite/trt/kernel/face_restoration_postprocess_manager.h +++ b/lite/trt/kernel/face_restoration_postprocess_manager.h @@ -11,7 +11,7 @@ void launch_face_restoration_postprocess( float* trt_outputs, - unsigned char* output_final, + float* output_final, // HWC, BGR, float [0,255] int channel, int height, int width From 75ec4de5e493d164053ec28b1a8c73c1f22974c5 Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Sun, 7 Jun 2026 16:00:11 +0800 Subject: [PATCH 21/30] perf(trt): GPU-resident NPP warp for face-restoration preprocess (device-pipeline brick) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the CPU cv::warpAffine in restoration preprocess with an NPP affine warp whose output stays on the GPU, feeding the fused bgr2rgb+normalize+CHW kernel directly — the 512 crop no longer makes a D2H/H2D round-trip. - face_utils: split estimate_affine_by_landmark_5 out of warp_face_by_face_landmark_5 so the affine estimate stays on CPU while the warp moves to the GPU. - WarpAffineNpp::warp_to_device: nppiWarpAffine into an internal device buffer on the caller's stream, returns the device crop pointer (no D2H, no sync). nppSetStream for correct ordering with the consumer kernel. - FaceRestorePreprocessGPU::run_device: launch the fused preprocess kernel straight on a device crop pointer (skips the H2D the cv::Mat path needed). - cmake: link NPP (nppc/nppig/nppidei), ships with CUDA — no new dependency. Validated on RTX 4090: output PSNR 55.63 dB vs the CPU-warp result (max|diff|=11, numerically identical), memory flat (no leak), 28.6 FPS. This is the first device- resident brick toward keeping frames GPU-resident across the whole pipeline. Co-Authored-By: Claude Opus 4.8 --- cmake/utils.cmake | 3 +- lite/ort/cv/face_utils.cpp | 13 ++- lite/ort/cv/face_utils.h | 4 + lite/trt/cv/trt_face_restoration.cpp | 21 ++-- lite/trt/cv/trt_face_restoration.h | 2 + .../face_restoration_preprocess_manager.cpp | 8 ++ .../face_restoration_preprocess_manager.h | 4 + lite/trt/kernel/warp_affine_npp.cpp | 99 +++++++++++++++++++ lite/trt/kernel/warp_affine_npp.h | 38 +++++++ 9 files changed, 179 insertions(+), 13 deletions(-) create mode 100644 lite/trt/kernel/warp_affine_npp.cpp create mode 100644 lite/trt/kernel/warp_affine_npp.h diff --git a/cmake/utils.cmake b/cmake/utils.cmake index 63157299..498b8c92 100644 --- a/cmake/utils.cmake +++ b/cmake/utils.cmake @@ -53,7 +53,8 @@ function(add_lite_ai_toolkit_shared_library version soversion) include(cmake/tensorrt.cmake) set(LITE_SRCS ${LITE_SRCS} ${TRT_SRCS}) set(LITE_DEPENDENCIES ${LITE_DEPENDENCIES} cuda cudart nvinfer nvonnxparser - nvinfer_plugin ddim_scheduler_cpp) + nvinfer_plugin ddim_scheduler_cpp + nppc nppig nppidei) # NPP: GPU warp/resize link_directories(${CMAKE_SOURCE_DIR}/lite/bin) endif () diff --git a/lite/ort/cv/face_utils.cpp b/lite/ort/cv/face_utils.cpp index 56df2ee6..68f0a77e 100644 --- a/lite/ort/cv/face_utils.cpp +++ b/lite/ort/cv/face_utils.cpp @@ -101,9 +101,8 @@ namespace face_utils } -std::pair -face_utils::warp_face_by_face_landmark_5(cv::Mat input_mat, std::vector face_landmark_5, - unsigned int type) { +cv::Mat +face_utils::estimate_affine_by_landmark_5(std::vector face_landmark_5, unsigned int type) { std::vector current_template_select; if (type == face_utils::ARCFACE_112_V2) @@ -144,6 +143,14 @@ face_utils::warp_face_by_face_landmark_5(cv::Mat input_mat, std::vector +face_utils::warp_face_by_face_landmark_5(cv::Mat input_mat, std::vector face_landmark_5, + unsigned int type) { + + cv::Mat affine_matrix = estimate_affine_by_landmark_5(face_landmark_5, type); // 进行仿射变换 cv::Mat crop_img; diff --git a/lite/ort/cv/face_utils.h b/lite/ort/cv/face_utils.h index 08f5b24b..4a712f25 100644 --- a/lite/ort/cv/face_utils.h +++ b/lite/ort/cv/face_utils.h @@ -27,6 +27,10 @@ namespace face_utils std::pair warp_face_by_face_landmark_5(cv::Mat input_mat, std::vector face_landmark_5,unsigned int type); + // Just the affine estimate (no CPU warp) — lets the warp run on the GPU (NPP) while keeping + // the same 2x3 source->template matrix used by warp_face_by_face_landmark_5. + cv::Mat estimate_affine_by_landmark_5(std::vector face_landmark_5, unsigned int type); + std::vector convert_face_landmark_68_to_5(const std::vector& landmark_68); cv::Mat blend_frame(const cv::Mat &target_image, const cv::Mat &paste_frame); diff --git a/lite/trt/cv/trt_face_restoration.cpp b/lite/trt/cv/trt_face_restoration.cpp index e7c6f707..0ddbc994 100644 --- a/lite/trt/cv/trt_face_restoration.cpp +++ b/lite/trt/cv/trt_face_restoration.cpp @@ -14,17 +14,18 @@ cv::Mat TRTFaceFusionFaceRestoration::restore(cv::Mat &face_swap_image, lite::bench::Profiler *prof) { auto ori_image = face_swap_image.clone(); - cv::Mat crop_image; cv::Mat affine_matrix; cv::Mat box_mask; - // ---------------- preprocess (CPU): warp + bgr2rgb + normalize + build tensor ---------------- + // ---------------- preprocess: estimate affine (CPU) -> GPU warp (NPP) -> fused CHW tensor ------ { LITE_CPU_SCOPE_OPT(prof, "preprocess"); { - LITE_CPU_SCOPE_OPT(prof, " warp"); - std::tie(crop_image, affine_matrix) = face_utils::warp_face_by_face_landmark_5( - face_swap_image, target_landmarks_5, face_utils::FFHQ_512); + // Only the affine estimate stays on the CPU; the warp itself runs on the GPU and the + // warped 512 crop stays device-resident (no D2H/H2D round-trip for the crop). + LITE_CPU_SCOPE_OPT(prof, " estimate_affine"); + affine_matrix = face_utils::estimate_affine_by_landmark_5( + target_landmarks_5, face_utils::FFHQ_512); } { // the static box mask only depends on the (fixed) 512 crop size, so build it @@ -36,10 +37,12 @@ cv::Mat TRTFaceFusionFaceRestoration::restore(cv::Mat &face_swap_image, } { - // GPU fused: bgr2rgb + normalize + HWC->CHW written straight into the inference - // input buffer (buffers[0]) — also removes the separate H2D below. - LITE_CPU_SCOPE_OPT(prof, " to_chw(gpu)"); - preprocess_gpu_.run(crop_image, static_cast(buffers[0]), stream); + // GPU NPP warp -> device crop, then fused bgr2rgb+normalize+HWC->CHW straight into the + // inference input buffer (buffers[0]). The crop never leaves the GPU between the two. + LITE_CPU_SCOPE_OPT(prof, " warp+to_chw(gpu)"); + const unsigned char *d_crop = warp_npp_.warp_to_device( + face_swap_image, affine_matrix, 512, stream); + preprocess_gpu_.run_device(d_crop, 512, 512, static_cast(buffers[0]), stream); } } diff --git a/lite/trt/cv/trt_face_restoration.h b/lite/trt/cv/trt_face_restoration.h index ea46ea55..e5b798b7 100644 --- a/lite/trt/cv/trt_face_restoration.h +++ b/lite/trt/cv/trt_face_restoration.h @@ -12,6 +12,7 @@ #include "lite/trt/kernel/face_restoration_preprocess_manager.h" #include "lite/trt/kernel/bgr2rgb_manager.h" #include "lite/trt/kernel/paste_back_manager.h" +#include "lite/trt/kernel/warp_affine_npp.h" // Forward declaration for benchmark timing; library passes nullptr by default (zero overhead) namespace lite { namespace bench { class Profiler; } } @@ -33,6 +34,7 @@ namespace trtcv{ private: PasteBackGPU paste_back_gpu_; // GPU fused paste_back, reuses device buffers FaceRestorePreprocessGPU preprocess_gpu_; // GPU fused bgr2rgb+normalize+CHW into input buffer + WarpAffineNpp warp_npp_; // GPU (NPP) affine warp; crop stays device-resident cv::Mat box_mask_cache_; // static box mask is size-only; compute once and reuse }; diff --git a/lite/trt/kernel/face_restoration_preprocess_manager.cpp b/lite/trt/kernel/face_restoration_preprocess_manager.cpp index e0f46af1..4a582516 100644 --- a/lite/trt/kernel/face_restoration_preprocess_manager.cpp +++ b/lite/trt/kernel/face_restoration_preprocess_manager.cpp @@ -35,3 +35,11 @@ void FaceRestorePreprocessGPU::run(const cv::Mat& crop_bgr_u8, float* d_out, cud cudaStreamSynchronize(stream); } + +void FaceRestorePreprocessGPU::run_device(const unsigned char* d_crop, int H, int W, + float* d_out, cudaStream_t stream) { + dim3 block(16, 16); + dim3 grid((W + block.x - 1) / block.x, (H + block.y - 1) / block.y); + face_restoration_preprocess_kernel<<>>(d_crop, d_out, H, W); + cudaStreamSynchronize(stream); +} diff --git a/lite/trt/kernel/face_restoration_preprocess_manager.h b/lite/trt/kernel/face_restoration_preprocess_manager.h index 9d5a076f..864c5558 100644 --- a/lite/trt/kernel/face_restoration_preprocess_manager.h +++ b/lite/trt/kernel/face_restoration_preprocess_manager.h @@ -18,6 +18,10 @@ class FaceRestorePreprocessGPU { // crop_bgr_u8: CV_8UC3 (e.g. 512x512). d_out: device float CHW buffer (the inference input). void run(const cv::Mat& crop_bgr_u8, float* d_out, cudaStream_t stream = nullptr); + // Device-resident variant: the crop is already on the GPU (e.g. NPP warp output), so skip the + // H2D — just launch the fused kernel reading d_crop -> d_out on `stream`. Syncs before return. + void run_device(const unsigned char* d_crop, int H, int W, float* d_out, cudaStream_t stream = nullptr); + private: void ensure_capacity(size_t bytes); diff --git a/lite/trt/kernel/warp_affine_npp.cpp b/lite/trt/kernel/warp_affine_npp.cpp new file mode 100644 index 00000000..04e8528b --- /dev/null +++ b/lite/trt/kernel/warp_affine_npp.cpp @@ -0,0 +1,99 @@ +#include "warp_affine_npp.h" +#include +#include +#include + +// Fill the NPP 2x3 coefficient array from a cv 2x3 affine (same forward src->dst convention). +static void fill_coeffs(const cv::Mat& affine_2x3, double aCoeffs[2][3]) { + cv::Mat M64; + affine_2x3.convertTo(M64, CV_64F); + for (int i = 0; i < 2; ++i) + for (int j = 0; j < 3; ++j) aCoeffs[i][j] = M64.at(i, j); +} + +WarpAffineNpp::~WarpAffineNpp() { + if (d_src_) cudaFree(d_src_); + if (d_dst_) cudaFree(d_dst_); + if (h_src_pinned_) cudaFreeHost(h_src_pinned_); + if (h_dst_pinned_) cudaFreeHost(h_dst_pinned_); +} + +void WarpAffineNpp::ensure(size_t src_bytes, size_t dst_bytes) { + if (src_bytes > cap_src_) { + if (d_src_) cudaFree(d_src_); + if (h_src_pinned_) cudaFreeHost(h_src_pinned_); + cudaMalloc(&d_src_, src_bytes); + cudaMallocHost(&h_src_pinned_, src_bytes); + cap_src_ = src_bytes; + } + if (dst_bytes > cap_dst_) { + if (d_dst_) cudaFree(d_dst_); + if (h_dst_pinned_) cudaFreeHost(h_dst_pinned_); + cudaMalloc(&d_dst_, dst_bytes); + cudaMallocHost(&h_dst_pinned_, dst_bytes); + cap_dst_ = dst_bytes; + } +} + +cv::Mat WarpAffineNpp::warp(const cv::Mat& frame_bgr_u8, const cv::Mat& affine_2x3, int out_size) { + cv::Mat src = frame_bgr_u8; + if (src.type() != CV_8UC3) src.convertTo(src, CV_8UC3); + if (!src.isContinuous()) src = src.clone(); + + const int SW = src.cols, SH = src.rows; + const int DW = out_size, DH = out_size; + const size_t src_bytes = static_cast(SW) * SH * 3; + const size_t dst_bytes = static_cast(DW) * DH * 3; + ensure(src_bytes, dst_bytes); + + std::memcpy(h_src_pinned_, src.data, src_bytes); + cudaMemcpy(d_src_, h_src_pinned_, src_bytes, cudaMemcpyHostToDevice); + cudaMemset(d_dst_, 0, dst_bytes); + + double aCoeffs[2][3]; + fill_coeffs(affine_2x3, aCoeffs); + + NppiSize srcSize{SW, SH}; + NppiRect srcROI{0, 0, SW, SH}; + NppiRect dstROI{0, 0, DW, DH}; + // Same convention as cv::warpAffine(src,dst,M) (no WARP_INVERSE_MAP): forward src->dst M. + nppiWarpAffine_8u_C3R(d_src_, srcSize, SW * 3, srcROI, + d_dst_, DW * 3, dstROI, + aCoeffs, NPPI_INTER_LINEAR); + + cudaMemcpy(h_dst_pinned_, d_dst_, dst_bytes, cudaMemcpyDeviceToHost); + cv::Mat dst(DH, DW, CV_8UC3, h_dst_pinned_); + return dst.clone(); +} + +const unsigned char* WarpAffineNpp::warp_to_device(const cv::Mat& frame_bgr_u8, + const cv::Mat& affine_2x3, + int out_size, cudaStream_t stream) { + cv::Mat src = frame_bgr_u8; + if (src.type() != CV_8UC3) src.convertTo(src, CV_8UC3); + if (!src.isContinuous()) src = src.clone(); + + const int SW = src.cols, SH = src.rows; + const int DW = out_size, DH = out_size; + const size_t src_bytes = static_cast(SW) * SH * 3; + const size_t dst_bytes = static_cast(DW) * DH * 3; + ensure(src_bytes, dst_bytes); + + // frame H2D on the caller's stream (still needed — frame lives on host); the warped crop then + // stays on device (d_dst_) to feed the next GPU stage directly. No D2H, no sync here. + std::memcpy(h_src_pinned_, src.data, src_bytes); + cudaMemcpyAsync(d_src_, h_src_pinned_, src_bytes, cudaMemcpyHostToDevice, stream); + cudaMemsetAsync(d_dst_, 0, dst_bytes, stream); + + double aCoeffs[2][3]; + fill_coeffs(affine_2x3, aCoeffs); + + NppiSize srcSize{SW, SH}; + NppiRect srcROI{0, 0, SW, SH}; + NppiRect dstROI{0, 0, DW, DH}; + nppSetStream(stream); + nppiWarpAffine_8u_C3R(d_src_, srcSize, SW * 3, srcROI, + d_dst_, DW * 3, dstROI, + aCoeffs, NPPI_INTER_LINEAR); + return d_dst_; +} diff --git a/lite/trt/kernel/warp_affine_npp.h b/lite/trt/kernel/warp_affine_npp.h new file mode 100644 index 00000000..346ac445 --- /dev/null +++ b/lite/trt/kernel/warp_affine_npp.h @@ -0,0 +1,38 @@ +#ifndef LITE_AI_TOOLKIT_WARP_AFFINE_NPP_H +#define LITE_AI_TOOLKIT_WARP_AFFINE_NPP_H + +#include +#include + +// GPU affine warp via NPP (nppiWarpAffine), reusing device + pinned staging buffers. +// Replaces cv::warpAffine for the face-crop warps. Same affine convention as +// cv::warpAffine(src, dst, M): M is the 2x3 source->template transform from +// estimateAffinePartial2D. (Device-resident variant returning a device pointer comes next; +// this host->host version is the de-risk step to validate the NPP convention + quality.) +class WarpAffineNpp { +public: + WarpAffineNpp() = default; + ~WarpAffineNpp(); + WarpAffineNpp(const WarpAffineNpp&) = delete; + WarpAffineNpp& operator=(const WarpAffineNpp&) = delete; + + // frame_bgr_u8: CV_8UC3 full frame. affine_2x3: 2x3 (CV_32F/64F). out_size: crop is out_size x out_size. + cv::Mat warp(const cv::Mat& frame_bgr_u8, const cv::Mat& affine_2x3, int out_size); + + // Device-resident variant: warps into the internal device buffer and returns a device pointer + // to the out_size x out_size interleaved BGR uint8 crop (no D2H). The frame H2D + NPP warp run + // on `stream`; caller must use the same stream for the consumer (no sync here). The returned + // pointer is owned by this object and valid until the next warp_to_device/warp call. + const unsigned char* warp_to_device(const cv::Mat& frame_bgr_u8, const cv::Mat& affine_2x3, + int out_size, cudaStream_t stream = nullptr); + +private: + void ensure(size_t src_bytes, size_t dst_bytes); + unsigned char* d_src_ = nullptr; + unsigned char* d_dst_ = nullptr; + unsigned char* h_src_pinned_ = nullptr; + unsigned char* h_dst_pinned_ = nullptr; + size_t cap_src_ = 0, cap_dst_ = 0; +}; + +#endif // LITE_AI_TOOLKIT_WARP_AFFINE_NPP_H From c84235608909c3975163ccf4b78fc58a5fdd4ab3 Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Sun, 7 Jun 2026 16:10:56 +0800 Subject: [PATCH 22/30] perf(facefusion): ship mixed-precision GFPGAN by default (clean FP16, -3ms restoration) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The grey-block problem that forced GFPGAN to FP32 is solved: build_gfpgan_fp16_engine.py keeps only the StyleGAN style_conv/to_rgb layers in FP32 and runs the rest in FP16. That output is numerically identical to the FP32 engine (PSNR 57.8 dB on the test pair) while cutting the restoration stage 15.1->12.1 ms (pipeline 28.6->31.2 FPS on an RTX 4090). A naive --fp16 GFPGAN (or an over-aggressive pin) instead produces a grey halo around the pasted-back face (PSNR ~20 dB) — verified by A/B, so it is NOT used. - build_facefusion_engines.sh: GFPGAN now built via build_gfpgan_fp16_engine.py as gfpgan_1.4_mixed.engine; GFPGAN_FP32=1 falls back to the plain FP32 engine. - facefusion CLI: default restoration engine is gfpgan_1.4_mixed.engine, falling back to gfpgan_1.4_fp32.engine if the mixed one isn't present. - quickstart doc: document the mixed build + the TensorRT python-wheel requirement. Co-Authored-By: Claude Opus 4.8 --- build_facefusion_engines.sh | 21 +++++++++++++++---- docs/facefusion_quickstart.md | 16 ++++++++++---- examples/lite/cv/test_lite_facefusion_cli.cpp | 15 +++++++++++-- 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/build_facefusion_engines.sh b/build_facefusion_engines.sh index 2d1dab4c..ccedafbc 100755 --- a/build_facefusion_engines.sh +++ b/build_facefusion_engines.sh @@ -13,12 +13,11 @@ set -euo pipefail ONNX_DIR="${1:?usage: $0 }" ENGINE_DIR="${2:?usage: $0 }" TRTEXEC="${TRTEXEC:-trtexec}" +PYTHON="${PYTHON:-python3}" +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" mkdir -p "$ENGINE_DIR" -# onnx_basename engine_basename extra_flags -# GFPGAN stays FP32 on purpose: FP16 makes its StyleGAN modulated convs blow up -# (grey-block artifacts). The other four are fine in FP16. build() { local onnx="$ONNX_DIR/$1" engine="$ENGINE_DIR/$2"; shift 2 if [[ ! -f "$onnx" ]]; then @@ -35,6 +34,20 @@ build yoloface_8n.onnx yoloface_8n_fp16.engine --fp16 build 2dfan4.onnx 2dfan4_fp16.engine --fp16 build arcface_w600k_r50.onnx arcface_w600k_r50_fp16.engine --fp16 build inswapper_128.onnx inswapper_128_fp16.engine --fp16 -build gfpgan_1.4.onnx gfpgan_1.4_fp32.engine + +# GFPGAN: a naive --fp16 engine blows up its StyleGAN modulated convs (grey-block / a grey +# halo around the pasted-back face). The fix is mixed precision — FP16 everywhere except the +# style_conv/to_rgb layers, which stay FP32 (build_gfpgan_fp16_engine.py). That is numerically +# identical to the FP32 engine (PSNR ~58 dB) while cutting the restoration stage ~3 ms. +# Needs the TensorRT 10.x python wheel on $PYTHON; set GFPGAN_FP32=1 to fall back to plain FP32. +GFPGAN_ENGINE="$ENGINE_DIR/gfpgan_1.4_mixed.engine" +if [[ "${GFPGAN_FP32:-0}" == "1" ]]; then + build gfpgan_1.4.onnx gfpgan_1.4_fp32.engine +elif [[ -f "$GFPGAN_ENGINE" ]]; then + echo "[build_facefusion_engines] skip (exists): $GFPGAN_ENGINE" +else + echo "[build_facefusion_engines] gfpgan_1.4.onnx -> $GFPGAN_ENGINE (mixed fp16, style layers fp32)" + "$PYTHON" "$HERE/build_gfpgan_fp16_engine.py" "$ONNX_DIR/gfpgan_1.4.onnx" "$GFPGAN_ENGINE" +fi echo "[build_facefusion_engines] done -> $ENGINE_DIR" diff --git a/docs/facefusion_quickstart.md b/docs/facefusion_quickstart.md index f76ce521..a09fa550 100644 --- a/docs/facefusion_quickstart.md +++ b/docs/facefusion_quickstart.md @@ -34,10 +34,18 @@ Put all 5 in one directory, e.g. `~/ff_onnx/`. bash ./build_facefusion_engines.sh ~/ff_onnx ~/ff_engines ``` -This runs `trtexec` once per model and writes the 5 `.engine` files into `~/ff_engines/`. -GFPGAN is kept FP32 on purpose (FP16 produces grey-block artifacts on its StyleGAN -modulated convs); the other four are FP16. Engines are GPU/TensorRT-version specific — -rebuild them if you change GPU or TensorRT version. +This runs `trtexec` for four of the models (FP16) and writes the `.engine` files into +`~/ff_engines/`. GFPGAN is built as a **mixed-precision** engine via +`build_gfpgan_fp16_engine.py`: a naive `--fp16` GFPGAN blows up its StyleGAN modulated +convs (a grey halo around the pasted-back face), so the style_conv/to_rgb layers are kept +FP32 and the rest run FP16. That is numerically identical to the FP32 engine (PSNR ~58 dB) +while cutting the restoration stage ~3 ms (≈28.6 → 31 FPS on an RTX 4090). + +The mixed build needs the **TensorRT 10.x python wheel** on `python3` (ships in the TRT +tarball under `python/`, e.g. `pip install /usr/local/tensorrt/python/tensorrt-10.*-cp3*-*.whl`). +If you can't set that up, run `GFPGAN_FP32=1 bash ./build_facefusion_engines.sh ...` to fall +back to a plain FP32 GFPGAN engine. Engines are GPU/TensorRT-version specific — rebuild them +if you change GPU or TensorRT version. ## 4. Run diff --git a/examples/lite/cv/test_lite_facefusion_cli.cpp b/examples/lite/cv/test_lite_facefusion_cli.cpp index b48acf0b..5984604a 100644 --- a/examples/lite/cv/test_lite_facefusion_cli.cpp +++ b/examples/lite/cv/test_lite_facefusion_cli.cpp @@ -12,13 +12,17 @@ #include "lite/lite.h" #include #include +#include // Default engine filenames expected inside . static const char *kFaceDetectEngine = "yoloface_8n_fp16.engine"; static const char *kFaceLandmarksEngine = "2dfan4_fp16.engine"; static const char *kFaceRecognizerEngine = "arcface_w600k_r50_fp16.engine"; static const char *kFaceSwapEngine = "inswapper_128_fp16.engine"; -static const char *kFaceRestoreEngine = "gfpgan_1.4_fp32.engine"; +// Mixed-precision GFPGAN (style layers FP32, rest FP16) — clean + ~3 ms faster than plain FP32. +// Falls back to the plain FP32 engine name if the mixed one isn't present. +static const char *kFaceRestoreEngine = "gfpgan_1.4_mixed.engine"; +static const char *kFaceRestoreEngineFp32 = "gfpgan_1.4_fp32.engine"; static void usage(const char *prog) { @@ -53,12 +57,19 @@ int main(int argc, char *argv[]) (engine_dir.empty() || engine_dir.back() == '/') ? "" : "/"; auto engine = [&](const char *name) { return engine_dir + sep + name; }; + // Prefer the mixed-precision restoration engine; fall back to plain FP32 if only that exists. + std::string restore_engine = engine(kFaceRestoreEngine); + { + std::ifstream f(restore_engine); + if (!f.good()) restore_engine = engine(kFaceRestoreEngineFp32); + } + auto pipeline = lite::trt::cv::face::swap::FaceFusionPipeLine( engine(kFaceDetectEngine), engine(kFaceLandmarksEngine), engine(kFaceRecognizerEngine), engine(kFaceSwapEngine), - engine(kFaceRestoreEngine)); + restore_engine); pipeline.detect(source_img, src_idx, target_img, tgt_idx, output_img); std::cout << "[FaceFusion] wrote: " << output_img << std::endl; From 6af7f38b9c734cd0568b9b0045163c4eb04648b6 Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Sun, 7 Jun 2026 16:36:17 +0800 Subject: [PATCH 23/30] =?UTF-8?q?perf(trt):=20DeviceFrame=20brick=201=20?= =?UTF-8?q?=E2=80=94=20restoration=20uploads=20input=20frame=20once=20(sha?= =?UTF-8?q?red=20warp+paste)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First brick of the device pipeline: a DeviceFrame holds the input frame resident in device memory, so restoration's NPP warp and paste-back both read it straight from the device instead of each doing its own full-frame H2D. - DeviceFrame: upload-once / download-once full-frame (HWC BGR uint8) device buffer, reused device + pinned staging across frames. - WarpAffineNpp::warp_device_to_device: NPP warp whose source is already on the device (no H2D). - PasteBackGPU: overload taking a device temp pointer (skips the full-frame H2D); the existing cv::Mat path and the new device path now share a private run() tail. - restoration: upload face_swap_image once, warp + paste_back both read input_frame_.data(). Removes one full-frame H2D per frame. Output is bit-exact (PSNR 99 dB, max|diff|=0 vs the pre-brick result); restoration 12.1->11.7 ms, memory flat. Brick 2 (paste output stays on device + fold blend) and brick 3 (weld the swap->restoration seam) build on this. Co-Authored-By: Claude Opus 4.8 --- lite/trt/cv/trt_face_restoration.cpp | 23 ++++++++++----- lite/trt/cv/trt_face_restoration.h | 2 ++ lite/trt/kernel/device_frame.cpp | 39 ++++++++++++++++++++++++++ lite/trt/kernel/device_frame.h | 38 +++++++++++++++++++++++++ lite/trt/kernel/paste_back_manager.cpp | 37 ++++++++++++++++++++---- lite/trt/kernel/paste_back_manager.h | 13 +++++++++ lite/trt/kernel/warp_affine_npp.cpp | 23 +++++++++++++++ lite/trt/kernel/warp_affine_npp.h | 7 +++++ 8 files changed, 169 insertions(+), 13 deletions(-) create mode 100644 lite/trt/kernel/device_frame.cpp create mode 100644 lite/trt/kernel/device_frame.h diff --git a/lite/trt/cv/trt_face_restoration.cpp b/lite/trt/cv/trt_face_restoration.cpp index 0ddbc994..cf18dd79 100644 --- a/lite/trt/cv/trt_face_restoration.cpp +++ b/lite/trt/cv/trt_face_restoration.cpp @@ -37,11 +37,18 @@ cv::Mat TRTFaceFusionFaceRestoration::restore(cv::Mat &face_swap_image, } { - // GPU NPP warp -> device crop, then fused bgr2rgb+normalize+HWC->CHW straight into the - // inference input buffer (buffers[0]). The crop never leaves the GPU between the two. + // Upload the input frame to the device ONCE; both the NPP warp (here) and the + // paste_back (below) read it straight from device memory — no second full-frame H2D. + LITE_CPU_SCOPE_OPT(prof, " upload"); + input_frame_.upload(face_swap_image, stream); + } + { + // GPU NPP warp (reads the device-resident frame) -> device crop, then fused + // bgr2rgb+normalize+HWC->CHW straight into the inference input buffer (buffers[0]). LITE_CPU_SCOPE_OPT(prof, " warp+to_chw(gpu)"); - const unsigned char *d_crop = warp_npp_.warp_to_device( - face_swap_image, affine_matrix, 512, stream); + const unsigned char *d_crop = warp_npp_.warp_device_to_device( + input_frame_.data(), input_frame_.width(), input_frame_.height(), + affine_matrix, 512, stream); preprocess_gpu_.run_device(d_crop, 512, 512, static_cast(buffers[0]), stream); } } @@ -75,10 +82,12 @@ cv::Mat TRTFaceFusionFaceRestoration::restore(cv::Mat &face_swap_image, cv::Mat paste_frame; { - // GPU fused version: inverse-mapping sampling + blend in one kernel, reused - // device buffers, pinned + async copies (replaces the CPU warpAffine bottleneck). + // GPU fused: inverse-mapping sampling + blend in one kernel. temp frame is read + // straight from the device-resident input_frame_ (no full-frame H2D here). LITE_CPU_SCOPE_OPT(prof, " paste_back"); - paste_frame = paste_back_gpu_.paste_back(ori_image, mat, box_mask, affine_matrix, stream); + paste_frame = paste_back_gpu_.paste_back( + input_frame_.data(), input_frame_.width(), input_frame_.height(), + mat, box_mask, affine_matrix, stream); } { LITE_CPU_SCOPE_OPT(prof, " blend"); diff --git a/lite/trt/cv/trt_face_restoration.h b/lite/trt/cv/trt_face_restoration.h index e5b798b7..4af40df7 100644 --- a/lite/trt/cv/trt_face_restoration.h +++ b/lite/trt/cv/trt_face_restoration.h @@ -13,6 +13,7 @@ #include "lite/trt/kernel/bgr2rgb_manager.h" #include "lite/trt/kernel/paste_back_manager.h" #include "lite/trt/kernel/warp_affine_npp.h" +#include "lite/trt/kernel/device_frame.h" // Forward declaration for benchmark timing; library passes nullptr by default (zero overhead) namespace lite { namespace bench { class Profiler; } } @@ -35,6 +36,7 @@ namespace trtcv{ PasteBackGPU paste_back_gpu_; // GPU fused paste_back, reuses device buffers FaceRestorePreprocessGPU preprocess_gpu_; // GPU fused bgr2rgb+normalize+CHW into input buffer WarpAffineNpp warp_npp_; // GPU (NPP) affine warp; crop stays device-resident + DeviceFrame input_frame_; // input frame uploaded once, shared by warp + paste_back cv::Mat box_mask_cache_; // static box mask is size-only; compute once and reuse }; diff --git a/lite/trt/kernel/device_frame.cpp b/lite/trt/kernel/device_frame.cpp new file mode 100644 index 00000000..d08b19a0 --- /dev/null +++ b/lite/trt/kernel/device_frame.cpp @@ -0,0 +1,39 @@ +#include "device_frame.h" +#include + +DeviceFrame::~DeviceFrame() { + if (d_) cudaFree(d_); + if (h_pinned_) cudaFreeHost(h_pinned_); +} + +void DeviceFrame::ensure(int w, int h) { + const size_t bytes = static_cast(w) * h * 3; + if (bytes > cap_) { + if (d_) cudaFree(d_); + if (h_pinned_) cudaFreeHost(h_pinned_); + cudaMalloc(&d_, bytes); + cudaMallocHost(&h_pinned_, bytes); + cap_ = bytes; + } + w_ = w; h_ = h; +} + +void DeviceFrame::upload(const cv::Mat& bgr_u8, cudaStream_t stream) { + cv::Mat f = bgr_u8; + if (f.type() != CV_8UC3) f.convertTo(f, CV_8UC3); + if (!f.isContinuous()) f = f.clone(); + + ensure(f.cols, f.rows); + const size_t bytes = static_cast(w_) * h_ * 3; + std::memcpy(h_pinned_, f.data, bytes); + cudaMemcpyAsync(d_, h_pinned_, bytes, cudaMemcpyHostToDevice, stream); +} + +cv::Mat DeviceFrame::download(cudaStream_t stream) const { + cv::Mat out(h_, w_, CV_8UC3); + const size_t bytes = static_cast(w_) * h_ * 3; + cudaMemcpyAsync(h_pinned_, d_, bytes, cudaMemcpyDeviceToHost, stream); + cudaStreamSynchronize(stream); + std::memcpy(out.data, h_pinned_, bytes); + return out; +} diff --git a/lite/trt/kernel/device_frame.h b/lite/trt/kernel/device_frame.h new file mode 100644 index 00000000..1ff65eae --- /dev/null +++ b/lite/trt/kernel/device_frame.h @@ -0,0 +1,38 @@ +#ifndef LITE_AI_TOOLKIT_DEVICE_FRAME_H +#define LITE_AI_TOOLKIT_DEVICE_FRAME_H + +#include +#include + +// A full frame kept resident in device memory (HWC, BGR, uint8) for the device pipeline: +// upload once at entry, hand the raw device pointer between GPU stages (NPP warp, paste-back) +// without bouncing through host, download once at exit. Reuses its device + pinned staging +// buffers across frames (no per-frame cudaMalloc). +class DeviceFrame { +public: + DeviceFrame() = default; + ~DeviceFrame(); + DeviceFrame(const DeviceFrame&) = delete; + DeviceFrame& operator=(const DeviceFrame&) = delete; + + // Host BGR uint8 -> device (1 H2D, via pinned staging for true async). Does not sync; + // the consuming GPU op must run on the same stream. + void upload(const cv::Mat& bgr_u8, cudaStream_t stream = nullptr); + // Device -> host BGR uint8 (1 D2H, syncs). Returns a fresh CV_8UC3 Mat. + cv::Mat download(cudaStream_t stream = nullptr) const; + + unsigned char* data() { return d_; } + const unsigned char* data() const { return d_; } + int width() const { return w_; } + int height() const { return h_; } + bool empty() const { return d_ == nullptr; } + +private: + void ensure(int w, int h); + unsigned char* d_ = nullptr; + mutable unsigned char* h_pinned_ = nullptr; + size_t cap_ = 0; + int w_ = 0, h_ = 0; +}; + +#endif // LITE_AI_TOOLKIT_DEVICE_FRAME_H diff --git a/lite/trt/kernel/paste_back_manager.cpp b/lite/trt/kernel/paste_back_manager.cpp index a6fa361f..cd37d140 100644 --- a/lite/trt/kernel/paste_back_manager.cpp +++ b/lite/trt/kernel/paste_back_manager.cpp @@ -125,15 +125,43 @@ cv::Mat PasteBackGPU::paste_back(const cv::Mat& temp_vision_frame, const cv::Mat& crop_mask, const cv::Mat& affine_matrix, cudaStream_t stream) { - // normalize temp to a contiguous BGR uint8 frame + // normalize temp to a contiguous BGR uint8 frame, then H2D it into d_temp_ cv::Mat temp = temp_vision_frame; if (temp.type() != CV_8UC3) temp.convertTo(temp, CV_8UC3); if (!temp.isContinuous()) temp = temp.clone(); + const int W = temp.cols, H = temp.rows; + const size_t temp_bytes = static_cast(W) * H * 3; + // d_temp_ capacity is grown inside ensure_capacity (called by run); but we need it sized + // before the H2D, so size it here via a crop-agnostic ensure of the temp buffer. + if (temp_bytes > cap_temp_) { + if (d_temp_) cudaFree(d_temp_); + if (h_temp_pinned_) cudaFreeHost(h_temp_pinned_); + cudaMalloc(&d_temp_, temp_bytes); + cudaMallocHost(&h_temp_pinned_, temp_bytes); + cap_temp_ = temp_bytes; + } + std::memcpy(h_temp_pinned_, temp.data, temp_bytes); + cudaMemcpyAsync(d_temp_, h_temp_pinned_, temp_bytes, cudaMemcpyHostToDevice, stream); + + return run(d_temp_, W, H, crop_vision_frame, crop_mask, affine_matrix, stream); +} + +cv::Mat PasteBackGPU::paste_back(const unsigned char* d_temp, int W, int H, + const cv::Mat& crop_vision_frame, + const cv::Mat& crop_mask, + const cv::Mat& affine_matrix, + cudaStream_t stream) { + // temp is already on the device — no H2D for the full frame. + return run(d_temp, W, H, crop_vision_frame, crop_mask, affine_matrix, stream); +} + +cv::Mat PasteBackGPU::run(const unsigned char* d_temp, int W, int H, + const cv::Mat& crop_vision_frame, const cv::Mat& crop_mask, + const cv::Mat& affine_matrix, cudaStream_t stream) { cv::Mat crop = crop_vision_frame.isContinuous() ? crop_vision_frame : crop_vision_frame.clone(); cv::Mat mask = crop_mask.isContinuous() ? crop_mask : crop_mask.clone(); - const int W = temp.cols, H = temp.rows; const int Cw = crop.cols, Ch = crop.rows; const size_t temp_bytes = static_cast(W) * H * 3; const size_t out_bytes = temp_bytes; @@ -148,9 +176,6 @@ cv::Mat PasteBackGPU::paste_back(const cv::Mat& temp_vision_frame, float h_aff[6]; for (int i = 0; i < 6; ++i) h_aff[i] = static_cast(M64.at(i / 3, i % 3)); - // H2D (temp goes through a pinned staging buffer for true async transfer) - std::memcpy(h_temp_pinned_, temp.data, temp_bytes); - cudaMemcpyAsync(d_temp_, h_temp_pinned_, temp_bytes, cudaMemcpyHostToDevice, stream); cudaMemcpyAsync(d_crop_, crop.ptr(), crop_bytes, cudaMemcpyHostToDevice, stream); cudaMemcpyAsync(d_mask_, mask.ptr(), mask_bytes, cudaMemcpyHostToDevice, stream); cudaMemcpyAsync(d_affine_, h_aff, 6 * sizeof(float), cudaMemcpyHostToDevice, stream); @@ -158,7 +183,7 @@ cv::Mat PasteBackGPU::paste_back(const cv::Mat& temp_vision_frame, dim3 block(16, 16); dim3 grid((W + block.x - 1) / block.x, (H + block.y - 1) / block.y); paste_back_fused_kernel<<>>( - d_temp_, d_crop_, d_mask_, d_affine_, d_out_, W, H, Cw, Ch); + d_temp, d_crop_, d_mask_, d_affine_, d_out_, W, H, Cw, Ch); cudaMemcpyAsync(h_out_pinned_, d_out_, out_bytes, cudaMemcpyDeviceToHost, stream); cudaStreamSynchronize(stream); diff --git a/lite/trt/kernel/paste_back_manager.h b/lite/trt/kernel/paste_back_manager.h index 14ac677f..bd269015 100644 --- a/lite/trt/kernel/paste_back_manager.h +++ b/lite/trt/kernel/paste_back_manager.h @@ -26,9 +26,22 @@ class PasteBackGPU { const cv::Mat& affine_matrix, // 2x3, original->crop cudaStream_t stream = nullptr); + // Device-resident temp: the full frame is ALREADY on the device (e.g. a DeviceFrame), so the + // temp H2D is skipped — the kernel reads d_temp directly. crop/mask/affine still come from host. + cv::Mat paste_back(const unsigned char* d_temp, int W, int H, // device BGR uint8 full frame + const cv::Mat& crop_vision_frame, + const cv::Mat& crop_mask, + const cv::Mat& affine_matrix, + cudaStream_t stream = nullptr); + private: void ensure_capacity(size_t temp_bytes, size_t crop_bytes, size_t mask_bytes, size_t out_bytes); + // Shared tail: crop/mask/affine H2D, kernel into d_out_, D2H result. d_temp is the kernel's + // full-frame input (either d_temp_ after an H2D, or a caller-provided device pointer). + cv::Mat run(const unsigned char* d_temp, int W, int H, + const cv::Mat& crop_vision_frame, const cv::Mat& crop_mask, + const cv::Mat& affine_matrix, cudaStream_t stream); unsigned char* d_temp_ = nullptr; unsigned char* d_out_ = nullptr; diff --git a/lite/trt/kernel/warp_affine_npp.cpp b/lite/trt/kernel/warp_affine_npp.cpp index 04e8528b..dfe68697 100644 --- a/lite/trt/kernel/warp_affine_npp.cpp +++ b/lite/trt/kernel/warp_affine_npp.cpp @@ -97,3 +97,26 @@ const unsigned char* WarpAffineNpp::warp_to_device(const cv::Mat& frame_bgr_u8, aCoeffs, NPPI_INTER_LINEAR); return d_dst_; } + +const unsigned char* WarpAffineNpp::warp_device_to_device(const unsigned char* d_frame, + int SW, int SH, + const cv::Mat& affine_2x3, + int out_size, cudaStream_t stream) { + const int DW = out_size, DH = out_size; + const size_t dst_bytes = static_cast(DW) * DH * 3; + ensure(0, dst_bytes); // src is the caller's device frame; only the dst buffer is ours + + cudaMemsetAsync(d_dst_, 0, dst_bytes, stream); + + double aCoeffs[2][3]; + fill_coeffs(affine_2x3, aCoeffs); + + NppiSize srcSize{SW, SH}; + NppiRect srcROI{0, 0, SW, SH}; + NppiRect dstROI{0, 0, DW, DH}; + nppSetStream(stream); + nppiWarpAffine_8u_C3R(d_frame, srcSize, SW * 3, srcROI, + d_dst_, DW * 3, dstROI, + aCoeffs, NPPI_INTER_LINEAR); + return d_dst_; +} diff --git a/lite/trt/kernel/warp_affine_npp.h b/lite/trt/kernel/warp_affine_npp.h index 346ac445..8508c88f 100644 --- a/lite/trt/kernel/warp_affine_npp.h +++ b/lite/trt/kernel/warp_affine_npp.h @@ -26,6 +26,13 @@ class WarpAffineNpp { const unsigned char* warp_to_device(const cv::Mat& frame_bgr_u8, const cv::Mat& affine_2x3, int out_size, cudaStream_t stream = nullptr); + // Fully device-resident: source frame is ALREADY on the device (e.g. a DeviceFrame), so no + // H2D at all — NPP reads d_frame directly and warps into the internal device buffer, returning + // the device crop pointer. SW/SH are the source frame dimensions. + const unsigned char* warp_device_to_device(const unsigned char* d_frame, int SW, int SH, + const cv::Mat& affine_2x3, int out_size, + cudaStream_t stream = nullptr); + private: void ensure(size_t src_bytes, size_t dst_bytes); unsigned char* d_src_ = nullptr; From dc4446982122f09fc5a4eb60d82e43244b57d8ca Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Sun, 7 Jun 2026 16:44:45 +0800 Subject: [PATCH 24/30] =?UTF-8?q?perf(trt):=20DeviceFrame=20brick=202=20?= =?UTF-8?q?=E2=80=94=20fold=20restoration's=20blend=5Fframe=20into=20the?= =?UTF-8?q?=20paste=20kernel?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The face-enhancer blend_frame(target 0.2 / paste 0.8) is algebraically just the paste with the mask scaled by 0.8: result = temp*(1 - 0.8*mask) + crop*(0.8*mask). Add a blend_alpha to the paste-back kernel (default 1.0 = plain paste, swap unchanged) and have restoration pass 0.8, dropping the separate full-frame CPU cv::addWeighted and the now-dead ori_image clone. restoration 11.7->10.6 ms (the blend ran on the full ~1024x768 frame), TOTAL 31.8->31.0, 31.4->32.2 FPS. Output numerically identical (PSNR 56.5 dB, max|diff|=8 — float rounding order vs addWeighted), visually identical. Co-Authored-By: Claude Opus 4.8 --- lite/trt/cv/trt_face_restoration.cpp | 18 ++++++------------ lite/trt/kernel/paste_back.cu | 6 +++++- lite/trt/kernel/paste_back.cuh | 3 ++- lite/trt/kernel/paste_back_manager.cpp | 14 ++++++++------ lite/trt/kernel/paste_back_manager.h | 10 +++++++--- 5 files changed, 28 insertions(+), 23 deletions(-) diff --git a/lite/trt/cv/trt_face_restoration.cpp b/lite/trt/cv/trt_face_restoration.cpp index cf18dd79..287c4633 100644 --- a/lite/trt/cv/trt_face_restoration.cpp +++ b/lite/trt/cv/trt_face_restoration.cpp @@ -12,8 +12,6 @@ using trtcv::TRTFaceFusionFaceRestoration; cv::Mat TRTFaceFusionFaceRestoration::restore(cv::Mat &face_swap_image, std::vector &target_landmarks_5, lite::bench::Profiler *prof) { - auto ori_image = face_swap_image.clone(); - cv::Mat affine_matrix; cv::Mat box_mask; @@ -80,18 +78,14 @@ cv::Mat TRTFaceFusionFaceRestoration::restore(cv::Mat &face_swap_image, // aliases transposed_data (alive for the rest of this scope, i.e. through paste_back) cv::Mat mat(height, width, CV_32FC3, transposed_data.data()); - cv::Mat paste_frame; { - // GPU fused: inverse-mapping sampling + blend in one kernel. temp frame is read - // straight from the device-resident input_frame_ (no full-frame H2D here). - LITE_CPU_SCOPE_OPT(prof, " paste_back"); - paste_frame = paste_back_gpu_.paste_back( + // GPU fused: inverse-mapping sampling + paste + face-enhancer blend in ONE kernel. + // temp frame is read straight from the device-resident input_frame_ (no full-frame + // H2D). blend_alpha=0.8 folds the old CPU blend_frame(target 0.2 / paste 0.8) in. + LITE_CPU_SCOPE_OPT(prof, " paste_back+blend"); + dst_image = paste_back_gpu_.paste_back( input_frame_.data(), input_frame_.width(), input_frame_.height(), - mat, box_mask, affine_matrix, stream); - } - { - LITE_CPU_SCOPE_OPT(prof, " blend"); - dst_image = face_utils::blend_frame(ori_image, paste_frame); + mat, box_mask, affine_matrix, stream, /*blend_alpha=*/0.8f); } } diff --git a/lite/trt/kernel/paste_back.cu b/lite/trt/kernel/paste_back.cu index 2dbd63ff..6e65fe27 100644 --- a/lite/trt/kernel/paste_back.cu +++ b/lite/trt/kernel/paste_back.cu @@ -50,7 +50,8 @@ __global__ void paste_back_fused_kernel(const unsigned char* temp, const float* mask, const float* M, unsigned char* out, - int W, int H, int Cw, int Ch) { + int W, int H, int Cw, int Ch, + float blend_alpha) { int x = blockIdx.x * blockDim.x + threadIdx.x; int y = blockIdx.y * blockDim.y + threadIdx.y; if (x >= W || y >= H) return; @@ -64,6 +65,9 @@ __global__ void paste_back_fused_kernel(const unsigned char* temp, // pixels outside the crop get mask=0 and just copy temp (matches CPU BORDER_CONSTANT 0) float m = bilinear1(mask, Cw, Ch, u, v); m = fminf(fmaxf(m, 0.f), 1.f); + // fold the face-enhancer blend (result = temp*(1-a*mask) + crop*(a*mask)); a=1 -> plain paste, + // a=0.8 -> restoration's blend_frame(target,0.2 / paste,0.8) collapsed into the mask. + m *= blend_alpha; if (m > 0.f) { float w = 1.f - m; diff --git a/lite/trt/kernel/paste_back.cuh b/lite/trt/kernel/paste_back.cuh index 7ad0d03f..da6c813a 100644 --- a/lite/trt/kernel/paste_back.cuh +++ b/lite/trt/kernel/paste_back.cuh @@ -19,6 +19,7 @@ __global__ void paste_back_fused_kernel(const unsigned char* temp, const float* mask, const float* M, unsigned char* out, - int W, int H, int Cw, int Ch); + int W, int H, int Cw, int Ch, + float blend_alpha = 1.0f); #endif // PASTE_BACK_CUH diff --git a/lite/trt/kernel/paste_back_manager.cpp b/lite/trt/kernel/paste_back_manager.cpp index cd37d140..bf3c33f1 100644 --- a/lite/trt/kernel/paste_back_manager.cpp +++ b/lite/trt/kernel/paste_back_manager.cpp @@ -124,7 +124,8 @@ cv::Mat PasteBackGPU::paste_back(const cv::Mat& temp_vision_frame, const cv::Mat& crop_vision_frame, const cv::Mat& crop_mask, const cv::Mat& affine_matrix, - cudaStream_t stream) { + cudaStream_t stream, + float blend_alpha) { // normalize temp to a contiguous BGR uint8 frame, then H2D it into d_temp_ cv::Mat temp = temp_vision_frame; if (temp.type() != CV_8UC3) temp.convertTo(temp, CV_8UC3); @@ -144,21 +145,22 @@ cv::Mat PasteBackGPU::paste_back(const cv::Mat& temp_vision_frame, std::memcpy(h_temp_pinned_, temp.data, temp_bytes); cudaMemcpyAsync(d_temp_, h_temp_pinned_, temp_bytes, cudaMemcpyHostToDevice, stream); - return run(d_temp_, W, H, crop_vision_frame, crop_mask, affine_matrix, stream); + return run(d_temp_, W, H, crop_vision_frame, crop_mask, affine_matrix, stream, blend_alpha); } cv::Mat PasteBackGPU::paste_back(const unsigned char* d_temp, int W, int H, const cv::Mat& crop_vision_frame, const cv::Mat& crop_mask, const cv::Mat& affine_matrix, - cudaStream_t stream) { + cudaStream_t stream, + float blend_alpha) { // temp is already on the device — no H2D for the full frame. - return run(d_temp, W, H, crop_vision_frame, crop_mask, affine_matrix, stream); + return run(d_temp, W, H, crop_vision_frame, crop_mask, affine_matrix, stream, blend_alpha); } cv::Mat PasteBackGPU::run(const unsigned char* d_temp, int W, int H, const cv::Mat& crop_vision_frame, const cv::Mat& crop_mask, - const cv::Mat& affine_matrix, cudaStream_t stream) { + const cv::Mat& affine_matrix, cudaStream_t stream, float blend_alpha) { cv::Mat crop = crop_vision_frame.isContinuous() ? crop_vision_frame : crop_vision_frame.clone(); cv::Mat mask = crop_mask.isContinuous() ? crop_mask : crop_mask.clone(); @@ -183,7 +185,7 @@ cv::Mat PasteBackGPU::run(const unsigned char* d_temp, int W, int H, dim3 block(16, 16); dim3 grid((W + block.x - 1) / block.x, (H + block.y - 1) / block.y); paste_back_fused_kernel<<>>( - d_temp, d_crop_, d_mask_, d_affine_, d_out_, W, H, Cw, Ch); + d_temp, d_crop_, d_mask_, d_affine_, d_out_, W, H, Cw, Ch, blend_alpha); cudaMemcpyAsync(h_out_pinned_, d_out_, out_bytes, cudaMemcpyDeviceToHost, stream); cudaStreamSynchronize(stream); diff --git a/lite/trt/kernel/paste_back_manager.h b/lite/trt/kernel/paste_back_manager.h index bd269015..b2f1e111 100644 --- a/lite/trt/kernel/paste_back_manager.h +++ b/lite/trt/kernel/paste_back_manager.h @@ -20,11 +20,14 @@ class PasteBackGPU { PasteBackGPU(const PasteBackGPU&) = delete; PasteBackGPU& operator=(const PasteBackGPU&) = delete; + // blend_alpha folds an optional face-enhancer blend into the mask: 1.0 = plain paste, + // 0.8 = restoration's blend_frame(target 0.2 / paste 0.8) collapsed into one kernel. cv::Mat paste_back(const cv::Mat& temp_vision_frame, // CV_8UC3 (auto-converted otherwise) const cv::Mat& crop_vision_frame, // CV_32FC3, 0..255 const cv::Mat& crop_mask, // CV_32FC1, 0..1 const cv::Mat& affine_matrix, // 2x3, original->crop - cudaStream_t stream = nullptr); + cudaStream_t stream = nullptr, + float blend_alpha = 1.0f); // Device-resident temp: the full frame is ALREADY on the device (e.g. a DeviceFrame), so the // temp H2D is skipped — the kernel reads d_temp directly. crop/mask/affine still come from host. @@ -32,7 +35,8 @@ class PasteBackGPU { const cv::Mat& crop_vision_frame, const cv::Mat& crop_mask, const cv::Mat& affine_matrix, - cudaStream_t stream = nullptr); + cudaStream_t stream = nullptr, + float blend_alpha = 1.0f); private: void ensure_capacity(size_t temp_bytes, size_t crop_bytes, @@ -41,7 +45,7 @@ class PasteBackGPU { // full-frame input (either d_temp_ after an H2D, or a caller-provided device pointer). cv::Mat run(const unsigned char* d_temp, int W, int H, const cv::Mat& crop_vision_frame, const cv::Mat& crop_mask, - const cv::Mat& affine_matrix, cudaStream_t stream); + const cv::Mat& affine_matrix, cudaStream_t stream, float blend_alpha); unsigned char* d_temp_ = nullptr; unsigned char* d_out_ = nullptr; From f5d50d9a86ab8bd20f01336f1ef1872e40a2c8c6 Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Sun, 7 Jun 2026 20:13:40 +0800 Subject: [PATCH 25/30] =?UTF-8?q?perf(trt):=20DeviceFrame=20brick=203=20?= =?UTF-8?q?=E2=80=94=20weld=20the=20swap->restoration=20seam=20(no=20D2H/H?= =?UTF-8?q?2D=20round-trip)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The swapped full frame now stays GPU-resident across the swap->restoration boundary: swap pastes straight into a DeviceFrame (no D2H) and restoration reads it from the device (no upload), eliminating the fattest pair of full-frame copies in the pipeline. - PasteBackGPU: paste_back_to_device() writes into a caller DeviceFrame (no D2H); refactor into upload_temp() + run_core() shared by the host-Mat and device paths. FIX: run() now passes nullptr to run_core and resolves d_out_ AFTER ensure_capacity may have reallocated it — passing d_out_ directly used a stale/null pointer on the first frame (would write through a bad pointer and crash the host paste path). - face_swap: detect(...,DeviceFrame&) overload (swap_core shared with the cv::Mat path); syncs its stream so restoration can safely read the frame on its own stream. - restoration: restore(const DeviceFrame&) overload (restore_core shared with the cv::Mat path). - pipeline: owns swapped_frame_; process() hands it swap -> restoration with no host bounce. The cv::Mat APIs (standalone swap/restoration tests, CLI) are unchanged. Verified on RTX 4090: no crash, PSNR 99 dB (max|diff|=0, bit-exact), memory flat; swap 9.8->9.2 ms, restoration 10.6->9.8 ms, 32.2->32.8 FPS. This is the device-pipeline v0 across the swap->restoration seam. Co-Authored-By: Claude Opus 4.8 --- lite/trt/cv/trt_face_restoration.cpp | 30 ++++++--- lite/trt/cv/trt_face_restoration.h | 8 +++ lite/trt/cv/trt_face_swap.cpp | 38 +++++++---- lite/trt/cv/trt_face_swap.h | 10 +++ lite/trt/cv/trt_facefusion_pipeline.cpp | 8 +-- lite/trt/cv/trt_facefusion_pipeline.h | 1 + lite/trt/kernel/device_frame.h | 4 ++ lite/trt/kernel/paste_back_manager.cpp | 89 +++++++++++++++++-------- lite/trt/kernel/paste_back_manager.h | 23 ++++++- 9 files changed, 155 insertions(+), 56 deletions(-) diff --git a/lite/trt/cv/trt_face_restoration.cpp b/lite/trt/cv/trt_face_restoration.cpp index 287c4633..76674764 100644 --- a/lite/trt/cv/trt_face_restoration.cpp +++ b/lite/trt/cv/trt_face_restoration.cpp @@ -12,6 +12,24 @@ using trtcv::TRTFaceFusionFaceRestoration; cv::Mat TRTFaceFusionFaceRestoration::restore(cv::Mat &face_swap_image, std::vector &target_landmarks_5, lite::bench::Profiler *prof) { + // upload the input frame to the device once, then run the shared device-resident body. + { + LITE_CPU_SCOPE_OPT(prof, " upload"); + input_frame_.upload(face_swap_image, stream); + } + return restore_core(input_frame_, target_landmarks_5, prof); +} + +cv::Mat TRTFaceFusionFaceRestoration::restore(const DeviceFrame &input_frame, + std::vector &target_landmarks_5, + lite::bench::Profiler *prof) { + // input frame is already on the device (e.g. swap's output) — no upload. + return restore_core(input_frame, target_landmarks_5, prof); +} + +cv::Mat TRTFaceFusionFaceRestoration::restore_core(const DeviceFrame &frame, + std::vector &target_landmarks_5, + lite::bench::Profiler *prof) { cv::Mat affine_matrix; cv::Mat box_mask; @@ -34,18 +52,12 @@ cv::Mat TRTFaceFusionFaceRestoration::restore(cv::Mat &face_swap_image, box_mask = box_mask_cache_; } - { - // Upload the input frame to the device ONCE; both the NPP warp (here) and the - // paste_back (below) read it straight from device memory — no second full-frame H2D. - LITE_CPU_SCOPE_OPT(prof, " upload"); - input_frame_.upload(face_swap_image, stream); - } { // GPU NPP warp (reads the device-resident frame) -> device crop, then fused // bgr2rgb+normalize+HWC->CHW straight into the inference input buffer (buffers[0]). LITE_CPU_SCOPE_OPT(prof, " warp+to_chw(gpu)"); const unsigned char *d_crop = warp_npp_.warp_device_to_device( - input_frame_.data(), input_frame_.width(), input_frame_.height(), + frame.data(), frame.width(), frame.height(), affine_matrix, 512, stream); preprocess_gpu_.run_device(d_crop, 512, 512, static_cast(buffers[0]), stream); } @@ -80,11 +92,11 @@ cv::Mat TRTFaceFusionFaceRestoration::restore(cv::Mat &face_swap_image, { // GPU fused: inverse-mapping sampling + paste + face-enhancer blend in ONE kernel. - // temp frame is read straight from the device-resident input_frame_ (no full-frame + // temp frame is read straight from the device-resident `frame` (no full-frame // H2D). blend_alpha=0.8 folds the old CPU blend_frame(target 0.2 / paste 0.8) in. LITE_CPU_SCOPE_OPT(prof, " paste_back+blend"); dst_image = paste_back_gpu_.paste_back( - input_frame_.data(), input_frame_.width(), input_frame_.height(), + frame.data(), frame.width(), frame.height(), mat, box_mask, affine_matrix, stream, /*blend_alpha=*/0.8f); } } diff --git a/lite/trt/cv/trt_face_restoration.h b/lite/trt/cv/trt_face_restoration.h index 4af40df7..f37aca13 100644 --- a/lite/trt/cv/trt_face_restoration.h +++ b/lite/trt/cv/trt_face_restoration.h @@ -32,7 +32,15 @@ namespace trtcv{ cv::Mat restore(cv::Mat &face_swap_image, std::vector &target_landmarks_5, lite::bench::Profiler *prof = nullptr); + // Device-pipeline variant: the input frame is ALREADY on the device (e.g. swap's output), + // so no upload — warp + paste read it straight from device memory. + cv::Mat restore(const DeviceFrame &input_frame, std::vector &target_landmarks_5, + lite::bench::Profiler *prof = nullptr); + private: + // shared body: estimate affine -> NPP warp (from `frame`) -> infer -> postprocess -> paste. + cv::Mat restore_core(const DeviceFrame &frame, std::vector &target_landmarks_5, + lite::bench::Profiler *prof); PasteBackGPU paste_back_gpu_; // GPU fused paste_back, reuses device buffers FaceRestorePreprocessGPU preprocess_gpu_; // GPU fused bgr2rgb+normalize+CHW into input buffer WarpAffineNpp warp_npp_; // GPU (NPP) affine warp; crop stays device-resident diff --git a/lite/trt/cv/trt_face_swap.cpp b/lite/trt/cv/trt_face_swap.cpp index b9bb392e..e2406021 100644 --- a/lite/trt/cv/trt_face_swap.cpp +++ b/lite/trt/cv/trt_face_swap.cpp @@ -22,9 +22,10 @@ void TRTFaceFusionFaceSwap::preprocess(cv::Mat &target_face, std::vector } -void TRTFaceFusionFaceSwap::detect(cv::Mat &target_image, std::vector source_face_embeding, - std::vector target_landmark_5, cv::Mat &face_swap_image) { - cv::Mat ori_image = target_image.clone(); +// infer + postprocess: produces the host BGR float[0,255] 128x128 swapped-face crop (owns its +// data) and sets affine_martix. Shared by both detect() overloads. +void TRTFaceFusionFaceSwap::swap_core(cv::Mat &target_image, std::vector &source_face_embeding, + std::vector &target_landmark_5, cv::Mat &mat_out) { std::vector source_embeding_input; cv::Mat model_input_mat; preprocess(target_image,source_face_embeding,target_landmark_5,source_embeding_input,model_input_mat); @@ -47,10 +48,7 @@ void TRTFaceFusionFaceSwap::detect(cv::Mat &target_image, std::vector sou cudaMemcpyAsync(output_vector.data(),buffers[2],1 * 3 * 128 * 128 * sizeof(float),cudaMemcpyDeviceToHost,stream); cudaStreamSynchronize(stream); - std::vector output_swap_image(1 * 3 * 128 * 128); - output_swap_image.assign(output_vector.begin(),output_vector.end()); - - // CHW float -> HWC uint8-range (denormalize); paste-back is done on the GPU below. + // CHW float -> HWC uint8-range (denormalize); paste-back is done on the GPU by the caller. std::vector transposed(3 * 128 * 128); const int channels = 3, height = 128, width = 128; #pragma omp parallel for collapse(3) @@ -59,7 +57,7 @@ void TRTFaceFusionFaceSwap::detect(cv::Mat &target_image, std::vector sou for (int w = 0; w < width; ++w) { int src_idx = c * (height * width) + h * width + w; // CHW int dst_idx = h * (width * channels) + w * channels + c; // HWC - transposed[dst_idx] = output_swap_image[src_idx]; + transposed[dst_idx] = output_vector[src_idx]; } } } @@ -69,9 +67,25 @@ void TRTFaceFusionFaceSwap::detect(cv::Mat &target_image, std::vector sou cv::Mat mat(height, width, CV_32FC3, transposed.data()); cv::cvtColor(mat, mat, cv::COLOR_RGB2BGR); + mat.copyTo(mat_out); // own the data (transposed is local) +} - // GPU-fused paste-back (reused device buffers, no per-frame cudaMalloc), reusing the - // exact kernel restoration uses. Numerically equivalent to launch_paste_back. - cv::Mat dst_image = paste_back_gpu_.paste_back(ori_image, mat, box_mask_, affine_martix, stream); - face_swap_image = dst_image; +void TRTFaceFusionFaceSwap::detect(cv::Mat &target_image, std::vector source_face_embeding, + std::vector target_landmark_5, cv::Mat &face_swap_image) { + cv::Mat ori_image = target_image.clone(); + cv::Mat mat; + swap_core(target_image, source_face_embeding, target_landmark_5, mat); + // GPU-fused paste-back (reused device buffers); numerically equivalent to launch_paste_back. + face_swap_image = paste_back_gpu_.paste_back(ori_image, mat, box_mask_, affine_martix, stream); +} + +void TRTFaceFusionFaceSwap::detect(cv::Mat &target_image, std::vector source_face_embeding, + std::vector target_landmark_5, DeviceFrame &out_frame) { + cv::Mat ori_image = target_image.clone(); + cv::Mat mat; + swap_core(target_image, source_face_embeding, target_landmark_5, mat); + // paste straight into the device-resident out_frame (no D2H) for restoration to consume. + paste_back_gpu_.paste_back_to_device(ori_image, mat, box_mask_, affine_martix, out_frame, stream); + // restoration reads out_frame on its OWN stream, so make sure this paste has completed. + cudaStreamSynchronize(stream); } diff --git a/lite/trt/cv/trt_face_swap.h b/lite/trt/cv/trt_face_swap.h index bbd8c780..d207a2d0 100644 --- a/lite/trt/cv/trt_face_swap.h +++ b/lite/trt/cv/trt_face_swap.h @@ -10,6 +10,7 @@ #include "lite/trt/core/trt_types.h" #include "lite/trt/kernel/face_swap_postproces_manager.h" #include "lite/trt/kernel/paste_back_manager.h" +#include "lite/trt/kernel/device_frame.h" namespace trtcv{ class LITE_EXPORTS TRTFaceFusionFaceSwap : BasicTRTHandler{ @@ -34,6 +35,15 @@ namespace trtcv{ void detect(cv::Mat &target_image,std::vector source_face_embeding,std::vector target_landmark_5, cv::Mat &face_swap_image); + // Device-pipeline variant: the swapped full frame stays GPU-resident in out_frame (paste + // writes straight to device, no D2H) so restoration can consume it without re-uploading. + void detect(cv::Mat &target_image,std::vector source_face_embeding,std::vector target_landmark_5, + DeviceFrame &out_frame); + + private: + // shared body: infer + postprocess into the host BGR float crop `mat` + its affine. + void swap_core(cv::Mat &target_image, std::vector &source_face_embeding, + std::vector &target_landmark_5, cv::Mat &mat_out); }; } diff --git a/lite/trt/cv/trt_facefusion_pipeline.cpp b/lite/trt/cv/trt_facefusion_pipeline.cpp index 901ea0fe..695543f7 100644 --- a/lite/trt/cv/trt_facefusion_pipeline.cpp +++ b/lite/trt/cv/trt_facefusion_pipeline.cpp @@ -99,14 +99,14 @@ cv::Mat TRTFaceFusionPipeLine::process(const cv::Mat &target_image, int target_i { LITE_CPU_SCOPE_OPT(prof, "landmark_tgt"); face_landmarks->detect(target_img_bgr, target_final_boxes[tgt_pick], target_face_landmark_5of68); } - // ---- swap + restore (restore() returns the frame; no disk write) ---- - cv::Mat face_swap_image; + // ---- swap + restore: the swapped frame stays GPU-resident in swapped_frame_, so restoration + // reads it from the device (no swap-D2H + restoration-H2D round-trip across the seam) ---- { LITE_CPU_SCOPE_OPT(prof, "swap"); - face_swap->detect(target_img_bgr, source_embedding_, target_face_landmark_5of68, face_swap_image); } + face_swap->detect(target_img_bgr, source_embedding_, target_face_landmark_5of68, swapped_frame_); } cv::Mat result; { LITE_CPU_SCOPE_OPT(prof, "restoration"); - result = face_restoration->restore(face_swap_image, target_face_landmark_5of68, nullptr); } + result = face_restoration->restore(swapped_frame_, target_face_landmark_5of68, nullptr); } return result; } diff --git a/lite/trt/cv/trt_facefusion_pipeline.h b/lite/trt/cv/trt_facefusion_pipeline.h index b30227ed..a79a2967 100644 --- a/lite/trt/cv/trt_facefusion_pipeline.h +++ b/lite/trt/cv/trt_facefusion_pipeline.h @@ -37,6 +37,7 @@ namespace trtcv{ std::vector source_embedding_; // cached by prepare_source() bool source_ready_ = false; + DeviceFrame swapped_frame_; // swap output stays GPU-resident -> restoration (no D2H/H2D) public: // ---- Split API (the right shape for video / server: the SOURCE face is usually diff --git a/lite/trt/kernel/device_frame.h b/lite/trt/kernel/device_frame.h index 1ff65eae..9c217fad 100644 --- a/lite/trt/kernel/device_frame.h +++ b/lite/trt/kernel/device_frame.h @@ -21,6 +21,10 @@ class DeviceFrame { // Device -> host BGR uint8 (1 D2H, syncs). Returns a fresh CV_8UC3 Mat. cv::Mat download(cudaStream_t stream = nullptr) const; + // Size the device buffer for a w*h*3 BGR uint8 frame WITHOUT uploading (e.g. as a kernel + // output target). Returns the device pointer. + unsigned char* prepare(int w, int h) { ensure(w, h); return d_; } + unsigned char* data() { return d_; } const unsigned char* data() const { return d_; } int width() const { return w_; } diff --git a/lite/trt/kernel/paste_back_manager.cpp b/lite/trt/kernel/paste_back_manager.cpp index bf3c33f1..b177645e 100644 --- a/lite/trt/kernel/paste_back_manager.cpp +++ b/lite/trt/kernel/paste_back_manager.cpp @@ -120,21 +120,12 @@ void PasteBackGPU::ensure_capacity(size_t temp_bytes, size_t crop_bytes, if (d_affine_ == nullptr) cudaMalloc(&d_affine_, 6 * sizeof(float)); } -cv::Mat PasteBackGPU::paste_back(const cv::Mat& temp_vision_frame, - const cv::Mat& crop_vision_frame, - const cv::Mat& crop_mask, - const cv::Mat& affine_matrix, - cudaStream_t stream, - float blend_alpha) { - // normalize temp to a contiguous BGR uint8 frame, then H2D it into d_temp_ +const unsigned char* PasteBackGPU::upload_temp(const cv::Mat& temp_vision_frame, cudaStream_t stream) { cv::Mat temp = temp_vision_frame; if (temp.type() != CV_8UC3) temp.convertTo(temp, CV_8UC3); if (!temp.isContinuous()) temp = temp.clone(); - const int W = temp.cols, H = temp.rows; - const size_t temp_bytes = static_cast(W) * H * 3; - // d_temp_ capacity is grown inside ensure_capacity (called by run); but we need it sized - // before the H2D, so size it here via a crop-agnostic ensure of the temp buffer. + const size_t temp_bytes = static_cast(temp.cols) * temp.rows * 3; if (temp_bytes > cap_temp_) { if (d_temp_) cudaFree(d_temp_); if (h_temp_pinned_) cudaFreeHost(h_temp_pinned_); @@ -144,33 +135,26 @@ cv::Mat PasteBackGPU::paste_back(const cv::Mat& temp_vision_frame, } std::memcpy(h_temp_pinned_, temp.data, temp_bytes); cudaMemcpyAsync(d_temp_, h_temp_pinned_, temp_bytes, cudaMemcpyHostToDevice, stream); - - return run(d_temp_, W, H, crop_vision_frame, crop_mask, affine_matrix, stream, blend_alpha); -} - -cv::Mat PasteBackGPU::paste_back(const unsigned char* d_temp, int W, int H, - const cv::Mat& crop_vision_frame, - const cv::Mat& crop_mask, - const cv::Mat& affine_matrix, - cudaStream_t stream, - float blend_alpha) { - // temp is already on the device — no H2D for the full frame. - return run(d_temp, W, H, crop_vision_frame, crop_mask, affine_matrix, stream, blend_alpha); + return d_temp_; } -cv::Mat PasteBackGPU::run(const unsigned char* d_temp, int W, int H, - const cv::Mat& crop_vision_frame, const cv::Mat& crop_mask, - const cv::Mat& affine_matrix, cudaStream_t stream, float blend_alpha) { +// crop/mask/affine H2D + kernel into d_out. No D2H. If d_out is null, the kernel writes into the +// internal d_out_ buffer (host path); pass nullptr — NOT d_out_ — so the resolve happens AFTER +// ensure_capacity, which may reallocate d_out_ (a stale/null pointer would otherwise be used). +void PasteBackGPU::run_core(const unsigned char* d_temp, int W, int H, + const cv::Mat& crop_vision_frame, const cv::Mat& crop_mask, + const cv::Mat& affine_matrix, cudaStream_t stream, float blend_alpha, + unsigned char* d_out) { cv::Mat crop = crop_vision_frame.isContinuous() ? crop_vision_frame : crop_vision_frame.clone(); cv::Mat mask = crop_mask.isContinuous() ? crop_mask : crop_mask.clone(); const int Cw = crop.cols, Ch = crop.rows; const size_t temp_bytes = static_cast(W) * H * 3; - const size_t out_bytes = temp_bytes; const size_t crop_bytes = static_cast(Cw) * Ch * 3 * sizeof(float); const size_t mask_bytes = static_cast(Cw) * Ch * sizeof(float); - ensure_capacity(temp_bytes, crop_bytes, mask_bytes, out_bytes); + ensure_capacity(temp_bytes, crop_bytes, mask_bytes, temp_bytes); + if (d_out == nullptr) d_out = d_out_; // host path: resolve AFTER ensure_capacity (re)allocates // affine -> float[6] (estimateAffinePartial2D usually returns CV_64F) cv::Mat M64; @@ -185,8 +169,16 @@ cv::Mat PasteBackGPU::run(const unsigned char* d_temp, int W, int H, dim3 block(16, 16); dim3 grid((W + block.x - 1) / block.x, (H + block.y - 1) / block.y); paste_back_fused_kernel<<>>( - d_temp, d_crop_, d_mask_, d_affine_, d_out_, W, H, Cw, Ch, blend_alpha); + d_temp, d_crop_, d_mask_, d_affine_, d_out, W, H, Cw, Ch, blend_alpha); +} + +cv::Mat PasteBackGPU::run(const unsigned char* d_temp, int W, int H, + const cv::Mat& crop_vision_frame, const cv::Mat& crop_mask, + const cv::Mat& affine_matrix, cudaStream_t stream, float blend_alpha) { + // pass nullptr so run_core writes into d_out_ AFTER ensure_capacity has (re)allocated it. + run_core(d_temp, W, H, crop_vision_frame, crop_mask, affine_matrix, stream, blend_alpha, nullptr); + const size_t out_bytes = static_cast(W) * H * 3; cudaMemcpyAsync(h_out_pinned_, d_out_, out_bytes, cudaMemcpyDeviceToHost, stream); cudaStreamSynchronize(stream); @@ -194,3 +186,42 @@ cv::Mat PasteBackGPU::run(const unsigned char* d_temp, int W, int H, std::memcpy(result.data, h_out_pinned_, out_bytes); return result; } + +cv::Mat PasteBackGPU::paste_back(const cv::Mat& temp_vision_frame, + const cv::Mat& crop_vision_frame, + const cv::Mat& crop_mask, + const cv::Mat& affine_matrix, + cudaStream_t stream, + float blend_alpha) { + const unsigned char* d_temp = upload_temp(temp_vision_frame, stream); + return run(d_temp, temp_vision_frame.cols, temp_vision_frame.rows, + crop_vision_frame, crop_mask, affine_matrix, stream, blend_alpha); +} + +cv::Mat PasteBackGPU::paste_back(const unsigned char* d_temp, int W, int H, + const cv::Mat& crop_vision_frame, + const cv::Mat& crop_mask, + const cv::Mat& affine_matrix, + cudaStream_t stream, + float blend_alpha) { + // temp is already on the device — no H2D for the full frame. + return run(d_temp, W, H, crop_vision_frame, crop_mask, affine_matrix, stream, blend_alpha); +} + +void PasteBackGPU::paste_back_to_device(const cv::Mat& temp_vision_frame, + const cv::Mat& crop_vision_frame, const cv::Mat& crop_mask, + const cv::Mat& affine_matrix, DeviceFrame& out, + cudaStream_t stream, float blend_alpha) { + const int W = temp_vision_frame.cols, H = temp_vision_frame.rows; + const unsigned char* d_temp = upload_temp(temp_vision_frame, stream); + run_core(d_temp, W, H, crop_vision_frame, crop_mask, affine_matrix, stream, blend_alpha, + out.prepare(W, H)); +} + +void PasteBackGPU::paste_back_to_device(const unsigned char* d_temp, int W, int H, + const cv::Mat& crop_vision_frame, const cv::Mat& crop_mask, + const cv::Mat& affine_matrix, DeviceFrame& out, + cudaStream_t stream, float blend_alpha) { + run_core(d_temp, W, H, crop_vision_frame, crop_mask, affine_matrix, stream, blend_alpha, + out.prepare(W, H)); +} diff --git a/lite/trt/kernel/paste_back_manager.h b/lite/trt/kernel/paste_back_manager.h index b2f1e111..3fba09b3 100644 --- a/lite/trt/kernel/paste_back_manager.h +++ b/lite/trt/kernel/paste_back_manager.h @@ -2,6 +2,7 @@ #define PASTE_BACK_MANAGER_H #include "paste_back.cuh" +#include "device_frame.h" #include // Old CPU-heavy version (two full-frame warpAffine + per-call malloc/sync copies); kept for A/B. @@ -38,11 +39,29 @@ class PasteBackGPU { cudaStream_t stream = nullptr, float blend_alpha = 1.0f); + // Device-OUTPUT: paste straight into a DeviceFrame (no D2H) so the result stays GPU-resident + // for the next stage. Host-temp variant (uploads temp) and device-temp variant. + void paste_back_to_device(const cv::Mat& temp_vision_frame, + const cv::Mat& crop_vision_frame, const cv::Mat& crop_mask, + const cv::Mat& affine_matrix, DeviceFrame& out, + cudaStream_t stream = nullptr, float blend_alpha = 1.0f); + void paste_back_to_device(const unsigned char* d_temp, int W, int H, + const cv::Mat& crop_vision_frame, const cv::Mat& crop_mask, + const cv::Mat& affine_matrix, DeviceFrame& out, + cudaStream_t stream = nullptr, float blend_alpha = 1.0f); + private: void ensure_capacity(size_t temp_bytes, size_t crop_bytes, size_t mask_bytes, size_t out_bytes); - // Shared tail: crop/mask/affine H2D, kernel into d_out_, D2H result. d_temp is the kernel's - // full-frame input (either d_temp_ after an H2D, or a caller-provided device pointer). + // Uploads temp into d_temp_ (grows it as needed) and returns the device pointer. + const unsigned char* upload_temp(const cv::Mat& temp_vision_frame, cudaStream_t stream); + // Core: crop/mask/affine H2D + kernel into d_out (no D2H). d_temp is the full-frame input + // (d_temp_ after an H2D, or a caller device pointer); d_out is the kernel's output buffer. + void run_core(const unsigned char* d_temp, int W, int H, + const cv::Mat& crop_vision_frame, const cv::Mat& crop_mask, + const cv::Mat& affine_matrix, cudaStream_t stream, float blend_alpha, + unsigned char* d_out); + // run_core into d_out_ then D2H -> a fresh host Mat. cv::Mat run(const unsigned char* d_temp, int W, int H, const cv::Mat& crop_vision_frame, const cv::Mat& crop_mask, const cv::Mat& affine_matrix, cudaStream_t stream, float blend_alpha); From dc2a4a5d1d437e4916f5f87e7be0acb662491919 Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Sun, 7 Jun 2026 20:26:25 +0800 Subject: [PATCH 26/30] =?UTF-8?q?perf(trt):=20DeviceFrame=20brick=204=20?= =?UTF-8?q?=E2=80=94=20restoration=20crop=20stays=20on=20device=20(postpro?= =?UTF-8?q?c=20->=20paste,=20no=20bounce)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The restored 512 crop no longer round-trips through host: the postprocess kernel writes into a reusable device buffer and paste-back reads it straight from there. restoration is now fully device-resident internally — warp(device) -> infer -> postproc(device crop) -> paste(device temp + device crop) -> a single final D2H. - FaceRestorePostprocessGPU: runs the transpose/cvtColor/denorm kernel into a reused device buffer (no per-call cudaMalloc, no D2H), returns the device crop pointer. - PasteBackGPU: device-crop overload + upload_crop(); run_core now takes a device crop pointer so the host-Mat and device paths share it (mask/affine are the only remaining H2D in paste). - restoration: postproc_gpu_ -> paste_back(device temp, device crop). Verified on RTX 4090: PSNR 99 dB (max|diff|=0, bit-exact — same kernel, just kept on device), memory flat. restoration 9.8->8.8 ms, 32.8->33.6 FPS. Co-Authored-By: Claude Opus 4.8 --- lite/trt/cv/trt_face_restoration.cpp | 23 +++---- lite/trt/cv/trt_face_restoration.h | 1 + .../face_restoration_postprocess_manager.cpp | 21 ++++++ .../face_restoration_postprocess_manager.h | 19 ++++++ lite/trt/kernel/paste_back_manager.cpp | 64 +++++++++++++------ lite/trt/kernel/paste_back_manager.h | 20 ++++-- 6 files changed, 108 insertions(+), 40 deletions(-) diff --git a/lite/trt/cv/trt_face_restoration.cpp b/lite/trt/cv/trt_face_restoration.cpp index 76674764..ec9f588d 100644 --- a/lite/trt/cv/trt_face_restoration.cpp +++ b/lite/trt/cv/trt_face_restoration.cpp @@ -74,30 +74,25 @@ cv::Mat TRTFaceFusionFaceRestoration::restore_core(const DeviceFrame &frame, cudaStreamSynchronize(stream); } - // ---------------- postprocess: transpose kernel + cvtColor + paste_back + blend ---------------- + // ---------------- postprocess: GPU transpose -> device crop -> paste+blend (all device) ------- cv::Mat dst_image; { LITE_CPU_SCOPE_OPT(prof, "postprocess"); - const int height = 512, width = 512; - std::vector transposed_data(1 * 3 * 512 * 512); + const float *d_crop = nullptr; { - // GPU kernel writes HWC, BGR, float[0,255] straight into transposed_data, - // folding the old CPU uint8->float conversion + cv::cvtColor(RGB2BGR). - LITE_CPU_SCOPE_OPT(prof, " transpose+dl"); - launch_face_restoration_postprocess( - static_cast(buffers[1]), transposed_data.data(), 3, 512, 512); + // GPU kernel writes HWC BGR float[0,255] into a reusable DEVICE buffer (no D2H, no + // per-call cudaMalloc) — fed straight to paste-back, so the 512 crop never hits host. + LITE_CPU_SCOPE_OPT(prof, " transpose(gpu)"); + d_crop = postproc_gpu_.run(static_cast(buffers[1]), 3, 512, 512, stream); } - // aliases transposed_data (alive for the rest of this scope, i.e. through paste_back) - cv::Mat mat(height, width, CV_32FC3, transposed_data.data()); - { // GPU fused: inverse-mapping sampling + paste + face-enhancer blend in ONE kernel. - // temp frame is read straight from the device-resident `frame` (no full-frame - // H2D). blend_alpha=0.8 folds the old CPU blend_frame(target 0.2 / paste 0.8) in. + // Both the temp frame and the crop are device-resident; only the box mask is H2D'd. + // blend_alpha=0.8 folds the old CPU blend_frame(target 0.2 / paste 0.8) in. LITE_CPU_SCOPE_OPT(prof, " paste_back+blend"); dst_image = paste_back_gpu_.paste_back( frame.data(), frame.width(), frame.height(), - mat, box_mask, affine_matrix, stream, /*blend_alpha=*/0.8f); + d_crop, 512, 512, box_mask, affine_matrix, stream, /*blend_alpha=*/0.8f); } } diff --git a/lite/trt/cv/trt_face_restoration.h b/lite/trt/cv/trt_face_restoration.h index f37aca13..d7132dcc 100644 --- a/lite/trt/cv/trt_face_restoration.h +++ b/lite/trt/cv/trt_face_restoration.h @@ -43,6 +43,7 @@ namespace trtcv{ lite::bench::Profiler *prof); PasteBackGPU paste_back_gpu_; // GPU fused paste_back, reuses device buffers FaceRestorePreprocessGPU preprocess_gpu_; // GPU fused bgr2rgb+normalize+CHW into input buffer + FaceRestorePostprocessGPU postproc_gpu_; // GPU transpose -> reusable device crop (no D2H) WarpAffineNpp warp_npp_; // GPU (NPP) affine warp; crop stays device-resident DeviceFrame input_frame_; // input frame uploaded once, shared by warp + paste_back cv::Mat box_mask_cache_; // static box mask is size-only; compute once and reuse diff --git a/lite/trt/kernel/face_restoration_postprocess_manager.cpp b/lite/trt/kernel/face_restoration_postprocess_manager.cpp index d21e37e3..45cf9ca5 100644 --- a/lite/trt/kernel/face_restoration_postprocess_manager.cpp +++ b/lite/trt/kernel/face_restoration_postprocess_manager.cpp @@ -33,4 +33,25 @@ void launch_face_restoration_postprocess( cudaMemcpy(output_final, d_output_final, vec_num * sizeof(float), cudaMemcpyDeviceToHost); cudaFree(d_output_final); +} + +FaceRestorePostprocessGPU::~FaceRestorePostprocessGPU() { + if (d_out_) cudaFree(d_out_); +} + +const float* FaceRestorePostprocessGPU::run(float* trt_outputs, int channel, int height, int width, + cudaStream_t stream) { + const int vec_num = channel * height * width; + const size_t bytes = static_cast(vec_num) * sizeof(float); + if (bytes > cap_) { + if (d_out_) cudaFree(d_out_); + cudaMalloc(&d_out_, bytes); + cap_ = bytes; + } + + const int block_size = 256; + const int grid_size = (vec_num + block_size - 1) / block_size; + face_restoration_postprocess<<>>( + trt_outputs, d_out_, channel, height, width); + return d_out_; // HWC BGR float[0,255], device-resident; caller consumes on the same stream } \ No newline at end of file diff --git a/lite/trt/kernel/face_restoration_postprocess_manager.h b/lite/trt/kernel/face_restoration_postprocess_manager.h index 00834e90..43b30bc9 100644 --- a/lite/trt/kernel/face_restoration_postprocess_manager.h +++ b/lite/trt/kernel/face_restoration_postprocess_manager.h @@ -7,6 +7,7 @@ #include #include #include +#include #include "face_restoration_postprocess.cuh" void launch_face_restoration_postprocess( @@ -17,5 +18,23 @@ void launch_face_restoration_postprocess( int width ); +// Device-resident variant: runs the postprocess kernel into a reusable device buffer (no per-call +// cudaMalloc, no D2H) and returns the device pointer to the HWC BGR float[0,255] crop, ready to +// feed paste-back directly. Launches on `stream` and does NOT sync. +class FaceRestorePostprocessGPU { +public: + FaceRestorePostprocessGPU() = default; + ~FaceRestorePostprocessGPU(); + FaceRestorePostprocessGPU(const FaceRestorePostprocessGPU&) = delete; + FaceRestorePostprocessGPU& operator=(const FaceRestorePostprocessGPU&) = delete; + + const float* run(float* trt_outputs, int channel, int height, int width, + cudaStream_t stream = nullptr); + +private: + float* d_out_ = nullptr; + size_t cap_ = 0; +}; + #endif //LITE_AI_TOOLKIT_FACE_RESTORATION_POSTPROCESS_MANAGER_H diff --git a/lite/trt/kernel/paste_back_manager.cpp b/lite/trt/kernel/paste_back_manager.cpp index b177645e..24ebd05b 100644 --- a/lite/trt/kernel/paste_back_manager.cpp +++ b/lite/trt/kernel/paste_back_manager.cpp @@ -138,22 +138,31 @@ const unsigned char* PasteBackGPU::upload_temp(const cv::Mat& temp_vision_frame, return d_temp_; } -// crop/mask/affine H2D + kernel into d_out. No D2H. If d_out is null, the kernel writes into the -// internal d_out_ buffer (host path); pass nullptr — NOT d_out_ — so the resolve happens AFTER -// ensure_capacity, which may reallocate d_out_ (a stale/null pointer would otherwise be used). +const float* PasteBackGPU::upload_crop(const cv::Mat& crop_vision_frame, cudaStream_t stream) { + cv::Mat crop = crop_vision_frame.isContinuous() ? crop_vision_frame : crop_vision_frame.clone(); + const size_t crop_bytes = static_cast(crop.cols) * crop.rows * 3 * sizeof(float); + if (crop_bytes > cap_crop_) { + if (d_crop_) cudaFree(d_crop_); + cudaMalloc(&d_crop_, crop_bytes); + cap_crop_ = crop_bytes; + } + cudaMemcpyAsync(d_crop_, crop.ptr(), crop_bytes, cudaMemcpyHostToDevice, stream); + return d_crop_; +} + +// mask/affine H2D + kernel into d_out. No D2H. d_temp and d_crop are device inputs. If d_out is null +// the kernel writes into the internal d_out_ (host path); pass nullptr — NOT d_out_ — so the resolve +// happens AFTER ensure_capacity, which may reallocate d_out_ (a stale pointer would otherwise be used). void PasteBackGPU::run_core(const unsigned char* d_temp, int W, int H, - const cv::Mat& crop_vision_frame, const cv::Mat& crop_mask, + const float* d_crop, int Cw, int Ch, const cv::Mat& crop_mask, const cv::Mat& affine_matrix, cudaStream_t stream, float blend_alpha, unsigned char* d_out) { - cv::Mat crop = crop_vision_frame.isContinuous() ? crop_vision_frame : crop_vision_frame.clone(); cv::Mat mask = crop_mask.isContinuous() ? crop_mask : crop_mask.clone(); - const int Cw = crop.cols, Ch = crop.rows; - const size_t temp_bytes = static_cast(W) * H * 3; - const size_t crop_bytes = static_cast(Cw) * Ch * 3 * sizeof(float); + const size_t out_bytes = static_cast(W) * H * 3; const size_t mask_bytes = static_cast(Cw) * Ch * sizeof(float); - - ensure_capacity(temp_bytes, crop_bytes, mask_bytes, temp_bytes); + // grows d_mask_/d_out_/h_out_pinned_ + allocs d_affine_; d_temp_/d_crop_ are managed by uploaders. + ensure_capacity(/*temp*/0, /*crop*/0, mask_bytes, out_bytes); if (d_out == nullptr) d_out = d_out_; // host path: resolve AFTER ensure_capacity (re)allocates // affine -> float[6] (estimateAffinePartial2D usually returns CV_64F) @@ -162,21 +171,20 @@ void PasteBackGPU::run_core(const unsigned char* d_temp, int W, int H, float h_aff[6]; for (int i = 0; i < 6; ++i) h_aff[i] = static_cast(M64.at(i / 3, i % 3)); - cudaMemcpyAsync(d_crop_, crop.ptr(), crop_bytes, cudaMemcpyHostToDevice, stream); cudaMemcpyAsync(d_mask_, mask.ptr(), mask_bytes, cudaMemcpyHostToDevice, stream); cudaMemcpyAsync(d_affine_, h_aff, 6 * sizeof(float), cudaMemcpyHostToDevice, stream); dim3 block(16, 16); dim3 grid((W + block.x - 1) / block.x, (H + block.y - 1) / block.y); paste_back_fused_kernel<<>>( - d_temp, d_crop_, d_mask_, d_affine_, d_out, W, H, Cw, Ch, blend_alpha); + d_temp, d_crop, d_mask_, d_affine_, d_out, W, H, Cw, Ch, blend_alpha); } cv::Mat PasteBackGPU::run(const unsigned char* d_temp, int W, int H, - const cv::Mat& crop_vision_frame, const cv::Mat& crop_mask, + const float* d_crop, int Cw, int Ch, const cv::Mat& crop_mask, const cv::Mat& affine_matrix, cudaStream_t stream, float blend_alpha) { // pass nullptr so run_core writes into d_out_ AFTER ensure_capacity has (re)allocated it. - run_core(d_temp, W, H, crop_vision_frame, crop_mask, affine_matrix, stream, blend_alpha, nullptr); + run_core(d_temp, W, H, d_crop, Cw, Ch, crop_mask, affine_matrix, stream, blend_alpha, nullptr); const size_t out_bytes = static_cast(W) * H * 3; cudaMemcpyAsync(h_out_pinned_, d_out_, out_bytes, cudaMemcpyDeviceToHost, stream); @@ -194,8 +202,10 @@ cv::Mat PasteBackGPU::paste_back(const cv::Mat& temp_vision_frame, cudaStream_t stream, float blend_alpha) { const unsigned char* d_temp = upload_temp(temp_vision_frame, stream); + const float* d_crop = upload_crop(crop_vision_frame, stream); return run(d_temp, temp_vision_frame.cols, temp_vision_frame.rows, - crop_vision_frame, crop_mask, affine_matrix, stream, blend_alpha); + d_crop, crop_vision_frame.cols, crop_vision_frame.rows, + crop_mask, affine_matrix, stream, blend_alpha); } cv::Mat PasteBackGPU::paste_back(const unsigned char* d_temp, int W, int H, @@ -204,8 +214,18 @@ cv::Mat PasteBackGPU::paste_back(const unsigned char* d_temp, int W, int H, const cv::Mat& affine_matrix, cudaStream_t stream, float blend_alpha) { - // temp is already on the device — no H2D for the full frame. - return run(d_temp, W, H, crop_vision_frame, crop_mask, affine_matrix, stream, blend_alpha); + // temp already on device; crop is host -> upload it. + const float* d_crop = upload_crop(crop_vision_frame, stream); + return run(d_temp, W, H, d_crop, crop_vision_frame.cols, crop_vision_frame.rows, + crop_mask, affine_matrix, stream, blend_alpha); +} + +cv::Mat PasteBackGPU::paste_back(const unsigned char* d_temp, int W, int H, + const float* d_crop, int Cw, int Ch, + const cv::Mat& crop_mask, const cv::Mat& affine_matrix, + cudaStream_t stream, float blend_alpha) { + // both temp and crop already on device — only mask/affine are H2D'd. + return run(d_temp, W, H, d_crop, Cw, Ch, crop_mask, affine_matrix, stream, blend_alpha); } void PasteBackGPU::paste_back_to_device(const cv::Mat& temp_vision_frame, @@ -214,14 +234,16 @@ void PasteBackGPU::paste_back_to_device(const cv::Mat& temp_vision_frame, cudaStream_t stream, float blend_alpha) { const int W = temp_vision_frame.cols, H = temp_vision_frame.rows; const unsigned char* d_temp = upload_temp(temp_vision_frame, stream); - run_core(d_temp, W, H, crop_vision_frame, crop_mask, affine_matrix, stream, blend_alpha, - out.prepare(W, H)); + const float* d_crop = upload_crop(crop_vision_frame, stream); + run_core(d_temp, W, H, d_crop, crop_vision_frame.cols, crop_vision_frame.rows, + crop_mask, affine_matrix, stream, blend_alpha, out.prepare(W, H)); } void PasteBackGPU::paste_back_to_device(const unsigned char* d_temp, int W, int H, const cv::Mat& crop_vision_frame, const cv::Mat& crop_mask, const cv::Mat& affine_matrix, DeviceFrame& out, cudaStream_t stream, float blend_alpha) { - run_core(d_temp, W, H, crop_vision_frame, crop_mask, affine_matrix, stream, blend_alpha, - out.prepare(W, H)); + const float* d_crop = upload_crop(crop_vision_frame, stream); + run_core(d_temp, W, H, d_crop, crop_vision_frame.cols, crop_vision_frame.rows, + crop_mask, affine_matrix, stream, blend_alpha, out.prepare(W, H)); } diff --git a/lite/trt/kernel/paste_back_manager.h b/lite/trt/kernel/paste_back_manager.h index 3fba09b3..efd9575b 100644 --- a/lite/trt/kernel/paste_back_manager.h +++ b/lite/trt/kernel/paste_back_manager.h @@ -39,6 +39,15 @@ class PasteBackGPU { cudaStream_t stream = nullptr, float blend_alpha = 1.0f); + // Both temp AND crop already on device (e.g. swap's output frame + restoration's postproc crop): + // only mask/affine are H2D'd. d_crop is HWC interleaved BGR float[0,255], Cw x Ch. + cv::Mat paste_back(const unsigned char* d_temp, int W, int H, + const float* d_crop, int Cw, int Ch, + const cv::Mat& crop_mask, + const cv::Mat& affine_matrix, + cudaStream_t stream = nullptr, + float blend_alpha = 1.0f); + // Device-OUTPUT: paste straight into a DeviceFrame (no D2H) so the result stays GPU-resident // for the next stage. Host-temp variant (uploads temp) and device-temp variant. void paste_back_to_device(const cv::Mat& temp_vision_frame, @@ -53,17 +62,18 @@ class PasteBackGPU { private: void ensure_capacity(size_t temp_bytes, size_t crop_bytes, size_t mask_bytes, size_t out_bytes); - // Uploads temp into d_temp_ (grows it as needed) and returns the device pointer. + // Upload temp/crop into the internal device buffers (grown as needed); return the device ptr. const unsigned char* upload_temp(const cv::Mat& temp_vision_frame, cudaStream_t stream); - // Core: crop/mask/affine H2D + kernel into d_out (no D2H). d_temp is the full-frame input - // (d_temp_ after an H2D, or a caller device pointer); d_out is the kernel's output buffer. + const float* upload_crop(const cv::Mat& crop_vision_frame, cudaStream_t stream); + // Core: mask/affine H2D + kernel into d_out (no D2H). d_temp and d_crop are device inputs + // (uploaded, or caller-provided); d_out is the kernel's output buffer (null -> internal d_out_). void run_core(const unsigned char* d_temp, int W, int H, - const cv::Mat& crop_vision_frame, const cv::Mat& crop_mask, + const float* d_crop, int Cw, int Ch, const cv::Mat& crop_mask, const cv::Mat& affine_matrix, cudaStream_t stream, float blend_alpha, unsigned char* d_out); // run_core into d_out_ then D2H -> a fresh host Mat. cv::Mat run(const unsigned char* d_temp, int W, int H, - const cv::Mat& crop_vision_frame, const cv::Mat& crop_mask, + const float* d_crop, int Cw, int Ch, const cv::Mat& crop_mask, const cv::Mat& affine_matrix, cudaStream_t stream, float blend_alpha); unsigned char* d_temp_ = nullptr; From ab102425274159dd42112e3d6ec0e636c55036a8 Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Sun, 7 Jun 2026 20:51:44 +0800 Subject: [PATCH 27/30] =?UTF-8?q?perf(trt):=20DeviceFrame=20brick=205=20?= =?UTF-8?q?=E2=80=94=20swap=20input=20device-resident=20via=20one=20shared?= =?UTF-8?q?=20target=20upload?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit swap now uploads the target frame ONCE into target_dev_ and both the NPP warp (128 crop) and the paste-back read it from there — no host-side warp, no redundant upload. This replaces the CPU warpAffine + cvtColor + convertTo + create_tensor + the 128 CHW-tensor H2D with: one full-frame H2D (which paste already did) -> GPU warp -> fused bgr2rgb+/255+CHW straight into buffers[0]. (An earlier attempt warped from a SECOND upload of the frame — uploading the full frame twice per swap — which was a net loss; sharing the single target_dev_ upload is the fix.) - preprocess kernel generalized to out = v*scale + bias (the reusable normalize+CHW template): restoration 1/127.5,-1; swap 1/255,0. - swap: target_dev_ + warp_npp_ + preprocess_gpu_; swap_core warps from device; both detect() overloads paste from target_dev_. Verified on RTX 4090: PSNR 57.0 dB vs the CPU-warp result (max|diff|=9, equivalent), clean eyeball, memory flat. swap input is now fully device-resident (a structural step toward sharing one target upload across detect/landmark/swap). Per-stage ms delta is within this box's ~1ms run-to-run clock jitter, so the win here is structural + less CPU work, not a measurable single-stage speedup. Co-Authored-By: Claude Opus 4.8 --- lite/trt/cv/trt_face_swap.cpp | 56 +++++++++---------- lite/trt/cv/trt_face_swap.h | 9 +-- .../trt/kernel/face_restoration_preprocess.cu | 12 ++-- .../kernel/face_restoration_preprocess.cuh | 3 +- .../face_restoration_preprocess_manager.cpp | 10 ++-- .../face_restoration_preprocess_manager.h | 7 ++- 6 files changed, 50 insertions(+), 47 deletions(-) diff --git a/lite/trt/cv/trt_face_swap.cpp b/lite/trt/cv/trt_face_swap.cpp index e2406021..548fd78b 100644 --- a/lite/trt/cv/trt_face_swap.cpp +++ b/lite/trt/cv/trt_face_swap.cpp @@ -5,36 +5,28 @@ #include "trt_face_swap.h" using trtcv::TRTFaceFusionFaceSwap; -void TRTFaceFusionFaceSwap::preprocess(cv::Mat &target_face, std::vector source_image_embeding, - std::vector target_landmark_5, - std::vector &processed_source_embeding, cv::Mat &preprocessed_mat) { - - std::tie(preprocessed_mat, affine_martix) = face_utils::warp_face_by_face_landmark_5(target_face,target_landmark_5,face_utils::ARCFACE_128_V2); - - cv::cvtColor(preprocessed_mat,preprocessed_mat,cv::COLOR_BGR2RGB); - preprocessed_mat.convertTo(preprocessed_mat,CV_32FC3,1.0 / 255.f); - preprocessed_mat.convertTo(preprocessed_mat,CV_32FC3,1.0 / 1.f,0); - - // model_matrix_ and box_mask_ are loaded/built once in the constructor (they are - // constant); they used to be load_npy'd from disk and rebuilt every frame. - processed_source_embeding = face_utils::dot_product(source_image_embeding, model_matrix_, 512); - face_utils::normalize(processed_source_embeding); -} - - // infer + postprocess: produces the host BGR float[0,255] 128x128 swapped-face crop (owns its -// data) and sets affine_martix. Shared by both detect() overloads. +// data) and sets affine_martix. Uploads the target frame ONCE into target_dev_ (shared by the +// warp here AND the caller's paste-back). Shared by both detect() overloads. void TRTFaceFusionFaceSwap::swap_core(cv::Mat &target_image, std::vector &source_face_embeding, std::vector &target_landmark_5, cv::Mat &mat_out) { - std::vector source_embeding_input; - cv::Mat model_input_mat; - preprocess(target_image,source_face_embeding,target_landmark_5,source_embeding_input,model_input_mat); + // upload the target frame to the device ONCE — warp (here) + paste-back (caller) both read it. + target_dev_.upload(target_image, stream); + + // source embedding (CPU, cheap): model_matrix_ loaded once in ctor. + std::vector source_embeding_input = face_utils::dot_product(source_face_embeding, model_matrix_, 512); + face_utils::normalize(source_embeding_input); - std::vector input_vector; - trtcv::utils::transform::create_tensor(model_input_mat,input_vector,input_node_dims,trtcv::utils::transform::CHW); + // image: estimate the ARCFACE-128 affine (CPU) -> NPP warp the 128 crop FROM the device frame + // -> fused bgr2rgb + /255 + HWC->CHW straight into buffers[0]. Replaces CPU warpAffine + cvtColor + // + convertTo + create_tensor + the CHW-tensor H2D, with no extra full-frame upload (reuses + // target_dev_). inswapper input is RGB normalized to [0,1]. + affine_martix = face_utils::estimate_affine_by_landmark_5(target_landmark_5, face_utils::ARCFACE_128_V2); + const unsigned char* d_crop = warp_npp_.warp_device_to_device( + target_dev_.data(), target_dev_.width(), target_dev_.height(), affine_martix, 128, stream); + preprocess_gpu_.run_device(d_crop, 128, 128, static_cast(buffers[0]), stream, + /*scale=*/1.0f / 255.f, /*bias=*/0.0f); - // input 0 = preprocessed target face, input 1 = source embedding - cudaMemcpyAsync(buffers[0],input_vector.data(),1 * 3 * 128 * 128 *sizeof(float ), cudaMemcpyHostToDevice,stream); cudaMemcpyAsync(buffers[1],source_embeding_input.data(),512 * sizeof(float), cudaMemcpyHostToDevice,stream); cudaStreamSynchronize(stream); @@ -72,20 +64,22 @@ void TRTFaceFusionFaceSwap::swap_core(cv::Mat &target_image, std::vector void TRTFaceFusionFaceSwap::detect(cv::Mat &target_image, std::vector source_face_embeding, std::vector target_landmark_5, cv::Mat &face_swap_image) { - cv::Mat ori_image = target_image.clone(); cv::Mat mat; swap_core(target_image, source_face_embeding, target_landmark_5, mat); - // GPU-fused paste-back (reused device buffers); numerically equivalent to launch_paste_back. - face_swap_image = paste_back_gpu_.paste_back(ori_image, mat, box_mask_, affine_martix, stream); + // paste-back reads the temp frame from the shared device-resident target_dev_ (no extra H2D). + face_swap_image = paste_back_gpu_.paste_back( + target_dev_.data(), target_dev_.width(), target_dev_.height(), + mat, box_mask_, affine_martix, stream); } void TRTFaceFusionFaceSwap::detect(cv::Mat &target_image, std::vector source_face_embeding, std::vector target_landmark_5, DeviceFrame &out_frame) { - cv::Mat ori_image = target_image.clone(); cv::Mat mat; swap_core(target_image, source_face_embeding, target_landmark_5, mat); - // paste straight into the device-resident out_frame (no D2H) for restoration to consume. - paste_back_gpu_.paste_back_to_device(ori_image, mat, box_mask_, affine_martix, out_frame, stream); + // paste straight into the device-resident out_frame (no D2H), temp read from shared target_dev_. + paste_back_gpu_.paste_back_to_device( + target_dev_.data(), target_dev_.width(), target_dev_.height(), + mat, box_mask_, affine_martix, out_frame, stream); // restoration reads out_frame on its OWN stream, so make sure this paste has completed. cudaStreamSynchronize(stream); } diff --git a/lite/trt/cv/trt_face_swap.h b/lite/trt/cv/trt_face_swap.h index d207a2d0..ad55db05 100644 --- a/lite/trt/cv/trt_face_swap.h +++ b/lite/trt/cv/trt_face_swap.h @@ -11,6 +11,8 @@ #include "lite/trt/kernel/face_swap_postproces_manager.h" #include "lite/trt/kernel/paste_back_manager.h" #include "lite/trt/kernel/device_frame.h" +#include "lite/trt/kernel/warp_affine_npp.h" +#include "lite/trt/kernel/face_restoration_preprocess_manager.h" namespace trtcv{ class LITE_EXPORTS TRTFaceFusionFaceSwap : BasicTRTHandler{ @@ -22,15 +24,14 @@ namespace trtcv{ model_matrix_ = face_utils::load_npy(std::string(SOURCE_PATH) + "/examples/lite/resources/model_matrix.npy"); box_mask_ = face_utils::create_static_box_mask(std::vector{128.0f, 128.0f}); }; - private: - void preprocess(cv::Mat &target_face,std::vector source_image_embeding,std::vector target_landmark_5, - std::vector &processed_source_embeding,cv::Mat &preprocessed_mat); - private: cv::Mat affine_martix; std::vector model_matrix_; // loaded once in ctor (was load_npy every frame) cv::Mat box_mask_; // cached static 128 box mask (was rebuilt every frame) PasteBackGPU paste_back_gpu_; // GPU-fused paste-back, reused device buffers (same as restoration) + WarpAffineNpp warp_npp_; // GPU (NPP) affine warp for the 128 crop (was CPU warpAffine) + FaceRestorePreprocessGPU preprocess_gpu_; // fused bgr2rgb+/255+CHW straight into buffers[0] + DeviceFrame target_dev_; // target frame uploaded ONCE; shared by warp + paste-back public: void detect(cv::Mat &target_image,std::vector source_face_embeding,std::vector target_landmark_5, cv::Mat &face_swap_image); diff --git a/lite/trt/kernel/face_restoration_preprocess.cu b/lite/trt/kernel/face_restoration_preprocess.cu index 66d94bb4..21af95af 100644 --- a/lite/trt/kernel/face_restoration_preprocess.cu +++ b/lite/trt/kernel/face_restoration_preprocess.cu @@ -1,8 +1,10 @@ #include "face_restoration_preprocess.cuh" // One thread per crop pixel. Reads interleaved BGR uint8, writes planar RGB float (CHW), -// normalized to [-1, 1] (v/127.5 - 1). Channel mapping: R->plane0, G->plane1, B->plane2. -__global__ void face_restoration_preprocess_kernel(const unsigned char* crop, float* out, int H, int W) { +// normalized as out = v*scale + bias. Channel mapping: R->plane0, G->plane1, B->plane2. +// restoration: scale=1/127.5, bias=-1 ([-1,1]); swap: scale=1/255, bias=0 ([0,1]). +__global__ void face_restoration_preprocess_kernel(const unsigned char* crop, float* out, int H, int W, + float scale, float bias) { int x = blockIdx.x * blockDim.x + threadIdx.x; int y = blockIdx.y * blockDim.y + threadIdx.y; if (x >= W || y >= H) return; @@ -14,7 +16,7 @@ __global__ void face_restoration_preprocess_kernel(const unsigned char* crop, fl int plane = H * W; int off = y * W + x; - out[0 * plane + off] = r / 127.5f - 1.f; - out[1 * plane + off] = g / 127.5f - 1.f; - out[2 * plane + off] = b / 127.5f - 1.f; + out[0 * plane + off] = r * scale + bias; + out[1 * plane + off] = g * scale + bias; + out[2 * plane + off] = b * scale + bias; } diff --git a/lite/trt/kernel/face_restoration_preprocess.cuh b/lite/trt/kernel/face_restoration_preprocess.cuh index 794f9fb3..b9ad5e8a 100644 --- a/lite/trt/kernel/face_restoration_preprocess.cuh +++ b/lite/trt/kernel/face_restoration_preprocess.cuh @@ -5,6 +5,7 @@ // Fused face-restoration preprocess: takes the HxW interleaved BGR uint8 crop and writes a // CHW (3,H,W) float tensor that is RGB and normalized by v/127.5 - 1 (i.e. (v/255)*2 - 1). -__global__ void face_restoration_preprocess_kernel(const unsigned char* crop, float* out, int H, int W); +__global__ void face_restoration_preprocess_kernel(const unsigned char* crop, float* out, int H, int W, + float scale, float bias); #endif // FACE_RESTORATION_PREPROCESS_CUH diff --git a/lite/trt/kernel/face_restoration_preprocess_manager.cpp b/lite/trt/kernel/face_restoration_preprocess_manager.cpp index 4a582516..369f33a8 100644 --- a/lite/trt/kernel/face_restoration_preprocess_manager.cpp +++ b/lite/trt/kernel/face_restoration_preprocess_manager.cpp @@ -17,7 +17,8 @@ void FaceRestorePreprocessGPU::ensure_capacity(size_t bytes) { } } -void FaceRestorePreprocessGPU::run(const cv::Mat& crop_bgr_u8, float* d_out, cudaStream_t stream) { +void FaceRestorePreprocessGPU::run(const cv::Mat& crop_bgr_u8, float* d_out, cudaStream_t stream, + float scale, float bias) { cv::Mat c = crop_bgr_u8; if (c.type() != CV_8UC3) c.convertTo(c, CV_8UC3); if (!c.isContinuous()) c = c.clone(); @@ -31,15 +32,16 @@ void FaceRestorePreprocessGPU::run(const cv::Mat& crop_bgr_u8, float* d_out, cud dim3 block(16, 16); dim3 grid((W + block.x - 1) / block.x, (H + block.y - 1) / block.y); - face_restoration_preprocess_kernel<<>>(d_crop_, d_out, H, W); + face_restoration_preprocess_kernel<<>>(d_crop_, d_out, H, W, scale, bias); cudaStreamSynchronize(stream); } void FaceRestorePreprocessGPU::run_device(const unsigned char* d_crop, int H, int W, - float* d_out, cudaStream_t stream) { + float* d_out, cudaStream_t stream, + float scale, float bias) { dim3 block(16, 16); dim3 grid((W + block.x - 1) / block.x, (H + block.y - 1) / block.y); - face_restoration_preprocess_kernel<<>>(d_crop, d_out, H, W); + face_restoration_preprocess_kernel<<>>(d_crop, d_out, H, W, scale, bias); cudaStreamSynchronize(stream); } diff --git a/lite/trt/kernel/face_restoration_preprocess_manager.h b/lite/trt/kernel/face_restoration_preprocess_manager.h index 864c5558..d2ea6594 100644 --- a/lite/trt/kernel/face_restoration_preprocess_manager.h +++ b/lite/trt/kernel/face_restoration_preprocess_manager.h @@ -16,11 +16,14 @@ class FaceRestorePreprocessGPU { FaceRestorePreprocessGPU& operator=(const FaceRestorePreprocessGPU&) = delete; // crop_bgr_u8: CV_8UC3 (e.g. 512x512). d_out: device float CHW buffer (the inference input). - void run(const cv::Mat& crop_bgr_u8, float* d_out, cudaStream_t stream = nullptr); + // out = v*scale + bias (default = restoration's [-1,1]; pass 1/255, 0 for swap's [0,1]). + void run(const cv::Mat& crop_bgr_u8, float* d_out, cudaStream_t stream = nullptr, + float scale = 1.0f / 127.5f, float bias = -1.0f); // Device-resident variant: the crop is already on the GPU (e.g. NPP warp output), so skip the // H2D — just launch the fused kernel reading d_crop -> d_out on `stream`. Syncs before return. - void run_device(const unsigned char* d_crop, int H, int W, float* d_out, cudaStream_t stream = nullptr); + void run_device(const unsigned char* d_crop, int H, int W, float* d_out, cudaStream_t stream = nullptr, + float scale = 1.0f / 127.5f, float bias = -1.0f); private: void ensure_capacity(size_t bytes); From 23c4932367ed6aa7a2b58cb4accf333665c228ab Mon Sep 17 00:00:00 2001 From: DefTruth <31974251+DefTruth@users.noreply.github.com> Date: Mon, 8 Jun 2026 08:19:52 +0800 Subject: [PATCH 28/30] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7bd4a4c1..7eb9d3ea 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ honestly with a reproducible benchmark harness. Welcome to 🌟 star this repo t taken from **78.2 ms → 17.7 ms (4.4×, 12.8 → 56.6 FPS)** on an RTX 4090 by moving paste-back and preprocessing into fused CUDA kernels. See [Benchmark](#benchmark) below. The rest of the pipeline (detect / landmark / swap, FP16) is being optimized stage by stage. -- [lite.ai.toolkit](https://github.com/xlite-dev/lite.ai.toolkit) is mainly maintained by 🎉[@wangzijian1010](https://github.com/wangzijian1010). +- Now, [lite.ai.toolkit](https://github.com/xlite-dev/lite.ai.toolkit) is mainly maintained by 🎉[@wangzijian1010](https://github.com/wangzijian1010). ## ⚡ Benchmark 🔥
From a9495d65115185b8a8e4b0c100be0e9d074930f3 Mon Sep 17 00:00:00 2001 From: wangzijian1010 Date: Sun, 14 Jun 2026 15:47:47 +0800 Subject: [PATCH 29/30] docs: reframe README/quickstart around the device-resident pipeline + real FP16 numbers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - README: lead with the GPU-resident FaceFusion pipeline (DeviceFrame carrying full frames between stages, CUDA/NPP hot kernels); drop the ONNXRuntime badge; fix the legacy tag link. - quickstart: update Performance to the real video-shaped benchmark — 23.6 ms/frame (42.3 FPS) on an RTX 4090 FP16, one full-frame H2D + one D2H, swap->restoration stays GPU-resident. - .gitignore: ignore *.jpg (the local demo result images). Co-Authored-By: Claude Opus 4.8 --- .gitignore | 2 +- README.md | 164 ++++++++++++++-------------------- docs/facefusion_quickstart.md | 19 ++-- 3 files changed, 81 insertions(+), 104 deletions(-) diff --git a/.gitignore b/.gitignore index 092e7000..838358f5 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,4 @@ third_party build/ lite.ai.toolkit.cmake TestExamples - +*.jpg diff --git a/README.md b/README.md index 7eb9d3ea..932709e6 100644 --- a/README.md +++ b/README.md @@ -8,66 +8,83 @@ - -🛠 **Lite.Ai.ToolKit** is a C++ toolkit for **extreme GPU inference**. The flagship is an end-to-end -**FaceFusion face-swap pipeline** (detect → landmark → recognize → swap → restore) running entirely on -**TensorRT**, with the CPU pre/post-processing glue rewritten as **hand-fused CUDA kernels**. The goal is -not breadth — it is to make one real pipeline as fast as a single GPU can make it, and to show the work -honestly with a reproducible benchmark harness. Welcome to 🌟 star this repo to support us ~ 🎉🎉 +🛠 **Lite.Ai.ToolKit** is a C++ toolkit focused on one flagship target: an end-to-end +**FaceFusion face-swap pipeline** (detect → landmark → recognize → swap → restore) running on +**TensorRT**. The current line is about keeping the real pipeline GPU-resident, not collecting model +wrappers: CUDA / NPP kernels handle the hot pre/post-processing, `DeviceFrame` carries full frames +between stages, and the benchmark reports the real per-frame path. > **Heads up (>= 0.3):** the active line targets **TensorRT only**. ONNXRuntime is kept as the numerical > reference + the host for the test suite. The legacy multi-backend build (MNN / NCNN / TNN, 300+ thin -> model wrappers) is frozen on tag **[`v0.2-all-backends`](https://github.com/xlite-dev/lite.ai.toolkit/tree/main)** — check it out if you need those backends. +> model wrappers) is frozen on tag **[`v0.2-all-backends`](https://github.com/xlite-dev/lite.ai.toolkit/tree/v0.2-all-backends)** — check it out if you need those backends. ## 📖 News 🔥🔥
-- **GPU-inference optimization in progress** — the FaceFusion face-restoration stage (GFPGAN 1.4) was - taken from **78.2 ms → 17.7 ms (4.4×, 12.8 → 56.6 FPS)** on an RTX 4090 by moving paste-back and - preprocessing into fused CUDA kernels. See [Benchmark](#benchmark) below. The rest of the pipeline - (detect / landmark / swap, FP16) is being optimized stage by stage. +- **Current FaceFusion pipeline:** **23.6 ms / frame, 42.3 FPS** on an RTX 4090, FP16 deployment, + source prepared once and target processed per frame. +- **Current full-frame copies:** one H2D upload of the target frame, one D2H download of the final + result. The swap → restoration boundary stays GPU-resident. - Now, [lite.ai.toolkit](https://github.com/xlite-dev/lite.ai.toolkit) is mainly maintained by 🎉[@wangzijian1010](https://github.com/wangzijian1010). ## ⚡ Benchmark 🔥
-GPU-inference optimization log. For each algorithm we profile it with a built-in, backend-agnostic harness ([`lite/bench/profiler.h`](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/lite/bench/profiler.h)), then move the CPU pre/post-processing (affine warp, color convert, normalize, tensor layout, paste-back, NMS …) into **fused CUDA kernels** with reused device buffers and pinned + async copies, so the algorithm spends its time on real inference instead of host glue and `cudaMalloc`/sync round-trips. All numbers are **RTX 4090 · TensorRT 10.1 · CUDA 12.4**, median (p50), compute-only, reproducible via the `lite_*_bench` binaries. - -| Algorithm | Before | After | Speedup | What changed | -|:--|:--:|:--:|:--:|:--| -| **FaceFusion · face restoration (GFPGAN 1.4)** | 78.2 ms
(12.8 FPS) | **17.7 ms
(56.6 FPS)** | **4.4×** | inverse-mapping paste-back kernel (replaces 2× CPU `warpAffine` + per-frame `cudaMalloc`); cached static mask; fused `bgr2rgb+normalize+CHW` straight into the input buffer | -| FaceFusion · face detect (YOLOv8-face) | 🚧 | 🚧 | — | bbox decode + NMS → CUDA | -| FaceFusion · 68 landmarks (2DFAN4) | 🚧 | 🚧 | — | warp + preprocess → CUDA | -| FaceFusion · face swap (InSwapper) | 🚧 | 🚧 | — | warp + paste → CUDA | -| FP16 / mixed-precision | 🚧 | 🚧 | — | layer-pinned style convs (keep sensitive layers FP32) | - -
-FaceFusion · face restoration — per-stage breakdown +Measured on **RTX 4090 · TensorRT 10.x · CUDA 12.x**, FP16 deployment, source prepared once and +60 per-frame target iterations. + +| Stage | Time | +|:--|--:| +| face detect | 3.94 ms | +| 68 landmarks | 3.46 ms | +| face swap | 4.96 ms | +| face restoration | 9.85 ms | +| **TOTAL** | **23.6 ms / frame** | +| **Throughput** | **42.3 FPS** | +| GPU memory | 1550 MiB | + +## Data Flow + +Current full-frame data movement is down to the intended minimum: + +| Copy | Direction | Purpose | +|:--|:--|:--| +| 1 | Host → Device | upload target frame once into `target_dev_` | +| 2 | Device → Host | download final restored result | + +The expensive swap → restoration boundary no longer bounces through host memory: + +```text +target host Mat + -> H2D once into target_dev_ + -> swap NPP warp + preprocess + infer + paste_back + -> swapped_frame_ DeviceFrame + -> restoration NPP warp + preprocess + infer + postprocess + paste_back + -> D2H final result +``` -| Stage | Baseline (ms) | Optimized (ms) | Speedup | -|:--|:--:|:--:|:--:| -| preprocess (warp + bgr2rgb + normalize + tensor) | 14.49 | 1.25 | **11.6×** | -| inference (TensorRT) | 11.30 | 10.79 | 1.05× | -| postprocess (incl. paste-back) | 52.02 | 5.32 | **9.8×** | -|   └ paste-back | 39.07 | 2.34 | **16.7×** | -| **End-to-end** | **78.17** | **17.66** | **4.4×** | +Remaining copies are small: detect letterbox input/output metadata, landmark crop/output points, swap's +128 crop transpose bounce, and restoration mask/affine uploads. -paste-back is numerically equivalent to the CPU path (max |diff| = 2/255). The static box mask used to be rebuilt every frame (a large-kernel Gaussian blur) although it only depends on the crop size. With pre/post off the critical path, inference is now ~60% of the stage — FP16 is the next lever. +## Headroom -
+Latency is now close to model-bound. The largest remaining block is GFPGAN inference inside restoration +(about 8 ms), so further single-frame latency gains are harder without quality-risky model changes such +as INT8 or a lighter restorer. The more realistic path toward 60+ FPS is throughput work: multi-stream +frame pipelining and CUDA Graphs, so different frames can overlap instead of running fully serial. ## Features 👏👋 - **GPU-first.** The whole FaceFusion pipeline runs on TensorRT; the pre/post-processing that usually - lingers on the CPU (warp / color-convert / normalize / layout / paste-back / NMS) is implemented as - **fused CUDA kernels** under [`lite/trt/kernel/`](https://github.com/xlite-dev/lite.ai.toolkit/tree/main/lite/trt/kernel), with reused device buffers and pinned + async copies. + lingers on the CPU (warp / color-convert / normalize / layout / paste-back / NMS) is being moved into + **CUDA / NPP kernels** under [`lite/trt/kernel/`](https://github.com/xlite-dev/lite.ai.toolkit/tree/main/lite/trt/kernel), with `DeviceFrame`, reused buffers, and pinned + async copies. - **Measured, not claimed.** A header-only profiler ([`lite/bench/profiler.h`](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/lite/bench/profiler.h)) gives CPU-chrono + GPU-cudaEvent timings (p50 / p99 / FPS / CSV). Every optimization ships with a before/after `lite_*_bench` binary. +- **Video-shaped API.** `prepare_source()` caches the fixed source face embedding once; `process()` is the per-frame target path. The old one-shot `detect()` API remains for images and demos. - **Multi-threaded TRT path.** `_mt` pipelines (e.g. `trt_face_restoration_mt`) run a thread pool with one `IExecutionContext` + `cudaStream_t` + buffer set per thread and an async task queue. -- **Consistent C++ API.** Same `lite::trt::cv::Type::Class` syntax across models, e.g. `lite::trt::cv::detection::YOLOV5`. ## Build 👇👇 @@ -78,7 +95,6 @@ build downloads third-party libs into `third_party/` automatically. git clone --depth=1 https://github.com/xlite-dev/lite.ai.toolkit.git cd lite.ai.toolkit bash ./build.sh tensorrt # GPU / TensorRT backend -# bash ./build.sh # ONNXRuntime backend (CPU reference + 100+ CV models, builds the tests) ``` See [tensorrt-linux-x86_64.zh.md](./docs/tensorrt/tensorrt-linux-x86_64.zh.md) for the TensorRT/CUDA setup. @@ -108,46 +124,19 @@ auto pipeline = lite::trt::cv::face::swap::FaceFusionPipeLine( face_recognizer_engine, // arcface_w600k_r50 face_swap_engine, // inswapper_128 face_restoration_engine); // gfpgan_1.4 -// swap face #0 of the source onto face #0 of the target, then write the result -pipeline.detect(source_image_path, 0, target_image_path, 0, save_image_path); -``` - -#### Single model on the GPU (YOLOv5) -```c++ -#include "lite/lite.h" -// trtexec --onnx=yolov5s.onnx --saveEngine=yolov5s.engine -auto *yolov5 = new lite::trt::cv::detection::YOLOV5(engine_path); -std::vector boxes; -cv::Mat img = cv::imread(test_img_path); -yolov5->detect(img, boxes); -lite::utils::draw_boxes_inplace(img, boxes); -cv::imwrite(save_img_path, img); -delete yolov5; -``` +// Video/server path: prepare the fixed source face once. +cv::Mat source = cv::imread(source_image_path); +pipeline.prepare_source(source, 0); -## Quick Setup 👀 +// Per target frame: process() reuses the cached source embedding. +cv::Mat target = cv::imread(target_image_path); +cv::Mat result = pipeline.process(target, 0); +cv::imwrite(save_image_path, result); -To use the installed library from your own project, point `find_package` at the install dir: - -```cmake -set(lite.ai.toolkit_DIR YOUR-PATH-TO-LITE-INSTALL) -find_package(lite.ai.toolkit REQUIRED PATHS ${lite.ai.toolkit_DIR}) -add_executable(lite_yolov5 test_lite_yolov5.cpp) -target_link_libraries(lite_yolov5 ${lite.ai.toolkit_LIBS}) +// One-shot image convenience is still available: +// pipeline.detect(source_image_path, 0, target_image_path, 0, save_image_path); ``` -## Supported Models (TensorRT) 🚀 -
- -|Class|Class|Class|Class|Class| System | Engine | -|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -|✅[YOLOv5](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolov5.cpp)|✅[YOLOv6](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolov6.cpp)|✅[YOLOv8](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolov8.cpp)|✅[YOLOv8Face](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolov8face.cpp)|✅[YOLOv5Face](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolo5face.cpp)| Linux | TensorRT | -|✅[YOLOX](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolox.cpp)|✅[YOLOv5BlazeFace](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_yolov5_blazeface.cpp)|✅[StableDiffusion](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/sd/test_lite_sd_pipeline.cpp)|✅[FaceFusion](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/examples/lite/cv/test_lite_facefusion_pipeline.cpp)| / | Linux | TensorRT | - -> Also includes **100+ CPU / ONNXRuntime CV models** (detection, face recognition, segmentation, matting, -> classification, …) behind the same `lite::cv::Type::Class` API. They are not the focus of the active -> line but remain available — see the [ONNX Hub](https://github.com/xlite-dev/lite.ai.toolkit/tree/main/docs/hub/lite.ai.toolkit.hub.onnx.md) for the full catalog and weights, or tag [`v0.2-all-backends`](https://github.com/xlite-dev/lite.ai.toolkit/tree/main) for the legacy multi-backend matrix. - ## Architecture 🧩 ``` @@ -157,37 +146,22 @@ lite/ │ ├── cv/ # one .h/.cpp per model + the facefusion pipeline (+ _mt variants) │ ├── kernel/ # hand-written fused CUDA kernels (.cu/.cuh) + host-side managers │ └── sd/ # Stable Diffusion components (clip / unet / vae / scheduler) -├── ort/ # ONNXRuntime backend — numerical reference + test host (100+ CV models) +├── ort/ # ONNXRuntime backend — numerical reference + test host ├── bench/ # header-only profiler (CPU chrono + GPU cudaEvent, p50/p99/FPS/CSV) └── lite.h # single public include ``` `lite::cv` is a compile-time namespace alias resolved in [`lite/models.h`](https://github.com/xlite-dev/lite.ai.toolkit/blob/main/lite/models.h). Pin a backend explicitly with `lite::trt::cv::...` (GPU) or `lite::onnxruntime::cv::...` (CPU reference). -## Citations 🎉🎉 -```BibTeX -@misc{lite.ai.toolkit@2021, - title={lite.ai.toolkit: A lite C++ toolkit of 100+ Awesome AI models.}, - url={https://github.com/xlite-dev/lite.ai.toolkit}, - note={Open-source software available at https://github.com/xlite-dev/lite.ai.toolkit}, - author={xlite-dev, wangzijian1010 etc}, - year={2021} -} -``` - ## ©️License GNU General Public License v3.0 -## 🎉Contribute -Please consider ⭐ this repo if you like it, as it is the simplest way to support us. +## Star History - - diff --git a/docs/facefusion_quickstart.md b/docs/facefusion_quickstart.md index a09fa550..8c3c4aa2 100644 --- a/docs/facefusion_quickstart.md +++ b/docs/facefusion_quickstart.md @@ -52,19 +52,22 @@ if you change GPU or TensorRT version. ```bash ./build/install/bin/lite_facefusion_cli \ ~/ff_engines \ - source.jpg \ # face to take - target.jpg \ # image to paste it onto - output.jpg # result + source.jpg \ + target.jpg \ + output.jpg # optionally pick which detected face to use on each side (default 0 0): # ... output.jpg ``` -That's it — `output.jpg` is the swapped + restored result. +That's it — `output.jpg` is the swapped + restored result. `source.jpg` is the face to take, +and `target.jpg` is the image to paste it onto. ## Performance -The face-restoration stage is GPU-fused (paste-back + preprocess moved into CUDA -kernels): **78.2 ms → 17.7 ms (4.4×)** on an RTX 4090. See the -[Benchmark](../README.md#benchmark) section. Other stages are being optimized stage -by stage. +The benchmark path is video-shaped: prepare the fixed source face once, then time +per-frame `process(target)`. On an RTX 4090, FP16 deployment, the current pipeline +runs at **23.6 ms / frame (42.3 FPS)**. The pipeline now does one full-frame H2D +upload for the target and one full-frame D2H download for the final result; the +swap → restoration boundary stays GPU-resident. See the +[Benchmark](../README.md#benchmark) section. From 2019bd6498763fb653e6fc717e243cae2e3377bc Mon Sep 17 00:00:00 2001 From: DefTruth <31974251+DefTruth@users.noreply.github.com> Date: Tue, 16 Jun 2026 12:06:34 +0800 Subject: [PATCH 30/30] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 932709e6..9ac3c89b 100644 --- a/README.md +++ b/README.md @@ -160,8 +160,8 @@ GNU General Public License v3.0 - - - Star History Chart + + + Star History Chart