diff --git a/.cargo/config.toml b/.cargo/config.toml index 8528209..0e82fb3 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -3,16 +3,4 @@ target = "wasm32-wasi" [target.wasm32-wasi] -runner = "./scripts/wasmedge-runner.sh" - -[env] -# ffmpeg install path (default is ```assets/ffmpeg-lib```) -FFMPEG_DIR = { value = "assets/ffmpeg-lib", relative = true } -# wasi sysroot path (default is ```assets/wasi-sysroot```) -WASI_SYSROOT = { value = "assets/wasi-sysroot", relative = true } -# lib clang rt path (default is ```assets/clang-rt```) -CLANG_RT = { value = "assets/clang-rt", relative = true } - -# config wasi-sysroot path (default is ```./asset/wasi-sysroot```) -#BINDGEN_EXTRA_CLANG_ARGS = "--sysroot=./asset/wasi-sysroot --target=wasm32-wasi -fvisibility=default" -BINDGEN_EXTRA_CLANG_ARGS = "--sysroot=/opt/wasi-sdk/share/wasi-sysroot --target=wasm32-wasi -fvisibility=default" +runner = "./scripts/wasmedge-runner.sh" \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 776220d..e4002dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,10 +39,9 @@ lazy_static = { version = "^1", optional = true } symphonia-core = { version = "^0", optional = true } [dependencies.ffmpeg-next] -features = ["static"] optional = true -git = "https://github.com/yanghaku/rust-ffmpeg.git" -branch = "wasm32-wasi" +git = "https://github.com/Hrushi20/rust-ffmpeg" +branch = "master" [features] diff --git a/README.md b/README.md index 6963cab..62b9f9c 100644 --- a/README.md +++ b/README.md @@ -269,7 +269,7 @@ from https://storage.googleapis.com/mediapipe-tasks/gesture_recognizer/victory.j Example output in console: ```console -$ cargo run --release --example gesture_recognition -- ./assets/models/gesture_recognition/gesture_recognizer.task ./assets/testdata/img/gesture_recognition_google_samples/victory.jpg +$ cargo run --release --example gesture_recognition -- ./assets/models/gesture_recognition/gesture_recognizer.task ./assets/testdata/img/victory.jpg Finished release [optimized] target(s) in 0.02s Running `/mediapipe-rs/./scripts/wasmedge-runner.sh target/wasm32-wasi/release/examples/gesture_recognition.wasm ./assets/models/gesture_recognition/gesture_recognizer.task ./assets/testdata/img/gesture_recognition_google_samples/victory.jpg` Category name: "Victory" @@ -389,21 +389,10 @@ fn inference( ## Use the FFMPEG feature to process video and audio. -When building the library with ```ffmpeg``` feature using cargo, users must set the following environment variables: +When building the library with ```ffmpeg``` feature using cargo, users must build WasmEdge by enabling FFmpeg Plugin: -* ```FFMPEG_DIR```: the pre-built FFmpeg library path. You can download it from - https://github.com/yanghaku/ffmpeg-wasm32-wasi/releases. -* ```WASI_SDK``` or (```WASI_SYSROOT``` and ```CLANG_RT```), You can download it from - https://github.com/WebAssembly/wasi-sdk/releases -* ```BINDGEN_EXTRA_CLANG_ARGS```: set **sysroot** and **target** and **function visibility** for libclang. - (The sysroot must be **absolute path**). - -Example: - -```shell -export FFMPEG_DIR=/path/to/ffmpeg/library -export WASI_SDK=/opt/wasi-sdk -export BINDGEN_EXTRA_CLANG_ARGS="--sysroot=/opt/wasi-sdk/share/wasi-sysroot --target=wasm32-wasi -fvisibility=default" +```console +$ cmake -DCMAKE_BUILD_TYPE=Release -GNinja -DWASMEDGE_PLUGIN_WASI_NN_BACKEND="TensorflowLite" -DWASMEDGE_PLUGIN_FFMPEG=ON .. # Then run cargo ``` diff --git a/examples/audio_classification.rs b/examples/audio_classification.rs index e9041fe..979bc83 100644 --- a/examples/audio_classification.rs +++ b/examples/audio_classification.rs @@ -32,7 +32,7 @@ fn read_audio_using_symphonia(audio_path: String) -> SymphoniaAudioData { } #[cfg(feature = "ffmpeg")] -fn read_video_using_ffmpeg(audio_path: String) -> FFMpegAudioData { +fn read_audio_using_ffmpeg(audio_path: String) -> FFMpegAudioData { ffmpeg_next::init().unwrap(); FFMpegAudioData::new(ffmpeg_next::format::input(&audio_path.as_str()).unwrap()).unwrap() } @@ -43,7 +43,7 @@ fn main() -> Result<(), Box> { #[cfg(not(feature = "ffmpeg"))] let audio = read_audio_using_symphonia(audio_path); #[cfg(feature = "ffmpeg")] - let audio = read_video_using_ffmpeg(audio_path); + let audio = read_audio_using_ffmpeg(audio_path); let classification_results = AudioClassifierBuilder::new() .max_results(3) // set max result diff --git a/scripts/ffmpeg-deps-init.sh b/scripts/ffmpeg-deps-init.sh deleted file mode 100755 index 358e85c..0000000 --- a/scripts/ffmpeg-deps-init.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/bash - -set -ex - -# install libclang and clang -apt install -y libclang1-14 clang - -FFMPEG_VERSION="v0.0.1" -FFMPEG_FILENAME="ffmpeg-6.0-wasm32-wasi-v0.0.1.tar.gz" - -WASI_SDK_VERSION="wasi-sdk-19" -LIB_CLANG_RT_FILENAME="libclang_rt.builtins-wasm32-wasi-19.0.tar.gz" -WASI_SYSROOT_FILENAME="wasi-sysroot-19.0.tar.gz" - -OUTPUT_ROOT="$(realpath "$(dirname -- "$0")")/../assets" -TEMP_DIR="/tmp" - -download_ffmpeg_lib() { - pushd "${TEMP_DIR}" - - BASE_URL="https://github.com/yanghaku/ffmpeg-wasm32-wasi/releases/download/" - - OUTPUT_DIR="${OUTPUT_ROOT:?}/ffmpeg-lib/" - mkdir -p "${OUTPUT_DIR}" - - # clean old if exist - if [[ -d "${OUTPUT_DIR:?}/include" ]]; then - rm -rf "${OUTPUT_DIR:?}/include" - fi - if [[ -d "${OUTPUT_DIR:?}/lib" ]]; then - rm -rf "${OUTPUT_DIR:?}/lib" - fi - - curl -sLO "${BASE_URL}/${FFMPEG_VERSION}/${FFMPEG_FILENAME}" - tar -zxvf "${FFMPEG_FILENAME}" - folder_name=$(basename "${FFMPEG_FILENAME}" .tar.gz) - mv "${folder_name:?}/include" "${OUTPUT_DIR:?}/include" - mv "${folder_name:?}/lib" "${OUTPUT_DIR:?}/lib" - - rm -rf "${FFMPEG_FILENAME}" "${folder_name}" - - popd -} - -download_wasi_sysroot() { - pushd "${TEMP_DIR}" - - BASE_URL="https://github.com/WebAssembly/wasi-sdk/releases/download/" - - OUTPUT_DIR="${OUTPUT_ROOT:?}/wasi-sysroot" - # clean old if exist - if [[ -d "${OUTPUT_DIR:?}" ]]; then - rm -rf "${OUTPUT_DIR:?}" - fi - - curl -sLO "${BASE_URL}/${WASI_SDK_VERSION}/${WASI_SYSROOT_FILENAME}" - tar -zxvf "${WASI_SYSROOT_FILENAME}" - mv wasi-sysroot "${OUTPUT_ROOT:?}/" - rm "${WASI_SYSROOT_FILENAME}" - - popd -} - -download_lib_clang_rt() { - pushd "${TEMP_DIR}" - - BASE_URL="https://github.com/WebAssembly/wasi-sdk/releases/download/" - - LIB_NAME="libclang_rt.builtins-wasm32.a" - - OUTPUT_DIR="${OUTPUT_ROOT:?}/clang-rt" - mkdir -p "${OUTPUT_DIR}" - # clean old if exist - if [[ -f "${OUTPUT_DIR:?}/${LIB_NAME}" ]]; then - rm -rf "${OUTPUT_DIR:?}/${LIB_NAME}" - fi - - curl -sLO "${BASE_URL}/${WASI_SDK_VERSION}/${LIB_CLANG_RT_FILENAME}" - - folder_name=$(basename "${LIB_CLANG_RT_FILENAME}" .tar.gz) - mkdir -p "${folder_name}" - tar -zxvf ${LIB_CLANG_RT_FILENAME} -C "${folder_name}" - mv "${folder_name}/lib/wasi/${LIB_NAME}" "${OUTPUT_DIR}/${LIB_NAME}" - - rm -rf "${LIB_CLANG_RT_FILENAME}" "${folder_name}" - popd -} - -download_ffmpeg_lib -download_wasi_sysroot -download_lib_clang_rt diff --git a/src/preprocess/vision/ffmpeg.rs b/src/preprocess/vision/ffmpeg.rs index 88ee039..0b80c80 100644 --- a/src/preprocess/vision/ffmpeg.rs +++ b/src/preprocess/vision/ffmpeg.rs @@ -2,6 +2,7 @@ use super::*; use std::cell::RefCell; use std::collections::hash_map::Entry; use std::collections::HashMap; +use ffmpeg_next::avUtilTypes::AVPixelFormat; type FFMpegVideoInput = common::ffmpeg_input::FFMpegInput; @@ -33,11 +34,12 @@ impl FFMpegVideoData { } else { source.decoder.time_base().numerator() }; + let filter_desc_in = format!( "buffer=video_size={}x{}:pix_fmt={}:time_base={}/{}:pixel_aspect={}/{}", source.decoder.width(), source.decoder.height(), - ffmpeg_next::ffi::AVPixelFormat::from(source.decoder.format()) as u32, + ffmpeg_next::util::format::Pixel::from(source.decoder.format() as AVPixelFormat).mask(), time_base_num, source.decoder.time_base().denominator(), source.decoder.aspect_ratio().numerator(), @@ -105,16 +107,16 @@ impl<'a> ImageToTensor for FFMpegFrame<'a> { } let out_format = match to_tensor_info.color_space { ImageColorSpaceType::GRAYSCALE => { - ffmpeg_next::ffi::AVPixelFormat::AV_PIX_FMT_GRAY8 as u32 + ffmpeg_next::format::Pixel::GRAY8 } - _ => ffmpeg_next::ffi::AVPixelFormat::AV_PIX_FMT_RGB24 as u32, + _ => ffmpeg_next::format::Pixel::RGB24, }; desc.extend( format!( "[s_in];[s_in]scale={}:{}[f];[f]format=pix_fmts={}[out];[out]buffersink", to_tensor_info.width(), to_tensor_info.height(), - out_format + out_format.mask() ) .chars(), ); @@ -158,7 +160,7 @@ impl<'a> ImageToTensor for FFMpegFrame<'a> { let img = image::ImageBuffer::, &[u8]>::from_raw( to_tensor_info.width(), to_tensor_info.height(), - data, + &data, ) .unwrap(); image::rgb8_image_buffer_to_tensor(&img, to_tensor_info, output_buffer)?;