From 0d1c5f04e08ea55a53e4f465b3b559a74a0b76f1 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 17 Jun 2023 20:16:31 +0200 Subject: [PATCH 01/12] build embeddings database --- .gitignore | 3 + examples/CMakeLists.txt | 2 + examples/image-search/CMakeLists.txt | 19 ++++ examples/image-search/build.cpp | 155 +++++++++++++++++++++++++++ 4 files changed, 179 insertions(+) create mode 100644 examples/image-search/CMakeLists.txt create mode 100644 examples/image-search/build.cpp diff --git a/.gitignore b/.gitignore index f785746..38a2532 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ build/ .vscode +.cache/ +*.swp +models/*.bin diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 61650f9..76132c9 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -3,6 +3,8 @@ add_library(common-clip STATIC common-clip.cpp) target_link_libraries(common-clip PRIVATE ggml) target_include_directories(common-clip PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +add_subdirectory(./image-search) + add_executable(main main.cpp) target_link_libraries(main PRIVATE clip common-clip ggml) diff --git a/examples/image-search/CMakeLists.txt b/examples/image-search/CMakeLists.txt new file mode 100644 index 0000000..f0b303f --- /dev/null +++ b/examples/image-search/CMakeLists.txt @@ -0,0 +1,19 @@ +project(image-search) + +set(CXX_STANDARD_REQUIRED ON) + +include(FetchContent) +FetchContent_Declare(usearch + GIT_REPOSITORY https://github.com/unum-cloud/usearch.git + GIT_TAG v0.18.8 +) +FetchContent_MakeAvailable(usearch) + +add_executable(image-search-build + build.cpp +) + +target_link_libraries(image-search-build PRIVATE clip ggml usearch) + +target_compile_features(image-search-build PUBLIC cxx_std_17) + diff --git a/examples/image-search/build.cpp b/examples/image-search/build.cpp new file mode 100644 index 0000000..68c582e --- /dev/null +++ b/examples/image-search/build.cpp @@ -0,0 +1,155 @@ +#include "clip.h" +#include "usearch/index.hpp" + +#include +#include + +struct my_app_params { + int32_t n_threads {4}; + std::string model = "../models/ggml-model-f16.bin"; + std::vector image_directories; +}; + +void my_print_help(int argc, char **argv, my_app_params ¶ms) { + printf("Usage: %s [options] dir/with/pictures [more/dirs]\n", argv[0]); + printf("\nOptions:"); + printf(" -h, --help: Show this message and exit\n"); + printf(" -m , --model : path to model. Default: %s\n", params.model.c_str()); + printf(" -t N, --threads N: Number of threads to use for inference. Default: %d\n", params.n_threads); +} + +// returns success +bool my_app_params_parse(int argc, char **argv, my_app_params ¶ms) { + bool invalid_param = false; + for (int i = 1; i < argc; i++) { + + std::string arg = argv[i]; + + if (arg == "-m" || arg == "--model") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.model = argv[i]; + } else if (arg == "-t" || arg == "--threads") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.n_threads = std::stoi(argv[i]); + } else if (arg == "-h" || arg == "--help") { + my_print_help(argc, argv, params); + exit(0); + } else if (arg.starts_with('-')) { + if (i != 0) { + printf("%s: unrecognized argument: %s\n", __func__, arg.c_str()); + return false; + } + } else { + // assume image directory + params.image_directories.push_back(argv[i]); + } + } + + return !(invalid_param || params.image_directories.empty()); +} + +bool is_image_file_extension(std::string_view ext) { + if (ext == ".jpg") return true; + if (ext == ".JPG") return true; + + if (ext == ".jpeg") return true; + if (ext == ".JPEG") return true; + + if (ext == ".gif") return true; + if (ext == ".GIF") return true; + + if (ext == ".png") return true; + if (ext == ".PNG") return true; + + // TODO: add all the supported files for std_image + + return false; +} + +int main(int argc, char** argv) { + my_app_params params; + if (!my_app_params_parse(argc, argv, params)) { + my_print_help(argc, argv, params); + return 1; + } + + auto clip_ctx = clip_model_load(params.model.c_str(), 1); // TODO: verbosity as cli arg + if (!clip_ctx) { + printf("%s: Unable to load model from %s\n", __func__, params.model.c_str()); + return 1; + } + + std::vector image_file_index; + unum::usearch::index_gt> embd_index; + + const size_t vec_dim = clip_ctx->vision_model.hparams.projection_dim; + + size_t label = 0; + + // search for images in path and create embedding + for (const auto& base_dir : params.image_directories) { + fprintf(stdout, "%s: starting base dir scan of '%s'\n", __func__, base_dir.c_str()); + + for (auto const& dir_entry : std::filesystem::recursive_directory_iterator(base_dir)) { + if (!dir_entry.is_regular_file()) { + continue; + } + + // check for image file + const auto& ext = dir_entry.path().extension(); + if (ext.empty()) { + continue; + } + if (!is_image_file_extension(ext.c_str())) { + continue; + } + + std::string img_path {dir_entry.path()}; + fprintf(stdout, "%s: found image file '%s'\n", __func__, img_path.c_str()); + + clip_image_u8 img0; + clip_image_f32 img_res; + if (!clip_image_load_from_file(img_path, img0)) { + fprintf(stderr, "%s: failed to load image from '%s'\n", __func__, img_path.c_str()); + continue; + } + + clip_image_preprocess(&img0, &img_res); + + std::vector vec(vec_dim); + if (!clip_image_encode(clip_ctx, params.n_threads, img_res, vec.data())) { + fprintf(stderr, "%s: failed to encode image from '%s'\n", __func__, img_path.c_str()); + continue; + } + + if (embd_index.capacity() == embd_index.size()) { + embd_index.reserve(embd_index.size() + 32); + } + + // add the image + embd_index.add(label++, {vec.data(), vec.size()}); + image_file_index.push_back(std::filesystem::canonical(dir_entry.path())); + } + } + + clip_free(clip_ctx); + + // save to disk + + embd_index.save("images.usearch"); + // TODO: figrue out why usearch leaks + + std::ofstream image_file_index_file("images.paths", std::ios::binary | std::ios::trunc); + for (const auto& i_path : image_file_index) { + image_file_index_file << i_path << "\n"; + } + + return 0; +} + From 70ec88f49a285ab09d8b4d143eaa31c9483ded63 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 17 Jun 2023 20:41:45 +0200 Subject: [PATCH 02/12] add search --- examples/image-search/CMakeLists.txt | 10 ++- examples/image-search/search.cpp | 115 +++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 examples/image-search/search.cpp diff --git a/examples/image-search/CMakeLists.txt b/examples/image-search/CMakeLists.txt index f0b303f..b163c86 100644 --- a/examples/image-search/CMakeLists.txt +++ b/examples/image-search/CMakeLists.txt @@ -5,7 +5,7 @@ set(CXX_STANDARD_REQUIRED ON) include(FetchContent) FetchContent_Declare(usearch GIT_REPOSITORY https://github.com/unum-cloud/usearch.git - GIT_TAG v0.18.8 + GIT_TAG v0.18.8 ) FetchContent_MakeAvailable(usearch) @@ -14,6 +14,12 @@ add_executable(image-search-build ) target_link_libraries(image-search-build PRIVATE clip ggml usearch) - target_compile_features(image-search-build PUBLIC cxx_std_17) +add_executable(image-search + search.cpp +) + +target_link_libraries(image-search PRIVATE clip ggml usearch) +target_compile_features(image-search PUBLIC cxx_std_11) + diff --git a/examples/image-search/search.cpp b/examples/image-search/search.cpp new file mode 100644 index 0000000..10aff82 --- /dev/null +++ b/examples/image-search/search.cpp @@ -0,0 +1,115 @@ +#include "clip.h" +#include "usearch/index.hpp" + +#include + +struct my_app_params { + int32_t n_threads {4}; + std::string model = "../models/ggml-model-f16.bin"; + // TODO: index dir + + // TODO: search by image + std::string search_text; +}; + +void my_print_help(int argc, char **argv, my_app_params ¶ms) { + printf("Usage: %s [options] \n", argv[0]); + printf("\nOptions:"); + printf(" -h, --help: Show this message and exit\n"); + printf(" -m , --model : path to model. Default: %s\n", params.model.c_str()); + printf(" -t N, --threads N: Number of threads to use for inference. Default: %d\n", params.n_threads); +} + +// returns success +bool my_app_params_parse(int argc, char **argv, my_app_params ¶ms) { + bool invalid_param = false; + for (int i = 1; i < argc; i++) { + std::string arg = argv[i]; + + if (arg == "-m" || arg == "--model") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.model = argv[i]; + } else if (arg == "-t" || arg == "--threads") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.n_threads = std::stoi(argv[i]); + } else if (arg == "-h" || arg == "--help") { + my_print_help(argc, argv, params); + exit(0); + } else if (arg.starts_with('-')) { + if (i != 0) { + printf("%s: unrecognized argument: %s\n", __func__, arg.c_str()); + return false; + } + } else { + // assume search string from here on out + params.search_text = arg; + for (++i; i < argc; i++) { + params.search_text += " "; + params.search_text += argv[i]; + } + } + } + + return !(invalid_param || params.search_text.empty()); +} + +int main(int argc, char** argv) { + my_app_params params; + if (!my_app_params_parse(argc, argv, params)) { + my_print_help(argc, argv, params); + return 1; + } + + auto clip_ctx = clip_model_load(params.model.c_str(), 1); // TODO: verbosity via cli arg + if (!clip_ctx) { + printf("%s: Unable to load model from %s\n", __func__, params.model.c_str()); + return 1; + } + + std::vector image_file_index; + unum::usearch::index_gt> embd_index; + + embd_index.view("images.usearch"); + + // load paths + std::ifstream image_file_index_file("images.paths", std::ios::binary); + std::string line; + do { + std::getline(image_file_index_file, line); + if (line.empty()) { + break; + } + image_file_index.push_back(line); + } while(image_file_index_file.good()); + + if (image_file_index.size() != embd_index.size()) { + printf("%s: index files size missmatch\n", __func__); + } + + const size_t vec_dim = clip_ctx->vision_model.hparams.projection_dim; + + auto tokens = clip_tokenize(clip_ctx, params.search_text); + + std::vector txt_vec(vec_dim); + + clip_text_encode(clip_ctx, params.n_threads, tokens, txt_vec.data()); + + auto results = embd_index.search({txt_vec.data(), txt_vec.size()}, 5); + + printf("search results:\n"); + printf("similarity path\n"); + for (std::size_t i = 0; i != results.size(); ++i) { + printf(" %f %s\n", results[i].distance, image_file_index.at(results[i].element.label).c_str()); + } + + clip_free(clip_ctx); + + return 0; +} + From ae23063c19796ada529055327b58e1e36df50e53 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sun, 18 Jun 2023 19:41:57 +0200 Subject: [PATCH 03/12] add verbosity and number of results cli args --- examples/image-search/build.cpp | 16 +++++++++++++--- examples/image-search/search.cpp | 24 +++++++++++++++++++++--- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/examples/image-search/build.cpp b/examples/image-search/build.cpp index 68c582e..ef241f9 100644 --- a/examples/image-search/build.cpp +++ b/examples/image-search/build.cpp @@ -6,7 +6,8 @@ struct my_app_params { int32_t n_threads {4}; - std::string model = "../models/ggml-model-f16.bin"; + std::string model {"../models/ggml-model-f16.bin"}; + int32_t verbose {1}; std::vector image_directories; }; @@ -16,6 +17,7 @@ void my_print_help(int argc, char **argv, my_app_params ¶ms) { printf(" -h, --help: Show this message and exit\n"); printf(" -m , --model : path to model. Default: %s\n", params.model.c_str()); printf(" -t N, --threads N: Number of threads to use for inference. Default: %d\n", params.n_threads); + printf(" -v , --verbose : Control the level of verbosity. 0 = minimum, 2 = maximum. Default: %d\n", params.verbose); } // returns success @@ -37,6 +39,12 @@ bool my_app_params_parse(int argc, char **argv, my_app_params ¶ms) { break; } params.n_threads = std::stoi(argv[i]); + } else if (arg == "-v" || arg == "--verbose") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.verbose = std::stoi(argv[i]); } else if (arg == "-h" || arg == "--help") { my_print_help(argc, argv, params); exit(0); @@ -79,7 +87,7 @@ int main(int argc, char** argv) { return 1; } - auto clip_ctx = clip_model_load(params.model.c_str(), 1); // TODO: verbosity as cli arg + auto clip_ctx = clip_model_load(params.model.c_str(), params.verbose); if (!clip_ctx) { printf("%s: Unable to load model from %s\n", __func__, params.model.c_str()); return 1; @@ -111,7 +119,9 @@ int main(int argc, char** argv) { } std::string img_path {dir_entry.path()}; - fprintf(stdout, "%s: found image file '%s'\n", __func__, img_path.c_str()); + if (params.verbose >= 1) { + fprintf(stdout, "%s: found image file '%s'\n", __func__, img_path.c_str()); + } clip_image_u8 img0; clip_image_f32 img_res; diff --git a/examples/image-search/search.cpp b/examples/image-search/search.cpp index 10aff82..b56e9f5 100644 --- a/examples/image-search/search.cpp +++ b/examples/image-search/search.cpp @@ -5,11 +5,14 @@ struct my_app_params { int32_t n_threads {4}; - std::string model = "../models/ggml-model-f16.bin"; + std::string model {"../models/ggml-model-f16.bin"}; + int32_t verbose {1}; // TODO: index dir // TODO: search by image std::string search_text; + + int32_t n_results {5}; }; void my_print_help(int argc, char **argv, my_app_params ¶ms) { @@ -18,6 +21,9 @@ void my_print_help(int argc, char **argv, my_app_params ¶ms) { printf(" -h, --help: Show this message and exit\n"); printf(" -m , --model : path to model. Default: %s\n", params.model.c_str()); printf(" -t N, --threads N: Number of threads to use for inference. Default: %d\n", params.n_threads); + printf(" -v , --verbose : Control the level of verbosity. 0 = minimum, 2 = maximum. Default: %d\n", params.verbose); + + printf(" -n N, --results N: Number of results to display. Default: %d\n", params.n_results); } // returns success @@ -38,6 +44,18 @@ bool my_app_params_parse(int argc, char **argv, my_app_params ¶ms) { break; } params.n_threads = std::stoi(argv[i]); + } else if (arg == "-n" || arg == "--results") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.n_results = std::stoi(argv[i]); + } else if (arg == "-v" || arg == "--verbose") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.verbose = std::stoi(argv[i]); } else if (arg == "-h" || arg == "--help") { my_print_help(argc, argv, params); exit(0); @@ -66,7 +84,7 @@ int main(int argc, char** argv) { return 1; } - auto clip_ctx = clip_model_load(params.model.c_str(), 1); // TODO: verbosity via cli arg + auto clip_ctx = clip_model_load(params.model.c_str(), params.verbose); if (!clip_ctx) { printf("%s: Unable to load model from %s\n", __func__, params.model.c_str()); return 1; @@ -100,7 +118,7 @@ int main(int argc, char** argv) { clip_text_encode(clip_ctx, params.n_threads, tokens, txt_vec.data()); - auto results = embd_index.search({txt_vec.data(), txt_vec.size()}, 5); + auto results = embd_index.search({txt_vec.data(), txt_vec.size()}, params.n_results); printf("search results:\n"); printf("similarity path\n"); From 2c024f29b60512252d7b2ebb0b426f34cd6366c9 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Tue, 20 Jun 2023 12:47:38 +0200 Subject: [PATCH 04/12] make examples and image-search be build conditionally (edit: why did you have to invert the cmake options <.<) --- CMakeLists.txt | 12 ++++++++++-- examples/CMakeLists.txt | 5 +++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f8b8f3b..d73aa60 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,14 +10,21 @@ endif() set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + set(CLIP_STANDALONE ON) +else() + set(CLIP_STANDALONE OFF) +endif() + # # Option list # # general option(CLIP_STATIC "CLIP: static link libraries" OFF) -option(CLIP_NO_EXAMPLES "CLIP: do not build examples" OFF) -option(CLIP_NO_TESTS "CLIP: do not build tests" OFF) +option(CLIP_NO_EXAMPLES "CLIP: do not build examples" NOT ${CLIP_STANDALONE}) +option(CLIP_BUILD_IMAGE_SEARCH "CLIP: build image-search" OFF) +option(CLIP_NO_TEST "CLIP: do not build tests" NOT ${CLIP_STANDALONE}) option(CLIP_NATIVE "CLIP: enable -march=native flag" ON) option(CLIP_LTO "CLIP: enable link time optimization" OFF) @@ -48,6 +55,7 @@ endif() option(CLIP_ACCELERATE "CLIP: enable Accelerate framework" ON) option(CLIP_OPENBLAS "CLIP: use OpenBLAS" OFF) + # # Compile flags # diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 76132c9..c9b89f5 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1,9 +1,10 @@ - add_library(common-clip STATIC common-clip.cpp) target_link_libraries(common-clip PRIVATE ggml) target_include_directories(common-clip PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) -add_subdirectory(./image-search) +if (CLIP_BUILD_IMAGE_SEARCH) + add_subdirectory(./image-search) +endif() add_executable(main main.cpp) target_link_libraries(main PRIVATE clip common-clip ggml) From b1ed0fe9e4ac8f10110327450c9bb32cc577356f Mon Sep 17 00:00:00 2001 From: Green Sky Date: Fri, 23 Jun 2023 11:31:57 +0200 Subject: [PATCH 05/12] save model path to image index and load from image index --- examples/image-search/CMakeLists.txt | 2 +- examples/image-search/build.cpp | 11 ++++++----- examples/image-search/search.cpp | 20 +++++++++++++++----- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/examples/image-search/CMakeLists.txt b/examples/image-search/CMakeLists.txt index b163c86..8368300 100644 --- a/examples/image-search/CMakeLists.txt +++ b/examples/image-search/CMakeLists.txt @@ -5,7 +5,7 @@ set(CXX_STANDARD_REQUIRED ON) include(FetchContent) FetchContent_Declare(usearch GIT_REPOSITORY https://github.com/unum-cloud/usearch.git - GIT_TAG v0.18.8 + GIT_TAG v0.19.0 ) FetchContent_MakeAvailable(usearch) diff --git a/examples/image-search/build.cpp b/examples/image-search/build.cpp index ef241f9..834e6ed 100644 --- a/examples/image-search/build.cpp +++ b/examples/image-search/build.cpp @@ -75,7 +75,7 @@ bool is_image_file_extension(std::string_view ext) { if (ext == ".png") return true; if (ext == ".PNG") return true; - // TODO: add all the supported files for std_image + // TODO(green-sky): determine if we should add more formats from stbi. tga/hdr/pnm seem kinda niche. return false; } @@ -100,7 +100,7 @@ int main(int argc, char** argv) { size_t label = 0; - // search for images in path and create embedding + // search for images in path and write embedding to database for (const auto& base_dir : params.image_directories) { fprintf(stdout, "%s: starting base dir scan of '%s'\n", __func__, base_dir.c_str()); @@ -124,12 +124,12 @@ int main(int argc, char** argv) { } clip_image_u8 img0; - clip_image_f32 img_res; if (!clip_image_load_from_file(img_path, img0)) { fprintf(stderr, "%s: failed to load image from '%s'\n", __func__, img_path.c_str()); continue; } + clip_image_f32 img_res; clip_image_preprocess(&img0, &img_res); std::vector vec(vec_dim); @@ -142,7 +142,7 @@ int main(int argc, char** argv) { embd_index.reserve(embd_index.size() + 32); } - // add the image + // add the image to the database embd_index.add(label++, {vec.data(), vec.size()}); image_file_index.push_back(std::filesystem::canonical(dir_entry.path())); } @@ -153,9 +153,10 @@ int main(int argc, char** argv) { // save to disk embd_index.save("images.usearch"); - // TODO: figrue out why usearch leaks std::ofstream image_file_index_file("images.paths", std::ios::binary | std::ios::trunc); + // first line is model + image_file_index_file << params.model << "\n"; for (const auto& i_path : image_file_index) { image_file_index_file << i_path << "\n"; } diff --git a/examples/image-search/search.cpp b/examples/image-search/search.cpp index b56e9f5..3914db0 100644 --- a/examples/image-search/search.cpp +++ b/examples/image-search/search.cpp @@ -5,7 +5,7 @@ struct my_app_params { int32_t n_threads {4}; - std::string model {"../models/ggml-model-f16.bin"}; + std::string model; int32_t verbose {1}; // TODO: index dir @@ -19,7 +19,7 @@ void my_print_help(int argc, char **argv, my_app_params ¶ms) { printf("Usage: %s [options] \n", argv[0]); printf("\nOptions:"); printf(" -h, --help: Show this message and exit\n"); - printf(" -m , --model : path to model. Default: %s\n", params.model.c_str()); + printf(" -m , --model : overwrite path to model. Read from images.paths by default.\n"); printf(" -t N, --threads N: Number of threads to use for inference. Default: %d\n", params.n_threads); printf(" -v , --verbose : Control the level of verbosity. 0 = minimum, 2 = maximum. Default: %d\n", params.verbose); @@ -84,20 +84,30 @@ int main(int argc, char** argv) { return 1; } + // load model path + std::ifstream image_file_index_file("images.paths", std::ios::binary); + std::string line; + std::getline(image_file_index_file, line); + if (params.model.empty()) { + params.model = line; + } else { + printf("%s: using alternative model from %s. Make sure you use the same model you used for indexing, or the embeddings wont work.\n", __func__, params.model.c_str()); + } + + // load model auto clip_ctx = clip_model_load(params.model.c_str(), params.verbose); if (!clip_ctx) { printf("%s: Unable to load model from %s\n", __func__, params.model.c_str()); return 1; } + // load paths and embeddings database std::vector image_file_index; unum::usearch::index_gt> embd_index; embd_index.view("images.usearch"); - // load paths - std::ifstream image_file_index_file("images.paths", std::ios::binary); - std::string line; + // load image paths do { std::getline(image_file_index_file, line); if (line.empty()) { From 20bb778e197670928b13d0796347c79079318ff2 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Mon, 26 Jun 2023 21:55:44 +0200 Subject: [PATCH 06/12] fix api change from main --- examples/image-search/build.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/image-search/build.cpp b/examples/image-search/build.cpp index 834e6ed..0279724 100644 --- a/examples/image-search/build.cpp +++ b/examples/image-search/build.cpp @@ -130,7 +130,7 @@ int main(int argc, char** argv) { } clip_image_f32 img_res; - clip_image_preprocess(&img0, &img_res); + clip_image_preprocess(clip_ctx, &img0, &img_res); std::vector vec(vec_dim); if (!clip_image_encode(clip_ctx, params.n_threads, img_res, vec.data())) { From 592147e26c2d8c94784c41ac0399324f9abf1639 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Fri, 7 Jul 2023 19:03:07 +0200 Subject: [PATCH 07/12] cmake has no negation (except NOT in conditional expressions) --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d73aa60..81babee 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,9 +22,9 @@ endif() # general option(CLIP_STATIC "CLIP: static link libraries" OFF) -option(CLIP_NO_EXAMPLES "CLIP: do not build examples" NOT ${CLIP_STANDALONE}) +option(CLIP_BUILD_TEST "CLIP: build tests" ${CLIP_STANDALONE}) +option(CLIP_BUILD_EXAMPLES "CLIP: build examples" ${CLIP_STANDALONE}) option(CLIP_BUILD_IMAGE_SEARCH "CLIP: build image-search" OFF) -option(CLIP_NO_TEST "CLIP: do not build tests" NOT ${CLIP_STANDALONE}) option(CLIP_NATIVE "CLIP: enable -march=native flag" ON) option(CLIP_LTO "CLIP: enable link time optimization" OFF) From 1eb480e4a31808f1231c8421c45d6149b9fe095d Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 8 Jul 2023 02:41:10 +0200 Subject: [PATCH 08/12] add readme --- examples/README.md | 56 ++++++++++++++++++++++++++++ examples/image-search/CMakeLists.txt | 4 +- 2 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 examples/README.md diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..0c620a5 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,56 @@ +# Image search + +This example implements basic semantic image search using [usearch](https://github.com/unum-cloud/usearch) as a similartiy accelerated database. + +Use `image-search-build` to build the database of images and their embeddings beforehand. Currently it does not support updating. + +Use `image-search` to seach for indexed images by semantic similarity. + +### examples + +#### build db + +help: +```sh +./image-search-build -h +Usage: ./image-search-build [options] dir/with/pictures [more/dirs] + +Options: -h, --help: Show this message and exit + -m , --model : path to model. Default: ../models/ggml-model-f16.bin + -t N, --threads N: Number of threads to use for inference. Default: 4 + -v , --verbose : Control the level of verbosity. 0 = minimum, 2 = maximum. Default: 1 +``` + +creating db for `tests/`: +```sh +./image-search-build -m models/openai_clip-vit-base-patch32.ggmlv0.f16.bin ./tests/ +``` + +#### search by text + +help: +```sh +./image-search -h +Usage: ./image-search [options] + +Options: -h, --help: Show this message and exit + -m , --model : overwrite path to model. Read from images.paths by default. + -t N, --threads N: Number of threads to use for inference. Default: 4 + -v , --verbose : Control the level of verbosity. 0 = minimum, 2 = maximum. Default: 1 + -n N, --results N: Number of results to display. Default: 5 +``` + +searching for `apple` in the db in the current directory: +```sh +./image-search apple +clip_model_load: loading model from 'models/openai_clip-vit-base-patch32.ggmlv0.f16.bin' - please wait....................................................clip_model_load: model size = 288.93 MB / num tensors = 397 +clip_model_load: model loaded + +search results: +similarity path + 0.674587 /home/xxxx/tests/red_apple.jpg + 0.785591 /home/xxxx/tests/white.jpg +``` + +note: lower score for search results is better. + diff --git a/examples/image-search/CMakeLists.txt b/examples/image-search/CMakeLists.txt index 8368300..243b4e9 100644 --- a/examples/image-search/CMakeLists.txt +++ b/examples/image-search/CMakeLists.txt @@ -5,7 +5,7 @@ set(CXX_STANDARD_REQUIRED ON) include(FetchContent) FetchContent_Declare(usearch GIT_REPOSITORY https://github.com/unum-cloud/usearch.git - GIT_TAG v0.19.0 + GIT_TAG v0.19.3 ) FetchContent_MakeAvailable(usearch) @@ -17,7 +17,7 @@ target_link_libraries(image-search-build PRIVATE clip ggml usearch) target_compile_features(image-search-build PUBLIC cxx_std_17) add_executable(image-search - search.cpp + search.cpp ) target_link_libraries(image-search PRIVATE clip ggml usearch) From a1f34ebe25745de46e308d7ac4cb506b89d7c62b Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 8 Jul 2023 02:43:50 +0200 Subject: [PATCH 09/12] move vector out of loop --- examples/image-search/build.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/image-search/build.cpp b/examples/image-search/build.cpp index 0279724..97a22ac 100644 --- a/examples/image-search/build.cpp +++ b/examples/image-search/build.cpp @@ -100,6 +100,8 @@ int main(int argc, char** argv) { size_t label = 0; + std::vector vec(vec_dim); + // search for images in path and write embedding to database for (const auto& base_dir : params.image_directories) { fprintf(stdout, "%s: starting base dir scan of '%s'\n", __func__, base_dir.c_str()); @@ -132,7 +134,6 @@ int main(int argc, char** argv) { clip_image_f32 img_res; clip_image_preprocess(clip_ctx, &img0, &img_res); - std::vector vec(vec_dim); if (!clip_image_encode(clip_ctx, params.n_threads, img_res, vec.data())) { fprintf(stderr, "%s: failed to encode image from '%s'\n", __func__, img_path.c_str()); continue; From 9b719c4e4ea573c899fe28b89abe718df478e66c Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 8 Jul 2023 02:45:32 +0200 Subject: [PATCH 10/12] move image search reame --- examples/{ => image-search}/README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/{ => image-search}/README.md (100%) diff --git a/examples/README.md b/examples/image-search/README.md similarity index 100% rename from examples/README.md rename to examples/image-search/README.md From e605bdd2f9fb20b3a453212b1ba7eed971ce2acd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sat, 8 Jul 2023 08:56:50 +0300 Subject: [PATCH 11/12] Fix some typos and notes in readme --- examples/image-search/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/image-search/README.md b/examples/image-search/README.md index 0c620a5..8f54c46 100644 --- a/examples/image-search/README.md +++ b/examples/image-search/README.md @@ -1,10 +1,10 @@ # Image search -This example implements basic semantic image search using [usearch](https://github.com/unum-cloud/usearch) as a similartiy accelerated database. +This example implements basic semantic image search using [usearch](https://github.com/unum-cloud/usearch) as a vector database for accelerated similarity search. Use `image-search-build` to build the database of images and their embeddings beforehand. Currently it does not support updating. -Use `image-search` to seach for indexed images by semantic similarity. +Use `image-search` to search for indexed images by semantic similarity. ### examples @@ -47,10 +47,10 @@ clip_model_load: loading model from 'models/openai_clip-vit-base-patch32.ggmlv0. clip_model_load: model loaded search results: -similarity path +distance path 0.674587 /home/xxxx/tests/red_apple.jpg 0.785591 /home/xxxx/tests/white.jpg ``` -note: lower score for search results is better. +note: lower score for search results is better as it indicates the distance, not the similarity. From 181b44b54c0ec8d43488484abca7b1a276c7f635 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sat, 8 Jul 2023 08:59:07 +0300 Subject: [PATCH 12/12] similarity -> distance --- examples/image-search/search.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/image-search/search.cpp b/examples/image-search/search.cpp index 3914db0..d975e69 100644 --- a/examples/image-search/search.cpp +++ b/examples/image-search/search.cpp @@ -131,7 +131,7 @@ int main(int argc, char** argv) { auto results = embd_index.search({txt_vec.data(), txt_vec.size()}, params.n_results); printf("search results:\n"); - printf("similarity path\n"); + printf("distance path\n"); for (std::size_t i = 0; i != results.size(); ++i) { printf(" %f %s\n", results[i].distance, image_file_index.at(results[i].element.label).c_str()); }