hnswlib/{bruteforce,hnswalg}.h: Added new generic serial / deserialization interfaces for HierarchicalNSW and BruteforceSearch...

README.md: Noted new serial / deserialization interfaces...
setup.py: Bumped patch version because new interfaces introduced...

Author: kiplingw

README.md

@@ -3,6 +3,10 @@ Header-only C++ HNSW implementation with python bindings, insertions and updates
 
 **NEWS:**
 
+**version 0.8.1**
+
+* Added generic serialization / deserialization interfaces for HierarchicalNSW and BruteforceSearch that take `std::ostream` / `std::istream` arguments
+
 **version 0.8.0** 
 
 * Multi-vector document search and epsilon search (for now, only in C++)

hnswlib/bruteforce.h

@@ -7,6 +7,7 @@
 #include <mutex>
 #include <algorithm>
 #include <assert.h>
+#include <iostream>
 
 namespace hnswlib {
 template<typename dist_t>
@@ -130,25 +131,30 @@ class BruteforceSearch : public AlgorithmInterface<dist_t> {
     }
 
 
-    Status saveIndexNoExceptions(const std::string &location) override {
-        std::ofstream output(location, std::ios::binary);
-        std::streampos position;
-
+    Status saveIndexNoExceptions(std::ostream &output) {
         writeBinaryPOD(output, maxelements_);
         writeBinaryPOD(output, size_per_element_);
         writeBinaryPOD(output, cur_element_count);
 
         output.write(data_, maxelements_ * size_per_element_);
+        return OkStatus();
+    }
+
+
+    Status saveIndexNoExceptions(const std::string &location) override {
+        std::ofstream output(location, std::ios::binary);
+
+        Status status = saveIndexNoExceptions(output);
+        if (!status.ok()) {
+            HNSWLIB_THROW_RUNTIME_ERROR(status.message());
+        }
 
         output.close();
         return OkStatus();
     }
 
 
-    void loadIndex(const std::string &location, SpaceInterface<dist_t> *s) {
-        std::ifstream input(location, std::ios::binary);
-        std::streampos position;
-
+    void loadIndex(std::istream &input, SpaceInterface<dist_t> *s) {
         readBinaryPOD(input, maxelements_);
         readBinaryPOD(input, size_per_element_);
         readBinaryPOD(input, cur_element_count);
@@ -162,6 +168,13 @@ class BruteforceSearch : public AlgorithmInterface<dist_t> {
             HNSWLIB_THROW_RUNTIME_ERROR("Not enough memory: loadIndex failed to allocate data");
 
         input.read(data_, maxelements_ * size_per_element_);
+    }
+
+
+    void loadIndex(const std::string &location, SpaceInterface<dist_t> *s) {
+        std::ifstream input(location, std::ios::binary);
+
+        loadIndex(input, s);
 
         input.close();
     }

hnswlib/hnswalg.h

@@ -7,6 +7,7 @@
 #include <stdlib.h>
 
 #include <atomic>
+#include <iostream>
 #include <limits>
 #include <list>
 #include <memory>
@@ -723,10 +724,7 @@ class HierarchicalNSW : public AlgorithmInterface<dist_t> {
         return size;
     }
 
-    Status saveIndexNoExceptions(const std::string &location) override {
-
-        std::ofstream output(location, std::ios::binary);
-
+    Status saveIndexNoExceptions(std::ostream &output) {
         writeBinaryPOD(output, offsetLevel0_);
         writeBinaryPOD(output, max_elements_);
         writeBinaryPOD(output, cur_element_count);
@@ -750,24 +748,22 @@ class HierarchicalNSW : public AlgorithmInterface<dist_t> {
             if (linkListSize)
                 output.write(linkLists_[i], linkListSize);
         }
-        output.close();
         return OkStatus();
     }
 
 
-    void loadIndex(const std::string &location, SpaceInterface<dist_t> *s, size_t max_elements_i = 0) {
-        Status status = loadIndexNoExceptions(location, s, max_elements_i);
+    Status saveIndexNoExceptions(const std::string &location) override {
+        std::ofstream output(location, std::ios::binary);
+        Status status = saveIndexNoExceptions(output);
         if (!status.ok()) {
             HNSWLIB_THROW_RUNTIME_ERROR(status.message());
         }
+        output.close();
+        return OkStatus();
     }
 
-    Status loadIndexNoExceptions(const std::string &location, SpaceInterface<dist_t> *s, size_t max_elements_i = 0) {
-        std::ifstream input(location, std::ios::binary);
-
-        if (!input.is_open())
-            return Status("Cannot open file");
 
+    Status loadIndexNoExceptions(std::istream &input, SpaceInterface<dist_t> *s, size_t max_elements_i = 0) {
         clear();
         // get file size:
         input.seekg(0, input.end);
@@ -865,11 +861,28 @@ class HierarchicalNSW : public AlgorithmInterface<dist_t> {
             }
         }
 
+        return OkStatus();
+    }
+
+
+
+    void loadIndex(const std::string &location, SpaceInterface<dist_t> *s, size_t max_elements_i = 0) {
+        std::ifstream input(location, std::ios::binary);
+
+        if (!input.is_open())
+            throw std::runtime_error("Cannot open file");
+
+        Status status = loadIndexNoExceptions(input, s, max_elements_i);
+        if (!status.ok()) {
+            HNSWLIB_THROW_RUNTIME_ERROR(status.message());
+        }
+
         input.close();
 
-        return OkStatus();
+        return;
     }
 
+
     template<typename data_t>
     std::vector<data_t>
     getDataByLabel(labeltype label) const {

setup.py

@@ -8,7 +8,7 @@
 from setuptools import Extension, setup
 from setuptools.command.build_ext import build_ext
 
-__version__ = '0.8.0'
+__version__ = '0.8.1'
 
 
 include_dirs = [