NSFS | Improve list objects performance on top of NS FS

Signed-off-by: naveenpaul1 <[email protected]>

Author: naveenpaul1

config.js

@@ -843,6 +843,9 @@ config.NSFS_GLACIER_MIGRATE_INTERVAL = 15 * 60 * 1000;
 // of `manage_nsfs glacier restore`
 config.NSFS_GLACIER_RESTORE_INTERVAL = 15 * 60 * 1000;
 
+// enable/disable unsorted listing application level
+config.NSFS_LIST_OBJECTS_V2_UNSORTED_ENABLED = false;
+
 // NSFS_GLACIER_EXPIRY_RUN_TIME must be of the format hh:mm which specifies
 // when NooBaa should allow running glacier expiry process
 // NOTE: This will also be in the same timezone as specified in

src/api/object_api.js

@@ -723,6 +723,9 @@ module.exports = {
                     limit: {
                         type: 'integer'
                     },
+                    list_type: {
+                        type: 'string',
+                    },
                 }
             },
             reply: {
@@ -777,6 +780,9 @@ module.exports = {
                     limit: {
                         type: 'integer'
                     },
+                    list_type: {
+                        type: 'string',
+                    },
                 }
             },
             reply: {

src/endpoint/s3/ops/s3_get_bucket.js

@@ -41,6 +41,7 @@ async function get_bucket(req) {
         bucket: req.params.bucket,
         prefix: req.query.prefix,
         delimiter: req.query.delimiter,
+        list_type: list_type,
         limit: Math.min(max_keys_received, 1000),
         key_marker: list_type === '2' ?
             (s3_utils.cont_tok_to_key_marker(cont_tok) || start_after) : req.query.marker,

src/native/fs/fs_napi.cpp

@@ -2105,7 +2105,7 @@ struct TellDir : public FSWrapWorker<DirWrap>
     }
     virtual void OnOK()
     {
-        DBG0("FS::Telldir::OnOK: " << DVAL(_wrap->_path) << DVAL(_tell_res));
+        DBG1("FS::Telldir::OnOK: " << DVAL(_wrap->_path) << DVAL(_tell_res));
         Napi::Env env = Env();
         auto res = Napi::BigInt::New(env, _tell_res);
         _deferred.Resolve(res);

src/sdk/keymarker_fs.js

@@ -0,0 +1,50 @@
+/* Copyright (C) 2020 NooBaa */
+'use strict';
+
+class KeyMarkerFS {
+    constructor({ marker, marker_pos, pre_dir, pre_dir_pos }, is_unsorted = false) {
+        this.marker = marker;
+        this.marker_pos = marker_pos.toString();
+        this.pre_dir = pre_dir;
+        this.pre_dir_pos = pre_dir_pos;
+        this.key_marker_value = marker;
+        this.current_dir = '';
+        this.is_unsorted = is_unsorted;
+        this.last_pre_dir = '';
+        this.last_pre_dir_position = '';
+        if (is_unsorted) {
+            this.current_dir = pre_dir.length > 0 && marker.includes('/') ?
+                            marker.substring(0, marker.lastIndexOf('/') + 1) : '';
+        }
+    }
+
+    async update_key_marker(marker, marker_pos) {
+        this.marker = marker;
+        this.marker_pos = marker_pos;
+        this.key_marker_value = marker;
+    }
+
+    async get_previour_dir_length() {
+        return this.pre_dir.length;
+    }
+
+    async get_previour_dir_info() {
+        return {
+            pre_dir_path: this.pre_dir.pop(),
+            pre_dir_position: this.pre_dir_pos.pop(),
+        };
+    }
+
+    async add_previour_dir(pre_dir, pre_dir_pos) {
+        this.pre_dir.push(pre_dir);
+        this.pre_dir_pos.push(pre_dir_pos.toString());
+    }
+
+    async update_last_previour_dir(last_pre_dir, last_pre_dir_position) {
+        this.last_pre_dir = last_pre_dir;
+        this.last_pre_dir_position = last_pre_dir_position;
+    }
+}
+
+// EXPORTS
+module.exports = KeyMarkerFS;

src/sdk/list_object_fs.js

@@ -0,0 +1,48 @@
+/* Copyright (C) 2020 NooBaa */
+'use strict';
+
+
+class ListObjectFS {
+    constructor({fs_context, list_versions, keymarker, prefix_dir_key, is_truncated, delimiter, prefix,
+        version_id_marker, list_type, results, limit, skip_list, key_marker}) {
+        this.fs_context = fs_context;
+        this.keymarker = keymarker;
+        this.prefix_dir_key = prefix_dir_key;
+        this.is_truncated = is_truncated;
+        this.delimiter = delimiter;
+        this.prefix = prefix;
+        this.version_id_marker = version_id_marker;
+        this.list_type = list_type;
+        this.results = results;
+        this.limit = limit;
+        this.skip_list = skip_list;
+        this.prefix_ent = '';
+        this.marker_dir = '';
+        this.param_key_marker = key_marker;
+        this.list_versions = list_versions;
+    }
+
+    async update_process_dir_properties(prefix_ent, marker_curr, dir_path) {
+        this.prefix_ent = prefix_ent;
+        this.dir_path = dir_path;
+    }
+
+    async update_is_truncated(is_truncated) {
+        this.is_truncated = is_truncated;
+    }
+
+    async get_is_truncated() {
+        return this.is_truncated;
+    }
+
+    async update_keymarker(keymarker) {
+        this.keymarker = keymarker;
+    }
+
+    async get_keymarker() {
+        return this.keymarker;
+    }
+}
+
+// EXPORTS
+module.exports = ListObjectFS;