NC | Adding support of user bucket path

config.js

@@ -1022,6 +1022,9 @@ config.NSFS_LIST_IGNORE_ENTRY_ON_EACCES = true;
 // we will for now handle the same way also EINVAL error - for gpfs stat issues on list (.snapshots)
 config.NSFS_LIST_IGNORE_ENTRY_ON_EINVAL = true;
 
+config.NSFS_CUSTOM_BUCKET_PATH_HTTP_HEADER = 'x-noobaa-custom-bucket-path';
+config.NSFS_CUSTOM_BUCKET_PATH_ALLOWED_LIST = ''; // colon separated list of paths prefixes
+
 ////////////////////////////
 // NSFS NON CONTAINERIZED //
 ////////////////////////////

docs/NooBaaNonContainerized/AccountsAndBuckets.md

@@ -38,6 +38,8 @@ See all available account properties - [NC Account Schema](../../src/server/syst
 
   - `new_buckets_path` - When an account creates a bucket using the S3 protocol, NooBaa will create the underlying file system directory. This directory will be created under new_buckets_path. Note that the account must have read and write access to its `new_buckets_path`.  Must be an absolute path.  
 
+  - `custom_bucket_path_allowed_list` - When an account creates a bucket using the S3 protocol, He can override the default bucket path location (under new_buckets_path) using `x-noobaa-custom-bucket-path` HTTP header. This directory will be created only if this path will be under one of the provided allowed list paths in custom_bucket_path_allowed_list. Must be a list of absolute paths (divided by colons).  
+
 ### Account configuration  
 Currently, an account can be configured via NooBaa CLI, see - [NooBaa CLI](./NooBaaCLI.md).  
 

docs/NooBaaNonContainerized/NooBaaCLI.md

@@ -76,7 +76,7 @@ The `account add` command is used to create a new account with customizable opti
 ```sh
 noobaa-cli account add --name <account_name> --uid <uid> --gid <gid> [--user]
 [--new_buckets_path][--access_key][--secret_key][--fs_backend]
-[--allow_bucket_creation][--force_md5_etag][--anonymous][--from_file][--iam_operate_on_root_account][--default_connection]
+[--allow_bucket_creation][--force_md5_etag][--anonymous][--from_file][--iam_operate_on_root_account][--default_connection][--custom_bucket_path_allowed_list]
 ```
 #### Flags -
 - `name` (Required)
@@ -140,6 +140,11 @@ noobaa-cli account add --name <account_name> --uid <uid> --gid <gid> [--user]
     - Type: String
     - Description: A default account for Kafka external servers. See bucket-notifications.md.
 
+- `custom_bucket_path_allowed_list`
+    - Type: String
+    - Description: Specifies an allowed list where this account can create buckets in using 
+    x-noobaa-custom-bucket-path header in create_bucket
+
 ### Update Account
 
 The `account update` command is used to update an existing account with customizable options.
@@ -149,6 +154,7 @@ The `account update` command is used to update an existing account with customiz
 noobaa-cli account update --name <account_name> [--new_name][--uid][--gid][--user]
 [--new_buckets_path][--access_key][--secret_key][--regenerate][--fs_backend]
 [--allow_bucket_creation][--force_md5_etag][--anonymous][--iam_operate_on_root_account][--default_connection]
+[--custom_bucket_path_allowed_list]
 ```
 #### Flags -
 - `name` (Required)
@@ -216,6 +222,11 @@ noobaa-cli account update --name <account_name> [--new_name][--uid][--gid][--use
     - Type: String
     - Description: A default account for Kafka external servers. See bucket-notifications.md.
 
+- `custom_bucket_path_allowed_list`
+    - Type: String
+    - Description: Specifies an allowed list where this account can create buckets in using 
+    x-noobaa-custom-bucket-path header in create_bucket
+
 ### Account Status
 
 The `account status` command is used to print the status of the account.

src/api/account_api.js

@@ -308,6 +308,7 @@ module.exports = {
                             supplemental_groups: {
                                 $ref: 'common_api#/definitions/supplemental_groups'
                             },
+                            custom_bucket_path_allowed_list: { type: 'string' },
                         }
                     },
                 },

src/api/bucket_api.js

@@ -33,6 +33,7 @@ module.exports = {
                     },
                     bucket_claim: { $ref: '#/definitions/bucket_claim' },
                     force_md5_etag: { type: 'boolean' },
+                    custom_bucket_path: { type: 'string' }
                 }
             },
             reply: {

src/api/common_api.js

@@ -1454,14 +1454,16 @@ module.exports = {
                     supplemental_groups: {
                         $ref: '#/definitions/supplemental_groups'
                     },
+                    custom_bucket_path_allowed_list: { type: 'string' },
                 }
             }, {
                 type: 'object',
                 required: ['distinguished_name', 'new_buckets_path', 'nsfs_only'],
                 properties: {
                     distinguished_name: { wrapper: SensitiveString },
                     new_buckets_path: { type: 'string' },
-                    nsfs_only: { type: 'boolean' }
+                    nsfs_only: { type: 'boolean' },
+                    custom_bucket_path_allowed_list: { type: 'string' },
                 }
             }]
         },

src/cmd/manage_nsfs.js

@@ -503,7 +503,8 @@ async function fetch_account_data(action, user_input) {
             uid: user_input.user ? undefined : user_input.uid,
             gid: user_input.user ? undefined : user_input.gid,
             new_buckets_path: user_input.new_buckets_path,
-            fs_backend: user_input.fs_backend ? String(user_input.fs_backend) : config.NSFS_NC_STORAGE_BACKEND
+            fs_backend: user_input.fs_backend ? String(user_input.fs_backend) : config.NSFS_NC_STORAGE_BACKEND,
+            custom_bucket_path_allowed_list: user_input.custom_bucket_path_allowed_list,
         },
         default_connection: user_input.default_connection === undefined ? undefined : String(user_input.default_connection)
     };
@@ -542,6 +543,8 @@ async function fetch_account_data(action, user_input) {
     } else { // string of true or false
         data.allow_bucket_creation = user_input.allow_bucket_creation.toLowerCase() === 'true';
     }
+    // custom_bucket_path_allowed_list deletion specified with empty string ''
+    data.nsfs_account_config.custom_bucket_path_allowed_list = data.nsfs_account_config.custom_bucket_path_allowed_list || undefined;
 
     return data;
 }

src/endpoint/s3/ops/s3_put_bucket.js

@@ -9,7 +9,8 @@ const config = require('../../../../config');
 async function put_bucket(req, res) {
     const lock_enabled = config.WORM_ENABLED ? req.headers['x-amz-bucket-object-lock-enabled'] &&
         req.headers['x-amz-bucket-object-lock-enabled'].toUpperCase() === 'TRUE' : undefined;
-    await req.object_sdk.create_bucket({ name: req.params.bucket, lock_enabled: lock_enabled });
+    const custom_bucket_path = req.headers[config.NSFS_CUSTOM_BUCKET_PATH_HTTP_HEADER];
+    await req.object_sdk.create_bucket({ name: req.params.bucket, lock_enabled, custom_bucket_path });
     if (config.allow_anonymous_access_in_test && req.headers['x-amz-acl'] === 'public-read') { // For now we will enable only for tests
         const policy = {
             Version: '2012-10-17',

src/manage_nsfs/manage_nsfs_constants.js

@@ -46,8 +46,8 @@ const FROM_FILE = 'from_file';
 const ANONYMOUS = 'anonymous';
 
 const VALID_OPTIONS_ACCOUNT = {
-    'add': new Set(['name', 'uid', 'gid', 'supplemental_groups', 'new_buckets_path', 'user', 'access_key', 'secret_key', 'fs_backend', 'allow_bucket_creation', 'force_md5_etag', 'iam_operate_on_root_account', 'default_connection', FROM_FILE, ...CLI_MUTUAL_OPTIONS]),
-    'update': new Set(['name', 'uid', 'gid', 'supplemental_groups', 'new_buckets_path', 'user', 'access_key', 'secret_key', 'fs_backend', 'allow_bucket_creation', 'force_md5_etag', 'iam_operate_on_root_account', 'new_name', 'regenerate', 'default_connection', ...CLI_MUTUAL_OPTIONS]),
+    'add': new Set(['name', 'uid', 'gid', 'supplemental_groups', 'new_buckets_path', 'custom_bucket_path_allowed_list', 'user', 'access_key', 'secret_key', 'fs_backend', 'allow_bucket_creation', 'force_md5_etag', 'iam_operate_on_root_account', 'default_connection', FROM_FILE, ...CLI_MUTUAL_OPTIONS]),
+    'update': new Set(['name', 'uid', 'gid', 'supplemental_groups', 'new_buckets_path', 'custom_bucket_path_allowed_list', 'user', 'access_key', 'secret_key', 'fs_backend', 'allow_bucket_creation', 'force_md5_etag', 'iam_operate_on_root_account', 'new_name', 'regenerate', 'default_connection', ...CLI_MUTUAL_OPTIONS]),
     'delete': new Set(['name', ...CLI_MUTUAL_OPTIONS]),
     'list': new Set(['wide', 'show_secrets', 'gid', 'uid', 'user', 'name', 'access_key', ...CLI_MUTUAL_OPTIONS]),
     'status': new Set(['name', 'access_key', 'show_secrets', ...CLI_MUTUAL_OPTIONS]),
@@ -123,6 +123,7 @@ const OPTION_TYPE = {
     gid: 'number',
     supplemental_groups: 'string',
     new_buckets_path: 'string',
+    custom_bucket_path_allowed_list: 'string',
     user: 'string',
     access_key: 'string',
     secret_key: 'string',
@@ -196,6 +197,7 @@ const UNSETTABLE_OPTIONS_OBJ = Object.freeze({
     'force_md5_etag': CLI_EMPTY_STRING,
     'supplemental_groups': CLI_EMPTY_STRING,
     'new_buckets_path': CLI_EMPTY_STRING,
+    'custom_bucket_path_allowed_list': CLI_EMPTY_STRING,
     'ips': CLI_EMPTY_STRING_ARRAY,
 });
 

src/manage_nsfs/manage_nsfs_help_utils.js

@@ -143,6 +143,7 @@ Flags:
     --force_md5_etag <true | false>                       (optional)        Set the account to force md5 etag calculation. (unset with '') (will override default config.NSFS_NC_STORAGE_BACKEND)
     --iam_operate_on_root_account <true | false>          (optional)        Set the account to create root accounts instead of IAM users in IAM API requests.
     --from_file <string>                                  (optional)        Use details from the JSON file, there is no need to mention all the properties individually in the CLI
+    --custom_bucket_path_allowed_list <string>            (optional)        Set the list of allowed custom bucket paths, separated by colons (:) example: '/gpfs/data/custom1/:/gpfs/data/custom2/'
 `;
 
 const ACCOUNT_FLAGS_UPDATE = `
@@ -170,6 +171,7 @@ Flags:
     --allow_bucket_creation <true | false>                (optional)        Update the account to explicitly allow or block bucket creation
     --force_md5_etag <true | false>                       (optional)        Update the account to force md5 etag calculation (unset with '') (will override default config.NSFS_NC_STORAGE_BACKEND)
     --iam_operate_on_root_account <true | false>          (optional)        Update the account to create root accounts instead of IAM users in IAM API requests.
+    --custom_bucket_path_allowed_list <string>            (optional)        Update the list of allowed custom bucket paths, separated by colons (:) example: '/gpfs/data/custom1/:/gpfs/data/custom2/' (override;unset with '')
 `;
 
 const ACCOUNT_FLAGS_DELETE = `

src/sdk/accountspace_fs.js

@@ -608,6 +608,7 @@ class AccountSpaceFS {
                 supplemental_groups: requesting_account.nsfs_account_config.supplemental_groups,
                 new_buckets_path: requesting_account.nsfs_account_config.new_buckets_path,
                 fs_backend: requesting_account.nsfs_account_config.fs_backend,
+                custom_bucket_path_allowed_list: requesting_account.nsfs_account_config.custom_bucket_path_allowed_list,
             }
         };
         if (requesting_account.iam_operate_on_root_account) {

src/sdk/bucketspace_fs.js

@@ -303,9 +303,19 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
             const fs_context = this.prepare_fs_context(sdk);
             validate_bucket_creation(params);
 
-            const { name } = params;
+            const { name, custom_bucket_path } = params;
             const bucket_config_path = this.config_fs.get_bucket_path_by_name(name);
-            const bucket_storage_path = path.join(sdk.requesting_account.nsfs_account_config.new_buckets_path, name);
+            if (custom_bucket_path) {
+                const allowed_list = sdk.requesting_account.nsfs_account_config.custom_bucket_path_allowed_list ||
+                    config.NSFS_CUSTOM_BUCKET_PATH_ALLOWED_LIST;
+                const allowed_path_prefixes = allowed_list ? allowed_list.split(':').map(p => p.trim()).filter(p => p) : [];
+                if (!allowed_path_prefixes.length || !allowed_path_prefixes.some(prefix => custom_bucket_path.startsWith(prefix))) {
+                    const message = `Not allowed to create new buckets: ${custom_bucket_path} outside of the custom_bucket_path_allowed_list: ${allowed_list}`;
+                    dbg.error(`BucketSpaceFS.create_bucket: ${message}`);
+                    throw new RpcError('UNAUTHORIZED', message);
+                }
+            }
+            const bucket_storage_path = custom_bucket_path || path.join(sdk.requesting_account.nsfs_account_config.new_buckets_path, name);
 
             dbg.log0(`BucketSpaceFS.create_bucket
                 requesting_account=${util.inspect(sdk.requesting_account)},

src/server/system_services/schemas/nsfs_account_schema.js

@@ -87,7 +87,8 @@ module.exports = {
                     new_buckets_path: { type: 'string' },
                     fs_backend: {
                         $ref: 'common_api#/definitions/fs_backend'
-                    }
+                    },
+                    custom_bucket_path_allowed_list: { type: 'string' },
                 }
             }, {
                 type: 'object',
@@ -97,7 +98,8 @@ module.exports = {
                     new_buckets_path: { type: 'string' },
                     fs_backend: {
                         $ref: 'common_api#/definitions/fs_backend'
-                    }
+                    },
+                    custom_bucket_path_allowed_list: { type: 'string' },
                 }
             }]
         },

src/test/integration_tests/nc/cli/test_nc_account_cli.test.js

@@ -657,6 +657,20 @@ describe('manage nsfs cli account flow', () => {
             assert_account(account, account_options, false);
             expect(account.default_connection).toBe(default_connection);
         });
+
+        it('cli account add - with custom_bucket_path_allowed_list', async function() {
+            const action = ACTIONS.ADD;
+            const { type, name, new_buckets_path, uid, gid } = defaults;
+            const custom_bucket_path_allowed_list = '/root/bla:/my_buckets:/data/buckets';
+            const account_options = { config_root, name, new_buckets_path, uid, gid, custom_bucket_path_allowed_list };
+            await fs_utils.create_fresh_path(new_buckets_path);
+            await fs_utils.file_must_exist(new_buckets_path);
+            await set_path_permissions_and_owner(new_buckets_path, account_options, 0o700);
+            await exec_manage_cli(type, action, account_options);
+            const account = await config_fs.get_account_by_name(name, config_fs_account_options);
+            assert_account(account, account_options, false);
+            expect(account.nsfs_account_config.custom_bucket_path_allowed_list).toBe(custom_bucket_path_allowed_list);
+        });
     });
 
     describe('cli update account', () => {
@@ -1226,6 +1240,22 @@ describe('manage nsfs cli account flow', () => {
                 const updated_account = await config_fs.get_account_by_name(name, config_fs_account_options);
                 expect(updated_account.nsfs_account_config.supplemental_groups).toStrictEqual(expected_groups);
             });
+
+            it('cli update account unset custom_bucket_path_allowed_list', async () => {
+                const { name } = defaults;
+                //in exec_manage_cli an empty string is passed as no input. so operation fails on invalid type. use the string '' instead 
+                const account_options = { config_root, name, custom_bucket_path_allowed_list: CLI_UNSET_EMPTY_STRING};
+                const action = ACTIONS.UPDATE;
+                await exec_manage_cli(type, action, account_options);
+                let new_account_details = await config_fs.get_account_by_name(name, config_fs_account_options);
+                expect(new_account_details.nsfs_account_config.custom_bucket_path_allowed_list).toBeUndefined();
+
+                //set new_buckets_path value back to its original value
+                account_options.custom_bucket_path_allowed_list = "/some/path/:/another/path/";
+                await exec_manage_cli(type, action, account_options);
+                new_account_details = await config_fs.get_account_by_name(name, config_fs_account_options);
+                expect(new_account_details.nsfs_account_config.custom_bucket_path_allowed_list).toBe("/some/path/:/another/path/");
+            });
         });
 
         describe('cli update account (has distinguished name)', () => {