[update] Update the PR according to the latest version and disable shuffle

nabenabe0928 · nabenabe0928 · commit 1e004133f6b0 · 2021-05-10T11:14:36.000+09:00
Since the shuffle is performed in the split functions, we do not need
shuffle before the splitting.
For this reason, I disabled the shuffle argument from BaseDataset and
added the shuffle keyword for resampling_strategy_args.
diff --git a/autoPyTorch/api/base_task.py b/autoPyTorch/api/base_task.py
@@ -34,7 +34,7 @@
     STRING_TO_TASK_TYPES,
 )
 from autoPyTorch.datasets.base_dataset import BaseDataset
-from autoPyTorch.datasets.resampling_strategy import CrossValTypes, HoldoutValTypes
+from autoPyTorch.datasets.resampling_strategy import CrossValTypes, HoldoutTypes
 from autoPyTorch.ensemble.ensemble_builder import EnsembleBuilderManager
 from autoPyTorch.ensemble.ensemble_selection import EnsembleSelection
 from autoPyTorch.ensemble.singlebest_ensemble import SingleBest
@@ -138,7 +138,7 @@ def __init__(
         include_components: Optional[Dict] = None,
         exclude_components: Optional[Dict] = None,
         backend: Optional[Backend] = None,
-        resampling_strategy: Union[CrossValTypes, HoldoutValTypes] = HoldoutValTypes.holdout_validation,
+        resampling_strategy: Union[CrossValTypes, HoldoutTypes] = HoldoutTypes.holdout,
         resampling_strategy_args: Optional[Dict[str, Any]] = None,
         search_space_updates: Optional[HyperparameterSearchSpaceUpdates] = None,
         task_type: Optional[str] = None
@@ -1171,7 +1171,7 @@ def predict(
         assert self.ensemble_ is not None, "Load models should error out if no ensemble"
         self.ensemble_ = cast(Union[SingleBest, EnsembleSelection], self.ensemble_)
 
-        if isinstance(self.resampling_strategy, HoldoutValTypes):
+        if isinstance(self.resampling_strategy, HoldoutTypes):
             models = self.models_
         elif isinstance(self.resampling_strategy, CrossValTypes):
             models = self.cv_models_
diff --git a/autoPyTorch/api/tabular_classification.py b/autoPyTorch/api/tabular_classification.py
@@ -15,7 +15,7 @@
 from autoPyTorch.datasets.base_dataset import BaseDataset
 from autoPyTorch.datasets.resampling_strategy import (
     CrossValTypes,
-    HoldoutValTypes,
+    HoldoutTypes,
 )
 from autoPyTorch.datasets.tabular_dataset import TabularDataset
 from autoPyTorch.pipeline.tabular_classification import TabularClassificationPipeline
@@ -72,7 +72,7 @@ def __init__(
         delete_output_folder_after_terminate: bool = True,
         include_components: Optional[Dict] = None,
         exclude_components: Optional[Dict] = None,
-        resampling_strategy: Union[CrossValTypes, HoldoutValTypes] = HoldoutValTypes.holdout_validation,
+        resampling_strategy: Union[CrossValTypes, HoldoutTypes] = HoldoutTypes.holdout,
         resampling_strategy_args: Optional[Dict[str, Any]] = None,
         backend: Optional[Backend] = None,
         search_space_updates: Optional[HyperparameterSearchSpaceUpdates] = None
diff --git a/autoPyTorch/api/tabular_regression.py b/autoPyTorch/api/tabular_regression.py
@@ -15,7 +15,7 @@
 from autoPyTorch.datasets.base_dataset import BaseDataset
 from autoPyTorch.datasets.resampling_strategy import (
     CrossValTypes,
-    HoldoutValTypes,
+    HoldoutTypes,
 )
 from autoPyTorch.datasets.tabular_dataset import TabularDataset
 from autoPyTorch.pipeline.tabular_regression import TabularRegressionPipeline
@@ -64,7 +64,7 @@ def __init__(
             delete_output_folder_after_terminate: bool = True,
             include_components: Optional[Dict] = None,
             exclude_components: Optional[Dict] = None,
-            resampling_strategy: Union[CrossValTypes, HoldoutValTypes] = HoldoutValTypes.holdout_validation,
+            resampling_strategy: Union[CrossValTypes, HoldoutTypes] = HoldoutTypes.holdout,
             resampling_strategy_args: Optional[Dict[str, Any]] = None,
             backend: Optional[Backend] = None,
             search_space_updates: Optional[HyperparameterSearchSpaceUpdates] = None
diff --git a/autoPyTorch/datasets/base_dataset.py b/autoPyTorch/datasets/base_dataset.py
@@ -71,7 +71,6 @@ def __init__(
         test_tensors: Optional[BaseDatasetInputType] = None,
         resampling_strategy: Union[CrossValTypes, HoldoutTypes] = HoldoutTypes.holdout,
         resampling_strategy_args: Optional[Dict[str, Any]] = None,
-        shuffle: Optional[bool] = True,
         seed: Optional[int] = 42,
         train_transforms: Optional[torchvision.transforms.Compose] = None,
         val_transforms: Optional[torchvision.transforms.Compose] = None,
@@ -89,10 +88,9 @@ def __init__(
             resampling_strategy (Union[CrossValTypes, HoldoutTypes]),
                 (default=HoldoutTypes.holdout):
                 strategy to split the training data.
-            resampling_strategy_args (Optional[Dict[str, Any]]): arguments
-                required for the chosen resampling strategy. If None, uses
-                the default values provided in DEFAULT_RESAMPLING_PARAMETERS
-                in ```datasets/resampling_strategy.py```.
+            resampling_strategy_args (Optional[Dict[str, Any]]):
+                arguments required for the chosen resampling strategy.
+                The details are provided in autoPytorch/datasets/resampling_strategy.py
             shuffle:  Whether to shuffle the data when performing splits
             seed (int), (default=1): seed to be used for reproducibility.
             train_transforms (Optional[torchvision.transforms.Compose]):
@@ -109,9 +107,9 @@ def __init__(
             type_check(train_tensors, val_tensors)
         self.train_tensors, self.val_tensors, self.test_tensors = train_tensors, val_tensors, test_tensors
         self.random_state = np.random.RandomState(seed=seed)
-        self.shuffle = shuffle
         self.resampling_strategy = resampling_strategy
         self.resampling_strategy_args = resampling_strategy_args
+        self.shuffle = self.resampling_strategy_args['shuffle']
         self.is_stratify = self.resampling_strategy.get('stratify', False)
 
         self.task_type: Optional[str] = None
diff --git a/autoPyTorch/datasets/image_dataset.py b/autoPyTorch/datasets/image_dataset.py
@@ -23,7 +23,7 @@
 from autoPyTorch.datasets.base_dataset import BaseDataset
 from autoPyTorch.datasets.resampling_strategy import (
     CrossValTypes,
-    HoldoutValTypes,
+    HoldoutTypes,
 )
 
 IMAGE_DATASET_INPUT = Union[Dataset, Tuple[Union[np.ndarray, List[str]], np.ndarray]]
@@ -39,13 +39,12 @@ class ImageDataset(BaseDataset):
             validation data
         test (Union[Dataset, Tuple[Union[np.ndarray, List[str]], np.ndarray]]):
             testing data
-        resampling_strategy (Union[CrossValTypes, HoldoutValTypes]),
-            (default=HoldoutValTypes.holdout_validation):
+        resampling_strategy (Union[CrossValTypes, HoldoutTypes]),
+            (default=HoldoutTypes.holdout):
             strategy to split the training data.
-        resampling_strategy_args (Optional[Dict[str, Any]]): arguments
-            required for the chosen resampling strategy. If None, uses
-            the default values provided in DEFAULT_RESAMPLING_PARAMETERS
-            in ```datasets/resampling_strategy.py```.
+        resampling_strategy_args (Optional[Dict[str, Any]]):
+            arguments required for the chosen resampling strategy.
+            The details are provided in autoPytorch/datasets/resampling_strategy.py
         shuffle:  Whether to shuffle the data before performing splits
         seed (int), (default=1): seed to be used for reproducibility.
         train_transforms (Optional[torchvision.transforms.Compose]):
@@ -57,9 +56,8 @@ def __init__(self,
                  train: IMAGE_DATASET_INPUT,
                  val: Optional[IMAGE_DATASET_INPUT] = None,
                  test: Optional[IMAGE_DATASET_INPUT] = None,
-                 resampling_strategy: Union[CrossValTypes, HoldoutValTypes] = HoldoutValTypes.holdout_validation,
+                 resampling_strategy: Union[CrossValTypes, HoldoutTypes] = HoldoutTypes.holdout,
                  resampling_strategy_args: Optional[Dict[str, Any]] = None,
-                 shuffle: Optional[bool] = True,
                  seed: Optional[int] = 42,
                  train_transforms: Optional[torchvision.transforms.Compose] = None,
                  val_transforms: Optional[torchvision.transforms.Compose] = None,
@@ -72,7 +70,7 @@ def __init__(self,
             test = _create_image_dataset(data=test)
         self.mean, self.std = _calc_mean_std(train=train)
 
-        super().__init__(train_tensors=train, val_tensors=val, test_tensors=test, shuffle=shuffle,
+        super().__init__(train_tensors=train, val_tensors=val, test_tensors=test,
                          resampling_strategy=resampling_strategy, resampling_strategy_args=resampling_strategy_args,
                          seed=seed,
                          train_transforms=train_transforms,
diff --git a/autoPyTorch/datasets/resampling_strategy.py b/autoPyTorch/datasets/resampling_strategy.py
@@ -1,6 +1,6 @@
 from enum import Enum
 from functools import partial
-from typing import List, Optional, Tuple, Union
+from typing import List, NamedTuple, Optional, Tuple, Union
 
 import numpy as np
 
@@ -16,6 +16,13 @@
 from torch.utils.data import Dataset
 
 
+class _ResamplingStrategyArgs(NamedTuple):
+    val_share: float = 0.33
+    num_splits: int = 5
+    shuffle: bool = False
+    stratify: bool = False
+
+
 class HoldoutFuncs():
     @staticmethod
     def holdout(
diff --git a/autoPyTorch/datasets/tabular_dataset.py b/autoPyTorch/datasets/tabular_dataset.py
@@ -20,7 +20,7 @@
 from autoPyTorch.datasets.base_dataset import BaseDataset
 from autoPyTorch.datasets.resampling_strategy import (
     CrossValTypes,
-    HoldoutValTypes,
+    HoldoutTypes,
 )
 
 
@@ -44,13 +44,12 @@ class TabularDataset(BaseDataset):
             Y (Union[np.ndarray, pd.Series]): training data targets.
             X_test (Optional[Union[np.ndarray, pd.DataFrame]]):  input testing data.
             Y_test (Optional[Union[np.ndarray, pd.DataFrame]]): testing data targets
-            resampling_strategy (Union[CrossValTypes, HoldoutValTypes]),
-                (default=HoldoutValTypes.holdout_validation):
+            resampling_strategy (Union[CrossValTypes, HoldoutTypes]),
+                (default=HoldoutTypes.holdout):
                 strategy to split the training data.
-            resampling_strategy_args (Optional[Dict[str, Any]]): arguments
-                required for the chosen resampling strategy. If None, uses
-                the default values provided in DEFAULT_RESAMPLING_PARAMETERS
-                in ```datasets/resampling_strategy.py```.
+            resampling_strategy_args (Optional[Dict[str, Any]]):
+                arguments required for the chosen resampling strategy.
+                The details are provided in autoPytorch/datasets/resampling_strategy.py
             shuffle:  Whether to shuffle the data before performing splits
             seed (int), (default=1): seed to be used for reproducibility.
             train_transforms (Optional[torchvision.transforms.Compose]):
@@ -67,9 +66,8 @@ def __init__(self,
                  Y: Union[np.ndarray, pd.Series],
                  X_test: Optional[Union[np.ndarray, pd.DataFrame]] = None,
                  Y_test: Optional[Union[np.ndarray, pd.DataFrame]] = None,
-                 resampling_strategy: Union[CrossValTypes, HoldoutValTypes] = HoldoutValTypes.holdout_validation,
+                 resampling_strategy: Union[CrossValTypes, HoldoutTypes] = HoldoutTypes.holdout,
                  resampling_strategy_args: Optional[Dict[str, Any]] = None,
-                 shuffle: Optional[bool] = True,
                  seed: Optional[int] = 42,
                  train_transforms: Optional[torchvision.transforms.Compose] = None,
                  val_transforms: Optional[torchvision.transforms.Compose] = None,
@@ -92,7 +90,7 @@ def __init__(self,
         self.num_features = validator.feature_validator.num_features
         self.categories = validator.feature_validator.categories
 
-        super().__init__(train_tensors=(X, Y), test_tensors=(X_test, Y_test), shuffle=shuffle,
+        super().__init__(train_tensors=(X, Y), test_tensors=(X_test, Y_test),
                          resampling_strategy=resampling_strategy,
                          resampling_strategy_args=resampling_strategy_args,
                          seed=seed, train_transforms=train_transforms,
diff --git a/autoPyTorch/datasets/time_series_dataset.py b/autoPyTorch/datasets/time_series_dataset.py
@@ -41,7 +41,6 @@ def __init__(self,
                  val: Optional[TIME_SERIES_FORECASTING_INPUT] = None,
                  resampling_strategy: Union[CrossValTypes, HoldoutTypes] = HoldoutTypes.holdout,
                  resampling_strategy_args: Optional[Dict[str, Any]] = None,
-                 shuffle: Optional[bool] = False,
                  seed: Optional[int] = 42,
                  train_transforms: Optional[torchvision.transforms.Compose] = None,
                  val_transforms: Optional[torchvision.transforms.Compose] = None,
@@ -69,7 +68,7 @@ def __init__(self,
                                                           target_variables=target_variables,
                                                           sequence_length=sequence_length,
                                                           n_steps=n_steps)
-        super().__init__(train_tensors=train, val_tensors=val, shuffle=shuffle,
+        super().__init__(train_tensors=train, val_tensors=val,
                          resampling_strategy=resampling_strategy, resampling_strategy_args=resampling_strategy_args,
                          seed=seed,
                          train_transforms=train_transforms,
@@ -129,15 +128,17 @@ def __init__(self,
         _check_time_series_inputs(train=train,
                                   val=val,
                                   task_type="time_series_classification")
-        super().__init__(train_tensors=train, val_tensors=val, shuffle=True)
+        resampling_strategy_args = {'shuffle': True}
+        super().__init__(train_tensors=train, val_tensors=val, resampling_strategy_args=resampling_strategy_args)
 
 
 class TimeSeriesRegressionDataset(BaseDataset):
     def __init__(self, train: Tuple[np.ndarray, np.ndarray], val: Optional[Tuple[np.ndarray, np.ndarray]] = None):
         _check_time_series_inputs(train=train,
                                   val=val,
                                   task_type="time_series_regression")
-        super().__init__(train_tensors=train, val_tensors=val, shuffle=True)
+        resampling_strategy_args = {'shuffle': True}
+        super().__init__(train_tensors=train, val_tensors=val, resampling_strategy_args=resampling_strategy_args)
 
 
 def _check_time_series_inputs(task_type: str,
diff --git a/autoPyTorch/optimizer/smbo.py b/autoPyTorch/optimizer/smbo.py
@@ -19,8 +19,7 @@
 from autoPyTorch.datasets.base_dataset import BaseDataset
 from autoPyTorch.datasets.resampling_strategy import (
     CrossValTypes,
-    DEFAULT_RESAMPLING_PARAMETERS,
-    HoldoutValTypes,
+    HoldoutTypes,
 )
 from autoPyTorch.ensemble.ensemble_builder import EnsembleBuilderManager
 from autoPyTorch.evaluation.tae import ExecuteTaFuncWithQueue, get_cost_of_crash
@@ -93,7 +92,7 @@ def __init__(self,
                  pipeline_config: typing.Dict[str, typing.Any],
                  start_num_run: int = 1,
                  seed: int = 1,
-                 resampling_strategy: typing.Union[HoldoutValTypes, CrossValTypes] = HoldoutValTypes.holdout_validation,
+                 resampling_strategy: typing.Union[HoldoutTypes, CrossValTypes] = HoldoutTypes.holdout,
                  resampling_strategy_args: typing.Optional[typing.Dict[str, typing.Any]] = None,
                  include: typing.Optional[typing.Dict[str, typing.Any]] = None,
                  exclude: typing.Optional[typing.Dict[str, typing.Any]] = None,
@@ -173,9 +172,7 @@ def __init__(self,
 
         # Evaluation
         self.resampling_strategy = resampling_strategy
-        if resampling_strategy_args is None:
-            resampling_strategy_args = DEFAULT_RESAMPLING_PARAMETERS[resampling_strategy]
-        self.resampling_strategy_args = resampling_strategy_args
+        self.resampling_strategy_args = resampling_strategy_args if resampling_strategy_args is None else {}
 
         # and a bunch of useful limits
         self.worst_possible_result = get_cost_of_crash(self.metric)
diff --git a/examples/tabular/40_advanced/example_resampling_strategy.py b/examples/tabular/40_advanced/example_resampling_strategy.py
@@ -24,7 +24,7 @@
 import sklearn.model_selection
 
 from autoPyTorch.api.tabular_classification import TabularClassificationTask
-from autoPyTorch.datasets.resampling_strategy import CrossValTypes, HoldoutValTypes
+from autoPyTorch.datasets.resampling_strategy import CrossValTypes, HoldoutTypes
 
 
 if __name__ == '__main__':
@@ -48,11 +48,11 @@
         # To maintain logs of the run, set the next two as False
         delete_tmp_folder_after_terminate=True,
         delete_output_folder_after_terminate=True,
-        # 'HoldoutValTypes.holdout_validation' with 'val_share': 0.33
+        # 'HoldoutTypes.holdout' with 'val_share': 0.33
         # is the default argument setting for TabularClassificationTask.
         # It is explicitly specified in this example for demonstrational
         # purpose.
-        resampling_strategy=HoldoutValTypes.holdout_validation,
+        resampling_strategy=HoldoutTypes.holdout,
         resampling_strategy_args={'val_share': 0.33}
     )
 
@@ -90,7 +90,7 @@
         # To maintain logs of the run, set the next two as False
         delete_tmp_folder_after_terminate=True,
         delete_output_folder_after_terminate=True,
-        resampling_strategy=CrossValTypes.k_fold_cross_validation,
+        resampling_strategy=CrossValTypes.k_fold,
         resampling_strategy_args={'num_splits': 3}
     )
 
@@ -130,9 +130,9 @@
         delete_output_folder_after_terminate=True,
         # For demonstration purposes, we use
         # Stratified hold out validation. However,
-        # one can also use CrossValTypes.stratified_k_fold_cross_validation.
-        resampling_strategy=HoldoutValTypes.stratified_holdout_validation,
-        resampling_strategy_args={'val_share': 0.33}
+        # one can also use CrossValTypes.k_fold.
+        resampling_strategy=HoldoutTypes.holdout,
+        resampling_strategy_args={'val_share': 0.33, 'stratify': True}
     )
 
     ############################################################################
diff --git a/test/test_api/test_api.py b/test/test_api/test_api.py
diff --git a/test/test_evaluation/evaluation_util.py b/test/test_evaluation/evaluation_util.py
diff --git a/test/test_evaluation/test_train_evaluator.py b/test/test_evaluation/test_train_evaluator.py
