forked from PracticumAI/deep_learning
-
Notifications
You must be signed in to change notification settings - Fork 2
Open
Description
Need to add this block
# Set the number of workers to use for data loading
num_workers = None # To manually set the number of workers, change this to an integer
if num_workers is None:
# If Slurm is being used, set the number of workers to a Slurm-provided value.
# If Slurm is not being used, set the number of workers to the number of available CPU cores.
if os.getenv("SLURM_CPUS_PER_TASK") is not None:
num_workers = int(os.getenv("SLURM_CPUS_PER_TASK"))
elif os.getenv("SLURM_NTASKS_PER_NODE") is not None:
num_workers = int(os.getenv("SLURM_NTASKS_PER_NODE"))
elif os.getenv("SLURM_NTASKS") is not None:
num_workers = int(os.getenv("SLURM_NTASKS"))
else:
num_workers = os.cpu_count()
print(f"Using {num_workers} workers for data loading.")And add num_workers=num_workers in the dataloader def code. See ARS version for details. Probably also add a note about how to launch Slurm sessions to ensure people don't use ntasks.
Metadata
Metadata
Assignees
Labels
No labels