RMDP_Algorithm/rl_hyperparameter_tuning.py at main · TristanKruse/RMDP_Algorithm · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
import random
import train_rl


def run_hyperparameter_tuning(num_experiments=20, episodes_per_experiment=500):
    hyperparameter_space = {
        "learning_rate": [0.0001, 0.0002, 0.0003, 0.0004, 0.0005],  #
        "batch_size": [32, 64, 96],
        "target_update_frequency": [25, 50, 75],
        "discount_factor": [0.9, 0.95, 0.99],
        "exploration_decay": [0.99],
        "min_exploration_rate": [0.01, 0.05],
        "postponement_penalty": [0.00],
        "bundling_reward": [0.00, 0.5, 1.0, 3.0, 5.0],  #  5.0, 10.0,
        "on_time_reward": [0.0],
    }

    results = []
    for i in range(num_experiments):
        # Randomly sample hyperparameters
        params = {key: random.choice(values) for key, values in hyperparameter_space.items()}
        print(f"Experiment {i+1}/{num_experiments}: {params}")

        # Define training phases
        phases = [
            {
                "name": "Simple Environment",
                "env_config": {
                    "num_vehicles": 10,
                    "num_restaurants": 20,
                    "service_area_dimensions": (6.0, 6.0),
                    "mean_interarrival_time": 8,
                },
                "performance_criteria": {},
                "min_episodes": 20,
                "max_episodes": episodes_per_experiment,
            }
        ]

        # Run training
        try:
            final_model_path = train_rl.train_rl_aca(
                phases=phases,
                save_interval=20,
                stability_window=10,
                stability_threshold=3.0,
                seed=42,
                visualize=False,
                reposition_idle_vehicles=True,
                model_dir=f"data/models/tuning_exp_{i+1}",
                resume_from_model=None,
                start_phase=0,
                start_episode=0,
                exploration_start=0.9,
                exploration_end=params["min_exploration_rate"],
                decay_method="exponential",
                decay_rate=params["exploration_decay"],
                rl_learning_rate=params["learning_rate"],
                rl_batch_size=params["batch_size"],
                rl_target_update_frequency=params["target_update_frequency"],
                rl_discount_factor=params["discount_factor"],
                rl_replay_buffer_capacity=50000,  # Fixed for now
                rl_bundling_reward=params["bundling_reward"],
                rl_postponement_penalty=params["postponement_penalty"],
                rl_on_time_reward=params["on_time_reward"],
            )

            # Evaluate the final model
            eval_stats = train_rl.evaluate_model(
                model_path=final_model_path,
                num_episodes=10,
                seed=100,
                visualize=False,
                env_config=phases[-1]["env_config"],
            )

            # Record the results
            result = {
                "params": params,
                "avg_reward": eval_stats["total_rewards"][-1],
                "avg_delay": eval_stats["total_delays"][-1],
                "avg_on_time_rate": eval_stats["on_time_rates"][-1],
                "avg_distance": eval_stats["total_distances"][-1],
                "avg_vehicle_utilization": eval_stats["vehicle_utilizations"][-1],
                "avg_travel_time": eval_stats["total_travel_times"][-1] if "total_travel_times" in eval_stats else 0.0,
            }
            results.append(result)
            print(f"Experiment {i+1} Results: {result}")

        except Exception as e:
            print(f"Experiment {i+1} failed: {e}")
            continue

    # Find the best combination
    if results:
        best_result = max(results, key=lambda x: x["avg_reward"])
        print(f"\nBest Hyperparameters: {best_result['params']}")
        print(f"Best Average Reward: {best_result['avg_reward']}")
        print(f"Best Average Delay: {best_result['avg_delay']}")
        print(f"Best On-Time Rate: {best_result['avg_on_time_rate']}")
        print(f"Best Average Distance: {best_result['avg_distance']:.2f} km")
        print(f"Best Vehicle Utilization: {best_result['avg_vehicle_utilization']:.2f}%")
        print(f"Best Average Travel Time: {best_result['avg_travel_time']:.2f} minutes")
    else:
        print("No successful experiments completed.")


if __name__ == "__main__":
    run_hyperparameter_tuning(num_experiments=40, episodes_per_experiment=500)