From 3c26199d32bbc2718d3522b19c0f177addef75ed Mon Sep 17 00:00:00 2001 From: jacques franc Date: Thu, 30 Oct 2025 12:08:09 +0100 Subject: [PATCH 01/22] starting import launcher handling --- .../src/geos/trame/app/io/simulation.py | 202 ++++++++++++++++++ .../geos/trame/app/ui/simulationStatusView.py | 78 +++++++ 2 files changed, 280 insertions(+) create mode 100644 geos-trame/src/geos/trame/app/io/simulation.py create mode 100644 geos-trame/src/geos/trame/app/ui/simulationStatusView.py diff --git a/geos-trame/src/geos/trame/app/io/simulation.py b/geos-trame/src/geos/trame/app/io/simulation.py new file mode 100644 index 00000000..e62ce2ed --- /dev/null +++ b/geos-trame/src/geos/trame/app/io/simulation.py @@ -0,0 +1,202 @@ + +from abc import ABC, abstractmethod +from pathlib import Path +from dataclasses import dataclass, field, fields +from enum import Enum, unique +from geos.trame.app.ui.simulationStatusView import SimulationStatus +from typing import Callable, Optional +import datetime +from trame_server.core import Server +from trame_server.state import State + +#TODO move outside +@dataclass(frozen=True) +class SimulationConstant: + SIMULATION_GEOS_PATH = "/some/path/" + SIMULATION_MACHINE_NAME = "p4log01" # Only run on P4 machine + + +@unique +class SlurmJobStatus(Enum): + PENDING = "PD" + RUNNING = "R" + COMPLETING = "CG" + COMPLETED = "CD" + SUSPENDED = "S" + UNKNOWN = "UNKNOWN" + + @classmethod + def from_string(cls, job_str) -> "SlurmJobStatus": + try: + return cls(job_str) + except ValueError: + return cls.UNKNOWN + +# TODO: dataclass_json +# @dataclass_json +@dataclass +class SimulationInformation: + pass + + def get_simulation_status( + self, + get_running_user_jobs_f: Callable[[], list[tuple[str, SlurmJobStatus]]], + ) -> SimulationStatus: + """ + Returns the simulation status given the current Jobs running for the current user. + Only runs the callback if the timeseries file is not already present in the done directory. + """ + if not self.geos_job_id: + return SimulationStatus.NOT_RUN + + done_sim_path = self.get_simulation_dir(SimulationStatus.DONE) + if self.get_timeseries_path(done_sim_path).exists(): + return SimulationStatus.DONE + + user_jobs = get_running_user_jobs_f() + if (self.geos_job_id, SlurmJobStatus.RUNNING) in user_jobs: + return SimulationStatus.RUNNING + + if (self.geos_job_id, SlurmJobStatus.COMPLETING) in user_jobs: + return SimulationStatus.COMPLETING + + if (self.copy_back_job_id, SlurmJobStatus.RUNNING) in user_jobs: + return SimulationStatus.COPY_BACK + + if (self.copy_job_id, SlurmJobStatus.RUNNING) in user_jobs: + return SimulationStatus.SCHEDULED + + return SimulationStatus.UNKNOWN + +@dataclass +class LauncherParams: + simulation_files_path: Optional[str] = None + simulation_cmd_filename: Optional[str] = None + simulation_job_name: Optional[str] = None + simulation_nb_process: int = 1 + + @classmethod + def from_server_state(cls, server_state: State) -> "LauncherParams": + state = cls() + for f in fields(cls): + setattr(state, f.name, server_state[f.name]) + return state + + def is_complete(self) -> bool: + return None not in [getattr(self, f.name) for f in fields(self)] + + def assert_is_complete(self) -> None: + if not self.is_complete(): + raise RuntimeError(f"Incomplete simulation launch parameters : {self}.") + + +def get_timestamp() -> str: + return datetime.utcnow().strftime("%Y-%m-%d_%H-%M-%S.%f")[:-3] + + +def get_simulation_output_file_name(timestamp: str, user_name: str = "user_name"): + return f"{user_name}_{timestamp}.json" + + +def parse_launcher_output(output: str) -> SimulationInformation: + split_output = output.split("\n") + + information = SimulationInformation() + information_dict = information.to_dict() # type: ignore + + content_to_parse = [ + ("Working directory: ", "working_directory"), + ("1. copy job id: ", "copy_job_id"), + ("2. geos job id: ", "geos_job_id"), + ("3. copy back job id: ", "copy_back_job_id"), + ("Run directory: ", "run_directory"), + ] + + for line in split_output: + for info_tuple in content_to_parse: + if info_tuple[0] in line: + split_line = line.split(info_tuple[0]) + if len(split_line) < 2: + continue + information_dict[info_tuple[1]] = split_line[-1] + + information_dict["timestamp"] = get_timestamp() + return SimulationInformation.from_dict(information_dict) # type: ignore + + +# def write_simulation_information_to_repo(info: SimulationInformation, sim_info_path: Path) -> Optional[Path]: +# return write_file( +# sim_info_path.as_posix(), +# get_simulation_output_file_name(info.timestamp, info.user_igg), +# json.dumps(info.to_dict()), # type: ignore +# ) + + +##TODO yay slurm +def get_launcher_command(launcher_params: LauncherParams) -> str: + launcher_cmd_args = ( + f"{SimulationConstant.SIMULATION_GEOS_PATH} " + f"--nprocs {launcher_params.simulation_nb_process} " + f"--fname {launcher_params.simulation_cmd_filename} " + f"--job_name {launcher_params.simulation_job_name}" + ) + + # state.simulation_nb_process is supposed to be an integer, but the UI present a VTextField, + # so if user changes it, then it can be defined as a str + if int(launcher_params.simulation_nb_process) > 1: + launcher_cmd_args += " --partition" + return launcher_cmd_args + + +# def get_simulation_screenshot_timestep(filename: str) -> int: +# """ +# From a given file name returns the time step. +# Filename is defined as: RenderView0_000000.png with 000000 the time step to parse and return +# """ +# if not filename: +# print("Simulation filename is not defined") +# return -1 + +# pattern = re.compile(r"RenderView[0-9]_[0-9]{6}\.png", re.IGNORECASE) +# if pattern.match(filename) is None: +# print("Simulation filename does not match the pattern: RenderView0_000000.png") +# return -1 + +# timestep = os.path.splitext(filename)[0].split("_")[-1] +# return int(timestep) if timestep else -1 + + +# def get_most_recent_file_from_list(files_list: list[str]) -> Optional[str]: +# if not files_list: +# return None +# return max(files_list, key=get_simulation_screenshot_timestep) + + +# def get_most_recent_simulation_screenshot(folder_path: Path) -> Optional[str]: +# return get_most_recent_file_from_list(os.listdir(folder_path)) if folder_path.exists() else None + + +class ISimRunner(ABC): + """ + Abstract interface for sim runner. + Provides methods to trigger simulation, get simulation output path and knowing if simulation is done or not. + """ + + @abstractmethod + def launch_simulation(self, launcher_params: LauncherParams) -> tuple[Path, SimulationInformation]: + pass + + @abstractmethod + def get_user_igg(self) -> str: + pass + + @abstractmethod + def get_running_user_jobs(self) -> list[tuple[str, SlurmJobStatus]]: + pass + + +class SimRunner(ISimRunner): + """ + Runs sim on HPC + """ + pass \ No newline at end of file diff --git a/geos-trame/src/geos/trame/app/ui/simulationStatusView.py b/geos-trame/src/geos/trame/app/ui/simulationStatusView.py new file mode 100644 index 00000000..84fc4d4b --- /dev/null +++ b/geos-trame/src/geos/trame/app/ui/simulationStatusView.py @@ -0,0 +1,78 @@ +from enum import Enum, auto, unique + +from trame_client.widgets.html import H3, Div +from trame_server import Server +from trame_vuetify.widgets.vuetify3 import VCard + +@unique +class SimulationStatus(Enum): + SCHEDULED = auto() + RUNNING = auto() + COMPLETING = auto() + COPY_BACK = auto() + DONE = auto() + NOT_RUN = auto() + UNKNOWN = auto() + + +class SimulationStatusView: + """ + Simple component containing simulation status in a VCard with some coloring depending on the status. + """ + + def __init__(self, server: Server): + def state_name(state_str): + return f"{type(self).__name__}_{state_str}_{id(self)}" + + self._text_state = state_name("text") + self._date_state = state_name("date") + self._time_state = state_name("time") + self._color_state = state_name("color") + self._state = server.state + + for s in [self._text_state, self._date_state, self._time_state, self._color_state]: + self._state.client_only(s) + + with VCard( + classes="p-8", + style=(f"`border: 4px solid ${{{self._color_state}}}; width: 300px; margin:auto; padding: 4px;`",), + ) as self.ui: + H3(f"{{{{{self._text_state}}}}}", style="text-align:center;") + Div(f"{{{{{self._date_state}}}}} {{{{{self._time_state}}}}}", style="text-align:center;") + + self.set_status(SimulationStatus.NOT_RUN) + self.set_time_stamp("") + + def set_status(self, status: SimulationStatus): + self._state[self._text_state] = status.name + self._state[self._color_state] = self.status_color(status) + self._state.flush() + + def set_time_stamp(self, time_stamp: str): + date, time = self.split_time_stamp(time_stamp) + self._state[self._time_state] = time + self._state[self._date_state] = date + self._state.flush() + + @staticmethod + def split_time_stamp(time_stamp: str) -> tuple[str, str]: + default_time_stamp = "", "" + if not time_stamp: + return default_time_stamp + + time_stamp = time_stamp.split("_") + if len(time_stamp) < 2: + return default_time_stamp + + return time_stamp[0].replace("-", "/"), time_stamp[1].split(".")[0].replace("-", ":") + + @staticmethod + def status_color(status: SimulationStatus) -> str: + return { + SimulationStatus.DONE: "#4CAF50", + SimulationStatus.RUNNING: "#3F51B5", + SimulationStatus.SCHEDULED: "#FFC107", + SimulationStatus.COMPLETING: "#C5E1A5", + SimulationStatus.COPY_BACK: "#C5E1A5", + SimulationStatus.UNKNOWN: "#E53935", + }.get(status, "#607D8B") \ No newline at end of file From abfd58f5ff38b977364101339ab3cdda835c511c Mon Sep 17 00:00:00 2001 From: jacques franc Date: Thu, 30 Oct 2025 12:08:09 +0100 Subject: [PATCH 02/22] starting import launcher handling --- .../src/geos/trame/app/io/simulation.py | 319 ++++++++++++++++++ .../geos/trame/app/ui/simulationStatusView.py | 78 +++++ 2 files changed, 397 insertions(+) create mode 100644 geos-trame/src/geos/trame/app/io/simulation.py create mode 100644 geos-trame/src/geos/trame/app/ui/simulationStatusView.py diff --git a/geos-trame/src/geos/trame/app/io/simulation.py b/geos-trame/src/geos/trame/app/io/simulation.py new file mode 100644 index 00000000..77178bdb --- /dev/null +++ b/geos-trame/src/geos/trame/app/io/simulation.py @@ -0,0 +1,319 @@ + +from abc import ABC, abstractmethod +from pathlib import Path +from dataclasses import dataclass, field, fields +from enum import Enum, unique +from geos.trame.app.ui.simulationStatusView import SimulationStatus +from typing import Callable, Optional +import datetime +from trame_server.core import Server +from trame_server.state import State + +#TODO move outside +#TODO use Jinja on real launcher + +@dataclass(frozen=True) +class SimulationConstant: + SIMULATION_GEOS_PATH = "/some/path/" + SIMULATION_MACHINE_NAME = "p4log01" # Only run on P4 machine + +@unique +class SlurmJobStatus(Enum): + PENDING = "PD" + RUNNING = "R" + COMPLETING = "CG" + COMPLETED = "CD" + SUSPENDED = "S" + UNKNOWN = "UNKNOWN" + + @classmethod + def from_string(cls, job_str) -> "SlurmJobStatus": + try: + return cls(job_str) + except ValueError: + return cls.UNKNOWN + +# TODO: dataclass_json +# @dataclass_json +@dataclass +class SimulationInformation: + pass + + def get_simulation_status( + self, + get_running_user_jobs_f: Callable[[], list[tuple[str, SlurmJobStatus]]], + ) -> SimulationStatus: + """ + Returns the simulation status given the current Jobs running for the current user. + Only runs the callback if the timeseries file is not already present in the done directory. + """ + if not self.geos_job_id: + return SimulationStatus.NOT_RUN + + done_sim_path = self.get_simulation_dir(SimulationStatus.DONE) + if self.get_timeseries_path(done_sim_path).exists(): + return SimulationStatus.DONE + + user_jobs = get_running_user_jobs_f() + if (self.geos_job_id, SlurmJobStatus.RUNNING) in user_jobs: + return SimulationStatus.RUNNING + + if (self.geos_job_id, SlurmJobStatus.COMPLETING) in user_jobs: + return SimulationStatus.COMPLETING + + if (self.copy_back_job_id, SlurmJobStatus.RUNNING) in user_jobs: + return SimulationStatus.COPY_BACK + + if (self.copy_job_id, SlurmJobStatus.RUNNING) in user_jobs: + return SimulationStatus.SCHEDULED + + return SimulationStatus.UNKNOWN + +@dataclass +class LauncherParams: + simulation_files_path: Optional[str] = None + simulation_cmd_filename: Optional[str] = None + simulation_job_name: Optional[str] = None + simulation_nb_process: int = 1 + + @classmethod + def from_server_state(cls, server_state: State) -> "LauncherParams": + state = cls() + for f in fields(cls): + setattr(state, f.name, server_state[f.name]) + return state + + def is_complete(self) -> bool: + return None not in [getattr(self, f.name) for f in fields(self)] + + def assert_is_complete(self) -> None: + if not self.is_complete(): + raise RuntimeError(f"Incomplete simulation launch parameters : {self}.") + + +def get_timestamp() -> str: + return datetime.utcnow().strftime("%Y-%m-%d_%H-%M-%S.%f")[:-3] + + +def get_simulation_output_file_name(timestamp: str, user_name: str = "user_name"): + return f"{user_name}_{timestamp}.json" + + +def parse_launcher_output(output: str) -> SimulationInformation: + split_output = output.split("\n") + + information = SimulationInformation() + information_dict = information.to_dict() # type: ignore + + content_to_parse = [ + ("Working directory: ", "working_directory"), + ("1. copy job id: ", "copy_job_id"), + ("2. geos job id: ", "geos_job_id"), + ("3. copy back job id: ", "copy_back_job_id"), + ("Run directory: ", "run_directory"), + ] + + for line in split_output: + for info_tuple in content_to_parse: + if info_tuple[0] in line: + split_line = line.split(info_tuple[0]) + if len(split_line) < 2: + continue + information_dict[info_tuple[1]] = split_line[-1] + + information_dict["timestamp"] = get_timestamp() + return SimulationInformation.from_dict(information_dict) # type: ignore + + +# def write_simulation_information_to_repo(info: SimulationInformation, sim_info_path: Path) -> Optional[Path]: +# return write_file( +# sim_info_path.as_posix(), +# get_simulation_output_file_name(info.timestamp, info.user_igg), +# json.dumps(info.to_dict()), # type: ignore +# ) + + +##TODO yay slurm +def get_launcher_command(launcher_params: LauncherParams) -> str: + launcher_cmd_args = ( + f"{SimulationConstant.SIMULATION_GEOS_PATH} " + f"--nprocs {launcher_params.simulation_nb_process} " + f"--fname {launcher_params.simulation_cmd_filename} " + f"--job_name {launcher_params.simulation_job_name}" + ) + + # state.simulation_nb_process is supposed to be an integer, but the UI present a VTextField, + # so if user changes it, then it can be defined as a str + if int(launcher_params.simulation_nb_process) > 1: + launcher_cmd_args += " --partition" + return launcher_cmd_args + + +# def get_simulation_screenshot_timestep(filename: str) -> int: +# """ +# From a given file name returns the time step. +# Filename is defined as: RenderView0_000000.png with 000000 the time step to parse and return +# """ +# if not filename: +# print("Simulation filename is not defined") +# return -1 + +# pattern = re.compile(r"RenderView[0-9]_[0-9]{6}\.png", re.IGNORECASE) +# if pattern.match(filename) is None: +# print("Simulation filename does not match the pattern: RenderView0_000000.png") +# return -1 + +# timestep = os.path.splitext(filename)[0].split("_")[-1] +# return int(timestep) if timestep else -1 + + +# def get_most_recent_file_from_list(files_list: list[str]) -> Optional[str]: +# if not files_list: +# return None +# return max(files_list, key=get_simulation_screenshot_timestep) + + +# def get_most_recent_simulation_screenshot(folder_path: Path) -> Optional[str]: +# return get_most_recent_file_from_list(os.listdir(folder_path)) if folder_path.exists() else None + + +class ISimRunner(ABC): + """ + Abstract interface for sim runner. + Provides methods to trigger simulation, get simulation output path and knowing if simulation is done or not. + """ + + @abstractmethod + def launch_simulation(self, launcher_params: LauncherParams) -> tuple[Path, SimulationInformation]: + pass + + @abstractmethod + def get_user_igg(self) -> str: + pass + + @abstractmethod + def get_running_user_jobs(self) -> list[tuple[str, SlurmJobStatus]]: + pass + + +class SimRunner(ISimRunner): + """ + Runs sim on HPC + """ + pass + +class Simulation: + """ + Simulation component. + Fills the UI with the screenshot as read from the simulation outputs folder and a graph with the time series + from the simulation. + + Requires a simulation runner providing information on the output path of the simulation to monitor and ways to + trigger the simulation. + """ + + def __init__(self, sim_runner: ISimRunner, server: Server, sim_info_dir: Optional[Path] = None) -> None: + self._server = server + self._sim_runner = sim_runner + self._sim_info_dir = sim_info_dir or SIMULATIONS_INFORMATION_FOLDER_PATH + + self._job_status_watcher: Optional[AsyncPeriodicRunner] = None + self._job_status_watcher_period_ms = 2000 + + self.start_result_streams() + + def __del__(self): + self.stop_result_streams() + + def set_status_watcher_period_ms(self, period_ms): + self._job_status_watcher_period_ms = period_ms + if self._job_status_watcher: + self._job_status_watcher.set_period_ms(period_ms) + + def _update_screenshot_display(self, screenshots_folder_path: Path) -> None: + newer_file = get_most_recent_simulation_screenshot(screenshots_folder_path) + if not newer_file: + return + + f_name = Path(newer_file).name + if not f_name: + return + + self._server.state.active_screenshot_folder_path = str(screenshots_folder_path) + self._server.state.dirty("active_screenshot_folder_path") + self._server.state.active_screenshot_relative_path = f_name + self._server.state.dirty("active_screenshot_relative_path") + self._server.state.flush() + + def _update_job_status(self) -> None: + sim_info = self.get_last_user_simulation_info() + job_status = sim_info.get_simulation_status(self._sim_runner.get_running_user_jobs) + sim_path = sim_info.get_simulation_dir(job_status) + + self._server.controller.set_simulation_status(job_status) + self._server.controller.set_simulation_time_stamp(sim_info.timestamp) + + self._update_screenshot_display(sim_info.get_screenshot_path(sim_path)) + self._update_plots(sim_info.get_timeseries_path(sim_path)) + + # Stop results stream if job is done + if job_status == SimulationStatus.DONE: + self.stop_result_streams() + + def get_last_user_simulation_info(self) -> SimulationInformation: + last_sim_information = self.get_last_information_path() + return SimulationInformation.from_file(last_sim_information) + + def get_last_information_path(self) -> Optional[Path]: + user_igg = self._sim_runner.get_user_igg() + + user_files = list(reversed(sorted(self._sim_info_dir.glob(f"{user_igg}*.json")))) + if not user_files: + return None + + return user_files[0] + + def stop_result_streams(self): + if self._job_status_watcher is not None: + self._job_status_watcher.stop() + + def start_result_streams(self) -> None: + self.stop_result_streams() + + self._job_status_watcher = AsyncPeriodicRunner( + self._update_job_status, period_ms=self._job_status_watcher_period_ms + ) + + def start_simulation(self) -> None: + state = self._server.state + script_path = None + try: + launcher_params = LauncherParams.from_server_state(self._server.state) + launcher_params.assert_is_complete() + + script_path, sim_info = self._sim_runner.launch_simulation(launcher_params) + self._write_sim_info(launcher_params, sim_info) + self.start_result_streams() + state.simulation_error = "" + except Exception as e: + print("Error occurred: ", e) + state.simulation_error = str(e) + finally: + state.avoid_rewriting = False + if isinstance(script_path, Path) and script_path.is_file(): + os.remove(script_path) + + def _write_sim_info(self, launcher_params: LauncherParams, sim_info: Optional[SimulationInformation]) -> None: + if sim_info is None: + raise RuntimeError("Error parsing simulation launcher output.") + + # Make sure to save the absolute path to the working directory used by the launcher in case parsed information + # is a relative Path + if not Path(sim_info.working_directory).is_absolute(): + sim_info.working_directory = path_to_string( + launcher_params.simulation_files_path + "/" + sim_info.working_directory + ) + print("simulation information", sim_info) + + sim_info.user_igg = self._sim_runner.get_user_igg() + write_simulation_information_to_repo(sim_info, self._sim_info_dir) diff --git a/geos-trame/src/geos/trame/app/ui/simulationStatusView.py b/geos-trame/src/geos/trame/app/ui/simulationStatusView.py new file mode 100644 index 00000000..84fc4d4b --- /dev/null +++ b/geos-trame/src/geos/trame/app/ui/simulationStatusView.py @@ -0,0 +1,78 @@ +from enum import Enum, auto, unique + +from trame_client.widgets.html import H3, Div +from trame_server import Server +from trame_vuetify.widgets.vuetify3 import VCard + +@unique +class SimulationStatus(Enum): + SCHEDULED = auto() + RUNNING = auto() + COMPLETING = auto() + COPY_BACK = auto() + DONE = auto() + NOT_RUN = auto() + UNKNOWN = auto() + + +class SimulationStatusView: + """ + Simple component containing simulation status in a VCard with some coloring depending on the status. + """ + + def __init__(self, server: Server): + def state_name(state_str): + return f"{type(self).__name__}_{state_str}_{id(self)}" + + self._text_state = state_name("text") + self._date_state = state_name("date") + self._time_state = state_name("time") + self._color_state = state_name("color") + self._state = server.state + + for s in [self._text_state, self._date_state, self._time_state, self._color_state]: + self._state.client_only(s) + + with VCard( + classes="p-8", + style=(f"`border: 4px solid ${{{self._color_state}}}; width: 300px; margin:auto; padding: 4px;`",), + ) as self.ui: + H3(f"{{{{{self._text_state}}}}}", style="text-align:center;") + Div(f"{{{{{self._date_state}}}}} {{{{{self._time_state}}}}}", style="text-align:center;") + + self.set_status(SimulationStatus.NOT_RUN) + self.set_time_stamp("") + + def set_status(self, status: SimulationStatus): + self._state[self._text_state] = status.name + self._state[self._color_state] = self.status_color(status) + self._state.flush() + + def set_time_stamp(self, time_stamp: str): + date, time = self.split_time_stamp(time_stamp) + self._state[self._time_state] = time + self._state[self._date_state] = date + self._state.flush() + + @staticmethod + def split_time_stamp(time_stamp: str) -> tuple[str, str]: + default_time_stamp = "", "" + if not time_stamp: + return default_time_stamp + + time_stamp = time_stamp.split("_") + if len(time_stamp) < 2: + return default_time_stamp + + return time_stamp[0].replace("-", "/"), time_stamp[1].split(".")[0].replace("-", ":") + + @staticmethod + def status_color(status: SimulationStatus) -> str: + return { + SimulationStatus.DONE: "#4CAF50", + SimulationStatus.RUNNING: "#3F51B5", + SimulationStatus.SCHEDULED: "#FFC107", + SimulationStatus.COMPLETING: "#C5E1A5", + SimulationStatus.COPY_BACK: "#C5E1A5", + SimulationStatus.UNKNOWN: "#E53935", + }.get(status, "#607D8B") \ No newline at end of file From 2d1c395322164d93695192ffb69d9eda9da5643c Mon Sep 17 00:00:00 2001 From: jacques franc Date: Mon, 17 Nov 2025 16:34:26 +0100 Subject: [PATCH 03/22] some more imports --- geos-trame/src/geos/trame/app/core.py | 6 + .../src/geos/trame/app/io/simulation.py | 51 +++++--- ...tatusView.py => simulation_status_view.py} | 0 .../trame/app/utils/async_file_watcher.py | 113 ++++++++++++++++++ 4 files changed, 156 insertions(+), 14 deletions(-) rename geos-trame/src/geos/trame/app/ui/{simulationStatusView.py => simulation_status_view.py} (100%) create mode 100644 geos-trame/src/geos/trame/app/utils/async_file_watcher.py diff --git a/geos-trame/src/geos/trame/app/core.py b/geos-trame/src/geos/trame/app/core.py index 0a8f4097..7020bbfe 100644 --- a/geos-trame/src/geos/trame/app/core.py +++ b/geos-trame/src/geos/trame/app/core.py @@ -23,6 +23,7 @@ from geos.trame.app.ui.timeline import TimelineEditor from geos.trame.app.ui.viewer.viewer import DeckViewer from geos.trame.app.components.alertHandler import AlertHandler +from geos.trame.app.io.simulation import Simulation, SimRunner import sys @@ -38,6 +39,7 @@ def __init__( self, server: Server, file_name: str ) -> None: self.deckEditor: DeckEditor | None = None self.timelineEditor: TimelineEditor | None = None self.deckInspector: DeckInspector | None = None + self.simulationLauncher : Simulation | None = None self.server = server server.enable_module( module ) @@ -67,6 +69,10 @@ def __init__( self, server: Server, file_name: str ) -> None: self.region_viewer = RegionViewer() self.well_viewer = WellViewer( 5, 5 ) + # Simulation runner + self.sim_runner : SimRunner = SimRunner(self.state.user_id) + self.simulationLauncher = Simulation(self.sim_runner, server=server) + # Data loader self.data_loader = DataLoader( self.tree, self.region_viewer, self.well_viewer, trame_server=server ) diff --git a/geos-trame/src/geos/trame/app/io/simulation.py b/geos-trame/src/geos/trame/app/io/simulation.py index 77178bdb..01b09d6d 100644 --- a/geos-trame/src/geos/trame/app/io/simulation.py +++ b/geos-trame/src/geos/trame/app/io/simulation.py @@ -3,11 +3,12 @@ from pathlib import Path from dataclasses import dataclass, field, fields from enum import Enum, unique -from geos.trame.app.ui.simulationStatusView import SimulationStatus -from typing import Callable, Optional +from geos.trame.app.ui.simulation_status_view import SimulationStatus +from typing import Callable, Optional, Union import datetime from trame_server.core import Server from trame_server.state import State +from geos.trame.app.utils.async_file_watcher import AsyncPeriodicRunner #TODO move outside #TODO use Jinja on real launcher @@ -230,20 +231,20 @@ def set_status_watcher_period_ms(self, period_ms): if self._job_status_watcher: self._job_status_watcher.set_period_ms(period_ms) - def _update_screenshot_display(self, screenshots_folder_path: Path) -> None: - newer_file = get_most_recent_simulation_screenshot(screenshots_folder_path) - if not newer_file: - return + # def _update_screenshot_display(self, screenshots_folder_path: Path) -> None: + # newer_file = get_most_recent_simulation_screenshot(screenshots_folder_path) + # if not newer_file: + # return - f_name = Path(newer_file).name - if not f_name: - return + # f_name = Path(newer_file).name + # if not f_name: + # return - self._server.state.active_screenshot_folder_path = str(screenshots_folder_path) - self._server.state.dirty("active_screenshot_folder_path") - self._server.state.active_screenshot_relative_path = f_name - self._server.state.dirty("active_screenshot_relative_path") - self._server.state.flush() + # self._server.state.active_screenshot_folder_path = str(screenshots_folder_path) + # self._server.state.dirty("active_screenshot_folder_path") + # self._server.state.active_screenshot_relative_path = f_name + # self._server.state.dirty("active_screenshot_relative_path") + # self._server.state.flush() def _update_job_status(self) -> None: sim_info = self.get_last_user_simulation_info() @@ -317,3 +318,25 @@ def _write_sim_info(self, launcher_params: LauncherParams, sim_info: Optional[Si sim_info.user_igg = self._sim_runner.get_user_igg() write_simulation_information_to_repo(sim_info, self._sim_info_dir) + + +def path_to_string(p: Union[str, Path]) -> str: + return Path(p).as_posix() + +def write_simulation_information_to_repo(info: SimulationInformation, sim_info_path: Path) -> Optional[Path]: + return write_file( + sim_info_path.as_posix(), + get_simulation_output_file_name(info.timestamp, info.user_igg), + json.dumps(info.to_dict()), # type: ignore + ) + +def write_file(folder_path: str, filename: str, file_content: str) -> Optional[Path]: + try: + Path(folder_path).mkdir(exist_ok=True) + file_path = Path(f"{folder_path}/{filename}") + with open(file_path, "w") as f: + f.write(file_content) + return file_path.absolute() + except Exception as e: + print("error occurred when copying file to", folder_path, e) + return None \ No newline at end of file diff --git a/geos-trame/src/geos/trame/app/ui/simulationStatusView.py b/geos-trame/src/geos/trame/app/ui/simulation_status_view.py similarity index 100% rename from geos-trame/src/geos/trame/app/ui/simulationStatusView.py rename to geos-trame/src/geos/trame/app/ui/simulation_status_view.py diff --git a/geos-trame/src/geos/trame/app/utils/async_file_watcher.py b/geos-trame/src/geos/trame/app/utils/async_file_watcher.py new file mode 100644 index 00000000..d5ad532f --- /dev/null +++ b/geos-trame/src/geos/trame/app/utils/async_file_watcher.py @@ -0,0 +1,113 @@ +import asyncio +import os +from asyncio import CancelledError, ensure_future +from io import TextIOWrapper +from pathlib import Path +from typing import Callable, Optional, Union + +from trame_server.utils import asynchronous + + +class AsyncPeriodicRunner: + """ + While started, runs given callback at given period. + """ + + def __init__(self, callback: Callable, period_ms=100): + self.last_m_time = None + self.callback = callback + self.period_ms = period_ms + self.task = None + self.start() + + def __del__(self): + self.stop() + + def set_period_ms(self, period_ms): + self.period_ms = period_ms + + def start(self): + self.stop() + self.task = asynchronous.create_task(self._runner()) + + def stop(self): + if not self.task: + return + + ensure_future(self._wait_for_cancel()) + + async def _wait_for_cancel(self): + """ + Cancel and await cancel error for the task. + If cancel is done outside async, it may raise warnings as cancelled exception may be triggered outside async + loop. + """ + if not self.task or self.task.done() or self.task.cancelled(): + self.task = None + return + + try: + self.task.cancel() + await self.task + except CancelledError: + self.task = None + + async def _runner(self): + while True: + self.callback() + await asyncio.sleep(self.period_ms / 1000.0) + + +class AsyncFileWatcher(AsyncPeriodicRunner): + def __init__(self, path_to_watch: Path, on_modified_callback: Callable, check_time_out_ms=100): + super().__init__(self._check_modified, check_time_out_ms) + self.path_to_watch = Path(path_to_watch) + self.last_m_time = None + self.on_modified_callback = on_modified_callback + + def get_m_time(self): + if not self.path_to_watch.exists(): + return None + return os.stat(self.path_to_watch).st_mtime + + def _check_modified(self): + if self.get_m_time() != self.last_m_time: + self.last_m_time = self.get_m_time() + self.on_modified_callback() + + +class AsyncSubprocess: + def __init__( + self, + args, + timeout: Union[float, None] = None, + ) -> None: + self.args = args + self.timeout = timeout + self._writer: Optional[TextIOWrapper] = None + + self.stdout: Optional[bytes] = None + self.stderr: Optional[bytes] = None + self.process: Optional[asyncio.subprocess.Process] = None + self.exception: Optional[RuntimeError] = None + + async def run(self) -> None: + cmd = " ".join(map(str, self.args)) + self.process = await self._init_subprocess(cmd) + + try: + self.stdout, self.stderr = await asyncio.wait_for(self.process.communicate(), timeout=self.timeout) + except asyncio.exceptions.TimeoutError: + self.process.kill() + self.stdout, self.stderr = await self.process.communicate() + self.exception = RuntimeError("Process timed out") + finally: + if self.process.returncode != 0: + self.exception = RuntimeError(f"Process exited with code {self.process.returncode}") + + async def _init_subprocess(self, cmd: str) -> asyncio.subprocess.Process: + return await asyncio.create_subprocess_shell( + cmd=cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) From 46899b4483a2b736ca28952607d64ba2ca2f244a Mon Sep 17 00:00:00 2001 From: jacques franc Date: Tue, 18 Nov 2025 15:18:31 +0100 Subject: [PATCH 04/22] wip --- geos-trame/src/geos/trame/app/core.py | 57 +++-- .../src/geos/trame/app/io/simulation.py | 221 ++++++++++++++++-- geos-trame/src/geos/trame/app/main.py | 3 + .../src/geos/trame/app/ui/simulation_view.py | 141 +++++++++++ geos-trame/src/geos/trame/app/ui/timeline.py | 26 +-- geos-trame/src/geos/trame/assets/cluster.json | 24 ++ 6 files changed, 421 insertions(+), 51 deletions(-) create mode 100644 geos-trame/src/geos/trame/app/ui/simulation_view.py create mode 100644 geos-trame/src/geos/trame/assets/cluster.json diff --git a/geos-trame/src/geos/trame/app/core.py b/geos-trame/src/geos/trame/app/core.py index 7020bbfe..17fdd41a 100644 --- a/geos-trame/src/geos/trame/app/core.py +++ b/geos-trame/src/geos/trame/app/core.py @@ -23,7 +23,12 @@ from geos.trame.app.ui.timeline import TimelineEditor from geos.trame.app.ui.viewer.viewer import DeckViewer from geos.trame.app.components.alertHandler import AlertHandler + + from geos.trame.app.io.simulation import Simulation, SimRunner +from geos.trame.app.ui.simulation_view import define_simulation_view + + import sys @@ -44,6 +49,7 @@ def __init__( self, server: Server, file_name: str ) -> None: server.enable_module( module ) self.state.input_file = file_name + self.state.user_id = None # TODO handle hot_reload @@ -69,9 +75,9 @@ def __init__( self, server: Server, file_name: str ) -> None: self.region_viewer = RegionViewer() self.well_viewer = WellViewer( 5, 5 ) - # Simulation runner + ######## Simulation runner self.sim_runner : SimRunner = SimRunner(self.state.user_id) - self.simulationLauncher = Simulation(self.sim_runner, server=server) + self.simulation = Simulation(self.sim_runner, server=server) # Data loader self.data_loader = DataLoader( self.tree, self.region_viewer, self.well_viewer, trame_server=server ) @@ -183,23 +189,23 @@ def build_ui( self ) -> None: ): vuetify.VIcon( "mdi-content-save-outline" ) - with html.Div( - style= - "height: 100%; width: 300px; display: flex; align-items: center; justify-content: space-between;", - v_if=( "tab_idx == 1", ), - ): - vuetify.VBtn( - "Run", - style="z-index: 1;", - ) - vuetify.VBtn( - "Kill", - style="z-index: 1;", - ) - vuetify.VBtn( - "Clear", - style="z-index: 1;", - ) + # with html.Div( + # style= + # "height: 100%; width: 300px; display: flex; align-items: center; justify-content: space-between;", + # v_if=( "tab_idx == 1", ), + # ): + # vuetify.VBtn( + # "Run", + # style="z-index: 1;", + # ) + # vuetify.VBtn( + # "Kill", + # style="z-index: 1;", + # ) + # vuetify.VBtn( + # "Clear", + # style="z-index: 1;", + # ) # input file editor with vuetify.VCol( v_show=( "tab_idx == 0", ), classes="flex-grow-1 pa-0 ma-0" ): @@ -214,3 +220,16 @@ def build_ui( self ) -> None: "The file " + self.state.input_file + " cannot be parsed.", file=sys.stderr, ) + + with vuetify.VCol( v_show=( "tab_idx == 1"), classes="flex-grow-1 pa-0 ma-0") : + if self.simulation is not None: + define_simulation_view(self.server) + else: + self.ctrl.on_add_error( + "Error", + "The execution context " + self.state.exec_context + " is not consistent.", + ) + print( + "The execution context " + self.state.exec_context + " is not consistent.", + file=sys.stderr, + ) diff --git a/geos-trame/src/geos/trame/app/io/simulation.py b/geos-trame/src/geos/trame/app/io/simulation.py index 01b09d6d..4a108a18 100644 --- a/geos-trame/src/geos/trame/app/io/simulation.py +++ b/geos-trame/src/geos/trame/app/io/simulation.py @@ -10,13 +10,46 @@ from trame_server.state import State from geos.trame.app.utils.async_file_watcher import AsyncPeriodicRunner +import jinja2 +import paramiko + #TODO move outside #TODO use Jinja on real launcher @dataclass(frozen=True) class SimulationConstant: - SIMULATION_GEOS_PATH = "/some/path/" - SIMULATION_MACHINE_NAME = "p4log01" # Only run on P4 machine + SIMULATION_GEOS_PATH = "/workrd/users/" + HOST = "p4log01" # Only run on P4 machine + PORT = 22 + SIMULATIONS_INFORMATION_FOLDER_PATH= "/workrd/user" + SIMULATION_DEFAULT_FILE_NAME="geosDeck.xml" + +class Authentificator:#namespacing more than anything eler + + @staticmethod + def get_key(login:str, passphrase = "trameisrunning"): + + try: + PRIVATE_KEY = paramiko.RSAKey.from_private_key_file("~/.ssh/id_trame") + except paramiko.SSHException as e: + print(f"Error loading private key: {e}\n") + except FileNotFoundError as e: + print(f"Private key not found: {e}\n Generating key ...") + PRIVATE_KEY = Authentificator.gen_key(login, SimulationConstant.HOST, passphrase) + return PRIVATE_KEY + + return PRIVATE_KEY + + @staticmethod + def gen_key(login:str, host: str, passphrase: str): + file_path = "~/.ssh/id_trame" + cmd = f"ssh-keygen -t rsa -b 4096 -C {login}@{host} -f {file_path} -N \"{passphrase}\" " + import subprocess + print(f"Running: {''.join(cmd)}") + subprocess.run(cmd, shell=True) + print(f"SSH key generated at: {file_path}") + print(f"Public key: {file_path}.pub") + SIMULATION_DEFAULT_FILE_NAME = "geosDeck.xml" @unique class SlurmJobStatus(Enum): @@ -183,25 +216,170 @@ class ISimRunner(ABC): Abstract interface for sim runner. Provides methods to trigger simulation, get simulation output path and knowing if simulation is done or not. """ + pass + # @abstractmethod + # def launch_simulation(self, launcher_params: LauncherParams) -> tuple[Path, SimulationInformation]: + # pass - @abstractmethod - def launch_simulation(self, launcher_params: LauncherParams) -> tuple[Path, SimulationInformation]: - pass - - @abstractmethod - def get_user_igg(self) -> str: - pass + # @abstractmethod + # def get_user_igg(self) -> str: + # pass - @abstractmethod - def get_running_user_jobs(self) -> list[tuple[str, SlurmJobStatus]]: - pass + # @abstractmethod + # def get_running_user_jobs(self) -> list[tuple[str, SlurmJobStatus]]: + # pass class SimRunner(ISimRunner): """ - Runs sim on HPC + Runs sim on HPC. Wrap paramiko use """ - pass + + def __init__(self, user): + super().__init__() + + ssh_client = self._create_ssh_client(SimulationConstant.HOST, SimulationConstant.PORT, username=user, key=Authentificator.get_key(user)) + print(ssh_client) + + # early test + self.local_upload_file = "test_upload.txt" + import time + with open(self.local_upload_file, "w") as f: + f.write(f"This file was uploaded at {time.ctime()}\n") + print(f"Created local file: {self.local_upload_file}") + + @staticmethod + def _create_ssh_client( host, port, username, password=None, key=None): + """ + Initializes and returns an SSH client connection. + Uses context manager for automatic cleanup. + """ + client = paramiko.SSHClient() + # Automatically adds the hostname and new host keys to the host files (~/.ssh/known_hosts) + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + try: + if key: + print(f"Connecting to {host} using key-based authentication...") + client.connect(host, port, username, pkey=key, timeout=10) + else: + raise paramiko.SSHException("No Key Found") + + return client + except paramiko.AuthenticationException: + print("Authentication failed. Check your credentials or key.") + return None + except paramiko.SSHException as e: + print(f"Could not establish SSH connection: {e}") + return None + except Exception as e: + print(f"An unexpected error occurred: {e}") + return None + + + @staticmethod + def _execute_remote_command(client, command): + """ + Executes a single command on the remote server and prints the output. + """ + if not client: + return + + print(f"\n--- Executing Command: '{command}' ---") + try: + # Executes the command. stdin, stdout, and stderr are file-like objects. + # Ensure command ends with a newline character for some shell environments. + stdin, stdout, stderr = client.exec_command(command) + + # Wait for the command to finish and read the output + exit_status = stdout.channel.recv_exit_status() + + # Print standard output + stdout_data = stdout.read().decode().strip() + if stdout_data: + print("STDOUT:") + print(stdout_data) + + # Print standard error (if any) + stderr_data = stderr.read().decode().strip() + if stderr_data: + print("STDERR:") + print(stderr_data) + + print(f"Command exited with status: {exit_status}") + return exit_status + + except Exception as e: + print(f"Error executing command: {e}") + return -1 + + @staticmethod + def _transfer_file_sftp(client, local_path, remote_path, direction="put"): + """ + Transfers a file using SFTP (Secure File Transfer Protocol). + Direction can be 'put' (upload) or 'get' (download). + """ + if not client: + return + + print(f"\n--- Starting SFTP Transfer ({direction.upper()}) ---") + + try: + # Establish an SFTP connection session + sftp = client.open_sftp() + + if direction == "put": + print(f"Uploading '{local_path}' to '{remote_path}'...") + sftp.put(local_path, remote_path) + print("Upload complete.") + elif direction == "get": + print(f"Downloading '{remote_path}' to '{local_path}'...") + sftp.get(remote_path, local_path) + print("Download complete.") + else: + print("Invalid transfer direction. Use 'put' or 'get'.") + + sftp.close() + return True + + except FileNotFoundError: + print(f"Error: Local file '{local_path}' not found.") + return False + except IOError as e: + print(f"Error accessing remote file or path: {e}") + return False + except Exception as e: + print(f"An error occurred during SFTP: {e}") + return False + + + def launch_simulation(self): + + if self.ssh_client: + try: + # --- 3. Execute a Remote Command --- + self._execute_remote_command(self.ssh_client, "ls -l /tmp") + + # --- 4. Upload a File (PUT) --- + remote_path_upload = f"/tmp/{self.local_upload_file}" + self._transfer_file_sftp(self.ssh_client, self.local_upload_file, remote_path_upload, direction="put") + + # --- 5. Verify Upload by Listing Remote Directory --- + self._execute_remote_command(self.ssh_client, f"ls -l /tmp") + + # --- 6. Download a File (GET) --- + remote_download_file = f"/workrd/{self.local_upload_file}" # Use a known remote file + local_download_file = "downloaded_hostname.txt" + self._transfer_file_sftp(self.ssh_client, local_download_file, remote_download_file, direction="get") + + # --- 7. Clean up the uploaded file (Optional) --- + self._execute_remote_command(self.ssh_client, f"rm {remote_path_upload}") + + finally: + # --- 8. Close the connection --- + self.ssh_client.close() + print("\nSSH Connection closed.") + class Simulation: """ @@ -216,7 +394,7 @@ class Simulation: def __init__(self, sim_runner: ISimRunner, server: Server, sim_info_dir: Optional[Path] = None) -> None: self._server = server self._sim_runner = sim_runner - self._sim_info_dir = sim_info_dir or SIMULATIONS_INFORMATION_FOLDER_PATH + self._sim_info_dir = sim_info_dir or SimulationConstant.SIMULATIONS_INFORMATION_FOLDER_PATH self._job_status_watcher: Optional[AsyncPeriodicRunner] = None self._job_status_watcher_period_ms = 2000 @@ -235,6 +413,10 @@ def set_status_watcher_period_ms(self, period_ms): # newer_file = get_most_recent_simulation_screenshot(screenshots_folder_path) # if not newer_file: # return + # def _update_screenshot_display(self, screenshots_folder_path: Path) -> None: + # newer_file = get_most_recent_simulation_screenshot(screenshots_folder_path) + # if not newer_file: + # return # f_name = Path(newer_file).name # if not f_name: @@ -279,11 +461,12 @@ def stop_result_streams(self): self._job_status_watcher.stop() def start_result_streams(self) -> None: - self.stop_result_streams() + pass + # self.stop_result_streams() - self._job_status_watcher = AsyncPeriodicRunner( - self._update_job_status, period_ms=self._job_status_watcher_period_ms - ) + # self._job_status_watcher = AsyncPeriodicRunner( + # self._update_job_status, period_ms=self._job_status_watcher_period_ms + # ) def start_simulation(self) -> None: state = self._server.state diff --git a/geos-trame/src/geos/trame/app/main.py b/geos-trame/src/geos/trame/app/main.py index 2ad3b293..5840dbeb 100644 --- a/geos-trame/src/geos/trame/app/main.py +++ b/geos-trame/src/geos/trame/app/main.py @@ -7,6 +7,9 @@ from trame.app import get_server # type: ignore from trame_server import Server +import sys +sys.path.insert(0,"/data/pau901/SIM_CS/users/jfranc/geosPythonPackages/geos-trame/src") + from geos.trame.app.core import GeosTrame diff --git a/geos-trame/src/geos/trame/app/ui/simulation_view.py b/geos-trame/src/geos/trame/app/ui/simulation_view.py new file mode 100644 index 00000000..b6b35282 --- /dev/null +++ b/geos-trame/src/geos/trame/app/ui/simulation_view.py @@ -0,0 +1,141 @@ +from trame.widgets import html +from trame.widgets import vuetify3 as vuetify + +from geos.trame.app.io.simulation import SimulationConstant +from geos.trame.app.ui.simulation_status_view import SimulationStatusView + + +def hint_config(): + + return ["P4: 1x12", "P4: 2x6"] + + +def define_simulation_view(server) -> None: + with vuetify.VContainer(): + with vuetify.VRow(): + with vuetify.VCol(cols=4): + vuetify.VTextField( + v_model=("login", None,), + label="Login", + dense=True, + hide_details=True, + clearable=True, + prepend_icon="mdi-login" + ) + with vuetify.VCol(cols=4): + vuetify.VTextField( + v_model=("password", None,), + label="Password", + type="password", + dense=True, + hide_details=True, + clearable=True, + prepend_icon="mdi-onepassword" + ) + + # + items = hint_config() + vuetify.VDivider(vertical=True, thickness=5, classes="mx-4") + with vuetify.VCol(cols=2): + vuetify.VSelect(label="Cluster", + items=("items",items)) + + with vuetify.VRow(): + with vuetify.VCol(cols=8): + vuetify.VFileInput( + v_model=("key_path", None,), + label="Path to ssh key", + dense=True, + hide_details=True, + clearable=True, + prepend_icon="mdi-key-chain-variant" + ) + + # + vuetify.VDivider(vertical=True, thickness=5, classes="mx-4") + with vuetify.VCol(cols=2): + vuetify.VBtn("Log in", click="trigger('run_try_logging')"), # type: ignore + + + vuetify.VDivider(thickness=5, classes="my-4") + + with vuetify.VRow(): + with vuetify.VCol(): + vuetify.VFileInput( + v_model=("simulation_cmd_filename", SimulationConstant.SIMULATION_DEFAULT_FILE_NAME), + label="Simulation file name", + dense=True, + hide_details=True, + clearable=True, + ) + # with vuetify.VCol(cols=1): + # vuetify.VFileInput( + # v_model=("cmd_file", None), + # prepend_icon="mdi-file-upload-outline", + # hide_input=True, + # style="padding: 0;", + # disabled=("!simulation_files_path",), + # ) + + with vuetify.VRow(), vuetify.VCol(): + vuetify.VTextField( + v_model=( + "simulation_files_path", + None, + ), + label="Path where to write files and launch code", + prepend_icon="mdi-upload", + dense=True, + hide_details=True, + clearable=True, + # TODO callback validation of path + ) + + with vuetify.VRow(), vuetify.VCol(): + # must_be_greater_than_0 = ( + # "[value => Number.isInteger(Number(value)) && value > 0 || 'Must be an integer greater than 0']" + # ) + # vuetify.VTextField( + # v_model=("simulation_nb_process", 1), + # label="Processes number", + # dense=True, + # hide_details=True, + # clearable=True, + # rules=(must_be_greater_than_0,), + # ) + vuetify.VTextField( + v_model=("simulation_dl_path",), + label="Simulation download path", + dense=True, + clearable=True, + prepend_icon="mdi-download", + # TODO callback validation of path + ) + + with vuetify.VRow(), vuetify.VCol(): + vuetify.VTextField( + v_model=("simulation_job_name", "geosJob"), + label="Job Name", + dense=True, + hide_details=True, + clearable=True, + ) + with vuetify.VRow(): + vuetify.VSpacer() + with vuetify.VCol(cols=1): + vuetify.VBtn("Run", click="trigger('run_simulation')"), # type: ignore + with vuetify.VCol(cols=1): + vuetify.VBtn("Kill", click="trigger('kill_simulation')"), # type: ignore + # with vuetify.VCol(cols=1): + # vuetify.VBtn("Clear", click="trigger('clear_simulation')"), # type: ignore + + vuetify.VDivider(thickness=5, classes="my-4") + + with vuetify.VRow(): + with vuetify.VCol(cols=2): + SimulationStatusView(server=server) + + + + with vuetify.VRow(v_if="simulation_error"): + html.Div("An error occurred while running simulation :
{{simulation_error}}", style="color:red;") diff --git a/geos-trame/src/geos/trame/app/ui/timeline.py b/geos-trame/src/geos/trame/app/ui/timeline.py index d6961c0e..6d3559f9 100644 --- a/geos-trame/src/geos/trame/app/ui/timeline.py +++ b/geos-trame/src/geos/trame/app/ui/timeline.py @@ -3,7 +3,7 @@ # SPDX-FileContributor: Lionel Untereiner from typing import Any -from trame.widgets import gantt +# from trame.widgets import gantt from trame.widgets import vuetify3 as vuetify from trame_simput import get_simput_manager @@ -72,18 +72,18 @@ def __init__( self, source: DeckTree, **kwargs: Any ) -> None: vuetify.VAlert( "{{ item.summary }}" ) vuetify.Template( "{{ item.start_date }}", raw_attrs=[ "v-slot:opposite" ] ) - with vuetify.VContainer( "Events chart" ): - gantt.Gantt( - canEdit=True, - dateLimit=30, - startDate="2024-11-01 00:00", - endDate="2024-12-01 00:00", - # title='Gantt-pre-test', - fields=fields, - update=( self.update_from_js, "items" ), - items=( "items", items ), - classes="fill_height", - ) + # with vuetify.VContainer( "Events chart" ): + # gantt.Gantt( + # canEdit=True, + # dateLimit=30, + # startDate="2024-11-01 00:00", + # endDate="2024-12-01 00:00", + # # title='Gantt-pre-test', + # fields=fields, + # update=( self.update_from_js, "items" ), + # items=( "items", items ), + # classes="fill_height", + # ) def update_from_js( self, *items: tuple ) -> None: """Update method called from javascript.""" diff --git a/geos-trame/src/geos/trame/assets/cluster.json b/geos-trame/src/geos/trame/assets/cluster.json new file mode 100644 index 00000000..e3ba6a23 --- /dev/null +++ b/geos-trame/src/geos/trame/assets/cluster.json @@ -0,0 +1,24 @@ +{ + "clusters": [ + { + "name": "p4", + "simulation_default_path": "/www", + "geos_version_default": "daily_rhel", + "simulation_information_default_path": "/www", + "simulation_default_filename": "geosDeck.xml", + "n_nodes": 20, + "cpu": { "types": ["Intel Xeon"], "per_node": 64 }, + "gpu": { "types": ["NVIDIA A100"], "per_node": 8 } + }, + { + "name": "elba", + "n_nodes": 10, + "simulation_default_path": "/www", + "geos_version_default": "daily_rhel", + "simulation_information_default_path": "/www", + "simulation_default_filename": "geosDeck.xml", + "cpu": { "types": ["AMD EPYC"], "per_node": 32 }, + "gpu": { "types": ["NVIDIA V100"],"per_node": 4 } + } + ] +} From e9dd40db86b02aba3b6d37089b6127c9bbc60818 Mon Sep 17 00:00:00 2001 From: jacques franc Date: Wed, 19 Nov 2025 11:11:12 +0100 Subject: [PATCH 05/22] wip --- .../src/geos/trame/app/ui/simulation_view.py | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/geos-trame/src/geos/trame/app/ui/simulation_view.py b/geos-trame/src/geos/trame/app/ui/simulation_view.py index b6b35282..69347bcf 100644 --- a/geos-trame/src/geos/trame/app/ui/simulation_view.py +++ b/geos-trame/src/geos/trame/app/ui/simulation_view.py @@ -34,6 +34,7 @@ def define_simulation_view(server) -> None: ) # + access_granted = False # link to login button callback run_try_logging results items = hint_config() vuetify.VDivider(vertical=True, thickness=5, classes="mx-4") with vuetify.VCol(cols=2): @@ -67,6 +68,7 @@ def define_simulation_view(server) -> None: dense=True, hide_details=True, clearable=True, + disabled=("!access_granted") ) # with vuetify.VCol(cols=1): # vuetify.VFileInput( @@ -88,27 +90,18 @@ def define_simulation_view(server) -> None: dense=True, hide_details=True, clearable=True, + disabled=("!access_granted") # TODO callback validation of path ) with vuetify.VRow(), vuetify.VCol(): - # must_be_greater_than_0 = ( - # "[value => Number.isInteger(Number(value)) && value > 0 || 'Must be an integer greater than 0']" - # ) - # vuetify.VTextField( - # v_model=("simulation_nb_process", 1), - # label="Processes number", - # dense=True, - # hide_details=True, - # clearable=True, - # rules=(must_be_greater_than_0,), - # ) vuetify.VTextField( v_model=("simulation_dl_path",), label="Simulation download path", dense=True, clearable=True, prepend_icon="mdi-download", + disabled=("!access_granted") # TODO callback validation of path ) @@ -119,6 +112,7 @@ def define_simulation_view(server) -> None: dense=True, hide_details=True, clearable=True, + disabled=("!access_granted") ) with vuetify.VRow(): vuetify.VSpacer() @@ -126,8 +120,6 @@ def define_simulation_view(server) -> None: vuetify.VBtn("Run", click="trigger('run_simulation')"), # type: ignore with vuetify.VCol(cols=1): vuetify.VBtn("Kill", click="trigger('kill_simulation')"), # type: ignore - # with vuetify.VCol(cols=1): - # vuetify.VBtn("Clear", click="trigger('clear_simulation')"), # type: ignore vuetify.VDivider(thickness=5, classes="my-4") @@ -135,7 +127,5 @@ def define_simulation_view(server) -> None: with vuetify.VCol(cols=2): SimulationStatusView(server=server) - - with vuetify.VRow(v_if="simulation_error"): html.Div("An error occurred while running simulation :
{{simulation_error}}", style="color:red;") From 218c735247344f17d50fa7351585a8afe835e65a Mon Sep 17 00:00:00 2001 From: jacques franc Date: Wed, 19 Nov 2025 11:20:49 +0100 Subject: [PATCH 06/22] wip --- geos-trame/src/geos/trame/app/ui/simulation_view.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/geos-trame/src/geos/trame/app/ui/simulation_view.py b/geos-trame/src/geos/trame/app/ui/simulation_view.py index 69347bcf..f130d96b 100644 --- a/geos-trame/src/geos/trame/app/ui/simulation_view.py +++ b/geos-trame/src/geos/trame/app/ui/simulation_view.py @@ -70,14 +70,6 @@ def define_simulation_view(server) -> None: clearable=True, disabled=("!access_granted") ) - # with vuetify.VCol(cols=1): - # vuetify.VFileInput( - # v_model=("cmd_file", None), - # prepend_icon="mdi-file-upload-outline", - # hide_input=True, - # style="padding: 0;", - # disabled=("!simulation_files_path",), - # ) with vuetify.VRow(), vuetify.VCol(): vuetify.VTextField( @@ -114,6 +106,10 @@ def define_simulation_view(server) -> None: clearable=True, disabled=("!access_granted") ) + + + vuetify.VDivider(thickness=5, classes="my-4") + with vuetify.VRow(): vuetify.VSpacer() with vuetify.VCol(cols=1): @@ -121,7 +117,6 @@ def define_simulation_view(server) -> None: with vuetify.VCol(cols=1): vuetify.VBtn("Kill", click="trigger('kill_simulation')"), # type: ignore - vuetify.VDivider(thickness=5, classes="my-4") with vuetify.VRow(): with vuetify.VCol(cols=2): From 8a361d216901b1e3b7e05dbcfd7e2889c9f4be3c Mon Sep 17 00:00:00 2001 From: jacques franc Date: Wed, 19 Nov 2025 15:47:50 +0100 Subject: [PATCH 07/22] wip --- .../src/geos/trame/app/io/simulation.py | 4 +- .../src/geos/trame/app/ui/simulation_view.py | 106 +++++++++++++++--- geos-trame/src/geos/trame/assets/cluster.json | 16 +-- 3 files changed, 94 insertions(+), 32 deletions(-) diff --git a/geos-trame/src/geos/trame/app/io/simulation.py b/geos-trame/src/geos/trame/app/io/simulation.py index 4a108a18..434e612f 100644 --- a/geos-trame/src/geos/trame/app/io/simulation.py +++ b/geos-trame/src/geos/trame/app/io/simulation.py @@ -21,10 +21,10 @@ class SimulationConstant: SIMULATION_GEOS_PATH = "/workrd/users/" HOST = "p4log01" # Only run on P4 machine PORT = 22 - SIMULATIONS_INFORMATION_FOLDER_PATH= "/workrd/user" + SIMULATIONS_INFORMATION_FOLDER_PATH= "/workrd/users/" SIMULATION_DEFAULT_FILE_NAME="geosDeck.xml" -class Authentificator:#namespacing more than anything eler +class Authentificator:#namespacing more than anything else @staticmethod def get_key(login:str, passphrase = "trameisrunning"): diff --git a/geos-trame/src/geos/trame/app/ui/simulation_view.py b/geos-trame/src/geos/trame/app/ui/simulation_view.py index f130d96b..c225512d 100644 --- a/geos-trame/src/geos/trame/app/ui/simulation_view.py +++ b/geos-trame/src/geos/trame/app/ui/simulation_view.py @@ -3,11 +3,77 @@ from geos.trame.app.io.simulation import SimulationConstant from geos.trame.app.ui.simulation_status_view import SimulationStatusView - - -def hint_config(): - - return ["P4: 1x12", "P4: 2x6"] +import json + +def suggest_decomposition(n_unknowns, + memory_per_unknown_bytes, + node_memory_gb, + cores_per_node, + min_unknowns_per_rank=10000, + strong_scaling=True): + """ + Suggests node/rank distribution for a cluster computation. + + Parameters: + - n_unknowns: total number of unknowns + - memory_per_unknown_bytes: estimated memory per unknown + - node_memory_gb: available memory per node + - cores_per_node: cores available per node + - min_unknowns_per_rank: minimum for efficiency + - strong_scaling: True if problem size is fixed + + Note: + - 10,000-100,000 unknowns per rank is often a sweet spot for many PDE solvers + - Use power-of-2 decompositions when possible (helps with communication patterns) + - For 3D problems, try to maintain cubic subdomains (minimizes surface-to-volume ratio, reducing communication) + - Don't oversubscribe: avoid using more ranks than provide parallel efficiency + + """ + + # Memory constraint + node_memory_bytes = node_memory_gb * 1e9 + max_unknowns_per_node = int(0.8 * node_memory_bytes / memory_per_unknown_bytes) + + # Compute minimum nodes needed + min_nodes = max(1, (n_unknowns + max_unknowns_per_node - 1) // max_unknowns_per_node) + + # Determine ranks per node + unknowns_per_node = n_unknowns // min_nodes + unknowns_per_rank = max(min_unknowns_per_rank, unknowns_per_node // cores_per_node) + + # Calculate total ranks needed + n_ranks = max(1, n_unknowns // unknowns_per_rank) + + # Distribute across nodes + ranks_per_node = min(cores_per_node, (n_ranks + min_nodes - 1) // min_nodes) + n_nodes = (n_ranks + ranks_per_node - 1) // ranks_per_node + + return { + 'nodes': n_nodes, + 'ranks_per_node': ranks_per_node, + 'total_ranks': n_nodes * ranks_per_node, + 'unknowns_per_rank': n_unknowns // (n_nodes * ranks_per_node) + } + +def hint_config(cluster_name, n_unknowns, job_type = 'cpu'): + + # return ["P4: 1x22", "P4: 2x11"] + with open('/data/pau901/SIM_CS/04_WORKSPACE/USERS/jfranc/geosPythonPackages/geos-trame/src/geos/trame/assets/cluster.json','r') as file: + all_cluster = json.load(file) + selected_cluster = list(filter(lambda d: d.get('name')==cluster_name, all_cluster["clusters"]))[0] + + if job_type == 'cpu': #make it an enum + sd = suggest_decomposition(n_unknowns, + 64, + selected_cluster['mem_per_node'], + selected_cluster['cpu']['per_node'] + ) + # elif job_type == 'gpu': + # selected_cluster['n_nodes']*selected_cluster['gpu']['per_node'] + + + return [ f"{selected_cluster['name']}: {sd['nodes']} x {sd['ranks_per_node']}", f"{selected_cluster['name']}: {sd['nodes'] * 2} x {sd['ranks_per_node'] // 2}" ] + def define_simulation_view(server) -> None: @@ -35,7 +101,7 @@ def define_simulation_view(server) -> None: # access_granted = False # link to login button callback run_try_logging results - items = hint_config() + items = hint_config('p4', 12e6) vuetify.VDivider(vertical=True, thickness=5, classes="mx-4") with vuetify.VCol(cols=2): vuetify.VSelect(label="Cluster", @@ -97,23 +163,29 @@ def define_simulation_view(server) -> None: # TODO callback validation of path ) - with vuetify.VRow(), vuetify.VCol(): - vuetify.VTextField( - v_model=("simulation_job_name", "geosJob"), - label="Job Name", - dense=True, - hide_details=True, - clearable=True, - disabled=("!access_granted") - ) + with vuetify.VRow(): + with vuetify.VCol(cols=4): + vuetify.VTextField( + v_model=("simulation_job_name", "geosJob"), + label="Job Name", + dense=True, + hide_details=True, + clearable=True, + disabled=("!access_granted") + ) + + vuetify.VSpacer() + with vuetify.VCol(cols=1): + vuetify.VBtn("Run", + click="trigger('run_simulation')", + disabled=("!access_granted"), + classes="ml-auto"), # type: ignore vuetify.VDivider(thickness=5, classes="my-4") with vuetify.VRow(): vuetify.VSpacer() - with vuetify.VCol(cols=1): - vuetify.VBtn("Run", click="trigger('run_simulation')"), # type: ignore with vuetify.VCol(cols=1): vuetify.VBtn("Kill", click="trigger('kill_simulation')"), # type: ignore diff --git a/geos-trame/src/geos/trame/assets/cluster.json b/geos-trame/src/geos/trame/assets/cluster.json index e3ba6a23..d8bfa4e3 100644 --- a/geos-trame/src/geos/trame/assets/cluster.json +++ b/geos-trame/src/geos/trame/assets/cluster.json @@ -6,19 +6,9 @@ "geos_version_default": "daily_rhel", "simulation_information_default_path": "/www", "simulation_default_filename": "geosDeck.xml", - "n_nodes": 20, - "cpu": { "types": ["Intel Xeon"], "per_node": 64 }, - "gpu": { "types": ["NVIDIA A100"], "per_node": 8 } - }, - { - "name": "elba", - "n_nodes": 10, - "simulation_default_path": "/www", - "geos_version_default": "daily_rhel", - "simulation_information_default_path": "/www", - "simulation_default_filename": "geosDeck.xml", - "cpu": { "types": ["AMD EPYC"], "per_node": 32 }, - "gpu": { "types": ["NVIDIA V100"],"per_node": 4 } + "n_nodes": 212, + "cpu": { "types": ["AMD EPYC 4th gen"], "per_node": 192 }, + "mem_per_node": 768 } ] } From cad9c39bcfbc7196aa54b9caed38904a3cd98623 Mon Sep 17 00:00:00 2001 From: jacques franc Date: Thu, 20 Nov 2025 13:59:37 +0100 Subject: [PATCH 08/22] wip --- .../src/geos/trame/app/io/simulation.py | 29 ++++++++++--------- .../src/geos/trame/app/ui/simulation_view.py | 8 ++++- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/geos-trame/src/geos/trame/app/io/simulation.py b/geos-trame/src/geos/trame/app/io/simulation.py index 434e612f..51d7c5c8 100644 --- a/geos-trame/src/geos/trame/app/io/simulation.py +++ b/geos-trame/src/geos/trame/app/io/simulation.py @@ -12,6 +12,7 @@ import jinja2 import paramiko +import os #TODO move outside #TODO use Jinja on real launcher @@ -22,12 +23,11 @@ class SimulationConstant: HOST = "p4log01" # Only run on P4 machine PORT = 22 SIMULATIONS_INFORMATION_FOLDER_PATH= "/workrd/users/" - SIMULATION_DEFAULT_FILE_NAME="geosDeck.xml" - + SIMULATION_DEFAULT_FILE_NAME = "geosDeck.xml" class Authentificator:#namespacing more than anything else @staticmethod - def get_key(login:str, passphrase = "trameisrunning"): + def get_key(): try: PRIVATE_KEY = paramiko.RSAKey.from_private_key_file("~/.ssh/id_trame") @@ -35,21 +35,25 @@ def get_key(login:str, passphrase = "trameisrunning"): print(f"Error loading private key: {e}\n") except FileNotFoundError as e: print(f"Private key not found: {e}\n Generating key ...") - PRIVATE_KEY = Authentificator.gen_key(login, SimulationConstant.HOST, passphrase) + PRIVATE_KEY = Authentificator.gen_key() return PRIVATE_KEY return PRIVATE_KEY @staticmethod - def gen_key(login:str, host: str, passphrase: str): + def gen_key(): file_path = "~/.ssh/id_trame" - cmd = f"ssh-keygen -t rsa -b 4096 -C {login}@{host} -f {file_path} -N \"{passphrase}\" " - import subprocess - print(f"Running: {''.join(cmd)}") - subprocess.run(cmd, shell=True) - print(f"SSH key generated at: {file_path}") - print(f"Public key: {file_path}.pub") - SIMULATION_DEFAULT_FILE_NAME = "geosDeck.xml" + key = paramiko.RSAKey.generate(bits=4096) + + # Get public key in OpenSSH format + public_key = f"{key.get_name()} {key.get_base64()}" + with open(file_path, "w") as pub_file: + pub_file.write(public_key) + + print("SSH key pair generated: id_trame (private), id_trame.pub (public)") + + + @unique class SlurmJobStatus(Enum): @@ -71,7 +75,6 @@ def from_string(cls, job_str) -> "SlurmJobStatus": # @dataclass_json @dataclass class SimulationInformation: - pass def get_simulation_status( self, diff --git a/geos-trame/src/geos/trame/app/ui/simulation_view.py b/geos-trame/src/geos/trame/app/ui/simulation_view.py index c225512d..8cf167e2 100644 --- a/geos-trame/src/geos/trame/app/ui/simulation_view.py +++ b/geos-trame/src/geos/trame/app/ui/simulation_view.py @@ -73,7 +73,13 @@ def hint_config(cluster_name, n_unknowns, job_type = 'cpu'): return [ f"{selected_cluster['name']}: {sd['nodes']} x {sd['ranks_per_node']}", f"{selected_cluster['name']}: {sd['nodes'] * 2} x {sd['ranks_per_node'] // 2}" ] - + + +class Login: + + @controller.trigger("run_try_login") + def try_logging(): + pass def define_simulation_view(server) -> None: From 84f44529813d3355adb093bdb8638a68df7ddad0 Mon Sep 17 00:00:00 2001 From: jacques franc Date: Thu, 20 Nov 2025 18:18:29 +0100 Subject: [PATCH 09/22] start login backend --- geos-trame/src/geos/trame/app/core.py | 17 ----------- .../src/geos/trame/app/io/simulation.py | 29 +++++++++++++++---- .../src/geos/trame/app/ui/simulation_view.py | 8 +---- 3 files changed, 25 insertions(+), 29 deletions(-) diff --git a/geos-trame/src/geos/trame/app/core.py b/geos-trame/src/geos/trame/app/core.py index 17fdd41a..4d74d7aa 100644 --- a/geos-trame/src/geos/trame/app/core.py +++ b/geos-trame/src/geos/trame/app/core.py @@ -189,23 +189,6 @@ def build_ui( self ) -> None: ): vuetify.VIcon( "mdi-content-save-outline" ) - # with html.Div( - # style= - # "height: 100%; width: 300px; display: flex; align-items: center; justify-content: space-between;", - # v_if=( "tab_idx == 1", ), - # ): - # vuetify.VBtn( - # "Run", - # style="z-index: 1;", - # ) - # vuetify.VBtn( - # "Kill", - # style="z-index: 1;", - # ) - # vuetify.VBtn( - # "Clear", - # style="z-index: 1;", - # ) # input file editor with vuetify.VCol( v_show=( "tab_idx == 0", ), classes="flex-grow-1 pa-0 ma-0" ): diff --git a/geos-trame/src/geos/trame/app/io/simulation.py b/geos-trame/src/geos/trame/app/io/simulation.py index 51d7c5c8..44371d40 100644 --- a/geos-trame/src/geos/trame/app/io/simulation.py +++ b/geos-trame/src/geos/trame/app/io/simulation.py @@ -24,13 +24,18 @@ class SimulationConstant: PORT = 22 SIMULATIONS_INFORMATION_FOLDER_PATH= "/workrd/users/" SIMULATION_DEFAULT_FILE_NAME = "geosDeck.xml" + + + + + class Authentificator:#namespacing more than anything else @staticmethod - def get_key(): + def get_key(id=os.environ.get("USER")): try: - PRIVATE_KEY = paramiko.RSAKey.from_private_key_file("~/.ssh/id_trame") + PRIVATE_KEY = paramiko.RSAKey.from_private_key_file(f"/users/{id}/.ssh/id_trame") except paramiko.SSHException as e: print(f"Error loading private key: {e}\n") except FileNotFoundError as e: @@ -41,8 +46,8 @@ def get_key(): return PRIVATE_KEY @staticmethod - def gen_key(): - file_path = "~/.ssh/id_trame" + def gen_key(id=os.environ.get("USER")): + file_path = f"/users/{id}/.ssh/id_trame" key = paramiko.RSAKey.generate(bits=4096) # Get public key in OpenSSH format @@ -241,7 +246,7 @@ class SimRunner(ISimRunner): def __init__(self, user): super().__init__() - ssh_client = self._create_ssh_client(SimulationConstant.HOST, SimulationConstant.PORT, username=user, key=Authentificator.get_key(user)) + ssh_client = self._create_ssh_client(SimulationConstant.HOST, SimulationConstant.PORT, username=user, key=Authentificator.get_key()) print(ssh_client) # early test @@ -396,6 +401,7 @@ class Simulation: def __init__(self, sim_runner: ISimRunner, server: Server, sim_info_dir: Optional[Path] = None) -> None: self._server = server + controller = server.controller self._sim_runner = sim_runner self._sim_info_dir = sim_info_dir or SimulationConstant.SIMULATIONS_INFORMATION_FOLDER_PATH @@ -403,6 +409,19 @@ def __init__(self, sim_runner: ISimRunner, server: Server, sim_info_dir: Optiona self._job_status_watcher_period_ms = 2000 self.start_result_streams() + + #define triggers + @controller.trigger("run_try_login") + def run_try_login() -> None: + print("login login login") + + @controller.trigger("run_simulation") + def run_simulation()-> None: + pass + + @controller.trigger("kill_simulation") + def kill_simulation(pid)->None: + pass def __del__(self): self.stop_result_streams() diff --git a/geos-trame/src/geos/trame/app/ui/simulation_view.py b/geos-trame/src/geos/trame/app/ui/simulation_view.py index 8cf167e2..0145b05f 100644 --- a/geos-trame/src/geos/trame/app/ui/simulation_view.py +++ b/geos-trame/src/geos/trame/app/ui/simulation_view.py @@ -75,12 +75,6 @@ def hint_config(cluster_name, n_unknowns, job_type = 'cpu'): return [ f"{selected_cluster['name']}: {sd['nodes']} x {sd['ranks_per_node']}", f"{selected_cluster['name']}: {sd['nodes'] * 2} x {sd['ranks_per_node'] // 2}" ] -class Login: - - @controller.trigger("run_try_login") - def try_logging(): - pass - def define_simulation_view(server) -> None: with vuetify.VContainer(): @@ -127,7 +121,7 @@ def define_simulation_view(server) -> None: # vuetify.VDivider(vertical=True, thickness=5, classes="mx-4") with vuetify.VCol(cols=2): - vuetify.VBtn("Log in", click="trigger('run_try_logging')"), # type: ignore + vuetify.VBtn("Log in", click="trigger('run_try_login')"), # type: ignore vuetify.VDivider(thickness=5, classes="my-4") From ce820e799c31605c52978e4f4718804fa0a41b8b Mon Sep 17 00:00:00 2001 From: jacques franc Date: Fri, 21 Nov 2025 15:22:11 +0100 Subject: [PATCH 10/22] ssh login --- .../src/geos/trame/app/io/simulation.py | 254 ++++++++++-------- 1 file changed, 141 insertions(+), 113 deletions(-) diff --git a/geos-trame/src/geos/trame/app/io/simulation.py b/geos-trame/src/geos/trame/app/io/simulation.py index 44371d40..670c726b 100644 --- a/geos-trame/src/geos/trame/app/io/simulation.py +++ b/geos-trame/src/geos/trame/app/io/simulation.py @@ -20,7 +20,8 @@ @dataclass(frozen=True) class SimulationConstant: SIMULATION_GEOS_PATH = "/workrd/users/" - HOST = "p4log01" # Only run on P4 machine + HOST = "fr-vmx00368.main.glb.corp.local" #"p4log01" # Only run on P4 machine + REMOTE_HOME_BASE = "/users" PORT = 22 SIMULATIONS_INFORMATION_FOLDER_PATH= "/workrd/users/" SIMULATION_DEFAULT_FILE_NAME = "geosDeck.xml" @@ -31,33 +32,153 @@ class SimulationConstant: class Authentificator:#namespacing more than anything else + ssh_client : paramiko.SSHClient + @staticmethod - def get_key(id=os.environ.get("USER")): + def get_key( id, pword ): try: - PRIVATE_KEY = paramiko.RSAKey.from_private_key_file(f"/users/{id}/.ssh/id_trame") + home = os.environ.get("HOME") + PRIVATE_KEY = paramiko.RSAKey.from_private_key_file(f"{home}/.ssh/id_trame") + return PRIVATE_KEY except paramiko.SSHException as e: print(f"Error loading private key: {e}\n") except FileNotFoundError as e: print(f"Private key not found: {e}\n Generating key ...") PRIVATE_KEY = Authentificator.gen_key() + temp_client = paramiko.SSHClient() + temp_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + temp_client.connect(SimulationConstant.HOST, SimulationConstant.PORT, username=id, password=pword, timeout=10) + Authentificator._transfer_file_sftp(temp_client,f"{home}/.ssh/id_trame.pub",f"{SimulationConstant.REMOTE_HOME_BASE}/{id}/.ssh/id_trame.pub") + Authentificator._execute_remote_command(temp_client,f" cat {SimulationConstant.REMOTE_HOME_BASE}/{id}/.ssh/id_trame.pub | tee -a {SimulationConstant.REMOTE_HOME_BASE}/{id}/.ssh/authorized_keys") + return PRIVATE_KEY - return PRIVATE_KEY @staticmethod - def gen_key(id=os.environ.get("USER")): - file_path = f"/users/{id}/.ssh/id_trame" + def gen_key(): + + home = os.environ.get("HOME") + file_path = f"{home}/.ssh/id_trame" key = paramiko.RSAKey.generate(bits=4096) + key.write_private_key_file(file_path) # Get public key in OpenSSH format public_key = f"{key.get_name()} {key.get_base64()}" - with open(file_path, "w") as pub_file: + with open(file_path + ".pub", "w") as pub_file: pub_file.write(public_key) print("SSH key pair generated: id_trame (private), id_trame.pub (public)") + + return key + @staticmethod + def _create_ssh_client( host, port, username, password=None, key=None) -> paramiko.SSHClient: + """ + Initializes and returns an SSH client connection. + Uses context manager for automatic cleanup. + """ + client = paramiko.SSHClient() + # Automatically adds the hostname and new host keys to the host files (~/.ssh/known_hosts) + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + try: + # if key: + print(f"Connecting to {host} using key-based authentication...") + client.connect(host, port, username, pkey=key, timeout=10) + # elif password: + # print(f"Connecting to {host} using uid-password authentication...") + # client.connect(host, port, username, password=password, timeout=10) + # else: + # raise paramiko.SSHException("No Key Found") + + return client + except paramiko.AuthenticationException: + print("Authentication failed. Check your credentials or key.") + return None + except paramiko.SSHException as e: + print(f"Could not establish SSH connection: {e}") + return None + except Exception as e: + print(f"An unexpected error occurred: {e}") + return None + + + @staticmethod + def _execute_remote_command(client, command): + """ + Executes a single command on the remote server and prints the output. + """ + if not client: + return + + print(f"\n--- Executing Command: '{command}' ---") + try: + # Executes the command. stdin, stdout, and stderr are file-like objects. + # Ensure command ends with a newline character for some shell environments. + stdin, stdout, stderr = client.exec_command(command) + + # Wait for the command to finish and read the output + exit_status = stdout.channel.recv_exit_status() + + # Print standard output + stdout_data = stdout.read().decode().strip() + if stdout_data: + print("STDOUT:") + print(stdout_data) + + # Print standard error (if any) + stderr_data = stderr.read().decode().strip() + if stderr_data: + print("STDERR:") + print(stderr_data) + + print(f"Command exited with status: {exit_status}") + return exit_status + + except Exception as e: + print(f"Error executing command: {e}") + return -1 + + @staticmethod + def _transfer_file_sftp(client, local_path, remote_path, direction="put"): + """ + Transfers a file using SFTP (Secure File Transfer Protocol). + Direction can be 'put' (upload) or 'get' (download). + """ + if not client: + return + + print(f"\n--- Starting SFTP Transfer ({direction.upper()}) ---") + + try: + # Establish an SFTP connection session + sftp = client.open_sftp() + + if direction == "put": + print(f"Uploading '{local_path}' to '{remote_path}'...") + sftp.put(local_path, remote_path) + print("Upload complete.") + elif direction == "get": + print(f"Downloading '{remote_path}' to '{local_path}'...") + sftp.get(remote_path, local_path) + print("Download complete.") + else: + print("Invalid transfer direction. Use 'put' or 'get'.") + + sftp.close() + return True + + except FileNotFoundError: + print(f"Error: Local file '{local_path}' not found.") + return False + except IOError as e: + print(f"Error accessing remote file or path: {e}") + return False + except Exception as e: + print(f"An error occurred during SFTP: {e}") + return False @unique @@ -246,9 +367,6 @@ class SimRunner(ISimRunner): def __init__(self, user): super().__init__() - ssh_client = self._create_ssh_client(SimulationConstant.HOST, SimulationConstant.PORT, username=user, key=Authentificator.get_key()) - print(ssh_client) - # early test self.local_upload_file = "test_upload.txt" import time @@ -256,109 +374,7 @@ def __init__(self, user): f.write(f"This file was uploaded at {time.ctime()}\n") print(f"Created local file: {self.local_upload_file}") - @staticmethod - def _create_ssh_client( host, port, username, password=None, key=None): - """ - Initializes and returns an SSH client connection. - Uses context manager for automatic cleanup. - """ - client = paramiko.SSHClient() - # Automatically adds the hostname and new host keys to the host files (~/.ssh/known_hosts) - client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - - try: - if key: - print(f"Connecting to {host} using key-based authentication...") - client.connect(host, port, username, pkey=key, timeout=10) - else: - raise paramiko.SSHException("No Key Found") - - return client - except paramiko.AuthenticationException: - print("Authentication failed. Check your credentials or key.") - return None - except paramiko.SSHException as e: - print(f"Could not establish SSH connection: {e}") - return None - except Exception as e: - print(f"An unexpected error occurred: {e}") - return None - - - @staticmethod - def _execute_remote_command(client, command): - """ - Executes a single command on the remote server and prints the output. - """ - if not client: - return - - print(f"\n--- Executing Command: '{command}' ---") - try: - # Executes the command. stdin, stdout, and stderr are file-like objects. - # Ensure command ends with a newline character for some shell environments. - stdin, stdout, stderr = client.exec_command(command) - - # Wait for the command to finish and read the output - exit_status = stdout.channel.recv_exit_status() - - # Print standard output - stdout_data = stdout.read().decode().strip() - if stdout_data: - print("STDOUT:") - print(stdout_data) - - # Print standard error (if any) - stderr_data = stderr.read().decode().strip() - if stderr_data: - print("STDERR:") - print(stderr_data) - - print(f"Command exited with status: {exit_status}") - return exit_status - - except Exception as e: - print(f"Error executing command: {e}") - return -1 - - @staticmethod - def _transfer_file_sftp(client, local_path, remote_path, direction="put"): - """ - Transfers a file using SFTP (Secure File Transfer Protocol). - Direction can be 'put' (upload) or 'get' (download). - """ - if not client: - return - - print(f"\n--- Starting SFTP Transfer ({direction.upper()}) ---") - - try: - # Establish an SFTP connection session - sftp = client.open_sftp() - - if direction == "put": - print(f"Uploading '{local_path}' to '{remote_path}'...") - sftp.put(local_path, remote_path) - print("Upload complete.") - elif direction == "get": - print(f"Downloading '{remote_path}' to '{local_path}'...") - sftp.get(remote_path, local_path) - print("Download complete.") - else: - print("Invalid transfer direction. Use 'put' or 'get'.") - - sftp.close() - return True - - except FileNotFoundError: - print(f"Error: Local file '{local_path}' not found.") - return False - except IOError as e: - print(f"Error accessing remote file or path: {e}") - return False - except Exception as e: - print(f"An error occurred during SFTP: {e}") - return False + def launch_simulation(self): @@ -413,6 +429,18 @@ def __init__(self, sim_runner: ISimRunner, server: Server, sim_info_dir: Optiona #define triggers @controller.trigger("run_try_login") def run_try_login() -> None: + + # if server.state.key: + Authentificator.ssh_client = Authentificator._create_ssh_client(SimulationConstant.HOST,#test + SimulationConstant.PORT, + server.state.login, + key=Authentificator.get_key(server.state.login, server.state.password)) + + if Authentificator.ssh_client : + home = os.environ.get('HOME') + server.state.key_path = f"{home}/.ssh/id_trame" + Authentificator._execute_remote_command(Authentificator.ssh_client, f"ls -l {home}") + print("login login login") @controller.trigger("run_simulation") From 727c9273665c5aa611f3da4f56d29244228ac361 Mon Sep 17 00:00:00 2001 From: jacques franc Date: Fri, 21 Nov 2025 16:53:41 +0100 Subject: [PATCH 11/22] Unlock runner --- .../src/geos/trame/app/io/simulation.py | 14 +++++--------- .../src/geos/trame/app/ui/simulation_view.py | 19 +++++++++++-------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/geos-trame/src/geos/trame/app/io/simulation.py b/geos-trame/src/geos/trame/app/io/simulation.py index 670c726b..92b39589 100644 --- a/geos-trame/src/geos/trame/app/io/simulation.py +++ b/geos-trame/src/geos/trame/app/io/simulation.py @@ -84,14 +84,8 @@ def _create_ssh_client( host, port, username, password=None, key=None) -> parami client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: - # if key: print(f"Connecting to {host} using key-based authentication...") client.connect(host, port, username, pkey=key, timeout=10) - # elif password: - # print(f"Connecting to {host} using uid-password authentication...") - # client.connect(host, port, username, password=password, timeout=10) - # else: - # raise paramiko.SSHException("No Key Found") return client except paramiko.AuthenticationException: @@ -437,10 +431,12 @@ def run_try_login() -> None: key=Authentificator.get_key(server.state.login, server.state.password)) if Authentificator.ssh_client : - home = os.environ.get('HOME') - server.state.key_path = f"{home}/.ssh/id_trame" - Authentificator._execute_remote_command(Authentificator.ssh_client, f"ls -l {home}") + id = os.environ.get('USER') + Authentificator._execute_remote_command(Authentificator.ssh_client, f"ls -l {SimulationConstant.REMOTE_HOME_BASE}/{id}") + # server.state.update({"access_granted" : True, "key_path" : f"{SimulationConstant.REMOTE_HOME_BASE}/{id}/.ssh/id_trame" }) + # server.state.flush() + server.state.access_granted = True print("login login login") @controller.trigger("run_simulation") diff --git a/geos-trame/src/geos/trame/app/ui/simulation_view.py b/geos-trame/src/geos/trame/app/ui/simulation_view.py index 0145b05f..8b86dc80 100644 --- a/geos-trame/src/geos/trame/app/ui/simulation_view.py +++ b/geos-trame/src/geos/trame/app/ui/simulation_view.py @@ -100,7 +100,7 @@ def define_simulation_view(server) -> None: ) # - access_granted = False # link to login button callback run_try_logging results + server.state.access_granted = False# link to login button callback run_try_logging results items = hint_config('p4', 12e6) vuetify.VDivider(vertical=True, thickness=5, classes="mx-4") with vuetify.VCol(cols=2): @@ -109,7 +109,7 @@ def define_simulation_view(server) -> None: with vuetify.VRow(): with vuetify.VCol(cols=8): - vuetify.VFileInput( + vuetify.VTextField( v_model=("key_path", None,), label="Path to ssh key", dense=True, @@ -121,7 +121,10 @@ def define_simulation_view(server) -> None: # vuetify.VDivider(vertical=True, thickness=5, classes="mx-4") with vuetify.VCol(cols=2): - vuetify.VBtn("Log in", click="trigger('run_try_login')"), # type: ignore + vuetify.VBtn("Log in", + click="trigger('run_try_login')", + disabled=("access_granted",) + ) # type: ignore vuetify.VDivider(thickness=5, classes="my-4") @@ -134,7 +137,7 @@ def define_simulation_view(server) -> None: dense=True, hide_details=True, clearable=True, - disabled=("!access_granted") + disabled=("!access_granted",) ) with vuetify.VRow(), vuetify.VCol(): @@ -148,7 +151,7 @@ def define_simulation_view(server) -> None: dense=True, hide_details=True, clearable=True, - disabled=("!access_granted") + disabled=("!access_granted",) # TODO callback validation of path ) @@ -159,7 +162,7 @@ def define_simulation_view(server) -> None: dense=True, clearable=True, prepend_icon="mdi-download", - disabled=("!access_granted") + disabled=("!access_granted",) # TODO callback validation of path ) @@ -171,14 +174,14 @@ def define_simulation_view(server) -> None: dense=True, hide_details=True, clearable=True, - disabled=("!access_granted") + disabled=("!access_granted",) ) vuetify.VSpacer() with vuetify.VCol(cols=1): vuetify.VBtn("Run", click="trigger('run_simulation')", - disabled=("!access_granted"), + disabled=("!access_granted",), classes="ml-auto"), # type: ignore From 3dc8a83d8b99065fa28d5aefec50892b8c5cd140 Mon Sep 17 00:00:00 2001 From: jacques franc Date: Fri, 21 Nov 2025 18:31:25 +0100 Subject: [PATCH 12/22] wip --- .../src/geos/trame/app/io/simulation.py | 56 ++++++- .../src/geos/trame/app/ui/simulation_view.py | 148 ++++++++++-------- 2 files changed, 136 insertions(+), 68 deletions(-) diff --git a/geos-trame/src/geos/trame/app/io/simulation.py b/geos-trame/src/geos/trame/app/io/simulation.py index 92b39589..1f9e6a6d 100644 --- a/geos-trame/src/geos/trame/app/io/simulation.py +++ b/geos-trame/src/geos/trame/app/io/simulation.py @@ -10,7 +10,7 @@ from trame_server.state import State from geos.trame.app.utils.async_file_watcher import AsyncPeriodicRunner -import jinja2 +from jinja2 import Template import paramiko import os @@ -27,6 +27,35 @@ class SimulationConstant: SIMULATION_DEFAULT_FILE_NAME = "geosDeck.xml" +# Load template from file +# with open("slurm_job_template.j2") as f: + # template = Template(f.read()) + +#TODO from private-assets +template_str = """#!/bin/sh +#SBATCH --job-name="{{ job_name }}" +#SBATCH --ntasks={{ ntasks }} +#SBATCH --partition={{ partition }} +#SBATCH --comment={{ comment }} +#SBACTH --account={{ account }} +#SBATCH --nodes={{ nodes }} +#SBATCH --time={{ time | default('24:00:00') }} +#SBATCH --mem={{ mem }} +#SBATCH --output=job_GEOS_%j.out +#SBATCH --error=job_GEOS_%j.err + +ulimit -s unlimited +ulimit -c unlimited + +#module purge +#module geos +#run --mpi=pmix_v3 --hint=nomultithread \ +# -n {{ ntasks }} geos \ +# -o Outputs_{{ slurm_jobid | default('${SLURM_JOBID}') }} \ +# -i {{ input_file | default('geosDeck.xml') }} + +echo "Hello world" >> hello.out +""" @@ -441,10 +470,35 @@ def run_try_login() -> None: @controller.trigger("run_simulation") def run_simulation()-> None: + + if server.state.access_granted and server.state.sd and server.state.simulation_xml_filename: + template = Template(template_str) + sdi = server.state.sd + ci ={'nodes': 2 , 'total_ranks': 96 } + rendered = template.render(job_name=server.state.simulation_job_name, + input_file=server.state.simulation_xml_filename, + nodes= ci['nodes'], ntasks=ci['total_ranks'], mem=f"{ci['nodes']*sdi.selected_cluster['mem_per_node']}GB", + commment='mycomment', partition='mypart', account='myaccount' ) + + with open('job.slurm','w') as f: + f.write(rendered) + + if Authentificator.ssh_client: + Authentificator._transfer_file_sftp(Authentificator.ssh_client, + local_path='job.slurm', + remote_path=server.state.simulation_remote_path) + Authentificator._transfer_file_sftp(Authentificator.ssh_client, + remote_path=server.state.simulation_remote_path+'/job.slurm', + local_path=server.state.simulation_dl_path+'/dl.test', + direction="get") + else: + raise paramiko.SSHException + pass @controller.trigger("kill_simulation") def kill_simulation(pid)->None: + # exec scancel jobid pass def __del__(self): diff --git a/geos-trame/src/geos/trame/app/ui/simulation_view.py b/geos-trame/src/geos/trame/app/ui/simulation_view.py index 8b86dc80..531b6517 100644 --- a/geos-trame/src/geos/trame/app/ui/simulation_view.py +++ b/geos-trame/src/geos/trame/app/ui/simulation_view.py @@ -5,78 +5,92 @@ from geos.trame.app.ui.simulation_status_view import SimulationStatusView import json -def suggest_decomposition(n_unknowns, +class SuggestDecomposition: + + def __init__(self, cluster_name, n_unknowns, job_type = 'cpu'): + + # return ["P4: 1x22", "P4: 2x11"] + with open('/data/pau901/SIM_CS/04_WORKSPACE/USERS/jfranc/geosPythonPackages/geos-trame/src/geos/trame/assets/cluster.json','r') as file: + all_cluster = json.load(file) + self.selected_cluster = list(filter(lambda d: d.get('name')==cluster_name, all_cluster["clusters"]))[0] + self.n_unknowns = n_unknowns + self.job_type = job_type + + # @property + # def selected_cluster(self): + # return self.selected_cluster + + @staticmethod + def compute( n_unknowns, memory_per_unknown_bytes, node_memory_gb, cores_per_node, min_unknowns_per_rank=10000, strong_scaling=True): - """ - Suggests node/rank distribution for a cluster computation. - - Parameters: - - n_unknowns: total number of unknowns - - memory_per_unknown_bytes: estimated memory per unknown - - node_memory_gb: available memory per node - - cores_per_node: cores available per node - - min_unknowns_per_rank: minimum for efficiency - - strong_scaling: True if problem size is fixed - - Note: - - 10,000-100,000 unknowns per rank is often a sweet spot for many PDE solvers - - Use power-of-2 decompositions when possible (helps with communication patterns) - - For 3D problems, try to maintain cubic subdomains (minimizes surface-to-volume ratio, reducing communication) - - Don't oversubscribe: avoid using more ranks than provide parallel efficiency - - """ - - # Memory constraint - node_memory_bytes = node_memory_gb * 1e9 - max_unknowns_per_node = int(0.8 * node_memory_bytes / memory_per_unknown_bytes) - - # Compute minimum nodes needed - min_nodes = max(1, (n_unknowns + max_unknowns_per_node - 1) // max_unknowns_per_node) - - # Determine ranks per node - unknowns_per_node = n_unknowns // min_nodes - unknowns_per_rank = max(min_unknowns_per_rank, unknowns_per_node // cores_per_node) - - # Calculate total ranks needed - n_ranks = max(1, n_unknowns // unknowns_per_rank) - - # Distribute across nodes - ranks_per_node = min(cores_per_node, (n_ranks + min_nodes - 1) // min_nodes) - n_nodes = (n_ranks + ranks_per_node - 1) // ranks_per_node - - return { - 'nodes': n_nodes, - 'ranks_per_node': ranks_per_node, - 'total_ranks': n_nodes * ranks_per_node, - 'unknowns_per_rank': n_unknowns // (n_nodes * ranks_per_node) - } - -def hint_config(cluster_name, n_unknowns, job_type = 'cpu'): + """ + Suggests node/rank distribution for a cluster computation. + + Parameters: + - n_unknowns: total number of unknowns + - memory_per_unknown_bytes: estimated memory per unknown + - node_memory_gb: available memory per node + - cores_per_node: cores available per node + - min_unknowns_per_rank: minimum for efficiency + - strong_scaling: True if problem size is fixed + + Note: + - 10,000-100,000 unknowns per rank is often a sweet spot for many PDE solvers + - Use power-of-2 decompositions when possible (helps with communication patterns) + - For 3D problems, try to maintain cubic subdomains (minimizes surface-to-volume ratio, reducing communication) + - Don't oversubscribe: avoid using more ranks than provide parallel efficiency + + """ + + # Memory constraint + node_memory_bytes = node_memory_gb * 1e9 + max_unknowns_per_node = int(0.8 * node_memory_bytes / memory_per_unknown_bytes) + + # Compute minimum nodes needed + min_nodes = max(1, (n_unknowns + max_unknowns_per_node - 1) // max_unknowns_per_node) + + # Determine ranks per node + unknowns_per_node = n_unknowns // min_nodes + unknowns_per_rank = max(min_unknowns_per_rank, unknowns_per_node // cores_per_node) + + # Calculate total ranks needed + n_ranks = max(1, n_unknowns // unknowns_per_rank) + + # Distribute across nodes + ranks_per_node = min(cores_per_node, (n_ranks + min_nodes - 1) // min_nodes) + n_nodes = (n_ranks + ranks_per_node - 1) // ranks_per_node + + + return { + 'nodes': n_nodes, + 'ranks_per_node': ranks_per_node, + 'total_ranks': n_nodes * ranks_per_node, + 'unknowns_per_rank': n_unknowns // (n_nodes * ranks_per_node) + } - # return ["P4: 1x22", "P4: 2x11"] - with open('/data/pau901/SIM_CS/04_WORKSPACE/USERS/jfranc/geosPythonPackages/geos-trame/src/geos/trame/assets/cluster.json','r') as file: - all_cluster = json.load(file) - selected_cluster = list(filter(lambda d: d.get('name')==cluster_name, all_cluster["clusters"]))[0] - if job_type == 'cpu': #make it an enum - sd = suggest_decomposition(n_unknowns, - 64, - selected_cluster['mem_per_node'], - selected_cluster['cpu']['per_node'] - ) - # elif job_type == 'gpu': - # selected_cluster['n_nodes']*selected_cluster['gpu']['per_node'] + def to_list(self): + + if self.job_type == 'cpu': #make it an enum + sd = SuggestDecomposition.compute(self.n_unknowns, + 64, + self.selected_cluster['mem_per_node'], + self.selected_cluster['cpu']['per_node'] + ) + # elif job_type == 'gpu': + # selected_cluster['n_nodes']*selected_cluster['gpu']['per_node'] - return [ f"{selected_cluster['name']}: {sd['nodes']} x {sd['ranks_per_node']}", f"{selected_cluster['name']}: {sd['nodes'] * 2} x {sd['ranks_per_node'] // 2}" ] + return [ f"{self.selected_cluster['name']}: {sd['nodes']} x {sd['ranks_per_node']}", f"{self.selected_cluster['name']}: {sd['nodes'] * 2} x {sd['ranks_per_node'] // 2}" ] def define_simulation_view(server) -> None: + with vuetify.VContainer(): with vuetify.VRow(): with vuetify.VCol(cols=4): @@ -101,7 +115,9 @@ def define_simulation_view(server) -> None: # server.state.access_granted = False# link to login button callback run_try_logging results - items = hint_config('p4', 12e6) + server.state.simulation_xml_filename = "geosDeck.xml" + server.state.sd = SuggestDecomposition('p4', 12e6) + items = server.state.sd.to_list() vuetify.VDivider(vertical=True, thickness=5, classes="mx-4") with vuetify.VCol(cols=2): vuetify.VSelect(label="Cluster", @@ -131,21 +147,19 @@ def define_simulation_view(server) -> None: with vuetify.VRow(): with vuetify.VCol(): - vuetify.VFileInput( - v_model=("simulation_cmd_filename", SimulationConstant.SIMULATION_DEFAULT_FILE_NAME), + vuetify.VTextField( + v_model=("simulation_xml_filename",), label="Simulation file name", dense=True, hide_details=True, clearable=True, + readonly=True, disabled=("!access_granted",) ) with vuetify.VRow(), vuetify.VCol(): vuetify.VTextField( - v_model=( - "simulation_files_path", - None, - ), + v_model=("simulation_remote_path",None), label="Path where to write files and launch code", prepend_icon="mdi-upload", dense=True, @@ -157,7 +171,7 @@ def define_simulation_view(server) -> None: with vuetify.VRow(), vuetify.VCol(): vuetify.VTextField( - v_model=("simulation_dl_path",), + v_model=("simulation_dl_path", None), label="Simulation download path", dense=True, clearable=True, From 077e6faa5314b688317c7692835b6097d0132e3a Mon Sep 17 00:00:00 2001 From: jacques franc Date: Fri, 21 Nov 2025 20:29:34 +0100 Subject: [PATCH 13/22] full PoC --- geos-trame/src/geos/trame/app/io/simulation.py | 7 ++++--- geos-trame/src/geos/trame/app/ui/simulation_view.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/geos-trame/src/geos/trame/app/io/simulation.py b/geos-trame/src/geos/trame/app/io/simulation.py index 1f9e6a6d..e6c34b77 100644 --- a/geos-trame/src/geos/trame/app/io/simulation.py +++ b/geos-trame/src/geos/trame/app/io/simulation.py @@ -486,10 +486,11 @@ def run_simulation()-> None: if Authentificator.ssh_client: Authentificator._transfer_file_sftp(Authentificator.ssh_client, local_path='job.slurm', - remote_path=server.state.simulation_remote_path) + remote_path=f'{server.state.simulation_remote_path}/job.slurm', + direction="put") Authentificator._transfer_file_sftp(Authentificator.ssh_client, - remote_path=server.state.simulation_remote_path+'/job.slurm', - local_path=server.state.simulation_dl_path+'/dl.test', + remote_path=f'{server.state.simulation_remote_path}/job.slurm', + local_path=f'{server.state.simulation_dl_path}/dl.test', direction="get") else: raise paramiko.SSHException diff --git a/geos-trame/src/geos/trame/app/ui/simulation_view.py b/geos-trame/src/geos/trame/app/ui/simulation_view.py index 531b6517..94bc1bc8 100644 --- a/geos-trame/src/geos/trame/app/ui/simulation_view.py +++ b/geos-trame/src/geos/trame/app/ui/simulation_view.py @@ -114,7 +114,7 @@ def define_simulation_view(server) -> None: ) # - server.state.access_granted = False# link to login button callback run_try_logging results + server.state.access_granted = False server.state.simulation_xml_filename = "geosDeck.xml" server.state.sd = SuggestDecomposition('p4', 12e6) items = server.state.sd.to_list() From 19c04e364953a1e44c3e25265e0268052b97a964 Mon Sep 17 00:00:00 2001 From: jacques franc Date: Sat, 22 Nov 2025 17:20:13 +0100 Subject: [PATCH 14/22] wip --- geos-trame/src/geos/trame/app/io/simulation.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/geos-trame/src/geos/trame/app/io/simulation.py b/geos-trame/src/geos/trame/app/io/simulation.py index e6c34b77..738f3cd9 100644 --- a/geos-trame/src/geos/trame/app/io/simulation.py +++ b/geos-trame/src/geos/trame/app/io/simulation.py @@ -461,7 +461,8 @@ def run_try_login() -> None: if Authentificator.ssh_client : id = os.environ.get('USER') - Authentificator._execute_remote_command(Authentificator.ssh_client, f"ls -l {SimulationConstant.REMOTE_HOME_BASE}/{id}") + Authentificator._execute_remote_command(Authentificator.ssh_client, f"ps aux") + # Authentificator._execute_remote_command(Authentificator.ssh_client, f"ls -l {SimulationConstant.REMOTE_HOME_BASE}/{id}") # server.state.update({"access_granted" : True, "key_path" : f"{SimulationConstant.REMOTE_HOME_BASE}/{id}/.ssh/id_trame" }) # server.state.flush() @@ -492,6 +493,20 @@ def run_simulation()-> None: remote_path=f'{server.state.simulation_remote_path}/job.slurm', local_path=f'{server.state.simulation_dl_path}/dl.test', direction="get") + + + # TODO later ASYNC and subprocess # Submit job using subprocess (local ssh call) + # import subprocess + # result = subprocess.run(["ssh", "user@remote.host", "sbatch /remote/path/job.slurm"], + # capture_output=True, text=True) + + # PARAMIKO >> subprocess + # # Execute command remotely + # stdin, stdout, stderr = client.exec_command("ls -l /tmp") + # print(stdout.read().decode()) + # parse stdout + + else: raise paramiko.SSHException From 3eaabe912f8f9a7a4e200914b091a7e0a9de3b7d Mon Sep 17 00:00:00 2001 From: jacques franc Date: Tue, 25 Nov 2025 15:48:42 +0100 Subject: [PATCH 15/22] authorship --- geos-trame/pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/geos-trame/pyproject.toml b/geos-trame/pyproject.toml index 4e98f6ce..a68dafe0 100644 --- a/geos-trame/pyproject.toml +++ b/geos-trame/pyproject.toml @@ -8,7 +8,8 @@ version = "1.0.0" description = "Geos Simulation Modeler" authors = [{name = "GEOS Contributors" }] maintainers = [{name = "Alexandre Benedicto", email = "alexandre.benedicto@external.totalenergies.com" }, - {name = "Paloma Martinez", email = "paloma.martinez@external.totalenergies.com" }] + {name = "Paloma Martinez", email = "paloma.martinez@external.totalenergies.com" }, + {name = "Jacques Franc", email = "jacques.franc@external.totalenergies.com" },] license = {text = "Apache-2.0"} classifiers = [ "Development Status :: 4 - Beta", From d54f8b3ea1f8aa53912e75c12fa69b86bb2bbffb Mon Sep 17 00:00:00 2001 From: jacques franc Date: Thu, 27 Nov 2025 10:06:00 +0100 Subject: [PATCH 16/22] change host --- geos-trame/src/geos/trame/app/io/simulation.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/geos-trame/src/geos/trame/app/io/simulation.py b/geos-trame/src/geos/trame/app/io/simulation.py index 738f3cd9..01f9d467 100644 --- a/geos-trame/src/geos/trame/app/io/simulation.py +++ b/geos-trame/src/geos/trame/app/io/simulation.py @@ -20,7 +20,7 @@ @dataclass(frozen=True) class SimulationConstant: SIMULATION_GEOS_PATH = "/workrd/users/" - HOST = "fr-vmx00368.main.glb.corp.local" #"p4log01" # Only run on P4 machine + HOST = "p4log01" # Only run on P4 machine REMOTE_HOME_BASE = "/users" PORT = 22 SIMULATIONS_INFORMATION_FOLDER_PATH= "/workrd/users/" @@ -478,8 +478,8 @@ def run_simulation()-> None: ci ={'nodes': 2 , 'total_ranks': 96 } rendered = template.render(job_name=server.state.simulation_job_name, input_file=server.state.simulation_xml_filename, - nodes= ci['nodes'], ntasks=ci['total_ranks'], mem=f"{ci['nodes']*sdi.selected_cluster['mem_per_node']}GB", - commment='mycomment', partition='mypart', account='myaccount' ) + nodes= ci['nodes'], ntasks=ci['total_ranks'], mem=f"{2}GB", + commment="GEOS,CCS,testTrame", partition='p4_general', account='myaccount' ) with open('job.slurm','w') as f: f.write(rendered) @@ -489,8 +489,13 @@ def run_simulation()-> None: local_path='job.slurm', remote_path=f'{server.state.simulation_remote_path}/job.slurm', direction="put") + + Authentificator._execute_remote_command(Authentificator.ssh_client, + f'cd {server.state.simulation_remote_path} && sbatch job.slurm') + Authentificator._execute_remote_command(Authentificator.ssh_client, + f'squeue -u $USER') Authentificator._transfer_file_sftp(Authentificator.ssh_client, - remote_path=f'{server.state.simulation_remote_path}/job.slurm', + remote_path=f'{server.state.simulation_remote_path}/hello.out', local_path=f'{server.state.simulation_dl_path}/dl.test', direction="get") From c1a8395b80b8adf734ea9186145af24793d3f343 Mon Sep 17 00:00:00 2001 From: jacques franc Date: Fri, 28 Nov 2025 15:17:56 +0100 Subject: [PATCH 17/22] wip --- .../src/geos/trame/app/io/simulation.py | 81 +++++++++++++++---- .../src/geos/trame/app/ui/simulation_view.py | 18 ++++- 2 files changed, 79 insertions(+), 20 deletions(-) diff --git a/geos-trame/src/geos/trame/app/io/simulation.py b/geos-trame/src/geos/trame/app/io/simulation.py index 01f9d467..745d4a4b 100644 --- a/geos-trame/src/geos/trame/app/io/simulation.py +++ b/geos-trame/src/geos/trame/app/io/simulation.py @@ -47,14 +47,18 @@ class SimulationConstant: ulimit -s unlimited ulimit -c unlimited -#module purge -#module geos -#run --mpi=pmix_v3 --hint=nomultithread \ -# -n {{ ntasks }} geos \ -# -o Outputs_{{ slurm_jobid | default('${SLURM_JOBID}') }} \ -# -i {{ input_file | default('geosDeck.xml') }} - -echo "Hello world" >> hello.out +module purge +module use /workrd/SCR/GEOS/l1092082/modules +module load geos-develop-d36028cb-hypreUpdate + +export HDF5_USE_FILE_LOCKING=FALSE +export OMP_NUM_THREADS=1 + +srun --mpi=pmix_v3 --hint=nomultithread \ + -n {{ ntasks }} geos \ + -o Outputs_{{ slurm_jobid | default('${SLURM_JOBID}') }} \ + -i {{ input_file | default('geosDeck.xml') }} | tee log.out + """ @@ -63,6 +67,29 @@ class Authentificator:#namespacing more than anything else ssh_client : paramiko.SSHClient + @staticmethod + def _sftp_copy_tree(ssh_client, local_root, remote_root): + # Connect to remote server + sftp = ssh_client.open_sftp() + + local_root = Path(local_root).resolve() + + for path in local_root.rglob("*"): + remote_path = f"{remote_root}/{path.relative_to(local_root)}" + + if path.is_dir(): + # Create remote directory if it doesn't exist + try: + sftp.mkdir(remote_path) + except IOError: + # Directory may already exist + pass + else: + # Upload file + sftp.put(str(path), remote_path) + + sftp.close() + @staticmethod def get_key( id, pword ): @@ -478,24 +505,44 @@ def run_simulation()-> None: ci ={'nodes': 2 , 'total_ranks': 96 } rendered = template.render(job_name=server.state.simulation_job_name, input_file=server.state.simulation_xml_filename, - nodes= ci['nodes'], ntasks=ci['total_ranks'], mem=f"{2}GB", - commment="GEOS,CCS,testTrame", partition='p4_general', account='myaccount' ) + nodes= ci['nodes'], ntasks=ci['total_ranks'], mem=f"0",#TODO profile to use the correct amount + commment=server.state.slurm_comment, partition='p4_general', account='myaccount' ) - with open('job.slurm','w') as f: - f.write(rendered) + # with open(Path(server.state.simulation_xml_filename).parent/Path('job.slurm'),'w') as f: + # f.write(rendered) if Authentificator.ssh_client: - Authentificator._transfer_file_sftp(Authentificator.ssh_client, - local_path='job.slurm', - remote_path=f'{server.state.simulation_remote_path}/job.slurm', - direction="put") + #write slurm directly on remote + try: + sftp = Authentificator.ssh_client.open_sftp() + remote_path = Path(server.state.simulation_xml_filename).parent/Path('job.slurm') + with sftp.file(remote_path,'w') as f: + f.write(rendered) + + # except FileExistsError: + # print(f"Error: Local file '{remote_path}' not found.") + except PermissionError as e: + print(f"Permission error: {e}") + except IOError as e: + print(f"Error accessing remote file or path: {e}") + except Exception as e: + print(f"An error occurred during SFTP: {e}") + + Authentificator._sftp_copy_tree(Authentificator.ssh_client, + Path(server.state.simulation_xml_filename).parent, + Path(server.state.simulation_remote_path)) + Authentificator._execute_remote_command(Authentificator.ssh_client, f'cd {server.state.simulation_remote_path} && sbatch job.slurm') + + Authentificator._execute_remote_command(Authentificator.ssh_client, f'squeue -u $USER') + + Authentificator._transfer_file_sftp(Authentificator.ssh_client, - remote_path=f'{server.state.simulation_remote_path}/hello.out', + remote_path=f'{server.state.simulation_remote_path}/log.out', local_path=f'{server.state.simulation_dl_path}/dl.test', direction="get") diff --git a/geos-trame/src/geos/trame/app/ui/simulation_view.py b/geos-trame/src/geos/trame/app/ui/simulation_view.py index 94bc1bc8..40b0c5c6 100644 --- a/geos-trame/src/geos/trame/app/ui/simulation_view.py +++ b/geos-trame/src/geos/trame/app/ui/simulation_view.py @@ -136,24 +136,36 @@ def define_simulation_view(server) -> None: # vuetify.VDivider(vertical=True, thickness=5, classes="mx-4") - with vuetify.VCol(cols=2): + with vuetify.VCol(cols=1): vuetify.VBtn("Log in", click="trigger('run_try_login')", disabled=("access_granted",) ) # type: ignore + # + vuetify.VDivider(vertical=True, thickness=5, classes="mx-4") + with vuetify.VCol(cols=1): + vuetify.VTextField( + v_model=("slurm_comment", None,), + label="Comment to slurm", + dense=True, + hide_details=True, + clearable=True, + ) # type: ignore + vuetify.VDivider(thickness=5, classes="my-4") with vuetify.VRow(): with vuetify.VCol(): - vuetify.VTextField( + vuetify.VFileInput( v_model=("simulation_xml_filename",), label="Simulation file name", dense=True, hide_details=True, clearable=True, - readonly=True, + multiple=True, + # readonly=True, disabled=("!access_granted",) ) From 85964724a34e77b9a2326a5129e323d3c3fc5bad Mon Sep 17 00:00:00 2001 From: jacques franc Date: Fri, 28 Nov 2025 17:56:23 +0100 Subject: [PATCH 18/22] update versions f/ VFileUpload --- geos-trame/pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/geos-trame/pyproject.toml b/geos-trame/pyproject.toml index a68dafe0..1c823a34 100644 --- a/geos-trame/pyproject.toml +++ b/geos-trame/pyproject.toml @@ -32,12 +32,12 @@ keywords = [ dependencies = [ "typing-extensions==4.12.2", "trame==3.6.5", - "trame-vuetify==2.7.1", + "trame-vuetify==3.1.0", "trame-code==1.0.1", "trame-server==3.2.3", - "trame-client==3.5.0", + "trame-client==3.11.2", "trame-simput==2.4.3", - "trame-vtk>=2.8.14", + "trame-vtk==2.10.0", "matplotlib==3.9.4", "trame-matplotlib==2.0.3", "trame-components==2.4.2", From db83e7a1206fec614153c4494a52ff5d6b13012e Mon Sep 17 00:00:00 2001 From: jacques franc Date: Mon, 1 Dec 2025 15:19:34 +0100 Subject: [PATCH 19/22] another view --- .../src/geos/trame/app/io/simulation.py | 5 +++-- .../src/geos/trame/app/ui/simulation_view.py | 21 ++++++++++++------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/geos-trame/src/geos/trame/app/io/simulation.py b/geos-trame/src/geos/trame/app/io/simulation.py index 745d4a4b..435e90f4 100644 --- a/geos-trame/src/geos/trame/app/io/simulation.py +++ b/geos-trame/src/geos/trame/app/io/simulation.py @@ -499,9 +499,10 @@ def run_try_login() -> None: @controller.trigger("run_simulation") def run_simulation()-> None: - if server.state.access_granted and server.state.sd and server.state.simulation_xml_filename: + # if server.state.access_granted and server.state.sd and server.state.simulation_xml_filename: + if server.state.access_granted and server.state.simulation_xml_filename: template = Template(template_str) - sdi = server.state.sd + # sdi = server.state.sd ci ={'nodes': 2 , 'total_ranks': 96 } rendered = template.render(job_name=server.state.simulation_job_name, input_file=server.state.simulation_xml_filename, diff --git a/geos-trame/src/geos/trame/app/ui/simulation_view.py b/geos-trame/src/geos/trame/app/ui/simulation_view.py index 40b0c5c6..be6ea390 100644 --- a/geos-trame/src/geos/trame/app/ui/simulation_view.py +++ b/geos-trame/src/geos/trame/app/ui/simulation_view.py @@ -115,9 +115,9 @@ def define_simulation_view(server) -> None: # server.state.access_granted = False - server.state.simulation_xml_filename = "geosDeck.xml" - server.state.sd = SuggestDecomposition('p4', 12e6) - items = server.state.sd.to_list() + server.state.simulation_xml_filename = [ ] + sd = SuggestDecomposition('p4', 12e6) + items = sd.to_list() vuetify.VDivider(vertical=True, thickness=5, classes="mx-4") with vuetify.VCol(cols=2): vuetify.VSelect(label="Cluster", @@ -157,17 +157,22 @@ def define_simulation_view(server) -> None: vuetify.VDivider(thickness=5, classes="my-4") with vuetify.VRow(): - with vuetify.VCol(): - vuetify.VFileInput( + with vuetify.VCol(cols=4): + vuetify.VFileUpload( v_model=("simulation_xml_filename",), - label="Simulation file name", - dense=True, + title="Simulation file name", + density='comfortable', hide_details=True, - clearable=True, + # clearable=True, multiple=True, + filter_by_type='.xml,.vtu,.vtm,.pvtu,.pvtm,.dat,.csv,.txt', # readonly=True, disabled=("!access_granted",) ) + with vuetify.VCol(cols=4): + with vuetify.VList(): + with vuetify.VListItem( v_for=(f"file in {server.state.simulation_xml_filename}"), key="i", value="file" ): + vuetify.VListItemTitle( "{{ file.name }}" ) with vuetify.VRow(), vuetify.VCol(): vuetify.VTextField( From 21b4492fdac379fc35c27bc427f549b172bed5e3 Mon Sep 17 00:00:00 2001 From: jacques franc Date: Mon, 1 Dec 2025 17:39:17 +0100 Subject: [PATCH 20/22] new list update --- .../src/geos/trame/app/ui/simulation_view.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/geos-trame/src/geos/trame/app/ui/simulation_view.py b/geos-trame/src/geos/trame/app/ui/simulation_view.py index be6ea390..a5295cc0 100644 --- a/geos-trame/src/geos/trame/app/ui/simulation_view.py +++ b/geos-trame/src/geos/trame/app/ui/simulation_view.py @@ -87,10 +87,19 @@ def to_list(self): return [ f"{self.selected_cluster['name']}: {sd['nodes']} x {sd['ranks_per_node']}", f"{self.selected_cluster['name']}: {sd['nodes'] * 2} x {sd['ranks_per_node'] // 2}" ] + def define_simulation_view(server) -> None: + @server.state.change("simulation_xml_temp") + def on_temp_change(simulation_xml_temp : list, **kw): + current_list = server.state.simulation_xml_filename + + new_list = current_list + simulation_xml_temp + server.state.simulation_xml_filename = new_list + server.state.simulation_xml_temp = [] + with vuetify.VContainer(): with vuetify.VRow(): with vuetify.VCol(cols=4): @@ -116,6 +125,9 @@ def define_simulation_view(server) -> None: # server.state.access_granted = False server.state.simulation_xml_filename = [ ] + # server.state.simulation_xml_temp = [ ] + + sd = SuggestDecomposition('p4', 12e6) items = sd.to_list() vuetify.VDivider(vertical=True, thickness=5, classes="mx-4") @@ -159,7 +171,7 @@ def define_simulation_view(server) -> None: with vuetify.VRow(): with vuetify.VCol(cols=4): vuetify.VFileUpload( - v_model=("simulation_xml_filename",), + v_model=("simulation_xml_temp",[]), title="Simulation file name", density='comfortable', hide_details=True, @@ -167,12 +179,13 @@ def define_simulation_view(server) -> None: multiple=True, filter_by_type='.xml,.vtu,.vtm,.pvtu,.pvtm,.dat,.csv,.txt', # readonly=True, - disabled=("!access_granted",) + disabled=("access_granted",) ) with vuetify.VCol(cols=4): with vuetify.VList(): - with vuetify.VListItem( v_for=(f"file in {server.state.simulation_xml_filename}"), key="i", value="file" ): + with vuetify.VListItem( v_for=("(file,i) in simulation_xml_filename"), key="i", value="file" ): vuetify.VListItemTitle( "{{ file.name }}" ) + vuetify.VListItemSubtitle("{{ file.size ? (file.size / 1024).toFixed(1) + ' KB' : 'URL' }}") with vuetify.VRow(), vuetify.VCol(): vuetify.VTextField( From 6c50a55e6d1562b2da313d8237b1bc9a4b681fb5 Mon Sep 17 00:00:00 2001 From: jacques franc Date: Tue, 2 Dec 2025 08:42:01 +0100 Subject: [PATCH 21/22] loading files complete --- .../src/geos/trame/app/io/simulation.py | 2 ++ .../src/geos/trame/app/ui/simulation_view.py | 34 ++++++++++++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/geos-trame/src/geos/trame/app/io/simulation.py b/geos-trame/src/geos/trame/app/io/simulation.py index 435e90f4..7bc879a0 100644 --- a/geos-trame/src/geos/trame/app/io/simulation.py +++ b/geos-trame/src/geos/trame/app/io/simulation.py @@ -496,6 +496,8 @@ def run_try_login() -> None: server.state.access_granted = True print("login login login") + + @controller.trigger("run_simulation") def run_simulation()-> None: diff --git a/geos-trame/src/geos/trame/app/ui/simulation_view.py b/geos-trame/src/geos/trame/app/ui/simulation_view.py index a5295cc0..f5f45de7 100644 --- a/geos-trame/src/geos/trame/app/ui/simulation_view.py +++ b/geos-trame/src/geos/trame/app/ui/simulation_view.py @@ -93,13 +93,37 @@ def to_list(self): def define_simulation_view(server) -> None: @server.state.change("simulation_xml_temp") - def on_temp_change(simulation_xml_temp : list, **kw): + def on_temp_change(simulation_xml_temp : list, **_): current_list = server.state.simulation_xml_filename new_list = current_list + simulation_xml_temp server.state.simulation_xml_filename = new_list server.state.simulation_xml_temp = [] + @server.state.change("simulation_xml_filename") + def on_simfiles_change(simulation_xml_filename : list, **_): + import re + pattern = re.compile(r"\.xml$", re.IGNORECASE) + has_xml = any(pattern.search(file if isinstance(file, str) else file.get("name", "")) for file in simulation_xml_filename) + server.state.is_valid_jobfiles = has_xml + + + + # @controller.trigger("run_remove_jobfile") + def run_remove_jobfile(index_to_remove : int) -> None: + # for now just check there is an xml + current_files = list(server.state.simulation_xml_filename) # On prend une copie de la liste + if 0 <= index_to_remove < len(current_files): + # 1. Supprimer l'élément de la copie de la liste + del current_files[index_to_remove] + + # 2. Remplacer la variable d'état par la nouvelle liste. + # Ceci est CRITIQUE pour la réactivité, car cela force Vue.js à se mettre à jour. + server.state.simulation_xml_filename = current_files + print(f"Fichier à l'index {index_to_remove} supprimé. Nouveaux fichiers: {len(current_files)}") + else: + print(f"Erreur: Index de suppression invalide ({index_to_remove}).") + with vuetify.VContainer(): with vuetify.VRow(): with vuetify.VCol(cols=4): @@ -124,9 +148,8 @@ def on_temp_change(simulation_xml_temp : list, **kw): # server.state.access_granted = False + server.state.is_valid_jobfiles = False server.state.simulation_xml_filename = [ ] - # server.state.simulation_xml_temp = [ ] - sd = SuggestDecomposition('p4', 12e6) items = sd.to_list() @@ -186,6 +209,9 @@ def on_temp_change(simulation_xml_temp : list, **kw): with vuetify.VListItem( v_for=("(file,i) in simulation_xml_filename"), key="i", value="file" ): vuetify.VListItemTitle( "{{ file.name }}" ) vuetify.VListItemSubtitle("{{ file.size ? (file.size / 1024).toFixed(1) + ' KB' : 'URL' }}") + with vuetify.VListItemAction(): + vuetify.VBtn(small=True, icon=True, children=[vuetify.VIcon("mdi-minus-circle-outline")], + click=(run_remove_jobfile, "[i]") ) with vuetify.VRow(), vuetify.VCol(): vuetify.VTextField( @@ -225,7 +251,7 @@ def on_temp_change(simulation_xml_temp : list, **kw): with vuetify.VCol(cols=1): vuetify.VBtn("Run", click="trigger('run_simulation')", - disabled=("!access_granted",), + disabled=("!is_valid_jobfiles",), classes="ml-auto"), # type: ignore From 45358900c6eb808b864221a5fef8e6b68b6ac310 Mon Sep 17 00:00:00 2001 From: jacques franc Date: Tue, 2 Dec 2025 14:00:45 +0100 Subject: [PATCH 22/22] first working v --- .../src/geos/trame/app/io/simulation.py | 93 ++++++++++++++----- .../src/geos/trame/app/ui/simulation_view.py | 4 +- 2 files changed, 74 insertions(+), 23 deletions(-) diff --git a/geos-trame/src/geos/trame/app/io/simulation.py b/geos-trame/src/geos/trame/app/io/simulation.py index 7bc879a0..290f8647 100644 --- a/geos-trame/src/geos/trame/app/io/simulation.py +++ b/geos-trame/src/geos/trame/app/io/simulation.py @@ -68,27 +68,45 @@ class Authentificator:#namespacing more than anything else ssh_client : paramiko.SSHClient @staticmethod - def _sftp_copy_tree(ssh_client, local_root, remote_root): + def _sftp_copy_tree(ssh_client, file_tree, remote_root): # Connect to remote server sftp = ssh_client.open_sftp() + + Authentificator.dfs_tree(file_tree["structure"], file_tree["root"], sftp=sftp, remote_root=remote_root) - local_root = Path(local_root).resolve() - - for path in local_root.rglob("*"): - remote_path = f"{remote_root}/{path.relative_to(local_root)}" + sftp.close() - if path.is_dir(): - # Create remote directory if it doesn't exist - try: - sftp.mkdir(remote_path) - except IOError: - # Directory may already exist - pass - else: - # Upload file - sftp.put(str(path), remote_path) + @staticmethod + def dfs_tree(node, path, sftp, remote_root): + + lp = Path(path) + rp = Path(remote_root)/lp + + if isinstance(node, list): + for file in node: + # sftp.put(lp/Path(file), rp/Path(file)) + with sftp.file( str(rp/Path(file.get('name'))), 'w') as f: + f.write(file.get('content')) + print(f"copying {lp/Path(file.get('name'))} to {rp/Path(file.get('name'))}") + elif isinstance(node, dict): + if "files" in node: + for file in node["files"]: + # sftp.put( str(lp/Path(file)), str(rp/Path(file)) ) + with sftp.file( str(rp/Path(file.get('name'))), 'w') as f: + f.write(file.get('content')) + print(f"copying {lp/Path(file.get('name'))} to {rp/Path(file.get('name'))}") + if "subfolders" in node: + for subfolder, content in node["subfolders"].items(): + sftp.mkdir( str(rp/Path(subfolder))) + print(f"creating {rp/Path(subfolder)}") + Authentificator.dfs_tree(content, lp/Path(subfolder), sftp, remote_root) + + for folder, content in node.items(): + if folder not in ["files", "subfolders"]: + sftp.mkdir( str(rp/Path(folder)) ) + print(f"creating {rp/Path(folder)}") + Authentificator.dfs_tree(content, lp/Path(folder), sftp, remote_root) - sftp.close() @staticmethod def get_key( id, pword ): @@ -496,7 +514,40 @@ def run_try_login() -> None: server.state.access_granted = True print("login login login") - + @staticmethod + def gen_tree(xml_filename): + + import re + xml_pattern = re.compile(r"\.xml$", re.IGNORECASE) + mesh_pattern = re.compile(r"\.(vtu|vtm|pvtu|pvtm)$", re.IGNORECASE) + table_pattern = re.compile(r"\.(txt|dat|csv)$", re.IGNORECASE) + xml_matches = [] + mesh_matches = [] + table_matches = [] + + for file in xml_filename: + if xml_pattern.search(file.get("name","")): + xml_matches.append(file) + elif mesh_pattern.search(file.get("name","")): + mesh_matches.append(file) + elif table_pattern.search(file.get("name","")): + table_matches.append(file) + + file_tree = { + 'root' : '.', + "structure": { + "files" : xml_matches, + "subfolders": { + "mesh": mesh_matches, + "tables": table_matches + # "subfolders": { + # "inner_tables_1": ["placeholder.txt"], + # "inner_tables_2": ["placeholder.txt"] + # } + } + } + } + return file_tree @controller.trigger("run_simulation") def run_simulation()-> None: @@ -518,8 +569,8 @@ def run_simulation()-> None: #write slurm directly on remote try: sftp = Authentificator.ssh_client.open_sftp() - remote_path = Path(server.state.simulation_xml_filename).parent/Path('job.slurm') - with sftp.file(remote_path,'w') as f: + remote_path = Path(server.state.simulation_remote_path)/Path('job.slurm') + with sftp.file( str(remote_path),'w' ) as f: f.write(rendered) # except FileExistsError: @@ -532,8 +583,8 @@ def run_simulation()-> None: print(f"An error occurred during SFTP: {e}") Authentificator._sftp_copy_tree(Authentificator.ssh_client, - Path(server.state.simulation_xml_filename).parent, - Path(server.state.simulation_remote_path)) + gen_tree(server.state.simulation_xml_filename), + server.state.simulation_remote_path) Authentificator._execute_remote_command(Authentificator.ssh_client, diff --git a/geos-trame/src/geos/trame/app/ui/simulation_view.py b/geos-trame/src/geos/trame/app/ui/simulation_view.py index f5f45de7..2f7fc95d 100644 --- a/geos-trame/src/geos/trame/app/ui/simulation_view.py +++ b/geos-trame/src/geos/trame/app/ui/simulation_view.py @@ -202,7 +202,7 @@ def run_remove_jobfile(index_to_remove : int) -> None: multiple=True, filter_by_type='.xml,.vtu,.vtm,.pvtu,.pvtm,.dat,.csv,.txt', # readonly=True, - disabled=("access_granted",) + disabled=("!access_granted",) ) with vuetify.VCol(cols=4): with vuetify.VList(): @@ -210,7 +210,7 @@ def run_remove_jobfile(index_to_remove : int) -> None: vuetify.VListItemTitle( "{{ file.name }}" ) vuetify.VListItemSubtitle("{{ file.size ? (file.size / 1024).toFixed(1) + ' KB' : 'URL' }}") with vuetify.VListItemAction(): - vuetify.VBtn(small=True, icon=True, children=[vuetify.VIcon("mdi-minus-circle-outline")], + vuetify.VBtn(small=True, icon="mdi-minus-circle-outline", click=(run_remove_jobfile, "[i]") ) with vuetify.VRow(), vuetify.VCol():