diff --git a/.gitignore b/.gitignore index 0712f6d7..d158df83 100644 --- a/.gitignore +++ b/.gitignore @@ -188,3 +188,6 @@ cache # IntelliJ IDEA files .idea *.iml + +# Visual Studio Code files +.vscode/ \ No newline at end of file diff --git a/benchmarks/600.java/601.hello-world/config.json b/benchmarks/600.java/601.hello-world/config.json new file mode 100644 index 00000000..e3d6f85f --- /dev/null +++ b/benchmarks/600.java/601.hello-world/config.json @@ -0,0 +1,6 @@ +{ + "timeout": 60, + "memory": 256, + "languages": ["java"] +} + diff --git a/benchmarks/600.java/601.hello-world/input.py b/benchmarks/600.java/601.hello-world/input.py new file mode 100644 index 00000000..52536abf --- /dev/null +++ b/benchmarks/600.java/601.hello-world/input.py @@ -0,0 +1,13 @@ +def buckets_count(): + return (0, 0) + +def generate_input( + data_dir, + size, + benchmarks_bucket, + input_paths, + output_paths, + upload_func, + nosql_func=None +): + return { } \ No newline at end of file diff --git a/benchmarks/600.java/601.hello-world/java/pom.xml b/benchmarks/600.java/601.hello-world/java/pom.xml new file mode 100644 index 00000000..d504d9bc --- /dev/null +++ b/benchmarks/600.java/601.hello-world/java/pom.xml @@ -0,0 +1,59 @@ + + 4.0.0 + + faas + benchmark + 1.0 + + + UTF-8 + ${env.JAVA_VERSION} + + + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + ${java.version} + ${java.version} + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.4 + + + package + shade + + + + *:* + + module-info.class + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + diff --git a/benchmarks/600.java/601.hello-world/java/src/main/java/faas/App.java b/benchmarks/600.java/601.hello-world/java/src/main/java/faas/App.java new file mode 100644 index 00000000..fe0b2096 --- /dev/null +++ b/benchmarks/600.java/601.hello-world/java/src/main/java/faas/App.java @@ -0,0 +1,13 @@ +package faas; +import java.util.HashMap; +import java.util.Map; + +public class App { + public Map handler(Map input) { + + Map result = new HashMap<>(); + result.put("Hello", "World"); + return result; + } +} + diff --git a/benchmarks/wrappers/aws/java/Handler.java b/benchmarks/wrappers/aws/java/Handler.java new file mode 100644 index 00000000..b51de44c --- /dev/null +++ b/benchmarks/wrappers/aws/java/Handler.java @@ -0,0 +1,75 @@ +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.core.type.TypeReference; + +import faas.App; + +import java.io.File; +import java.io.IOException; +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; + +public class Handler implements RequestHandler, String> { + private static final ObjectMapper mapper = new ObjectMapper(); + + @Override + public String handleRequest(Map event, Context context) { + + Map inputData = event; + + // Extract input if trigger is API Gateway (body is a string) + if (event.containsKey("body") && event.get("body") instanceof String) + try { + inputData = mapper.readValue((String) event.get("body"),new TypeReference>() {}); + } catch (IOException e) { + throw new RuntimeException("Failed to parse JSON body", e); + } + + App function = new App(); + + Instant begin = Instant.now(); + long start_nano = System.nanoTime(); + + Map functionOutput = function.handler(inputData); + + long end_nano = System.nanoTime(); + Instant end = Instant.now(); + + + long computeTime = end_nano - start_nano; + // Detect cold start + boolean isCold = false; + String fileName = "/tmp/cold_run"; + + File file = new File(fileName); + if (!file.exists()) { + isCold = true; + try { + file.createNewFile(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + // Convert to Unix timestamp in seconds.microseconds + String formattedBegin = String.format("%d.%06d", begin.getEpochSecond(), begin.getNano() / 1000); // Convert nanoseconds to microseconds + String formattedEnd = String.format("%d.%06d", end.getEpochSecond(), end.getNano() / 1000); + + + Map result = new HashMap<>(); + result.put("begin", formattedBegin); + result.put("end", formattedEnd); + result.put("request_id", context.getAwsRequestId()); + result.put("compute_time", computeTime); + result.put("is_cold", isCold); + result.put("result", functionOutput); + try { + return mapper.writeValueAsString(result); + } catch (IOException e) { + throw new RuntimeException("Failed to serialize result of benchmark to JSON in Wrapper", e); + } + + } +} diff --git a/benchmarks/wrappers/openwhisk/java/Main.java b/benchmarks/wrappers/openwhisk/java/Main.java new file mode 100644 index 00000000..59a33ee3 --- /dev/null +++ b/benchmarks/wrappers/openwhisk/java/Main.java @@ -0,0 +1,55 @@ +import faas.App; +import com.google.gson.JsonObject; +import java.time.Instant; +import java.time.Duration; +import java.io.File; +import java.io.IOException; + + +public class Main { + public static JsonObject main(JsonObject args) { + + App function = new App(); + + long start_nano = System.nanoTime(); + + Instant begin = Instant.now(); + JsonObject result = function.handler(args); + Instant end = Instant.now(); + + long end_nano = System.nanoTime(); + + // long computeTime = Duration.between(begin, end).toNanos() / 1000; // Convert nanoseconds to microseconds + + long computeTime = end_nano - start_nano; + boolean isCold = false; + String fileName = "/tmp/cold_run"; + + File file = new File(fileName); + if (!file.exists()) { + isCold = true; + try { + file.createNewFile(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + // Convert to Unix timestamp in seconds.microseconds + String formattedBegin = String.format("%d.%06d", begin.getEpochSecond(), begin.getNano() / 1000); // Convert nanoseconds to microseconds + String formattedEnd = String.format("%d.%06d", end.getEpochSecond(), end.getNano() / 1000); + + String requestId = System.getenv("__OW_ACTIVATION_ID"); + + JsonObject jsonResult = new JsonObject(); + jsonResult.addProperty("begin", formattedBegin); + jsonResult.addProperty("end", formattedEnd); + jsonResult.addProperty("request_id", requestId); + jsonResult.addProperty("compute_time", computeTime); + jsonResult.addProperty("is_cold", isCold); + jsonResult.addProperty("result", result.toString()); + return jsonResult; + } + +} + diff --git a/benchmarks/wrappers/openwhisk/java/Storage.java b/benchmarks/wrappers/openwhisk/java/Storage.java new file mode 100644 index 00000000..e69de29b diff --git a/config/example2.json b/config/example2.json new file mode 100644 index 00000000..3575d601 --- /dev/null +++ b/config/example2.json @@ -0,0 +1,69 @@ +{ + "experiments": { + "deployment": "openwhisk", + "update_code": false, + "update_storage": false, + "download_results": false, + "runtime": { + "language": "java", + "version": "8" + }, + "type": "invocation-overhead", + "perf-cost": { + "benchmark": "601.hello-world", + "experiments": ["cold", "warm", "burst", "sequential"], + "input-size": "test", + "repetitions": 50, + "concurrent-invocations": 50, + "memory-sizes": [128, 256] + }, + "network-ping-pong": { + "invocations": 50, + "repetitions": 1000, + "threads": 1 + }, + "invocation-overhead": { + "repetitions": 5, + "N": 20, + "type": "payload", + "payload_begin": 1024, + "payload_end": 6251000, + "payload_points": 20, + "code_begin": 1048576, + "code_end": 261619712, + "code_points": 20 + }, + "eviction-model": { + "invocations": 1, + "function_copy_idx": 0, + "repetitions": 5, + "sleep": 1 + } + }, + "deployment": { + "openwhisk": { + "shutdownStorage": false, + "removeCluster": false, + "wskBypassSecurity": "true", + "wskExec": "wsk", + "experimentalManifest": false, + "docker_registry": { + "registry": "", + "username": "", + "password": "" + }, + "storage": { + "address": "", + "mapped_port": 9011, + "access_key": "", + "secret_key": "", + "instance_id": "", + "output_buckets": [], + "input_buckets": [], + "type": "minio" + } + + } + } + } + \ No newline at end of file diff --git a/config/systems.json b/config/systems.json index 5a4077a2..b00fe237 100644 --- a/config/systems.json +++ b/config/systems.json @@ -121,10 +121,32 @@ "uuid": "3.4.0" } } - } + }, + "java": { + "base_images": { + "x64": { + "11": "amazon/aws-lambda-java:11" + }, + "arm64": { + "11": "amazon/aws-lambda-java:11" + } + }, + "images": [ + "build" + ], + "deployment": { + "files": [ + "Handler.java" + ], + "packages": { + "com.amazonaws:aws-lambda-java-core": "1.2.3", + "com.fasterxml.jackson.core:jackson-databind": "2.15.2" + } + } + } }, "architecture": ["x64", "arm64"], - "deployments": ["package", "container"] + "deployments": ["package"] }, "azure": { "languages": { @@ -316,6 +338,24 @@ "minio": "7.0.16" } } + }, + "java": { + "base_images": { + "8": "openwhisk/java8action" + }, + "images": [ + "function" + ], + "username": "docker_user", + "deployment": { + "files": [ + "Main.java", + "Storage.java" + ], + "packages": { + "minio": "8.5.9" + } + } } }, "architecture": ["x64"], diff --git a/dockerfiles/aws/java/Dockerfile.build b/dockerfiles/aws/java/Dockerfile.build new file mode 100644 index 00000000..2b2a09e0 --- /dev/null +++ b/dockerfiles/aws/java/Dockerfile.build @@ -0,0 +1,22 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} +ARG VERSION +ENV JAVA_VERSION=${VERSION} + + +# useradd, groupmod + maven +RUN yum install -y shadow-utils maven +ENV GOSU_VERSION 1.14 +# https://github.com/tianon/gosu/releases/tag/1.14 +# key https://keys.openpgp.org/search?q=tianon%40debian.org +RUN curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-amd64" \ + && chmod +x /usr/local/bin/gosu +RUN mkdir -p /sebs/ +COPY dockerfiles/java_installer.sh /sebs/installer.sh +COPY dockerfiles/entrypoint.sh /sebs/entrypoint.sh +RUN chmod +x /sebs/entrypoint.sh + +# useradd and groupmod is installed in /usr/sbin which is not in PATH +ENV PATH=/usr/sbin:$PATH +CMD /bin/bash /sebs/installer.sh +ENTRYPOINT ["/sebs/entrypoint.sh"] \ No newline at end of file diff --git a/dockerfiles/java_installer.sh b/dockerfiles/java_installer.sh new file mode 100644 index 00000000..1cc221f3 --- /dev/null +++ b/dockerfiles/java_installer.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +cd /mnt/function + +mvn clean install + + + diff --git a/dockerfiles/openwhisk/java/Dockerfile.function b/dockerfiles/openwhisk/java/Dockerfile.function new file mode 100644 index 00000000..b72ceb15 --- /dev/null +++ b/dockerfiles/openwhisk/java/Dockerfile.function @@ -0,0 +1,8 @@ +ARG BASE_IMAGE +FROM $BASE_IMAGE +COPY . /function/ + +# RUN apt-get update && apt-get install -y maven + +# # Check if pom.xml exists before running Maven +# RUN if [ -f ./pom.xml ]; then mvn clean install; else echo "pom.xml not found, aborting build." && exit 1; fi diff --git a/sebs.py b/sebs.py index 80fb11ed..9334c6f6 100755 --- a/sebs.py +++ b/sebs.py @@ -64,7 +64,7 @@ def simplified_common_params(func): @click.option( "--language", default=None, - type=click.Choice(["python", "nodejs"]), + type=click.Choice(["python", "nodejs", "java"]), help="Benchmark language", ) @click.option("--language-version", default=None, type=str, help="Benchmark language version") diff --git a/sebs/aws/aws.py b/sebs/aws/aws.py index 243a6f0f..abea416e 100644 --- a/sebs/aws/aws.py +++ b/sebs/aws/aws.py @@ -134,33 +134,46 @@ def package_code( directory, language_name, language_version, architecture, benchmark, is_cached ) - CONFIG_FILES = { - "python": ["handler.py", "requirements.txt", ".python_packages"], - "nodejs": ["handler.js", "package.json", "node_modules"], - } - package_config = CONFIG_FILES[language_name] - function_dir = os.path.join(directory, "function") - os.makedirs(function_dir) - # move all files to 'function' except handler.py - for file in os.listdir(directory): - if file not in package_config: - file = os.path.join(directory, file) - shutil.move(file, function_dir) - # FIXME: use zipfile - # create zip with hidden directory but without parent directory - execute("zip -qu -r9 {}.zip * .".format(benchmark), shell=True, cwd=directory) - benchmark_archive = "{}.zip".format(os.path.join(directory, benchmark)) - self.logging.info("Created {} archive".format(benchmark_archive)) - - bytes_size = os.path.getsize(os.path.join(directory, benchmark_archive)) - mbytes = bytes_size / 1024.0 / 1024.0 - self.logging.info("Zip archive size {:2f} MB".format(mbytes)) - - return ( - os.path.join(directory, "{}.zip".format(benchmark)), - bytes_size, - container_uri, - ) + if (language_name == 'java'): + + jar_path = os.path.join(directory, "target", "benchmark-1.0.jar") + bytes_size = os.path.getsize(jar_path) + + return ( + jar_path, + bytes_size, + container_uri, + ) + + else: + # so no need to add anything here + CONFIG_FILES = { + "python": ["handler.py", "requirements.txt", ".python_packages"], + "nodejs": ["handler.js", "package.json", "node_modules"], + } + package_config = CONFIG_FILES[language_name] + function_dir = os.path.join(directory, "function") + os.makedirs(function_dir) + # move all files to 'function' except handler.py + for file in os.listdir(directory): + if file not in package_config: + file = os.path.join(directory, file) + shutil.move(file, function_dir) + # FIXME: use zipfile + # create zip with hidden directory but without parent directory + execute("zip -qu -r9 {}.zip * .".format(benchmark), shell=True, cwd=directory) + benchmark_archive = "{}.zip".format(os.path.join(directory, benchmark)) + self.logging.info("Created {} archive".format(benchmark_archive)) + + bytes_size = os.path.getsize(os.path.join(directory, benchmark_archive)) + mbytes = bytes_size / 1024.0 / 1024.0 + self.logging.info("Zip archive size {:2f} MB".format(mbytes)) + + return ( + os.path.join(directory, "{}.zip".format(benchmark)), + bytes_size, + container_uri, + ) def _map_architecture(self, architecture: str) -> str: @@ -254,7 +267,10 @@ def create_function( create_function_params["Runtime"] = "{}{}".format( language, self._map_language_runtime(language, language_runtime) ) - create_function_params["Handler"] = "handler.handler" + if language == "java": + create_function_params["Handler"] = "Handler::handleRequest" + else: + create_function_params["Handler"] = "handler.handler" create_function_params = { k: v for k, v in create_function_params.items() if v is not None diff --git a/sebs/benchmark.py b/sebs/benchmark.py index 42adb4e7..48797228 100644 --- a/sebs/benchmark.py +++ b/sebs/benchmark.py @@ -1,6 +1,7 @@ import glob import hashlib import json +import subprocess import os import shutil import subprocess @@ -252,15 +253,18 @@ def hash_directory(directory: str, deployment: str, language: str): FILES = { "python": ["*.py", "requirements.txt*"], "nodejs": ["*.js", "package.json"], + # Use recursive Java scan since *.java files are located in subfolders. + "java": ["**/*.java", "pom.xml"], } - WRAPPERS = {"python": "*.py", "nodejs": "*.js"} + WRAPPERS = {"python": "*.py", "nodejs": "*.js", "java": "*.java"} NON_LANG_FILES = ["*.sh", "*.json"] selected_files = FILES[language] + NON_LANG_FILES for file_type in selected_files: - for f in glob.glob(os.path.join(directory, file_type)): - path = os.path.join(directory, f) - with open(path, "rb") as opened_file: - hash_sum.update(opened_file.read()) + for f in glob.glob(os.path.join(directory, file_type), recursive=True): + if os.path.isfile(f): + path = os.path.join(directory, f) + with open(path, "rb") as opened_file: + hash_sum.update(opened_file.read()) # wrappers wrappers = project_absolute_path( "benchmarks", "wrappers", deployment, language, WRAPPERS[language] @@ -313,19 +317,56 @@ def query_cache(self): self._is_cached_valid = False def copy_code(self, output_dir): + from sebs.faas.function import Language + FILES = { "python": ["*.py", "requirements.txt*"], "nodejs": ["*.js", "package.json"], + "java": ["pom.xml"], } path = os.path.join(self.benchmark_path, self.language_name) + for file_type in FILES[self.language_name]: for f in glob.glob(os.path.join(path, file_type)): shutil.copy2(os.path.join(path, f), output_dir) + + # copy src folder of java (java benchmarks are maven project and need directories) + if self.language == Language.JAVA: + output_src_dir = os.path.join(output_dir, "src") + + if os.path.exists(output_src_dir): + # If src dir in output exist, remove the directory and all its contents + shutil.rmtree(output_src_dir) + #To have contents of src directory in the direcory named src located in output + shutil.copytree(os.path.join(path, "src"), output_src_dir) + # support node.js benchmarks with language specific packages nodejs_package_json = os.path.join(path, f"package.json.{self.language_version}") if os.path.exists(nodejs_package_json): shutil.copy2(nodejs_package_json, os.path.join(output_dir, "package.json")) + #This is for making jar file and add it to docker directory + def add_java_output(self, code_dir): + from sebs.faas.function import Language + if self.language == Language.JAVA: + + # Step 1: Move Main.java o src directory + src_dir = os.path.join(code_dir, "src", "main", "java") + if os.path.exists(code_dir): + main_java_path = os.path.join(code_dir, "Main.java") + if os.path.exists(main_java_path): + shutil.move(main_java_path, src_dir) + + # Step 2: Run mvn clean install + try: + # Navigate to the code directory where the pom.xml file is located + subprocess.run(['mvn', 'clean', 'install'], cwd=code_dir, check=True, text=True, capture_output=True) + print("Maven build successful!") + except subprocess.CalledProcessError as e: + print(f"Error during Maven build:\n{e.stdout}\n{e.stderr}") + return + + def add_benchmark_data(self, output_dir): cmd = "/bin/bash {benchmark_path}/init.sh {output_dir} false {architecture}" paths = [ @@ -355,8 +396,18 @@ def add_deployment_files(self, output_dir): self._deployment_name, self.language_name ) ] + + final_path = output_dir + + # For Java, use Maven structure: put handler files in src/main/java/ + if self.language_name == 'java': + final_path = os.path.join(output_dir, 'src', 'main', 'java') + os.makedirs(final_path, exist_ok=True) # make sure the path exists + for file in handlers: - shutil.copy2(file, os.path.join(output_dir)) + shutil.copy2(file, final_path) + + def add_deployment_package_python(self, output_dir): @@ -399,6 +450,40 @@ def add_deployment_package_nodejs(self, output_dir): with open(package_config, "w") as package_file: json.dump(package_json, package_file, indent=2) + # Dependencies in system.json are in "group:artifact": version format; + # this function converts them to proper Maven blocks. + def format_maven_dependency(self, group_artifact: str, version: str) -> str: + group_id, artifact_id = group_artifact.split(":") + return f""" + + {group_id} + {artifact_id} + {version} + """ + + def add_deployment_package_java(self, output_dir): + + pom_path = os.path.join(output_dir, "pom.xml") + with open(pom_path, "r") as f: + pom_content = f.read() + + packages = self._system_config.deployment_packages(self._deployment_name, self.language_name) + + dependency_blocks = "" + if len(packages): + for key, val in packages.items(): + dependency_name = key.strip('"').strip("'") + dependency_version = val.strip('"').strip("'") + dependency_blocks += self.format_maven_dependency(dependency_name, dependency_version) + "\n" + + if "" not in pom_content: + raise ValueError("pom.xml template is missing placeholder") + + pom_content = pom_content.replace("", dependency_blocks.strip()) + + with open(pom_path, "w") as f: + f.write(pom_content) + def add_deployment_package(self, output_dir): from sebs.faas.function import Language @@ -406,6 +491,8 @@ def add_deployment_package(self, output_dir): self.add_deployment_package_python(output_dir) elif self.language == Language.NODEJS: self.add_deployment_package_nodejs(output_dir) + elif self.language == Language.JAVA: + self.add_deployment_package_java(output_dir) else: raise NotImplementedError @@ -463,7 +550,7 @@ def install_dependencies(self, output_dir): } # run Docker container to install packages - PACKAGE_FILES = {"python": "requirements.txt", "nodejs": "package.json"} + PACKAGE_FILES = {"python": "requirements.txt", "nodejs": "package.json", "java" : "pom.xml"} file = os.path.join(output_dir, PACKAGE_FILES[self.language_name]) if os.path.exists(file): try: @@ -592,6 +679,7 @@ def build( self.copy_code(self._output_dir) self.add_benchmark_data(self._output_dir) self.add_deployment_files(self._output_dir) +# self.add_java_output(self._output_dir) self.add_deployment_package(self._output_dir) self.install_dependencies(self._output_dir) diff --git a/sebs/faas/function.py b/sebs/faas/function.py index 0fab7bcf..9ddad97f 100644 --- a/sebs/faas/function.py +++ b/sebs/faas/function.py @@ -263,6 +263,7 @@ def deserialize(cached_config: dict) -> "Trigger": class Language(Enum): PYTHON = "python" NODEJS = "nodejs" + JAVA = "java" # FIXME: 3.7+ python with future annotations @staticmethod @@ -299,7 +300,7 @@ def serialize(self) -> dict: @staticmethod def deserialize(config: dict) -> Runtime: - languages = {"python": Language.PYTHON, "nodejs": Language.NODEJS} + languages = {"python": Language.PYTHON, "nodejs": Language.NODEJS, "java": Language.JAVA} return Runtime(language=languages[config["language"]], version=config["version"]) diff --git a/sebs/openwhisk/openwhisk.py b/sebs/openwhisk/openwhisk.py index 9c196fe2..0dc61b75 100644 --- a/sebs/openwhisk/openwhisk.py +++ b/sebs/openwhisk/openwhisk.py @@ -110,13 +110,14 @@ def package_code( directory, language_name, language_version, architecture, benchmark, is_cached ) - # We deploy Minio config in code package since this depends on local - # deployment - it cannnot be a part of Docker image - CONFIG_FILES = { - "python": ["__main__.py"], - "nodejs": ["index.js"], - } - package_config = CONFIG_FILES[language_name] + if language_name != 'java': + # We deploy Minio config in code package since this depends on local + # deployment - it cannnot be a part of Docker image + CONFIG_FILES = { + "python": ["__main__.py"], + "nodejs": ["index.js"], + } + package_config = CONFIG_FILES[language_name] benchmark_archive = os.path.join(directory, f"{benchmark}.zip") subprocess.run( @@ -207,6 +208,25 @@ def create_function( code_package.language_version, code_package.architecture, ) + run_arguments = [ + *self.get_wsk_cmd(), + "action", + "create", + func_name, + "--web", + "true", + "--docker", + docker_image, + "--memory", + str(code_package.benchmark_config.memory), + "--timeout", + str(code_package.benchmark_config.timeout * 1000), + *self.storage_arguments(), + code_package.code_location, + ] + if code_package.language_name == 'java': + run_arguments.extend(["--main", "Main"]) + subprocess.run( [ *self.get_wsk_cmd(), @@ -228,6 +248,7 @@ def create_function( stdout=subprocess.PIPE, check=True, ) + function_cfg.docker_image = docker_image res = OpenWhiskFunction( func_name, code_package.benchmark, code_package.hash, function_cfg