diff --git a/scripts/repo.sh b/scripts/repo.sh new file mode 100755 index 00000000000..607ad987a1a --- /dev/null +++ b/scripts/repo.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# USAGE: ./repo.sh [args...] +# +# EXAMPLE: ./repo.sh tests decrypt --json ./scripts/secrets/AI.json +# +# Wraps around the local "repo" swift package, and facilitates calls to it. +# The main purpose of this is to make calling "repo" easier, as you typically +# need to call "swift run" with the package path. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if [[ $# -eq 0 ]]; then + cat 1>&2 < [args...] +EOF + exit 1 +fi + +swift run --package-path "${ROOT}/repo" "$@" diff --git a/scripts/repo/Package.swift b/scripts/repo/Package.swift new file mode 100755 index 00000000000..1edeeb69aad --- /dev/null +++ b/scripts/repo/Package.swift @@ -0,0 +1,50 @@ +// swift-tools-version:6.0 +// The swift-tools-version declares the minimum version of Swift required to build this package. + +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import PackageDescription + +/// Package containing CLI executables for our larger scripts that are a bit harder to follow in +/// bash form, or that need more advanced flag/optional requirements. +let package = Package( + name: "RepoScripts", + platforms: [.macOS(.v15)], + products: [ + .executable(name: "tests", targets: ["Tests"]), + ], + dependencies: [ + .package(url: "https://github.com/apple/swift-argument-parser", exact: "1.6.2"), + .package(url: "https://github.com/apple/swift-log", exact: "1.6.2"), + ], + targets: [ + .executableTarget( + name: "Tests", + dependencies: [ + .product(name: "ArgumentParser", package: "swift-argument-parser"), + .product(name: "Logging", package: "swift-log"), + .byName(name: "Util"), + ] + ), + .target( + name: "Util", + dependencies: [ + .product(name: "Logging", package: "swift-log"), + ] + ), + ] +) diff --git a/scripts/repo/README.md b/scripts/repo/README.md new file mode 100644 index 00000000000..318c16c547d --- /dev/null +++ b/scripts/repo/README.md @@ -0,0 +1,13 @@ +# Firebase Apple repo commands + +This project includes commands that are too long and complicated to properly +maintain in a bash script, or that have unique option/flag constraints that +are better represented in a swift project. + +## Tests + +Commands for interacting with integration tests in the repo. + +```sh +./scripts/repo.sh tests --help +``` diff --git a/scripts/repo/Sources/Tests/Decrypt.swift b/scripts/repo/Sources/Tests/Decrypt.swift new file mode 100755 index 00000000000..b1836787be9 --- /dev/null +++ b/scripts/repo/Sources/Tests/Decrypt.swift @@ -0,0 +1,183 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import ArgumentParser +import Foundation +import Logging +import Util + +extension Tests { + /// Command for decrypting the secret files needed for a test run. + struct Decrypt: ParsableCommand { + nonisolated(unsafe) static var configuration = CommandConfiguration( + abstract: "Decrypt the secret files for a test run.", + usage: """ + tests decrypt [--json] [--overwrite] [] + tests decrypt [--password ] [--overwrite] [ ...] + + tests decrypt --json secret_files.json + tests decrypt --json --overwrite secret_files.json + tests decrypt --password "super_secret" \\ + scripts/gha-encrypted/FirebaseAI/TestApp-GoogleService-Info.plist.gpg:FirebaseAI/Tests/TestApp/Resources/GoogleService-Info.plist \\ + scripts/gha-encrypted/FirebaseAI/TestApp-GoogleService-Info-Spark.plist.gpg:FirebaseAI/Tests/TestApp/Resources/GoogleService-Info-Spark.plist + """, + discussion: """ + The happy path usage is saving the secret passphrase in the environment variable \ + 'secrets_passphrase', and passing a json file to the command. Although, you can also \ + pass everything inline via options. + + When using a json file, it's expected that the json file is an array of json elements \ + in the format of: + { encrypted: , destination: } + """, + ) + + @Argument( + help: """ + An array of secret files to decrypt. \ + The files should be in the format "encrypted:destination", where "encrypted" is a path to \ + the encrypted file and "destination" is a path to where the decrypted file should be saved. + """ + ) + var secretFiles: [String] = [] + + @Option( + help: """ + The secret to use when decrypting the files. \ + Defaults to the environment variable 'secrets_passphrase'. + """ + ) + var password: String = "" + + @Flag(help: "Overwrite existing decrypted secret files.") + var overwrite: Bool = false + + @Flag( + help: """ + Use a json file of secret file mappings instead. \ + When this flag is enabled, should be a single json file. + """ + ) + var json: Bool = false + + /// The parsed version of ``secretFiles``. + /// + /// Only populated after `validate()` runs. + var files: [SecretFile] = [] + + static let log = Logger(label: "Tests::Decrypt") + private var log: Logger { Decrypt.log } + + mutating func validate() throws { + try validatePassword() + + if json { + try validateJSON() + } else { + try validateFileString() + } + + if !overwrite { + log.info("Overwrite is disabled, so we're skipping generation for existing files.") + files = files.filter { file in + let keep = !FileManager.default.fileExists(atPath: file.destination) + if !keep { + log.debug( + "Skipping generation for existing file", + metadata: ["destination": "\(file.destination)"] + ) + } + return keep + } + } + + for file in files { + guard FileManager.default.fileExists(atPath: file.encrypted) else { + throw ValidationError("Encrypted secret file does not exist: \(file.encrypted)") + } + } + } + + private mutating func validatePassword() throws { + if password.isEmpty { + // when a password isn't provided, try to load one from the environment variable + guard + let secrets_passphrase = ProcessInfo.processInfo.environment["secrets_passphrase"] + else { + throw ValidationError( + "Either provide a passphrase via the password option or set the environvment variable 'secrets_passphrase' to the passphrase." + ) + } + password = secrets_passphrase + } + } + + private mutating func validateJSON() throws { + guard let jsonPath = secretFiles.first else { + throw ValidationError("Missing path to json file for secret files") + } + + let fileURL = URL( + filePath: jsonPath, directoryHint: .notDirectory, + relativeTo: URL.currentDirectory() + ) + + files = try SecretFile.parseArrayFrom(file: fileURL) + guard !files.isEmpty else { + throw ValidationError("Missing secret files in json file: \(jsonPath)") + } + } + + private mutating func validateFileString() throws { + guard !secretFiles.isEmpty else { + throw ValidationError("Missing paths to secret files") + } + for string in secretFiles { + try files.append(SecretFile(string: string)) + } + } + + mutating func run() throws { + log.info("Decrypting files...") + + for file in files { + let gpg = Process("gpg", inheritEnvironment: true) + let result = try gpg.runWithSignals([ + "--quiet", + "--batch", + "--yes", + "--decrypt", + "--passphrase=\(password)", + "--output", + file.destination, + file.encrypted, + ]) + + guard result == 0 else { + log.error("Failed to decrypt file", metadata: ["file": "\(file.encrypted)"]) + throw ExitCode(result) + } + + log.debug( + "File encrypted", + metadata: ["file": "\(file.encrypted)", "destination": "\(file.destination)"] + ) + } + + log.info("Files decrypted") + } + } +} diff --git a/scripts/repo/Sources/Tests/SecretFile.swift b/scripts/repo/Sources/Tests/SecretFile.swift new file mode 100755 index 00000000000..67d5e953981 --- /dev/null +++ b/scripts/repo/Sources/Tests/SecretFile.swift @@ -0,0 +1,73 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import ArgumentParser +import Foundation + +/// A representation of a secret file, which should be decrypted for an integration test. +struct SecretFile: Codable { + /// A relative path to the encrypted file. + let encrypted: String + + /// A relative path to where the decrypted file should be output to. + let destination: String +} + +extension SecretFile { + /// Parses a `SecretFile` from a string. + /// + /// The string should be in the format of "encrypted:destination". + /// If it's not, then a `ValidationError`will be thrown. + /// + /// - Parameters: + /// - string: A string in the format of "encrypted:destination". + init(string: String) throws { + let splits = string.split(separator: ":") + guard splits.count == 2 else { + throw ValidationError( + "Invalid secret file format. Format should be \"encrypted:destination\". Cause: \(string)" + ) + } + encrypted = String(splits[0]) + destination = String(splits[1]) + } + + /// Parses an array of `SecretFile` from a JSON file. + /// + /// It's expected that the secrets are encoded in the JSON file in the format of: + /// ```json + /// [ + /// { + /// "encrypted": "path-to-encrypted-file", + /// "destination": "where-to-output-decrypted-file" + /// } + /// ] + /// ``` + /// + /// - Parameters: + /// - file: The URL of a JSON file which contains an array of `SecretFile`, + /// encoded as JSON. + static func parseArrayFrom(file: URL) throws -> [SecretFile] { + do { + let data = try Data(contentsOf: file) + return try JSONDecoder().decode([SecretFile].self, from: data) + } catch { + throw ValidationError( + "Failed to load secret files from json file. Cause: \(error.localizedDescription)" + ) + } + } +} diff --git a/scripts/repo/Sources/Tests/main.swift b/scripts/repo/Sources/Tests/main.swift new file mode 100755 index 00000000000..0abb218e46a --- /dev/null +++ b/scripts/repo/Sources/Tests/main.swift @@ -0,0 +1,54 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import ArgumentParser +import Foundation +import Logging + +struct Tests: ParsableCommand { + nonisolated(unsafe) static var configuration = CommandConfiguration( + abstract: "Commands for running and interacting with integration tests.", + discussion: """ + A note on logging: by default, only log levels "info" and above are logged. For further \ + debugging, you can set the "LOG_LEVEL" environment variable to a different minimum level \ + (eg; "debug"). + """, + subcommands: [Decrypt.self] + // defaultSubcommand: Run.self + ) +} + +LoggingSystem.bootstrap { label in + var handler = StreamLogHandler.standardOutput(label: label) + if let level = ProcessInfo.processInfo.environment["LOG_LEVEL"] { + if let parsedLevel = Logger.Level(rawValue: String(level)) { + handler.logLevel = parsedLevel + return handler + } else { + print( + """ + [WARNING]: Unrecognized log level "\(level)"; defaulting to "info". + Valid values: \(Logger.Level.allCases.map(\.rawValue)) + """ + ) + } + } + + handler.logLevel = .info + return handler +} + +Tests.main() diff --git a/scripts/repo/Sources/Util/Process.swift b/scripts/repo/Sources/Util/Process.swift new file mode 100755 index 00000000000..2959d5f0abb --- /dev/null +++ b/scripts/repo/Sources/Util/Process.swift @@ -0,0 +1,120 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Dispatch +import Foundation + +public extension Process { + /// Creates a new `Process` instance without running it. + /// + /// - Parameters: + /// - exe: The executable to run. + /// - args: An array of arguments to pass to the executable. + /// - env: A map of environment variables to set for the process. + /// - inheritEnvironment: When enabled, the parent process' environvment will also be applied + /// to this process. Effectively, this means that any environvment variables declared within the + /// parent process will propogate down to this new process. + convenience init(_ exe: String, + _ args: [String] = [], + env: [String: String] = [:], + inheritEnvironment: Bool = false) { + self.init() + executableURL = URL(filePath: "/usr/bin/env") + arguments = [exe] + args + environment = env + if inheritEnvironment { + mergeEnvironment(ProcessInfo.processInfo.environment) + } + } + + /// Merges the provided environment variables with this process' existing environment variables. + /// + /// If an environment variable is already set, then it will **NOT** be overwritten. Only + /// environment variables not currently set on the process will be applied. + /// + /// - Parameters: + /// - env: The environment variables to merge with this process. + func mergeEnvironment(_ env: [String: String]) { + guard environment != nil else { + // if this process doesn't have an environment, we can just set it instead of merging + environment = env + return + } + + environment = environment?.merging(env) { current, _ in current } + } + + /// Run the process with signals from the parent process. + /// + /// The signals `SIGINT` and `SIGTERM` will both be propogated + /// down to the process from the parent process. + /// + /// This function will not return until the process is done running. + /// + /// - Parameters: + /// - args: Optionally provide an array of arguments to run the process with. + /// + /// - Returns: The exit code that the process completed with. + @discardableResult + func runWithSignals(_ args: [String]? = nil) throws -> Int32 { + if let args { + arguments = (arguments ?? []) + args + } + + let sigint = bindSignal(signal: SIGINT) { + if self.isRunning { + self.interrupt() + } + } + + let sigterm = bindSignal(signal: SIGTERM) { + if self.isRunning { + self.terminate() + } + } + + sigint.resume() + sigterm.resume() + + try run() + waitUntilExit() + + return terminationStatus + } +} + +/// Binds a callback to a signal from the parent process. +/// +/// ```swift +/// bindSignal(SIGINT) { +/// print("SIGINT was triggered") +/// } +/// ``` +/// +/// - Parameters: +/// - signal: The signal to listen for. +/// - callback: The function to invoke when the signal is received. +func bindSignal(signal value: Int32, + callback: @escaping DispatchSourceProtocol + .DispatchSourceHandler) -> any DispatchSourceSignal { + // allow the process to survive long enough to trigger the callback + signal(value, SIG_IGN) + + let dispatch = DispatchSource.makeSignalSource(signal: value, queue: .main) + dispatch.setEventHandler(handler: callback) + + return dispatch +} diff --git a/scripts/secrets/AI.json b/scripts/secrets/AI.json new file mode 100755 index 00000000000..1f675c8fc3e --- /dev/null +++ b/scripts/secrets/AI.json @@ -0,0 +1,14 @@ +[ + { + "encrypted": "scripts/gha-encrypted/FirebaseAI/TestApp-GoogleService-Info.plist.gpg", + "destination": "FirebaseAI/Tests/TestApp/Resources/GoogleService-Info.plist" + }, + { + "encrypted": "scripts/gha-encrypted/FirebaseAI/TestApp-GoogleService-Info-Spark.plist.gpg", + "destination": "FirebaseAI/Tests/TestApp/Resources/GoogleService-Info-Spark.plist" + }, + { + "encrypted": "scripts/gha-encrypted/FirebaseAI/TestApp-Credentials.swift.gpg", + "destination": "FirebaseAI/Tests/TestApp/Tests/Integration/Credentials.swift" + } +]