diff --git a/NethermindNode.Core/Helpers/CommandExecutor.cs b/NethermindNode.Core/Helpers/CommandExecutor.cs index 34b917b..fc633fb 100644 --- a/NethermindNode.Core/Helpers/CommandExecutor.cs +++ b/NethermindNode.Core/Helpers/CommandExecutor.cs @@ -11,7 +11,6 @@ public static void RemoveDirectory(string absolutePath, NLog.Logger logger) if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { processInfo = new ProcessStartInfo("rm", $"-r {absolutePath}"); - logger.Debug("Removing path: " + absolutePath); } else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { @@ -21,6 +20,7 @@ public static void RemoveDirectory(string absolutePath, NLog.Logger logger) { throw new ArgumentOutOfRangeException(); } + logger.Debug("Removing path: " + absolutePath); processInfo.CreateNoWindow = true; processInfo.UseShellExecute = false; @@ -35,6 +35,37 @@ public static void RemoveDirectory(string absolutePath, NLog.Logger logger) } } + public static void BackupDirectory(string absolutePath, string backupPath, NLog.Logger logger) + { + ProcessStartInfo processInfo; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + processInfo = new ProcessStartInfo("cp", $"-r {absolutePath} {backupPath}"); + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + processInfo = new ProcessStartInfo("xcopy", $"{ConvertFromWslPathToWindowsPath(absolutePath)} {ConvertFromWslPathToWindowsPath(backupPath)} /E /H /K /O /X"); + } + else + { + throw new ArgumentOutOfRangeException(); + } + logger.Debug("Creating a backup of: " + absolutePath + " in a path: " + backupPath); + + processInfo.CreateNoWindow = true; + processInfo.UseShellExecute = false; + + using (var process = new Process()) + { + process.StartInfo = processInfo; + process.Start(); + // long as it may be huge backup + process.WaitForExit(3000000); + + process.Close(); + } + } + public static string ConvertFromWslPathToWindowsPath(string wslPath) { string windowsPath = ""; diff --git a/NethermindNode.Core/Helpers/DockerCommands.cs b/NethermindNode.Core/Helpers/DockerCommands.cs index e802b5b..3be1e03 100644 --- a/NethermindNode.Core/Helpers/DockerCommands.cs +++ b/NethermindNode.Core/Helpers/DockerCommands.cs @@ -50,6 +50,11 @@ public static string GetDockerDetails(string containerName, string dataToFetch, return result; } + public static void RecreateDockerCompose(string containerName, string dockerComposePath, Logger logger) + { + DockerCommandExecute("compose -f " + dockerComposePath + " create --force-recreate " + containerName, logger); + } + public static string GetExecutionDataPath(Logger logger) { return GetDockerDetails(ConfigurationHelper.Instance["execution-container-name"], "{{ range .Mounts }}{{ if eq .Destination \\\"/nethermind/data\\\" }}{{ .Source }}{{ end }}{{ end }}", logger).Trim(); diff --git a/NethermindNode.Core/Helpers/DockerComposeHelper.cs b/NethermindNode.Core/Helpers/DockerComposeHelper.cs new file mode 100644 index 0000000..8bb5c76 --- /dev/null +++ b/NethermindNode.Core/Helpers/DockerComposeHelper.cs @@ -0,0 +1,55 @@ +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace NethermindNode.Core.Helpers +{ + public static class DockerComposeHelper + { + public static Dictionary ReadDockerCompose(string filePath) + { + var deserializer = new DeserializerBuilder() + .WithNamingConvention(UnderscoredNamingConvention.Instance) + .Build(); + + using var reader = new StreamReader(filePath); + var yaml = reader.ReadToEnd(); + var result = deserializer.Deserialize>(yaml); + + return result; + } + + public static void WriteDockerCompose(Dictionary compose, string filePath) + { + var serializer = new SerializerBuilder() + .WithNamingConvention(UnderscoredNamingConvention.Instance) + .Build(); + + var yaml = serializer.Serialize(compose); + using var writer = new StreamWriter(filePath); + Console.WriteLine("Writing at path: " + filePath); + writer.Write(yaml); + } + + public static void RemoveCommandFlag(Dictionary dockerCompose, string serviceName, string flagToRemove) + { + var services = (Dictionary)dockerCompose["services"]; + if (services.ContainsKey(serviceName)) + { + var service = (Dictionary)services[serviceName]; + var command = (List)service["command"]; + command.Remove(flagToRemove); + } + } + + public static void AddCommandFlag(Dictionary dockerCompose, string serviceName, string flagToAdd) + { + var services = (Dictionary)dockerCompose["services"]; + if (services.ContainsKey(serviceName)) + { + var service = (Dictionary)services[serviceName]; + var command = (List)service["command"]; + command.Add(flagToAdd); + } + } + } +} diff --git a/NethermindNode.Core/NethermindNode.Core.csproj b/NethermindNode.Core/NethermindNode.Core.csproj index 8128953..5b802ef 100644 --- a/NethermindNode.Core/NethermindNode.Core.csproj +++ b/NethermindNode.Core/NethermindNode.Core.csproj @@ -16,6 +16,7 @@ + diff --git a/NethermindNode.Tests/NethermindNode.Tests.csproj b/NethermindNode.Tests/NethermindNode.Tests.csproj index 70f59ec..70d03c3 100644 --- a/NethermindNode.Tests/NethermindNode.Tests.csproj +++ b/NethermindNode.Tests/NethermindNode.Tests.csproj @@ -35,7 +35,10 @@ runtime; build; native; contentfiles; analyzers; buildtransitive + + + diff --git a/NethermindNode.Tests/Tests/Pruning/JsonRpcPruning.cs b/NethermindNode.Tests/Tests/Pruning/JsonRpcPruning.cs index a9b0b2e..3489faa 100644 --- a/NethermindNode.Tests/Tests/Pruning/JsonRpcPruning.cs +++ b/NethermindNode.Tests/Tests/Pruning/JsonRpcPruning.cs @@ -44,7 +44,7 @@ public void ShouldPruneDbUsingAdminRpc() var parameters = $""; var result = HttpExecutor.ExecuteNethermindJsonRpcCommand("admin_prune", parameters, TestItems.RpcAddress, Logger).Result.Item1; - Assert.IsTrue(result.Contains("Starting"), $"Result should contains \"Starting\" but it doesn't. Result content: {result}"); + Assert.IsTrue(result.ToLowerInvariant().Contains("starting"), $"Result should contains \"starting\" but it doesn't. Result content: {result}"); // Wait for maximum 60 seconds for pruning to be properly started Stopwatch stopwatch = new Stopwatch(); diff --git a/NethermindNode.Tests/Tests/SyncingNode/BodiesAndReceiptsFinalization.cs b/NethermindNode.Tests/Tests/SyncingNode/BodiesAndReceiptsFinalization.cs new file mode 100644 index 0000000..8185137 --- /dev/null +++ b/NethermindNode.Tests/Tests/SyncingNode/BodiesAndReceiptsFinalization.cs @@ -0,0 +1,108 @@ +using NethermindNode.Core.Helpers; +using Polly; +using Polly.Contrib.WaitAndRetry; + +namespace NethermindNode.Tests.SyncingNode +{ + [TestFixture] + public class BodiesAndReceiptsTests : BaseTest + { + + private static readonly NLog.Logger Logger = NLog.LogManager.GetLogger(TestContext.CurrentContext.Test.Name); + + [TestCase(10)] + [Category("BodiesAndReceipts")] + [Description( + """ + !!!!! Test to be used when NonValidator mode is true. !!!!! + + 1. Wait for node to be synced (eth_syncing returns false) + 2. Stop a execution node + 3. Create a backup of database + 4. Restart node but without NonValidator node flags + 5. Wait until end of sync (eth_syncing returns false) + 6. Check if logs "Fast blocks bodies task completed." and "Fast blocks receipts task completed." are there - those two should always been present meaning that tasks are properly finished. + 7. Stop a node, restore backup, redo steps 5 and 6 + + WHY? + If there is no such logs, there is high chance we "lost" something and we should investigate what is missing + """ + )] + public void ShouldResyncBodiesAndReceiptsAfterNonValidator(int repeatCount) + { + Logger.Info("***Starting test: ShouldResyncBodiesAndReceiptsAfterNonValidator***"); + + // 1 + NodeInfo.WaitForNodeToBeReady(Logger); + + var delay = Backoff.ConstantBackoff(TimeSpan.FromSeconds(3), retryCount: 100); + + var retryPolicy = Policy + .HandleResult(s => !s.Contains("SnapSync") && !s.Contains("StateNodes")) + .WaitAndRetry(delay); + + string result = retryPolicy.Execute(() => NodeInfo.GetCurrentStage(Logger)); + + NodeInfo.WaitForNodeToBeSynced(Logger); + + // 2 + DockerCommands.StopDockerContainer(ConfigurationHelper.Instance["execution-container-name"], Logger); + + // 3 + var execPath = DockerCommands.GetExecutionDataPath(Logger); + CommandExecutor.BackupDirectory(execPath + "/nethermind_db", execPath + "/nethermind_db_backup" , Logger); + + // 4 + string[] flagsToRemove = + { + "--Sync.NonValidatorNode=true", + "--Sync.DownloadBodiesInFastSync=false", + "--Sync.DownloadReceiptsInFastSync=false" + }; + + Logger.Info("Reading docker compose file at path: " + execPath + "/../docker-compose.yml"); + var dockerCompose = DockerComposeHelper.ReadDockerCompose(execPath + "/../docker-compose.yml"); + foreach (var flag in flagsToRemove) + { + Console.WriteLine("Removing: " + flag); + DockerComposeHelper.RemoveCommandFlag(dockerCompose, "execution", flag); + } + DockerComposeHelper.WriteDockerCompose(dockerCompose, execPath + "/../docker-compose.yml"); + DockerCommands.RecreateDockerCompose("execution", execPath + "/../docker-compose.yml", Logger); + DockerCommands.StartDockerContainer(ConfigurationHelper.Instance["execution-container-name"], Logger); + + // 5-6-7 + for (int i = 0; i < repeatCount; i++) + { + NodeInfo.WaitForNodeToBeReady(Logger); + NodeInfo.WaitForNodeToBeSynced(Logger); + var bodiesLine = DockerCommands.GetDockerLogs(ConfigurationHelper.Instance["execution-container-name"], "Fast blocks bodies task completed."); + var receiptsLine = DockerCommands.GetDockerLogs(ConfigurationHelper.Instance["execution-container-name"], "Fast blocks receipts task completed."); + + Assert.IsTrue(bodiesLine.Count() > 0, "Bodies log line missing - verify with getBlockByNumber."); + Assert.IsTrue(receiptsLine.Count() > 0, "Receipts log line missing - verify with getReceipt"); + + DockerCommands.StopDockerContainer(ConfigurationHelper.Instance["execution-container-name"], Logger); + + CommandExecutor.RemoveDirectory(execPath + "/nethermind_db", Logger); + CommandExecutor.BackupDirectory(execPath + "/nethermind_db_backup", execPath + "/nethermind_db", Logger); + + // For logs cleanup purpose only + DockerCommands.RecreateDockerCompose("execution", execPath + "/../docker-compose.yml", Logger); + + DockerCommands.StartDockerContainer(ConfigurationHelper.Instance["execution-container-name"], Logger); + } + + // Restore to previous state + foreach (var flag in flagsToRemove) + { + Console.WriteLine("Adding: " + flag); + DockerComposeHelper.AddCommandFlag(dockerCompose, "execution", flag); + } + DockerComposeHelper.WriteDockerCompose(dockerCompose, execPath + "/../docker-compose.yml"); + DockerCommands.RecreateDockerCompose("execution", execPath + "/../docker-compose.yml", Logger); + DockerCommands.StopDockerContainer(ConfigurationHelper.Instance["execution-container-name"], Logger); + DockerCommands.StartDockerContainer(ConfigurationHelper.Instance["execution-container-name"], Logger); + } + } +}