Cleanup runner config when GetMessage returns 404

Author: nikola-jokic

src/Runner.Listener/Runner.cs

@@ -498,6 +498,7 @@ private async Task<int> RunAsync(RunnerSettings settings, bool runOnce = false,
                 bool skipSessionDeletion = false;
                 bool restartSession = false; // Flag to indicate session restart
                 bool restartSessionPending = false;
+                bool cleanupLocalConfigAfter404 = false;
                 try
                 {
                     var notification = HostContext.GetService<IJobNotification>();
@@ -813,6 +814,14 @@ await configUpdater.UpdateRunnerConfigAsync(
                                 Trace.Error($"Received message {message.MessageId} with unsupported message type {message.MessageType}.");
                             }
                         }
+                        catch (Exception ex) when (ex is TaskAgentNotFoundException || ex is RunnerNotFoundException)
+                        {
+                            Trace.Info($"Runner registration no longer exists while retrieving messages. {ex.Message}");
+                            _term.WriteError("The runner no longer exists on the server. Cleaning up local configuration.");
+                            skipSessionDeletion = true;
+                            cleanupLocalConfigAfter404 = true;
+                            break;
+                        }
                         finally
                         {
                             if (!skipMessageDeletion && message != null)
@@ -859,7 +868,7 @@ await configUpdater.UpdateRunnerConfigAsync(
 
                     messageQueueLoopTokenSource.Dispose();
 
-                    if (settings.Ephemeral && runOnceJobCompleted)
+                    if ((settings.Ephemeral && runOnceJobCompleted) || cleanupLocalConfigAfter404)
                     {
                         configManager.DeleteLocalRunnerConfig();
                     }

src/Test/L0/Listener/RunnerL0.cs

@@ -29,6 +29,7 @@ public sealed class RunnerL0 : IDisposable
         private Mock<ICredentialManager> _credentialManager;
         private Mock<IActionsRunServer> _actionsRunServer;
         private Mock<IRunServer> _runServer;
+        private Mock<IBrokerServer> _brokerServer;
         private readonly string _returnJobResultForHosted;
 
         public RunnerL0()
@@ -46,6 +47,7 @@ public RunnerL0()
             _credentialManager = new Mock<ICredentialManager>();
             _actionsRunServer = new Mock<IActionsRunServer>();
             _runServer = new Mock<IRunServer>();
+            _brokerServer = new Mock<IBrokerServer>();
 
             _returnJobResultForHosted = Environment.GetEnvironmentVariable("ACTIONS_RUNNER_RETURN_JOB_RESULT_FOR_HOSTED");
             Environment.SetEnvironmentVariable("ACTIONS_RUNNER_RETURN_JOB_RESULT_FOR_HOSTED", null);
@@ -175,6 +177,96 @@ public async Task TestRunAsync()
             }
         }
 
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Runner")]
+        public async Task TestRunAsyncCleanupLocalConfigWhenGetNextMessageReturnsNotFound()
+        {
+            using (var hc = new TestHostContext(this))
+            {
+                //Arrange
+                var runner = new Runner.Listener.Runner();
+                hc.SetSingleton<IConfigurationManager>(_configurationManager.Object);
+                hc.SetSingleton<IJobNotification>(_jobNotification.Object);
+                hc.SetSingleton<IPromptManager>(_promptManager.Object);
+                hc.SetSingleton<IRunnerServer>(_runnerServer.Object);
+                hc.SetSingleton<IBrokerServer>(_brokerServer.Object);
+                hc.SetSingleton<ICredentialManager>(_credentialManager.Object);
+                hc.SetSingleton<IConfigurationStore>(_configStore.Object);
+                hc.EnqueueInstance<IErrorThrottler>(_acquireJobThrottler.Object);
+                hc.EnqueueInstance<IJobDispatcher>(_jobDispatcher.Object);
+
+                var messageListener = new MessageListener();
+                messageListener.Initialize(hc);
+                hc.SetSingleton<IMessageListener>(messageListener);
+
+                runner.Initialize(hc);
+
+                var settings = new RunnerSettings
+                {
+                    AgentId = 1,
+                    AgentName = "myagent",
+                    PoolId = 43242,
+                    PoolName = "default",
+                    ServerUrl = "http://myserver",
+                    WorkFolder = "_work",
+                    Ephemeral = false,
+                };
+
+                _configurationManager.Setup(x => x.LoadSettings())
+                    .Returns(settings);
+                _configurationManager.Setup(x => x.IsConfigured())
+                    .Returns(true);
+                _credentialManager.Setup(x => x.LoadCredentials(false)).Returns(new VssCredentials());
+                _runnerServer.Setup(x => x.ConnectAsync(It.IsAny<Uri>(), It.IsAny<VssCredentials>()))
+                    .Returns(Task.CompletedTask);
+                _runnerServer.Setup(x => x.CreateAgentSessionAsync(
+                        settings.PoolId,
+                        It.Is<TaskAgentSession>(x => x != null),
+                        It.IsAny<CancellationToken>()))
+                    .Returns(Task.FromResult(new TaskAgentSession()));
+                _runnerServer.Setup(x => x.GetAgentMessageAsync(
+                        settings.PoolId,
+                        It.IsAny<Guid>(),
+                        It.IsAny<long?>(),
+                        TaskAgentStatus.Online,
+                        It.IsAny<string>(),
+                        It.IsAny<string>(),
+                        It.IsAny<string>(),
+                        It.IsAny<bool>(),
+                        It.IsAny<CancellationToken>()))
+                    .Throws(new TaskAgentNotFoundException("runner not found"));
+                _jobNotification.Setup(x => x.StartClient(It.IsAny<string>()));
+                _configStore.Setup(x => x.IsServiceConfigured()).Returns(false);
+
+                //Act
+                var command = new CommandSettings(hc, new string[] { "run" });
+                var result = await runner.ExecuteCommand(command);
+
+                //Assert
+                Assert.Equal(Constants.Runner.ReturnCode.Success, result);
+                _runnerServer.Verify(x => x.CreateAgentSessionAsync(
+                    settings.PoolId,
+                    It.Is<TaskAgentSession>(x => x != null),
+                    It.IsAny<CancellationToken>()), Times.Once());
+                _runnerServer.Verify(x => x.GetAgentMessageAsync(
+                    settings.PoolId,
+                    It.IsAny<Guid>(),
+                    It.IsAny<long?>(),
+                    TaskAgentStatus.Online,
+                    It.IsAny<string>(),
+                    It.IsAny<string>(),
+                    It.IsAny<string>(),
+                    It.IsAny<bool>(),
+                    It.IsAny<CancellationToken>()), Times.Once());
+                _runnerServer.Verify(x => x.DeleteAgentSessionAsync(
+                    It.IsAny<int>(),
+                    It.IsAny<Guid>(),
+                    It.IsAny<CancellationToken>()), Times.Never());
+                _configurationManager.Verify(x => x.DeleteLocalRunnerConfig(), Times.Once());
+            }
+        }
+
         public static TheoryData<string[], bool, Times> RunAsServiceTestData = new TheoryData<string[], bool, Times>()
         {
             // staring with run command, configured as run as service, should start the runner