Add retries in case jenkins job has a build already enqueued (#1203)

Add retry mechanism when triggering a new build of a Jenkins
job. This retry mechanism also allows to set a retry with
exponential backoff.

Author: mrodm

.buildkite/pipeline.package-storage-publish.yml

@@ -19,7 +19,7 @@ steps:
     command: ".buildkite/scripts/signAndPublishPackage.sh"
     depends_on:
       - build-package
-    timeout_in_minutes: 30
+    timeout_in_minutes: 90
     agents:
       provider: "gcp"
       image: family/core-ubuntu-2004

.buildkite/scripts/triggerJenkinsJob/jenkins/jenkins.go

@@ -17,6 +17,13 @@ type JenkinsClient struct {
 	client *gojenkins.Jenkins
 }
 
+type Options struct {
+	WaitingTime    time.Duration
+	MaxWaitingTime time.Duration
+	GrowthFactor   float64
+	Retries        int
+}
+
 func NewJenkinsClient(ctx context.Context, host, user, token string) (*JenkinsClient, error) {
 	jenkins, err := gojenkins.CreateJenkins(nil, host, user, token).Init(ctx)
 	if err != nil {
@@ -28,12 +35,28 @@ func NewJenkinsClient(ctx context.Context, host, user, token string) (*JenkinsCl
 	}, nil
 }
 
-func (j *JenkinsClient) RunJob(ctx context.Context, jobName string, async bool, params map[string]string) error {
-	queueId, err := j.client.BuildJob(ctx, jobName, params)
-	if err != nil {
-		fmt.Printf("error running job %s : %s\n", jobName, err)
+func (j *JenkinsClient) RunJob(ctx context.Context, jobName string, async bool, params map[string]string, opts Options) error {
+	log.Printf("Building job %s", jobName)
+	var queueId int64
+
+	r := retry(func(ctx context.Context) error {
+		var err error
+		queueId, err = j.client.BuildJob(ctx, jobName, params)
+		if err != nil {
+			return fmt.Errorf("error running job %s: %w", jobName, err)
+		}
+
+		if queueId != 0 {
+			return nil
+		}
+		return fmt.Errorf("already running %s?", jobName)
+
+	}, opts.Retries, opts.GrowthFactor, opts.WaitingTime, opts.MaxWaitingTime)
+
+	if err := r(ctx); err != nil {
 		return err
 	}
+
 	build, err := j.getBuildFromJobAndQueueID(ctx, jobName, queueId)
 	if err != nil {
 		return err

.buildkite/scripts/triggerJenkinsJob/jenkins/retry.go

@@ -0,0 +1,48 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package jenkins
+
+import (
+	"context"
+	"log"
+	"math"
+	"time"
+)
+
+type retryableFunction func(context.Context) error
+
+func minDuration(a, b time.Duration) time.Duration {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+func retry(f retryableFunction, retries int, growthFactor float64, delay, maxDelay time.Duration) retryableFunction {
+	return func(ctx context.Context) error {
+		delaySeconds := delay.Seconds()
+		for r := 0; ; r++ {
+			err := f(ctx)
+			if err == nil || r >= retries {
+				// Return when there is no error or the maximum amount
+				// of retries is reached.
+				return err
+			}
+
+			waitingTimeSeconds := math.Pow(growthFactor, float64(r)) * delaySeconds
+			waitingTime := time.Duration(waitingTimeSeconds) * time.Second
+			waitingTime = minDuration(waitingTime, maxDelay)
+
+			log.Printf("Function failed, retrying in %v -> %.2f", waitingTime, waitingTimeSeconds)
+
+			select {
+			case <-time.After(waitingTime):
+			case <-ctx.Done():
+				return ctx.Err()
+			}
+		}
+		return nil
+	}
+}

.buildkite/scripts/triggerJenkinsJob/main.go

@@ -11,6 +11,7 @@ import (
 	"log"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/elastic/trigger-jenkins-buildkite-plugin/jenkins"
 )
@@ -44,6 +45,11 @@ func jenkinsJobOptions() []string {
 
 func main() {
 	jenkinsJob := flag.String("jenkins-job", "", fmt.Sprintf("Jenkins job to trigger. Allowed values: %s", strings.Join(jenkinsJobOptions(), " ,")))
+	waitingTime := flag.Duration("waiting-time", 5*time.Second, fmt.Sprintf("Waiting period between each retry"))
+	growthFactor := flag.Float64("growth-factor", 1.25, fmt.Sprintf("Growth-Factor used for exponential backoff delays"))
+	retries := flag.Int("retries", 20, fmt.Sprintf("Number of retries to trigger the job"))
+	maxWaitingTime := flag.Duration("max-waiting-time", 60*time.Minute, fmt.Sprintf("Maximum waiting time per each retry"))
+
 	folderPath := flag.String("folder", "", "Path to artifacts folder")
 	zipPackagePath := flag.String("package", "", "Path to zip package file (*.zip)")
 	sigPackagePath := flag.String("signature", "", "Path to the signature file of the package file (*.zip.sig)")
@@ -62,11 +68,18 @@ func main() {
 		log.Fatalf("error creating jenkins client")
 	}
 
+	opts := jenkins.Options{
+		WaitingTime:    *waitingTime,
+		Retries:        *retries,
+		GrowthFactor:   *growthFactor,
+		MaxWaitingTime: *maxWaitingTime,
+	}
+
 	switch *jenkinsJob {
 	case publishJobKey:
-		err = runPublishingRemoteJob(ctx, client, *async, allowedJenkinsJobs[*jenkinsJob], *zipPackagePath, *sigPackagePath)
+		err = runPublishingRemoteJob(ctx, client, *async, allowedJenkinsJobs[*jenkinsJob], *zipPackagePath, *sigPackagePath, opts)
 	case signJobKey:
-		err = runSignPackageJob(ctx, client, *async, allowedJenkinsJobs[*jenkinsJob], *folderPath)
+		err = runSignPackageJob(ctx, client, *async, allowedJenkinsJobs[*jenkinsJob], *folderPath, opts)
 	default:
 		log.Fatal("unsupported jenkins job")
 	}
@@ -76,18 +89,18 @@ func main() {
 	}
 }
 
-func runSignPackageJob(ctx context.Context, client *jenkins.JenkinsClient, async bool, jobName, folderPath string) error {
+func runSignPackageJob(ctx context.Context, client *jenkins.JenkinsClient, async bool, jobName, folderPath string, opts jenkins.Options) error {
 	if folderPath == "" {
 		return fmt.Errorf("missing parameter --gcs_input_path for")
 	}
 	params := map[string]string{
 		"gcs_input_path": folderPath,
 	}
 
-	return client.RunJob(ctx, jobName, async, params)
+	return client.RunJob(ctx, jobName, async, params, opts)
 }
 
-func runPublishingRemoteJob(ctx context.Context, client *jenkins.JenkinsClient, async bool, jobName, packagePath, signaturePath string) error {
+func runPublishingRemoteJob(ctx context.Context, client *jenkins.JenkinsClient, async bool, jobName, packagePath, signaturePath string, opts jenkins.Options) error {
 	if packagePath == "" {
 		return fmt.Errorf("missing parameter --gs_package_build_zip_path")
 	}
@@ -102,5 +115,5 @@ func runPublishingRemoteJob(ctx context.Context, client *jenkins.JenkinsClient,
 		"gs_package_signature_path": signaturePath,
 	}
 
-	return client.RunJob(ctx, jobName, async, params)
+	return client.RunJob(ctx, jobName, async, params, opts)
 }