Feature/curation header new

commands/curation/curationaudit.go

@@ -13,6 +13,7 @@
 	"strings"
 	"sync"
 
+	"github.com/google/uuid"
 	"github.com/jfrog/gofrog/datastructures"
 	"github.com/jfrog/gofrog/parallel"
 	rtUtils "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils"
@@ -78,6 +79,53 @@
 	MinArtiNuGetSupport       = "7.93.0"
 	MinXrayPassThroughSupport = "3.92.0"
 	MinArtiGradleGemSupport   = "7.63.5"
+
+	// Curation request headers
+	waiverHeader = "X-Artifactory-Curation-Request-Waiver"
+)
+
+func generateWaiverHeaderValue() string {
+	batchID := uuid.New().String()
+	headerData := map[string]string{
+		"batch_id": batchID,
+	}
+	jsonData, _ := json.Marshal(headerData)
+	return string(jsonData)
+}
+
+// addFieldToWaiverHeader adds a field to the existing waiver header JSON and returns the updated JSON string
+func addFieldToWaiverHeader(fieldName string, fieldValue interface{}) (string, error) {
+	// Parse the existing header
+	var headerData map[string]interface{}
+	if err := json.Unmarshal([]byte(waiverHeaderValue), &headerData); err != nil {
+		return "", fmt.Errorf("failed to parse waiver header: %v", err)
+	}
+	headerData[fieldName] = fieldValue
+	updatedHeaderData, err := json.Marshal(headerData)
+	if err != nil {
+		return "", fmt.Errorf("failed to marshal updated header: %v", err)
+	}
+	return string(updatedHeaderData), nil
+}
+
+// createRequestDetails creates a copy of httpClientDetails for thread-safe header modification
+func (nc *treeAnalyzer) createRequestDetails() httputils.HttpClientDetails {
+	requestDetails := nc.httpClientDetails
+	if requestDetails.Headers == nil {
+		requestDetails.Headers = make(map[string]string)
+	} else {
+		// Create a copy of the headers map
+		requestDetails.Headers = make(map[string]string)
+		for k, v := range nc.httpClientDetails.Headers {
+			requestDetails.Headers[k] = v
+		}
+	}
+	return requestDetails
+}
+
+var (
+	waiverHeaderValue string
+	blockedPackageUrl string
 )
 
 var CurationOutputFormats = []string{string(outFormat.Table), string(outFormat.Json)}
@@ -490,6 +538,12 @@
 		parallelRequests:     ca.parallelRequests,
 		downloadUrls:         depTreeResult.DownloadUrls,
 	}
+	// Initialize the waiver header value with batch ID from Xray API
+	waiverHeaderValue = generateWaiverHeaderValue()
+	blockedPackageUrl = ""
+
+	// Add curation client header to all requests
+	analyzer.httpClientDetails.Headers[waiverHeader] = waiverHeaderValue
 
 	rootNodes := map[string]struct{}{}
 	for _, tree := range depTreeResult.FullDepTrees {
@@ -509,6 +563,7 @@
 		// We subtract 1 because the root node is not a package.
 		totalNumberOfPackages: len(depTreeResult.FlatTree.Nodes) - 1,
 	}
+
 	return err
 }
 
@@ -571,7 +626,7 @@
 		return nil, err
 	}
 	clientDetails := rtAuth.CreateHttpClientDetails()
-	clientDetails.Headers["X-Artifactory-Curation-Request-Waiver"] = msg
+	clientDetails.Headers[waiverHeader] = msg
 	for _, pkg := range pkgs {
 		response, body, _, err := rtManager.Client().SendGet(pkg.BlockedPackageUrl, true, &clientDetails)
 		if err != nil {
@@ -788,12 +843,14 @@
 	var multiErrors error
 	consumerProducer := parallel.NewBounedRunner(nc.parallelRequests, false)
 	errorsQueue := clientutils.NewErrorsQueue(1)
+
 	go func() {
 		defer consumerProducer.Done()
 		for _, node := range graph.Nodes {
 			if _, ok := rootNodeIds[node.Id]; ok {
 				continue
 			}
+
 			getTask := func(node xrayUtils.GraphNode) func(threadId int) error {
 				return func(threadId int) (err error) {
 					return nc.fetchNodeStatus(node, p)
@@ -808,6 +865,15 @@
 	if err := errorsQueue.GetError(); err != nil {
 		multiErrors = errors.Join(err, multiErrors)
 	}
+
+	// After all parallel processing is done, send the completed request
+	// Only send completed request if there are blocked packages
+	if blockedPackageUrl != "" {
+		if err := nc.sendCompletedRequest(blockedPackageUrl); err != nil {
+			multiErrors = errors.Join(err, multiErrors)
+		}
+	}
+
 	return multiErrors
 }
 
@@ -819,8 +885,15 @@
 	if scope != "" {
 		name = scope + "/" + name
 	}
+
 	for _, packageUrl := range packageUrls {
-		resp, _, err := nc.rtManager.Client().SendHead(packageUrl, &nc.httpClientDetails)
+		// Create a copy of httpClientDetails for this request to avoid modifying the shared one
+		requestDetails := nc.createRequestDetails()
+
+		// Set the header for regular HEAD request
+		requestDetails.Headers[waiverHeader] = waiverHeaderValue
+
+		resp, _, err := nc.rtManager.Client().SendHead(packageUrl, &requestDetails)
 		if err != nil {
 			if resp != nil && resp.StatusCode >= 400 {
 				return errorutils.CheckErrorf(errorTemplateHeadRequest, packageUrl, name, version, resp.StatusCode, err)
@@ -837,6 +910,11 @@
 			return errorutils.CheckErrorf(errorTemplateHeadRequest, packageUrl, name, version, resp.StatusCode, err)
 		}
 		if resp.StatusCode == http.StatusForbidden {
+			// Track this blocked package URL (only need one for completed request)
+			if blockedPackageUrl == "" {
+				blockedPackageUrl = packageUrl
+			}
+
 			pkStatus, err := nc.getBlockedPackageDetails(packageUrl, name, version)
 			if err != nil {
 				return err
@@ -857,8 +935,11 @@
 
 // We try to collect curation details from GET response after HEAD request got forbidden status code.
 func (nc *treeAnalyzer) getBlockedPackageDetails(packageUrl string, name string, version string) (*PackageStatus, error) {
-	nc.httpClientDetails.Headers["X-Artifactory-Curation-Request-Waiver"] = "syn"
-	getResp, respBody, _, err := nc.rtManager.Client().SendGet(packageUrl, true, &nc.httpClientDetails)
+	// Create a copy of httpClientDetails for this request to avoid modifying the shared one
+	requestDetails := nc.createRequestDetails()
+	requestDetails.Headers[waiverHeader] = "syn"
+
+	getResp, respBody, _, err := nc.rtManager.Client().SendGet(packageUrl, true, &requestDetails)
 	if err != nil {
 		if getResp == nil {
 			return nil, err
@@ -898,6 +979,17 @@
 	return nil, nil
 }
 
+func (nc *treeAnalyzer) sendCompletedRequest(packageUrl string) error {
+	requestDetails := nc.createRequestDetails()
+	completedHeaderData, err := addFieldToWaiverHeader("completed", true)
+	if err != nil {
+		return err
+	}
+	requestDetails.Headers[waiverHeader] = completedHeaderData
+	_, _, err = nc.rtManager.Client().SendHead(packageUrl, &requestDetails)
+	return err
+}
+
 // Return policies and conditions names from the FORBIDDEN HTTP error message.
 // Message structure: Package %s:%s download was blocked by JFrog Packages Curation service due to the following policies violated {%s, %s, %s, %s},{%s, %s, %s, %s}.
 func (nc *treeAnalyzer) extractPoliciesFromMsg(respError *ErrorsResp) []Policy {

commands/curation/curationaudit_test.go

@@ -20,6 +20,7 @@ import (
 
 	biutils "github.com/jfrog/build-info-go/utils"
 	"github.com/jfrog/gofrog/datastructures"
+	rtUtils "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils"
 	coreCommonTests "github.com/jfrog/jfrog-cli-core/v2/common/tests"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/config"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
@@ -484,6 +485,101 @@ func createTempHomeDirWithConfig(t *testing.T, basePathToTests string, testCase
 	}
 }
 
+func TestCurationHeaders(t *testing.T) {
+	// Test that batch ID generation works correctly
+	headerValue := generateWaiverHeaderValue()
+
+	// Verify it's valid JSON with batch_id
+	var headerData map[string]interface{}
+	err := json.Unmarshal([]byte(headerValue), &headerData)
+	require.NoError(t, err)
+	assert.Contains(t, headerData, "batch_id")
+	assert.NotEmpty(t, headerData["batch_id"])
+
+	// Test that completed request adds "completed" flag
+	var capturedHeader http.Header
+	testHandler := func(w http.ResponseWriter, r *http.Request) {
+		capturedHeader = r.Header.Clone()
+		w.WriteHeader(http.StatusOK)
+	}
+
+	mockServer, mockServerDetails, _ := coreCommonTests.CreateRtRestsMockServer(t, testHandler)
+	defer mockServer.Close()
+
+	// Create analyzer and send completed request
+	rtAuth, err := mockServerDetails.CreateArtAuthConfig()
+	require.NoError(t, err)
+	rtManager, err := rtUtils.CreateServiceManager(mockServerDetails, 2, 0, false)
+	require.NoError(t, err)
+	analyzer := &treeAnalyzer{
+		rtManager:         rtManager,
+		httpClientDetails: rtAuth.CreateHttpClientDetails(),
+	}
+	// Use JSON header format like: {"batch_id":"196c55b0-5724-4a94-aa45-9b0bfd658985"}
+	waiverHeaderValue = `{"batch_id":"196c55b0-5724-4a94-aa45-9b0bfd658985"}`
+	testUrl := mockServerDetails.GetArtifactoryUrl() + "/test/pkg"
+	err = analyzer.sendCompletedRequest(testUrl)
+	assert.NoError(t, err)
+	assert.NotNil(t, capturedHeader)
+	waiverHeader := capturedHeader["X-Artifactory-Curation-Request-Waiver"]
+
+	// Parse the completed header JSON
+	var completedHeaderData map[string]interface{}
+	err = json.Unmarshal([]byte(waiverHeader[0]), &completedHeaderData)
+	require.NoError(t, err)
+	assert.Equal(t, "196c55b0-5724-4a94-aa45-9b0bfd658985", completedHeaderData["batch_id"])
+	assert.Equal(t, true, completedHeaderData["completed"])
+}
+
+func TestCurationSynHeader(t *testing.T) {
+	// Test that syn field is added correctly to the header
+	headerValue := generateWaiverHeaderValue()
+
+	// Verify it's valid JSON with batch_id
+	var headerData map[string]interface{}
+	err := json.Unmarshal([]byte(headerValue), &headerData)
+	require.NoError(t, err)
+	assert.Contains(t, headerData, "batch_id")
+	assert.NotEmpty(t, headerData["batch_id"])
+
+	// Add syn field (simulating what happens in getBlockedPackageDetails)
+	headerData["syn"] = true
+	synHeaderData, err := json.Marshal(headerData)
+	require.NoError(t, err)
+
+	// Verify the final JSON contains both batch_id and syn
+	var finalHeaderData map[string]interface{}
+	err = json.Unmarshal(synHeaderData, &finalHeaderData)
+	require.NoError(t, err)
+	assert.Equal(t, true, finalHeaderData["syn"])
+	assert.Contains(t, finalHeaderData, "batch_id")
+}
+
+func TestAddFieldToWaiverHeader(t *testing.T) {
+	// Test the helper function directly
+	waiverHeaderValue = `{"batch_id":"test-batch-123"}`
+
+	// Add syn field
+	synHeader, err := addFieldToWaiverHeader("syn", true)
+	require.NoError(t, err)
+
+	var synData map[string]interface{}
+	err = json.Unmarshal([]byte(synHeader), &synData)
+	require.NoError(t, err)
+	assert.Equal(t, "test-batch-123", synData["batch_id"])
+	assert.Equal(t, true, synData["syn"])
+
+	// Add completed field
+	completedHeader, err := addFieldToWaiverHeader("completed", true)
+	require.NoError(t, err)
+
+	var completedData map[string]interface{}
+	err = json.Unmarshal([]byte(completedHeader), &completedData)
+	require.NoError(t, err)
+	assert.Equal(t, "test-batch-123", completedData["batch_id"])
+	assert.Equal(t, true, completedData["completed"])
+}
+
 func setCurationFlagsForTest(t *testing.T) func() {
 	callbackCurationFlag := clienttestutils.SetEnvWithCallbackAndAssert(t, utils.CurationSupportFlag, "true")
 	// Golang option to disable the use of the checksum database