diff --git a/commands/curation/curationaudit.go b/commands/curation/curationaudit.go index 77d3a0fc2..6a64169dc 100644 --- a/commands/curation/curationaudit.go +++ b/commands/curation/curationaudit.go @@ -13,6 +13,7 @@ import ( "strings" "sync" + "github.com/google/uuid" "github.com/jfrog/gofrog/datastructures" "github.com/jfrog/gofrog/parallel" rtUtils "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" @@ -78,6 +79,53 @@ const ( MinArtiNuGetSupport = "7.93.0" MinXrayPassThroughSupport = "3.92.0" MinArtiGradleGemSupport = "7.63.5" + + // Curation request headers + waiverHeader = "X-Artifactory-Curation-Request-Waiver" +) + +func generateWaiverHeaderValue() string { + batchID := uuid.New().String() + headerData := map[string]string{ + "batch_id": batchID, + } + jsonData, _ := json.Marshal(headerData) + return string(jsonData) +} + +// addFieldToWaiverHeader adds a field to the existing waiver header JSON and returns the updated JSON string +func addFieldToWaiverHeader(fieldName string, fieldValue interface{}) (string, error) { + // Parse the existing header + var headerData map[string]interface{} + if err := json.Unmarshal([]byte(waiverHeaderValue), &headerData); err != nil { + return "", fmt.Errorf("failed to parse waiver header: %v", err) + } + headerData[fieldName] = fieldValue + updatedHeaderData, err := json.Marshal(headerData) + if err != nil { + return "", fmt.Errorf("failed to marshal updated header: %v", err) + } + return string(updatedHeaderData), nil +} + +// createRequestDetails creates a copy of httpClientDetails for thread-safe header modification +func (nc *treeAnalyzer) createRequestDetails() httputils.HttpClientDetails { + requestDetails := nc.httpClientDetails + if requestDetails.Headers == nil { + requestDetails.Headers = make(map[string]string) + } else { + // Create a copy of the headers map + requestDetails.Headers = make(map[string]string) + for k, v := range nc.httpClientDetails.Headers { + requestDetails.Headers[k] = v + } + } + return requestDetails +} + +var ( + waiverHeaderValue string + blockedPackageUrl string ) var CurationOutputFormats = []string{string(outFormat.Table), string(outFormat.Json)} @@ -490,6 +538,12 @@ func (ca *CurationAuditCommand) auditTree(tech techutils.Technology, results map parallelRequests: ca.parallelRequests, downloadUrls: depTreeResult.DownloadUrls, } + // Initialize the waiver header value with batch ID from Xray API + waiverHeaderValue = generateWaiverHeaderValue() + blockedPackageUrl = "" + + // Add curation client header to all requests + analyzer.httpClientDetails.Headers[waiverHeader] = waiverHeaderValue rootNodes := map[string]struct{}{} for _, tree := range depTreeResult.FullDepTrees { @@ -509,6 +563,7 @@ func (ca *CurationAuditCommand) auditTree(tech techutils.Technology, results map // We subtract 1 because the root node is not a package. totalNumberOfPackages: len(depTreeResult.FlatTree.Nodes) - 1, } + return err } @@ -571,7 +626,7 @@ func (ca *CurationAuditCommand) sendWaiverRequests(pkgs []*PackageStatus, msg st return nil, err } clientDetails := rtAuth.CreateHttpClientDetails() - clientDetails.Headers["X-Artifactory-Curation-Request-Waiver"] = msg + clientDetails.Headers[waiverHeader] = msg for _, pkg := range pkgs { response, body, _, err := rtManager.Client().SendGet(pkg.BlockedPackageUrl, true, &clientDetails) if err != nil { @@ -788,12 +843,14 @@ func (nc *treeAnalyzer) fetchNodesStatus(graph *xrayUtils.GraphNode, p *sync.Map var multiErrors error consumerProducer := parallel.NewBounedRunner(nc.parallelRequests, false) errorsQueue := clientutils.NewErrorsQueue(1) + go func() { defer consumerProducer.Done() for _, node := range graph.Nodes { if _, ok := rootNodeIds[node.Id]; ok { continue } + getTask := func(node xrayUtils.GraphNode) func(threadId int) error { return func(threadId int) (err error) { return nc.fetchNodeStatus(node, p) @@ -808,6 +865,15 @@ func (nc *treeAnalyzer) fetchNodesStatus(graph *xrayUtils.GraphNode, p *sync.Map if err := errorsQueue.GetError(); err != nil { multiErrors = errors.Join(err, multiErrors) } + + // After all parallel processing is done, send the completed request + // Only send completed request if there are blocked packages + if blockedPackageUrl != "" { + if err := nc.sendCompletedRequest(blockedPackageUrl); err != nil { + multiErrors = errors.Join(err, multiErrors) + } + } + return multiErrors } @@ -819,8 +885,15 @@ func (nc *treeAnalyzer) fetchNodeStatus(node xrayUtils.GraphNode, p *sync.Map) e if scope != "" { name = scope + "/" + name } + for _, packageUrl := range packageUrls { - resp, _, err := nc.rtManager.Client().SendHead(packageUrl, &nc.httpClientDetails) + // Create a copy of httpClientDetails for this request to avoid modifying the shared one + requestDetails := nc.createRequestDetails() + + // Set the header for regular HEAD request + requestDetails.Headers[waiverHeader] = waiverHeaderValue + + resp, _, err := nc.rtManager.Client().SendHead(packageUrl, &requestDetails) if err != nil { if resp != nil && resp.StatusCode >= 400 { return errorutils.CheckErrorf(errorTemplateHeadRequest, packageUrl, name, version, resp.StatusCode, err) @@ -837,6 +910,11 @@ func (nc *treeAnalyzer) fetchNodeStatus(node xrayUtils.GraphNode, p *sync.Map) e return errorutils.CheckErrorf(errorTemplateHeadRequest, packageUrl, name, version, resp.StatusCode, err) } if resp.StatusCode == http.StatusForbidden { + // Track this blocked package URL (only need one for completed request) + if blockedPackageUrl == "" { + blockedPackageUrl = packageUrl + } + pkStatus, err := nc.getBlockedPackageDetails(packageUrl, name, version) if err != nil { return err @@ -857,8 +935,11 @@ func (nc *treeAnalyzer) fetchNodeStatus(node xrayUtils.GraphNode, p *sync.Map) e // We try to collect curation details from GET response after HEAD request got forbidden status code. func (nc *treeAnalyzer) getBlockedPackageDetails(packageUrl string, name string, version string) (*PackageStatus, error) { - nc.httpClientDetails.Headers["X-Artifactory-Curation-Request-Waiver"] = "syn" - getResp, respBody, _, err := nc.rtManager.Client().SendGet(packageUrl, true, &nc.httpClientDetails) + // Create a copy of httpClientDetails for this request to avoid modifying the shared one + requestDetails := nc.createRequestDetails() + requestDetails.Headers[waiverHeader] = "syn" + + getResp, respBody, _, err := nc.rtManager.Client().SendGet(packageUrl, true, &requestDetails) if err != nil { if getResp == nil { return nil, err @@ -898,6 +979,17 @@ func (nc *treeAnalyzer) getBlockedPackageDetails(packageUrl string, name string, return nil, nil } +func (nc *treeAnalyzer) sendCompletedRequest(packageUrl string) error { + requestDetails := nc.createRequestDetails() + completedHeaderData, err := addFieldToWaiverHeader("completed", true) + if err != nil { + return err + } + requestDetails.Headers[waiverHeader] = completedHeaderData + _, _, err = nc.rtManager.Client().SendHead(packageUrl, &requestDetails) + return err +} + // Return policies and conditions names from the FORBIDDEN HTTP error message. // Message structure: Package %s:%s download was blocked by JFrog Packages Curation service due to the following policies violated {%s, %s, %s, %s},{%s, %s, %s, %s}. func (nc *treeAnalyzer) extractPoliciesFromMsg(respError *ErrorsResp) []Policy { diff --git a/commands/curation/curationaudit_test.go b/commands/curation/curationaudit_test.go index 86ed6d6d2..31f15c25e 100644 --- a/commands/curation/curationaudit_test.go +++ b/commands/curation/curationaudit_test.go @@ -20,6 +20,7 @@ import ( biutils "github.com/jfrog/build-info-go/utils" "github.com/jfrog/gofrog/datastructures" + rtUtils "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" coreCommonTests "github.com/jfrog/jfrog-cli-core/v2/common/tests" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" @@ -484,6 +485,101 @@ func createTempHomeDirWithConfig(t *testing.T, basePathToTests string, testCase } } +func TestCurationHeaders(t *testing.T) { + // Test that batch ID generation works correctly + headerValue := generateWaiverHeaderValue() + + // Verify it's valid JSON with batch_id + var headerData map[string]interface{} + err := json.Unmarshal([]byte(headerValue), &headerData) + require.NoError(t, err) + assert.Contains(t, headerData, "batch_id") + assert.NotEmpty(t, headerData["batch_id"]) + + // Test that completed request adds "completed" flag + var capturedHeader http.Header + testHandler := func(w http.ResponseWriter, r *http.Request) { + capturedHeader = r.Header.Clone() + w.WriteHeader(http.StatusOK) + } + + mockServer, mockServerDetails, _ := coreCommonTests.CreateRtRestsMockServer(t, testHandler) + defer mockServer.Close() + + // Create analyzer and send completed request + rtAuth, err := mockServerDetails.CreateArtAuthConfig() + require.NoError(t, err) + rtManager, err := rtUtils.CreateServiceManager(mockServerDetails, 2, 0, false) + require.NoError(t, err) + analyzer := &treeAnalyzer{ + rtManager: rtManager, + httpClientDetails: rtAuth.CreateHttpClientDetails(), + } + // Use JSON header format like: {"batch_id":"196c55b0-5724-4a94-aa45-9b0bfd658985"} + waiverHeaderValue = `{"batch_id":"196c55b0-5724-4a94-aa45-9b0bfd658985"}` + testUrl := mockServerDetails.GetArtifactoryUrl() + "/test/pkg" + err = analyzer.sendCompletedRequest(testUrl) + assert.NoError(t, err) + assert.NotNil(t, capturedHeader) + waiverHeader := capturedHeader["X-Artifactory-Curation-Request-Waiver"] + + // Parse the completed header JSON + var completedHeaderData map[string]interface{} + err = json.Unmarshal([]byte(waiverHeader[0]), &completedHeaderData) + require.NoError(t, err) + assert.Equal(t, "196c55b0-5724-4a94-aa45-9b0bfd658985", completedHeaderData["batch_id"]) + assert.Equal(t, true, completedHeaderData["completed"]) +} + +func TestCurationSynHeader(t *testing.T) { + // Test that syn field is added correctly to the header + headerValue := generateWaiverHeaderValue() + + // Verify it's valid JSON with batch_id + var headerData map[string]interface{} + err := json.Unmarshal([]byte(headerValue), &headerData) + require.NoError(t, err) + assert.Contains(t, headerData, "batch_id") + assert.NotEmpty(t, headerData["batch_id"]) + + // Add syn field (simulating what happens in getBlockedPackageDetails) + headerData["syn"] = true + synHeaderData, err := json.Marshal(headerData) + require.NoError(t, err) + + // Verify the final JSON contains both batch_id and syn + var finalHeaderData map[string]interface{} + err = json.Unmarshal(synHeaderData, &finalHeaderData) + require.NoError(t, err) + assert.Equal(t, true, finalHeaderData["syn"]) + assert.Contains(t, finalHeaderData, "batch_id") +} + +func TestAddFieldToWaiverHeader(t *testing.T) { + // Test the helper function directly + waiverHeaderValue = `{"batch_id":"test-batch-123"}` + + // Add syn field + synHeader, err := addFieldToWaiverHeader("syn", true) + require.NoError(t, err) + + var synData map[string]interface{} + err = json.Unmarshal([]byte(synHeader), &synData) + require.NoError(t, err) + assert.Equal(t, "test-batch-123", synData["batch_id"]) + assert.Equal(t, true, synData["syn"]) + + // Add completed field + completedHeader, err := addFieldToWaiverHeader("completed", true) + require.NoError(t, err) + + var completedData map[string]interface{} + err = json.Unmarshal([]byte(completedHeader), &completedData) + require.NoError(t, err) + assert.Equal(t, "test-batch-123", completedData["batch_id"]) + assert.Equal(t, true, completedData["completed"]) +} + func setCurationFlagsForTest(t *testing.T) func() { callbackCurationFlag := clienttestutils.SetEnvWithCallbackAndAssert(t, utils.CurationSupportFlag, "true") // Golang option to disable the use of the checksum database