Merge pull request #54 from bitrise-io/skip-source-packages

feat: ACI-2805 Add flag to skip SPM

Author: zsolt-marta-bitrise

cmd/saveXcodeDerivedDataFiles.go

@@ -39,6 +39,7 @@ var saveXcodeDerivedDataFilesCmd = &cobra.Command{
 		ddPath, _ := cmd.Flags().GetString("deriveddata-path")
 		xcodeCachePath, _ := cmd.Flags().GetString("xcodecache-path")
 		followSymlinks, _ := cmd.Flags().GetBool("follow-symlinks")
+		skipSPM, _ := cmd.Flags().GetBool("skip-spm")
 
 		tracker := xcode.NewDefaultStepTracker("save-xcode-build-cache", os.Getenv, logger)
 		defer tracker.Wait()
@@ -58,6 +59,7 @@ var saveXcodeDerivedDataFilesCmd = &cobra.Command{
 			ddPath,
 			xcodeCachePath,
 			followSymlinks,
+			skipSPM,
 			logger,
 			tracker,
 			startT,
@@ -99,6 +101,7 @@ func init() {
 	}
 	saveXcodeDerivedDataFilesCmd.Flags().String("xcodecache-path", "", "Path to the Xcode cache directory folder to be saved. If not set, it will not be uploaded.")
 	saveXcodeDerivedDataFilesCmd.Flags().Bool("follow-symlinks", false, "Follow symlinks when calculating metadata and save referenced files to the cache (default: false)")
+	saveXcodeDerivedDataFilesCmd.Flags().Bool("skip-spm", false, "Skip saving files under \"DerivedData/*/SourcePackages\", i.e. skip SPM dependencies. Consider enabling this flag if using SPM cache steps. Default: false")
 }
 
 func saveXcodeDerivedDataFilesCmdFn(ctx context.Context,
@@ -109,6 +112,7 @@ func saveXcodeDerivedDataFilesCmdFn(ctx context.Context,
 	derivedDataPath,
 	xcodeCachePath string,
 	followSymlinks bool,
+	skipSPM bool,
 	logger log.Logger,
 	tracker xcode.StepAnalyticsTracker,
 	startT time.Time,
@@ -150,6 +154,7 @@ func saveXcodeDerivedDataFilesCmdFn(ctx context.Context,
 		XcodeCacheDirPath:  xcodeCachePath,
 		CacheKey:           cacheKey,
 		FollowSymlinks:     followSymlinks,
+		SkipSPM:            skipSPM,
 	}, envProvider, logger)
 	if err != nil {
 		return op, fmt.Errorf("create metadata: %w", err)

cmd/saveXcodeDerivedDataFiles_test.go

@@ -36,7 +36,7 @@ func Test_saveXcodeDerivedDataFilesCmdFn(t *testing.T) {
 		envVars := createEnvProvider(map[string]string{
 			"BITRISEIO_BITRISE_SERVICES_ACCESS_TOKEN": "ServiceAccessTokenValue",
 		})
-		_, err := saveXcodeDerivedDataFilesCmdFn(context.Background(), common.CacheAuthConfig{}, "", "", "", "", "", false, mockLogger, mockTracker, time.Now(), envVars)
+		_, err := saveXcodeDerivedDataFilesCmdFn(context.Background(), common.CacheAuthConfig{}, "", "", "", "", "", false, false, mockLogger, mockTracker, time.Now(), envVars)
 
 		// then
 		require.EqualError(t, err, "get cache key: cache key is required if BITRISE_GIT_BRANCH env var is not set")

internal/xcode/file_group_info.go

@@ -16,6 +16,8 @@ import (
 	"io"
 	"syscall"
 
+	"strings"
+
 	"github.com/bitrise-io/go-utils/v2/log"
 )
 
@@ -86,7 +88,11 @@ func (mc *fileGroupInfoCollector) isSeen(path string) bool {
 	return mc.seen[path]
 }
 
-func collectFileGroupInfo(cacheDirPath string, collectAttributes, followSymlinks bool, logger log.Logger) (FileGroupInfo, error) {
+func collectFileGroupInfo(cacheDirPath string,
+	collectAttributes,
+	followSymlinks bool,
+	skipSPM bool,
+	logger log.Logger) (FileGroupInfo, error) {
 	var dd FileGroupInfo
 
 	fgi := fileGroupInfoCollector{
@@ -120,7 +126,7 @@ func collectFileGroupInfo(cacheDirPath string, collectAttributes, followSymlinks
 			if !filepath.IsAbs(path) {
 				path = filepath.Join(cacheDirPath, path)
 			}
-			if err := collectFileMetadata(path, inf, inf.IsDir(), &fgi, collectAttributes, followSymlinks, logger); err != nil {
+			if err := collectFileMetadata(cacheDirPath, path, inf, inf.IsDir(), &fgi, collectAttributes, followSymlinks, skipSPM, logger); err != nil {
 				logger.Errorf("Failed to collect metadata: %s", err)
 			}
 		}(d)
@@ -145,7 +151,11 @@ func collectFileGroupInfo(cacheDirPath string, collectAttributes, followSymlinks
 }
 
 // nolint:wrapcheck
-func followSymlink(path string, target string, fgi *fileGroupInfoCollector, followSymlinks bool, logger log.Logger) error {
+func followSymlink(rootPath, path, target string,
+	fgi *fileGroupInfoCollector,
+	followSymlinks,
+	skipSPM bool,
+	logger log.Logger) error {
 	if !followSymlinks {
 		logger.Debugf("Skipping symbolic link: %s", path)
 
@@ -172,7 +182,7 @@ func followSymlink(path string, target string, fgi *fileGroupInfoCollector, foll
 	})
 
 	if !stat.IsDir() {
-		return collectFileMetadata(target, stat, false, fgi, false, followSymlinks, logger)
+		return collectFileMetadata(rootPath, target, stat, false, fgi, false, followSymlinks, skipSPM, logger)
 	}
 
 	logger.Debugf("Symlink target is a directory, walking it: %s", target)
@@ -191,11 +201,17 @@ func followSymlink(path string, target string, fgi *fileGroupInfoCollector, foll
 			path = filepath.Join(target, path)
 		}
 
-		return collectFileMetadata(path, inf, inf.IsDir(), fgi, false, followSymlinks, logger)
+		return collectFileMetadata(target, path, inf, inf.IsDir(), fgi, false, followSymlinks, skipSPM, logger)
 	})
 }
 
-func collectFileMetadata(path string, fileInfo fs.FileInfo, isDirectory bool, fgi *fileGroupInfoCollector, collectAttributes, followSymlinks bool, logger log.Logger) error {
+func collectFileMetadata(
+	rootPath, path string,
+	fileInfo fs.FileInfo,
+	isDirectory bool,
+	fgi *fileGroupInfoCollector,
+	collectAttributes, followSymlinks, skipSPM bool,
+	logger log.Logger) error {
 	if fgi.isSeen(path) {
 		logger.Debugf("Skipping path %s, already seen", path)
 
@@ -211,6 +227,17 @@ func collectFileMetadata(path string, fileInfo fs.FileInfo, isDirectory bool, fg
 		return nil
 	}
 
+	if skipSPM {
+		relPath, _ := filepath.Rel(rootPath, path)
+		parts := strings.Split(filepath.ToSlash(relPath), "/")
+
+		// Checking for SourcePackages/* or */SourcePackages/* under the directory
+		if len(parts) >= 1 && parts[0] == "SourcePackages" ||
+			len(parts) >= 2 && parts[1] == "SourcePackages" {
+			return nil
+		}
+	}
+
 	isSymlink := fileInfo.Mode()&os.ModeSymlink != 0
 
 	if isSymlink {
@@ -223,7 +250,7 @@ func collectFileMetadata(path string, fileInfo fs.FileInfo, isDirectory bool, fg
 			target = filepath.Join(filepath.Dir(path), target)
 		}
 
-		return followSymlink(path, target, fgi, followSymlinks, logger)
+		return followSymlink(rootPath, path, target, fgi, followSymlinks, skipSPM, logger)
 	}
 
 	file, err := os.Open(path)

internal/xcode/metadata.go

@@ -33,6 +33,7 @@ type CreateMetadataParams struct {
 	XcodeCacheDirPath  string
 	CacheKey           string
 	FollowSymlinks     bool
+	SkipSPM            bool
 }
 
 func CreateMetadata(params CreateMetadataParams, envProvider func(string) string, logger log.Logger) (*Metadata, error) {
@@ -43,6 +44,7 @@ func CreateMetadata(params CreateMetadataParams, envProvider func(string) string
 	projectFiles, err := collectFileGroupInfo(params.ProjectRootDirPath,
 		true,
 		params.FollowSymlinks,
+		false,
 		logger)
 	if err != nil {
 		return nil, fmt.Errorf("calculate project files info: %w", err)
@@ -53,6 +55,7 @@ func CreateMetadata(params CreateMetadataParams, envProvider func(string) string
 		derivedData, err = collectFileGroupInfo(params.DerivedDataPath,
 			false,
 			params.FollowSymlinks,
+			params.SkipSPM,
 			logger)
 		if err != nil {
 			return nil, fmt.Errorf("calculate derived data info: %w", err)
@@ -64,6 +67,7 @@ func CreateMetadata(params CreateMetadataParams, envProvider func(string) string
 		xcodeCacheDir, err = collectFileGroupInfo(params.XcodeCacheDirPath,
 			false,
 			params.FollowSymlinks,
+			params.SkipSPM,
 			logger)
 		if err != nil {
 			return nil, fmt.Errorf("calculate xcode cache dir info: %w", err)