From b7f6d335afff9042d8ccbaa7099decf998dd1711 Mon Sep 17 00:00:00 2001 From: Joana Hrotko Date: Wed, 12 Feb 2025 18:00:47 +0000 Subject: [PATCH 1/3] Process multiple paths for watch config Signed-off-by: Joana Hrotko --- go.mod | 3 + go.sum | 4 +- pkg/compose/watch.go | 205 +++++++++++++++++++++++++------------------ 3 files changed, 124 insertions(+), 88 deletions(-) diff --git a/go.mod b/go.mod index 29724e71df9..e8ce78231f7 100644 --- a/go.mod +++ b/go.mod @@ -168,6 +168,7 @@ require ( github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect + github.com/xhit/go-str2duration/v2 v2.1.0 // indirect github.com/zclconf/go-cty v1.16.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.56.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0 // indirect @@ -198,3 +199,5 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) + +replace github.com/compose-spec/compose-go/v2 v2.4.8 => ../compose-go diff --git a/go.sum b/go.sum index 6fe7eb2be55..8b72a43c3d1 100644 --- a/go.sum +++ b/go.sum @@ -81,8 +81,6 @@ github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004 h1:lkAMpLVBDaj17e github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004/go.mod h1:yMWuSON2oQp+43nFtAV/uvKQIFpSPerB57DCt9t8sSA= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4= -github.com/compose-spec/compose-go/v2 v2.4.8 h1:7Myl8wDRl/4mRz77S+eyDJymGGEHu0diQdGSSeyq90A= -github.com/compose-spec/compose-go/v2 v2.4.8/go.mod h1:lFN0DrMxIncJGYAXTfWuajfwj5haBJqrBkarHcnjJKc= github.com/containerd/cgroups/v3 v3.0.5 h1:44na7Ud+VwyE7LIoJ8JTNQOa549a8543BmzaJHo6Bzo= github.com/containerd/cgroups/v3 v3.0.5/go.mod h1:SA5DLYnXO8pTGYiAHXz94qvLQTKfVM5GEVisn4jpins= github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn4ro= @@ -494,6 +492,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= +github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= diff --git a/pkg/compose/watch.go b/pkg/compose/watch.go index 31c811e77e2..2a936f96834 100644 --- a/pkg/compose/watch.go +++ b/pkg/compose/watch.go @@ -84,9 +84,15 @@ type watchRule struct { service string } -func (r watchRule) Matches(event watch.FileEvent) *sync.PathMapping { +func (r watchRule) Matches(event watch.FileEvent) []*sync.PathMapping { hostPath := string(event) - if !pathutil.IsChild(r.Path, hostPath) { + childPaths := []string{} + for _, p := range r.Path { + if pathutil.IsChild(p, hostPath) { + childPaths = append(childPaths, p) + } + } + if len(childPaths) == 0 { return nil } isIgnored, err := r.ignore.Matches(hostPath) @@ -100,20 +106,29 @@ func (r watchRule) Matches(event watch.FileEvent) *sync.PathMapping { return nil } - var containerPath string - if r.Target != "" { - rel, err := filepath.Rel(r.Path, hostPath) + if r.Target == "" { + return []*sync.PathMapping{ + &sync.PathMapping{ + HostPath: hostPath, + }, + } + } + + var res []*sync.PathMapping + for _, p := range childPaths { + rel, err := filepath.Rel(p, hostPath) if err != nil { - logrus.Warnf("error making %s relative to %s: %v", hostPath, r.Path, err) + logrus.Warnf("error making %s relative to %s: %v", hostPath, p, err) return nil } // always use Unix-style paths for inside the container - containerPath = path.Join(r.Target, filepath.ToSlash(rel)) - } - return &sync.PathMapping{ - HostPath: hostPath, - ContainerPath: containerPath, + containerPath := path.Join(r.Target, filepath.ToSlash(rel)) + res = append(res, &sync.PathMapping{ + HostPath: hostPath, + ContainerPath: containerPath, + }) } + return res } func (s *composeService) watch(ctx context.Context, syncChannel chan bool, project *types.Project, services []string, options api.WatchOptions) error { //nolint: gocyclo @@ -161,9 +176,13 @@ func (s *composeService) watch(ctx context.Context, syncChannel chan bool, proje } for _, trigger := range config.Watch { - if isSync(trigger) && checkIfPathAlreadyBindMounted(trigger.Path, service.Volumes) { - logrus.Warnf("path '%s' also declared by a bind mount volume, this path won't be monitored!\n", trigger.Path) - continue + if isSync(trigger) { + for _, p := range trigger.Path { + if checkIfPathAlreadyBindMounted(p, service.Volumes) { + logrus.Warnf("path '%s' also declared by a bind mount volume, this path won't be monitored!\n", p) + continue + } + } } else { var initialSync bool success, err := trigger.Extensions.Get("x-initialSync", &initialSync) @@ -175,7 +194,7 @@ func (s *composeService) watch(ctx context.Context, syncChannel chan bool, proje } } } - paths = append(paths, trigger.Path) + paths = append(paths, trigger.Path...) } serviceWatchRules, err := getWatchRules(config, service) @@ -238,18 +257,21 @@ func getWatchRules(config *types.DevelopConfig, service types.ServiceConfig) ([] } for _, trigger := range config.Watch { - ignore, err := watch.NewDockerPatternMatcher(trigger.Path, trigger.Ignore) - if err != nil { - return nil, err + var ignorePaths []watch.PathMatcher + + for _, p := range trigger.Path { + ignore, err := watch.NewDockerPatternMatcher(p, trigger.Ignore) + if err != nil { + return nil, err + } + ignorePaths = append(ignorePaths, ignore) } + ignorePaths = append(ignorePaths, dockerIgnores, dotGitIgnore, watch.EphemeralPathMatcher()) rules = append(rules, watchRule{ Trigger: trigger, ignore: watch.NewCompositeMatcher( - dockerIgnores, - watch.EphemeralPathMatcher(), - dotGitIgnore, - ignore, + ignorePaths..., ), service: service.Name, }) @@ -305,16 +327,18 @@ func loadDevelopmentConfig(service types.ServiceConfig, project *types.Project) } for i, trigger := range config.Watch { - if !filepath.IsAbs(trigger.Path) { - trigger.Path = filepath.Join(baseDir, trigger.Path) - } - if p, err := filepath.EvalSymlinks(trigger.Path); err == nil { - // this might fail because the path doesn't exist, etc. - trigger.Path = p - } - trigger.Path = filepath.Clean(trigger.Path) - if trigger.Path == "" { - return nil, errors.New("watch rules MUST define a path") + for j, p := range trigger.Path { + if !filepath.IsAbs(p) { + trigger.Path[j] = filepath.Join(baseDir, p) + } + if p, err := filepath.EvalSymlinks(p); err == nil { + // this might fail because the path doesn't exist, etc. + trigger.Path[j] = p + } + trigger.Path[j] = filepath.Clean(p) + if trigger.Path[j] == "" { + return nil, errors.New("watch rules MUST define a path") + } } if trigger.Action == types.WatchActionRebuild && service.Build == nil { @@ -427,7 +451,7 @@ func (s *composeService) handleWatchBatch(ctx context.Context, project *types.Pr for _, event := range batch { for i, rule := range rules { mapping := rule.Matches(event) - if mapping == nil { + if len(mapping) == 0 { continue } @@ -435,14 +459,14 @@ func (s *composeService) handleWatchBatch(ctx context.Context, project *types.Pr case types.WatchActionRebuild: rebuild[rule.service] = true case types.WatchActionSync: - syncfiles[rule.service] = append(syncfiles[rule.service], mapping) + syncfiles[rule.service] = append(syncfiles[rule.service], mapping...) case types.WatchActionRestart: restart[rule.service] = true case types.WatchActionSyncRestart: - syncfiles[rule.service] = append(syncfiles[rule.service], mapping) + syncfiles[rule.service] = append(syncfiles[rule.service], mapping...) restart[rule.service] = true case types.WatchActionSyncExec: - syncfiles[rule.service] = append(syncfiles[rule.service], mapping) + syncfiles[rule.service] = append(syncfiles[rule.service], mapping...) // We want to run exec hooks only once after syncfiles if multiple file events match // as we can't compare ServiceHook to sort and compact a slice, collect rule indexes exec[rule.service] = append(exec[rule.service], i) @@ -607,26 +631,32 @@ func (s *composeService) pruneDanglingImagesOnRebuild(ctx context.Context, proje // Walks develop.watch.path and checks which files should be copied inside the container // ignores develop.watch.ignore, Dockerfile, compose files, bind mounted paths and .git func (s *composeService) initialSync(ctx context.Context, project *types.Project, service types.ServiceConfig, trigger types.Trigger, syncer sync.Syncer) error { + var allIgnores []watch.PathMatcher + dockerIgnores, err := watch.LoadDockerIgnore(service.Build) if err != nil { return err } + allIgnores = append(allIgnores, dockerIgnores) dotGitIgnore, err := watch.NewDockerPatternMatcher("/", []string{".git/"}) if err != nil { return err } + allIgnores = append(allIgnores, dotGitIgnore) + allIgnores = append(allIgnores, watch.EphemeralPathMatcher()) - triggerIgnore, err := watch.NewDockerPatternMatcher(trigger.Path, trigger.Ignore) - if err != nil { - return err + for _, p := range trigger.Path { + triggerIgnore, err := watch.NewDockerPatternMatcher(p, trigger.Ignore) + if err != nil { + return err + } + allIgnores = append(allIgnores, triggerIgnore) } // FIXME .dockerignore ignoreInitialSync := watch.NewCompositeMatcher( - dockerIgnores, - watch.EphemeralPathMatcher(), - dotGitIgnore, - triggerIgnore) + allIgnores..., + ) pathsToCopy, err := s.initialSyncFiles(ctx, project, service, trigger, ignoreInitialSync) if err != nil { @@ -640,63 +670,66 @@ func (s *composeService) initialSync(ctx context.Context, project *types.Project // //nolint:gocyclo func (s *composeService) initialSyncFiles(ctx context.Context, project *types.Project, service types.ServiceConfig, trigger types.Trigger, ignore watch.PathMatcher) ([]*sync.PathMapping, error) { - fi, err := os.Stat(trigger.Path) - if err != nil { - return nil, err - } timeImageCreated, err := s.imageCreatedTime(ctx, project, service.Name) if err != nil { return nil, err } var pathsToCopy []*sync.PathMapping - switch mode := fi.Mode(); { - case mode.IsDir(): - // process directory - err = filepath.WalkDir(trigger.Path, func(path string, d fs.DirEntry, err error) error { - if err != nil { - // handle possible path err, just in case... - return err - } - if trigger.Path == path { - // walk starts at the root directory - return nil - } - if shouldIgnore(filepath.Base(path), ignore) || checkIfPathAlreadyBindMounted(path, service.Volumes) { - // By definition sync ignores bind mounted paths - if d.IsDir() { - // skip folder - return fs.SkipDir + + for _, p := range trigger.Path { + fi, err := os.Stat(p) + if err != nil { + return nil, err + } + switch mode := fi.Mode(); { + case mode.IsDir(): + // process directory + err = filepath.WalkDir(p, func(path string, d fs.DirEntry, err error) error { + if err != nil { + // handle possible path err, just in case... + return err } - return nil // skip file - } - info, err := d.Info() - if err != nil { - return err - } - if !d.IsDir() { - if info.ModTime().Before(timeImageCreated) { - // skip file if it was modified before image creation + if p == path { + // walk starts at the root directory return nil } - rel, err := filepath.Rel(trigger.Path, path) + if shouldIgnore(filepath.Base(path), ignore) || checkIfPathAlreadyBindMounted(path, service.Volumes) { + // By definition sync ignores bind mounted paths + if d.IsDir() { + // skip folder + return fs.SkipDir + } + return nil // skip file + } + info, err := d.Info() if err != nil { return err } - // only copy files (and not full directories) + if !d.IsDir() { + if info.ModTime().Before(timeImageCreated) { + // skip file if it was modified before image creation + return nil + } + rel, err := filepath.Rel(p, path) + if err != nil { + return err + } + // only copy files (and not full directories) + pathsToCopy = append(pathsToCopy, &sync.PathMapping{ + HostPath: path, + ContainerPath: filepath.Join(trigger.Target, rel), + }) + } + return nil + }) + case mode.IsRegular(): + // process file + if fi.ModTime().After(timeImageCreated) && !shouldIgnore(filepath.Base(p), ignore) && !checkIfPathAlreadyBindMounted(p, service.Volumes) { pathsToCopy = append(pathsToCopy, &sync.PathMapping{ - HostPath: path, - ContainerPath: filepath.Join(trigger.Target, rel), + HostPath: p, + ContainerPath: trigger.Target, }) } - return nil - }) - case mode.IsRegular(): - // process file - if fi.ModTime().After(timeImageCreated) && !shouldIgnore(filepath.Base(trigger.Path), ignore) && !checkIfPathAlreadyBindMounted(trigger.Path, service.Volumes) { - pathsToCopy = append(pathsToCopy, &sync.PathMapping{ - HostPath: trigger.Path, - ContainerPath: trigger.Target, - }) } } return pathsToCopy, err From 8fd8d391c7697e6591543dc48409957128ecef76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joana=20Hrotk=C3=B3?= Date: Tue, 18 Feb 2025 18:17:55 +0000 Subject: [PATCH 2/3] replace with jhrotko fork MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Joana Hrotkó --- go.mod | 2 +- go.sum | 2 ++ pkg/compose/watch_test.go | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e8ce78231f7..5ea9ca59644 100644 --- a/go.mod +++ b/go.mod @@ -200,4 +200,4 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -replace github.com/compose-spec/compose-go/v2 v2.4.8 => ../compose-go +replace github.com/compose-spec/compose-go/v2 v2.4.8 => github.com/jhrotko/compose-go/v2 v2.0.0-20250217114813-b0bb892d9de8 diff --git a/go.sum b/go.sum index 8b72a43c3d1..c4ba5194594 100644 --- a/go.sum +++ b/go.sum @@ -249,6 +249,8 @@ github.com/in-toto/in-toto-golang v0.5.0/go.mod h1:/Rq0IZHLV7Ku5gielPT4wPHJfH1Gd github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jhrotko/compose-go/v2 v2.0.0-20250217114813-b0bb892d9de8 h1:oiXfjzHda6UTDxYRjQDGc+OVqWKNpDRa9A59bDTDDOI= +github.com/jhrotko/compose-go/v2 v2.0.0-20250217114813-b0bb892d9de8/go.mod h1:lFN0DrMxIncJGYAXTfWuajfwj5haBJqrBkarHcnjJKc= github.com/jinzhu/gorm v0.0.0-20170222002820-5409931a1bb8 h1:CZkYfurY6KGhVtlalI4QwQ6T0Cu6iuY3e0x5RLu96WE= github.com/jinzhu/gorm v0.0.0-20170222002820-5409931a1bb8/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= github.com/jinzhu/inflection v0.0.0-20170102125226-1c35d901db3d h1:jRQLvyVGL+iVtDElaEIDdKwpPqUIZJfzkNLV34htpEc= diff --git a/pkg/compose/watch_test.go b/pkg/compose/watch_test.go index e7492f2bce7..b0a7a253d7b 100644 --- a/pkg/compose/watch_test.go +++ b/pkg/compose/watch_test.go @@ -124,13 +124,13 @@ func TestWatch_Sync(t *testing.T) { rules, err := getWatchRules(&types.DevelopConfig{ Watch: []types.Trigger{ { - Path: "/sync", + Path: []string{"/sync"}, Action: "sync", Target: "/work", Ignore: []string{"ignore"}, }, { - Path: "/rebuild", + Path: []string{"/rebuild"}, Action: "rebuild", }, }, From 4396577c6d05a0378e0766c9c8eaf9aeb9d74fa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joana=20Hrotk=C3=B3?= Date: Tue, 25 Feb 2025 15:37:15 +0000 Subject: [PATCH 3/3] Use path as string and handle glob pattern MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Joana Hrotkó --- go.mod | 2 +- go.sum | 4 +- pkg/compose/watch.go | 246 +++++++++++++++++--------------------- pkg/compose/watch_test.go | 17 ++- pkg/watch/dockerignore.go | 17 +++ 5 files changed, 142 insertions(+), 144 deletions(-) diff --git a/go.mod b/go.mod index 5ea9ca59644..b769fa3f1e1 100644 --- a/go.mod +++ b/go.mod @@ -200,4 +200,4 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -replace github.com/compose-spec/compose-go/v2 v2.4.8 => github.com/jhrotko/compose-go/v2 v2.0.0-20250217114813-b0bb892d9de8 +replace github.com/compose-spec/compose-go/v2 v2.4.8 => github.com/jhrotko/compose-go/v2 v2.0.0-20250225153415-9ce61ad83a27 diff --git a/go.sum b/go.sum index c4ba5194594..66769290f40 100644 --- a/go.sum +++ b/go.sum @@ -249,8 +249,8 @@ github.com/in-toto/in-toto-golang v0.5.0/go.mod h1:/Rq0IZHLV7Ku5gielPT4wPHJfH1Gd github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jhrotko/compose-go/v2 v2.0.0-20250217114813-b0bb892d9de8 h1:oiXfjzHda6UTDxYRjQDGc+OVqWKNpDRa9A59bDTDDOI= -github.com/jhrotko/compose-go/v2 v2.0.0-20250217114813-b0bb892d9de8/go.mod h1:lFN0DrMxIncJGYAXTfWuajfwj5haBJqrBkarHcnjJKc= +github.com/jhrotko/compose-go/v2 v2.0.0-20250225153415-9ce61ad83a27 h1:zt9TD5EqlE4d/RQ6hspiLj2VaoviTBrETfS8kr2YT30= +github.com/jhrotko/compose-go/v2 v2.0.0-20250225153415-9ce61ad83a27/go.mod h1:lFN0DrMxIncJGYAXTfWuajfwj5haBJqrBkarHcnjJKc= github.com/jinzhu/gorm v0.0.0-20170222002820-5409931a1bb8 h1:CZkYfurY6KGhVtlalI4QwQ6T0Cu6iuY3e0x5RLu96WE= github.com/jinzhu/gorm v0.0.0-20170222002820-5409931a1bb8/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= github.com/jinzhu/inflection v0.0.0-20170102125226-1c35d901db3d h1:jRQLvyVGL+iVtDElaEIDdKwpPqUIZJfzkNLV34htpEc= diff --git a/pkg/compose/watch.go b/pkg/compose/watch.go index 2a936f96834..3c3e013dd1d 100644 --- a/pkg/compose/watch.go +++ b/pkg/compose/watch.go @@ -80,21 +80,30 @@ func (s *composeService) Watch(ctx context.Context, project *types.Project, serv type watchRule struct { types.Trigger - ignore watch.PathMatcher - service string + ignore watch.PathMatcher + service string + globPattern watch.PathMatcher } -func (r watchRule) Matches(event watch.FileEvent) []*sync.PathMapping { +func (r watchRule) Matches(event watch.FileEvent) *sync.PathMapping { hostPath := string(event) - childPaths := []string{} - for _, p := range r.Path { - if pathutil.IsChild(p, hostPath) { - childPaths = append(childPaths, p) - } - } - if len(childPaths) == 0 { + + isGlob := r.IsGlobPath() + if !isGlob && !pathutil.IsChild(r.Path, hostPath) { return nil } + + if isGlob { + isMatch, err := r.globPattern.Matches(hostPath) + if err != nil { + logrus.Warnf("error while pattern matching %q: %v", hostPath, err) + return nil + } + if !isMatch { + return nil + } + } + isIgnored, err := r.ignore.Matches(hostPath) if err != nil { logrus.Warnf("error ignore matching %q: %v", hostPath, err) @@ -106,29 +115,20 @@ func (r watchRule) Matches(event watch.FileEvent) []*sync.PathMapping { return nil } - if r.Target == "" { - return []*sync.PathMapping{ - &sync.PathMapping{ - HostPath: hostPath, - }, - } - } - - var res []*sync.PathMapping - for _, p := range childPaths { - rel, err := filepath.Rel(p, hostPath) + var containerPath string + if r.Target != "" { + rel, err := filepath.Rel(r.AnchorPath(), hostPath) if err != nil { - logrus.Warnf("error making %s relative to %s: %v", hostPath, p, err) + logrus.Warnf("error making %s relative to %s: %v", hostPath, r.AnchorPath(), err) return nil } // always use Unix-style paths for inside the container - containerPath := path.Join(r.Target, filepath.ToSlash(rel)) - res = append(res, &sync.PathMapping{ - HostPath: hostPath, - ContainerPath: containerPath, - }) + containerPath = path.Join(r.Target, filepath.ToSlash(rel)) + } + return &sync.PathMapping{ + HostPath: hostPath, + ContainerPath: containerPath, } - return res } func (s *composeService) watch(ctx context.Context, syncChannel chan bool, project *types.Project, services []string, options api.WatchOptions) error { //nolint: gocyclo @@ -176,17 +176,13 @@ func (s *composeService) watch(ctx context.Context, syncChannel chan bool, proje } for _, trigger := range config.Watch { - if isSync(trigger) { - for _, p := range trigger.Path { - if checkIfPathAlreadyBindMounted(p, service.Volumes) { - logrus.Warnf("path '%s' also declared by a bind mount volume, this path won't be monitored!\n", p) - continue - } - } + if trigger.IsSyncAction() && isPathBindMounted(trigger.AnchorPath(), service.Volumes) { + logrus.Warnf("path '%s' also declared by a bind mount volume, this path won't be monitored!\n", trigger.Path) + continue } else { var initialSync bool success, err := trigger.Extensions.Get("x-initialSync", &initialSync) - if err == nil && success && initialSync && isSync(trigger) { + if err == nil && success && initialSync && trigger.IsSyncAction() { // Need to check initial files are in container that are meant to be synched from watch action err := s.initialSync(ctx, project, service, trigger, syncer) if err != nil { @@ -194,7 +190,7 @@ func (s *composeService) watch(ctx context.Context, syncChannel chan bool, proje } } } - paths = append(paths, trigger.Path...) + paths = append(paths, trigger.AnchorPath()) } serviceWatchRules, err := getWatchRules(config, service) @@ -243,46 +239,37 @@ func (s *composeService) watch(ctx context.Context, syncChannel chan bool, proje func getWatchRules(config *types.DevelopConfig, service types.ServiceConfig) ([]watchRule, error) { var rules []watchRule - dockerIgnores, err := watch.LoadDockerIgnore(service.Build) - if err != nil { - return nil, err - } - - // add a hardcoded set of ignores on top of what came from .dockerignore - // some of this should likely be configurable (e.g. there could be cases - // where you want `.git` to be synced) but this is suitable for now - dotGitIgnore, err := watch.NewDockerPatternMatcher("/", []string{".git/"}) + general, err := watch.GeneralIgnorePatterns(service) if err != nil { return nil, err } for _, trigger := range config.Watch { - var ignorePaths []watch.PathMatcher - - for _, p := range trigger.Path { - ignore, err := watch.NewDockerPatternMatcher(p, trigger.Ignore) + ignore, err := watch.NewDockerPatternMatcher(trigger.Path, trigger.Ignore) + if err != nil { + return nil, err + } + var glob watch.PathMatcher = watch.EmptyMatcher{} + if trigger.IsGlobPath() { + glob, err = watch.NewDockerPatternMatcher(trigger.AnchorPath(), []string{trigger.Path}) if err != nil { return nil, err } - ignorePaths = append(ignorePaths, ignore) } - ignorePaths = append(ignorePaths, dockerIgnores, dotGitIgnore, watch.EphemeralPathMatcher()) rules = append(rules, watchRule{ Trigger: trigger, ignore: watch.NewCompositeMatcher( - ignorePaths..., + general, + ignore, ), - service: service.Name, + globPattern: glob, + service: service.Name, }) } return rules, nil } -func isSync(trigger types.Trigger) bool { - return trigger.Action == types.WatchActionSync || trigger.Action == types.WatchActionSyncRestart -} - func (s *composeService) watchEvents(ctx context.Context, project *types.Project, options api.WatchOptions, watcher watch.Notify, syncer sync.Syncer, rules []watchRule) error { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -327,18 +314,16 @@ func loadDevelopmentConfig(service types.ServiceConfig, project *types.Project) } for i, trigger := range config.Watch { - for j, p := range trigger.Path { - if !filepath.IsAbs(p) { - trigger.Path[j] = filepath.Join(baseDir, p) - } - if p, err := filepath.EvalSymlinks(p); err == nil { - // this might fail because the path doesn't exist, etc. - trigger.Path[j] = p - } - trigger.Path[j] = filepath.Clean(p) - if trigger.Path[j] == "" { - return nil, errors.New("watch rules MUST define a path") - } + if !filepath.IsAbs(trigger.Path) { + trigger.Path = filepath.Join(baseDir, trigger.Path) + } + if p, err := filepath.EvalSymlinks(trigger.Path); err == nil { + // this might fail because the path doesn't exist, etc. + trigger.Path = p + } + trigger.Path = filepath.Clean(trigger.Path) + if trigger.Path == "" { + return nil, errors.New("watch rules MUST define a path") } if trigger.Action == types.WatchActionRebuild && service.Build == nil { @@ -353,7 +338,7 @@ func loadDevelopmentConfig(service types.ServiceConfig, project *types.Project) return &config, nil } -func checkIfPathAlreadyBindMounted(watchPath string, volumes []types.ServiceVolumeConfig) bool { +func isPathBindMounted(watchPath string, volumes []types.ServiceVolumeConfig) bool { for _, volume := range volumes { if volume.Bind != nil && strings.HasPrefix(watchPath, volume.Source) { return true @@ -451,7 +436,7 @@ func (s *composeService) handleWatchBatch(ctx context.Context, project *types.Pr for _, event := range batch { for i, rule := range rules { mapping := rule.Matches(event) - if len(mapping) == 0 { + if mapping == nil { continue } @@ -459,14 +444,14 @@ func (s *composeService) handleWatchBatch(ctx context.Context, project *types.Pr case types.WatchActionRebuild: rebuild[rule.service] = true case types.WatchActionSync: - syncfiles[rule.service] = append(syncfiles[rule.service], mapping...) + syncfiles[rule.service] = append(syncfiles[rule.service], mapping) case types.WatchActionRestart: restart[rule.service] = true case types.WatchActionSyncRestart: - syncfiles[rule.service] = append(syncfiles[rule.service], mapping...) + syncfiles[rule.service] = append(syncfiles[rule.service], mapping) restart[rule.service] = true case types.WatchActionSyncExec: - syncfiles[rule.service] = append(syncfiles[rule.service], mapping...) + syncfiles[rule.service] = append(syncfiles[rule.service], mapping) // We want to run exec hooks only once after syncfiles if multiple file events match // as we can't compare ServiceHook to sort and compact a slice, collect rule indexes exec[rule.service] = append(exec[rule.service], i) @@ -631,32 +616,19 @@ func (s *composeService) pruneDanglingImagesOnRebuild(ctx context.Context, proje // Walks develop.watch.path and checks which files should be copied inside the container // ignores develop.watch.ignore, Dockerfile, compose files, bind mounted paths and .git func (s *composeService) initialSync(ctx context.Context, project *types.Project, service types.ServiceConfig, trigger types.Trigger, syncer sync.Syncer) error { - var allIgnores []watch.PathMatcher - - dockerIgnores, err := watch.LoadDockerIgnore(service.Build) + ignore, err := watch.GeneralIgnorePatterns(service) if err != nil { return err } - allIgnores = append(allIgnores, dockerIgnores) - dotGitIgnore, err := watch.NewDockerPatternMatcher("/", []string{".git/"}) + triggerIgnore, err := watch.NewDockerPatternMatcher(trigger.Path, trigger.Ignore) if err != nil { return err } - allIgnores = append(allIgnores, dotGitIgnore) - allIgnores = append(allIgnores, watch.EphemeralPathMatcher()) - - for _, p := range trigger.Path { - triggerIgnore, err := watch.NewDockerPatternMatcher(p, trigger.Ignore) - if err != nil { - return err - } - allIgnores = append(allIgnores, triggerIgnore) - } // FIXME .dockerignore ignoreInitialSync := watch.NewCompositeMatcher( - allIgnores..., - ) + ignore, + triggerIgnore) pathsToCopy, err := s.initialSyncFiles(ctx, project, service, trigger, ignoreInitialSync) if err != nil { @@ -670,66 +642,66 @@ func (s *composeService) initialSync(ctx context.Context, project *types.Project // //nolint:gocyclo func (s *composeService) initialSyncFiles(ctx context.Context, project *types.Project, service types.ServiceConfig, trigger types.Trigger, ignore watch.PathMatcher) ([]*sync.PathMapping, error) { + sourcePath := trigger.AnchorPath() + + fi, err := os.Stat(sourcePath) + if err != nil { + return nil, err + } timeImageCreated, err := s.imageCreatedTime(ctx, project, service.Name) if err != nil { return nil, err } var pathsToCopy []*sync.PathMapping + switch mode := fi.Mode(); { + case mode.IsDir(): + // process directory + err = filepath.WalkDir(sourcePath, func(path string, d fs.DirEntry, err error) error { + if err != nil { + // handle possible path err, just in case... + return err + } + if sourcePath == path { + // walk starts at the root directory + return nil + } - for _, p := range trigger.Path { - fi, err := os.Stat(p) - if err != nil { - return nil, err - } - switch mode := fi.Mode(); { - case mode.IsDir(): - // process directory - err = filepath.WalkDir(p, func(path string, d fs.DirEntry, err error) error { - if err != nil { - // handle possible path err, just in case... - return err + if shouldIgnore(filepath.Base(path), ignore) || isPathBindMounted(path, service.Volumes) { + // By definition sync ignores bind mounted paths + if d.IsDir() { + // skip folder + return fs.SkipDir } - if p == path { - // walk starts at the root directory + return nil // skip file + } + info, err := d.Info() + if err != nil { + return err + } + if !d.IsDir() { + if info.ModTime().Before(timeImageCreated) { + // skip file if it was modified before image creation return nil } - if shouldIgnore(filepath.Base(path), ignore) || checkIfPathAlreadyBindMounted(path, service.Volumes) { - // By definition sync ignores bind mounted paths - if d.IsDir() { - // skip folder - return fs.SkipDir - } - return nil // skip file - } - info, err := d.Info() + rel, err := filepath.Rel(sourcePath, path) if err != nil { return err } - if !d.IsDir() { - if info.ModTime().Before(timeImageCreated) { - // skip file if it was modified before image creation - return nil - } - rel, err := filepath.Rel(p, path) - if err != nil { - return err - } - // only copy files (and not full directories) - pathsToCopy = append(pathsToCopy, &sync.PathMapping{ - HostPath: path, - ContainerPath: filepath.Join(trigger.Target, rel), - }) - } - return nil - }) - case mode.IsRegular(): - // process file - if fi.ModTime().After(timeImageCreated) && !shouldIgnore(filepath.Base(p), ignore) && !checkIfPathAlreadyBindMounted(p, service.Volumes) { + // only copy files (and not full directories) pathsToCopy = append(pathsToCopy, &sync.PathMapping{ - HostPath: p, - ContainerPath: trigger.Target, + HostPath: path, + ContainerPath: filepath.Join(trigger.Target, rel), }) } + return nil + }) + case mode.IsRegular(): + // process file + if fi.ModTime().After(timeImageCreated) && !shouldIgnore(filepath.Base(sourcePath), ignore) && !isPathBindMounted(sourcePath, service.Volumes) { + pathsToCopy = append(pathsToCopy, &sync.PathMapping{ + HostPath: sourcePath, + ContainerPath: trigger.Target, + }) } } return pathsToCopy, err diff --git a/pkg/compose/watch_test.go b/pkg/compose/watch_test.go index b0a7a253d7b..4b9f80f2af7 100644 --- a/pkg/compose/watch_test.go +++ b/pkg/compose/watch_test.go @@ -124,13 +124,18 @@ func TestWatch_Sync(t *testing.T) { rules, err := getWatchRules(&types.DevelopConfig{ Watch: []types.Trigger{ { - Path: []string{"/sync"}, + Path: "/sync", Action: "sync", Target: "/work", Ignore: []string{"ignore"}, }, { - Path: []string{"/rebuild"}, + Path: "/restart/*/sub", + Action: "sync", + Target: "/foo", + }, + { + Path: "/rebuild", Action: "rebuild", }, }, @@ -147,7 +152,10 @@ func TestWatch_Sync(t *testing.T) { watcher.Events() <- watch.NewFileEvent("/sync/changed") watcher.Events() <- watch.NewFileEvent("/sync/changed/sub") - err := clock.BlockUntilContext(ctx, 3) + + watcher.Events() <- watch.NewFileEvent("/restart/changed") + watcher.Events() <- watch.NewFileEvent("/restart/changed/sub") + err := clock.BlockUntilContext(ctx, 5) assert.NilError(t, err) clock.Advance(watch.QuietPeriod) select { @@ -155,6 +163,7 @@ func TestWatch_Sync(t *testing.T) { require.ElementsMatch(t, []*sync.PathMapping{ {HostPath: "/sync/changed", ContainerPath: "/work/changed"}, {HostPath: "/sync/changed/sub", ContainerPath: "/work/changed/sub"}, + {HostPath: "/restart/changed/sub", ContainerPath: "/foo/changed/sub"}, }, actual) case <-time.After(100 * time.Millisecond): t.Error("timeout") @@ -162,7 +171,7 @@ func TestWatch_Sync(t *testing.T) { watcher.Events() <- watch.NewFileEvent("/rebuild") watcher.Events() <- watch.NewFileEvent("/sync/changed") - err = clock.BlockUntilContext(ctx, 4) + err = clock.BlockUntilContext(ctx, 7) assert.NilError(t, err) clock.Advance(watch.QuietPeriod) select { diff --git a/pkg/watch/dockerignore.go b/pkg/watch/dockerignore.go index c51b6fabf1f..182e50c2ea7 100644 --- a/pkg/watch/dockerignore.go +++ b/pkg/watch/dockerignore.go @@ -96,6 +96,23 @@ func LoadDockerIgnore(build *types.BuildConfig) (PathMatcher, error) { return NewDockerPatternMatcher(absRoot, patterns) } +func GeneralIgnorePatterns(service types.ServiceConfig) (PathMatcher, error) { + dockerIgnores, err := LoadDockerIgnore(service.Build) + if err != nil { + return nil, err + } + + // add a hardcoded set of ignores on top of what came from .dockerignore + // some of this should likely be configurable (e.g. there could be cases + // where you want `.git` to be synced) but this is suitable for now + dotGitIgnore, err := NewDockerPatternMatcher("/", []string{".git/"}) + if err != nil { + return nil, err + } + + return NewCompositeMatcher(dockerIgnores, dotGitIgnore, EphemeralPathMatcher()), nil +} + // Make all the patterns use absolute paths. func absPatterns(absRoot string, patterns []string) []string { absPatterns := make([]string, 0, len(patterns))