diff --git a/cmd/seq-db/seq-db.go b/cmd/seq-db/seq-db.go index b0a7c47e..59d05eaa 100644 --- a/cmd/seq-db/seq-db.go +++ b/cmd/seq-db/seq-db.go @@ -271,6 +271,7 @@ func startStore( TokenTableZstdLevel: cfg.Compression.SealedZstdCompressionLevel, DocBlocksZstdLevel: cfg.Compression.DocBlockZstdCompressionLevel, DocBlockSize: int(cfg.DocsSorting.DocBlockSize), + SkipFsync: cfg.Resources.SkipFsync, }, Fraction: frac.Config{ Search: frac.SearchConfig{ @@ -283,6 +284,7 @@ func startStore( }, SkipSortDocs: !cfg.DocsSorting.Enabled, KeepMetaFile: false, + SkipFsync: cfg.Resources.SkipFsync, }, OffloadingEnabled: cfg.Offloading.Enabled, OffloadingRetention: cfg.Offloading.Retention, diff --git a/frac/active.go b/frac/active.go index 7c3691c1..9f83b66f 100644 --- a/frac/active.go +++ b/frac/active.go @@ -83,9 +83,9 @@ func NewActive( cfg *Config, skipMaskProvider skipMaskProvider, ) *Active { - docsFile, docsStats := mustOpenFile(baseFileName+consts.DocsFileSuffix, config.SkipFsync) + docsFile, docsStats := mustOpenFile(baseFileName+consts.DocsFileSuffix, cfg.SkipFsync) - metaFile, writer, metaReader, walReader, metaSize := mustOpenMetaWriter(baseFileName, readLimiter, docsFile, docsStats) + metaFile, writer, metaReader, walReader, metaSize := mustOpenMetaWriter(baseFileName, readLimiter, docsFile, docsStats, cfg.SkipFsync) f := &Active{ TokenList: NewActiveTokenList(config.IndexWorkers), @@ -128,24 +128,25 @@ func mustOpenMetaWriter( baseFileName string, readLimiter *storage.ReadLimiter, docsFile *os.File, - docsStats os.FileInfo) (*os.File, *ActiveWriter, *storage.DocBlocksReader, *storage.WalReader, uint64) { + docsStats os.FileInfo, + skipFsync bool) (*os.File, *ActiveWriter, *storage.DocBlocksReader, *storage.WalReader, uint64) { legacyMetaFileName := baseFileName + consts.MetaFileSuffix if _, err := os.Stat(legacyMetaFileName); err == nil { // .meta file exists - metaFile, metaStats := mustOpenFile(legacyMetaFileName, config.SkipFsync) + metaFile, metaStats := mustOpenFile(legacyMetaFileName, skipFsync) metaSize := uint64(metaStats.Size()) metaReader := storage.NewDocBlocksReader(readLimiter, metaFile) - writer := NewActiveWriterLegacy(docsFile, metaFile, docsStats.Size(), metaStats.Size(), config.SkipFsync) + writer := NewActiveWriterLegacy(docsFile, metaFile, docsStats.Size(), metaStats.Size(), skipFsync) logger.Info("using legacy meta file format", zap.String("fraction", baseFileName)) return metaFile, writer, &metaReader, nil, metaSize } logger.Info("using new WAL format", zap.String("fraction", baseFileName)) walFileName := baseFileName + consts.WalFileSuffix - metaFile, metaStats := mustOpenFile(walFileName, config.SkipFsync) + metaFile, metaStats := mustOpenFile(walFileName, skipFsync) metaSize := uint64(metaStats.Size()) - writer := NewActiveWriter(docsFile, metaFile, docsStats.Size(), metaStats.Size(), config.SkipFsync) + writer := NewActiveWriter(docsFile, metaFile, docsStats.Size(), metaStats.Size(), skipFsync) walReader, err := storage.NewWalReader(readLimiter, metaFile, baseFileName) if err != nil { logger.Fatal("failed to initialize WAL reader", zap.String("fraction", baseFileName), zap.Error(err)) diff --git a/frac/active_sealing_source.go b/frac/active_sealing_source.go index 43ca0239..b0afd68e 100644 --- a/frac/active_sealing_source.go +++ b/frac/active_sealing_source.go @@ -57,6 +57,7 @@ type ActiveSealingSource struct { blocksOffsets []uint64 // Document block offsets docsReader *storage.DocsReader // Document storage reader lastErr error // Last error + skipFsync bool } // NewActiveSealingSource creates a new data source for sealing @@ -87,6 +88,7 @@ func NewActiveSealingSource(active *Active, params common.SealParams) (*ActiveSe docPosMap: active.DocsPositions.idToPos, blocksOffsets: active.DocBlocks.vals, docsReader: &active.sortReader, + skipFsync: active.Config.SkipFsync, } src.prepareInfo() @@ -371,9 +373,10 @@ func (src *ActiveSealingSource) SortDocs() error { } src.info.DocsOnDisk = uint64(stat.Size()) - // Synchronize and rename file - if err := sdocsFile.Sync(); err != nil { - return err + if !src.skipFsync { + if err := sdocsFile.Sync(); err != nil { + return err + } } if err := sdocsFile.Close(); err != nil { return err @@ -381,8 +384,10 @@ func (src *ActiveSealingSource) SortDocs() error { if err := os.Rename(sdocsFile.Name(), src.info.Path+consts.SdocsFileSuffix); err != nil { return err } - if err := util.SyncPath(filepath.Dir(src.info.Path)); err != nil { - return err + if !src.skipFsync { + if err := util.SyncPath(filepath.Dir(src.info.Path)); err != nil { + return err + } } // Log compression statistics diff --git a/frac/common/seal_params.go b/frac/common/seal_params.go index c19365f9..443179f1 100644 --- a/frac/common/seal_params.go +++ b/frac/common/seal_params.go @@ -9,4 +9,6 @@ type SealParams struct { DocBlocksZstdLevel int // DocBlocksZstdLevel is the zstd compress level of each document block. DocBlockSize int // DocBlockSize is decompressed payload size of document block. + + SkipFsync bool } diff --git a/frac/config.go b/frac/config.go index 3b1c1e97..ac315b41 100644 --- a/frac/config.go +++ b/frac/config.go @@ -5,6 +5,7 @@ type Config struct { SkipSortDocs bool KeepMetaFile bool + SkipFsync bool } type SearchConfig struct { diff --git a/frac/sealed/sealing/sealer.go b/frac/sealed/sealing/sealer.go index 3eb00761..9e45f4f7 100644 --- a/frac/sealed/sealing/sealer.go +++ b/frac/sealed/sealing/sealer.go @@ -59,9 +59,11 @@ func Seal(src Source, params common.SealParams) (*sealed.PreloadedData, error) { return nil, err } - // Ensure data is flushed to disk - if err := indexFile.Sync(); err != nil { - return nil, err + if !params.SkipFsync { + // Ensure data is flushed to disk + if err := indexFile.Sync(); err != nil { + return nil, err + } } // Get final file size for metadata @@ -81,8 +83,10 @@ func Seal(src Source, params common.SealParams) (*sealed.PreloadedData, error) { return nil, err } - // Ensure directory metadata is synced to disk - util.MustSyncPath(filepath.Dir(info.Path)) + if !params.SkipFsync { + // Ensure directory metadata is synced to disk + util.MustSyncPath(filepath.Dir(info.Path)) + } // Build preloaded data structure for fast query access lidsTable := indexSealer.LIDsTable()