2018-03-30 22:43:18 +02:00
|
|
|
package archiver
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"os"
|
|
|
|
"path"
|
|
|
|
"runtime"
|
|
|
|
"sort"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/restic/restic/internal/debug"
|
|
|
|
"github.com/restic/restic/internal/errors"
|
|
|
|
"github.com/restic/restic/internal/fs"
|
|
|
|
"github.com/restic/restic/internal/restic"
|
2022-05-27 19:08:50 +02:00
|
|
|
"golang.org/x/sync/errgroup"
|
2018-03-30 22:43:18 +02:00
|
|
|
)
|
|
|
|
|
2018-07-31 17:25:25 +02:00
|
|
|
// SelectByNameFunc returns true for all items that should be included (files and
|
|
|
|
// dirs). If false is returned, files are ignored and dirs are not even walked.
|
|
|
|
type SelectByNameFunc func(item string) bool
|
|
|
|
|
2018-03-30 22:43:18 +02:00
|
|
|
// SelectFunc returns true for all items that should be included (files and
|
|
|
|
// dirs). If false is returned, files are ignored and dirs are not even walked.
|
|
|
|
type SelectFunc func(item string, fi os.FileInfo) bool
|
|
|
|
|
|
|
|
// ErrorFunc is called when an error during archiving occurs. When nil is
|
|
|
|
// returned, the archiver continues, otherwise it aborts and passes the error
|
|
|
|
// up the call stack.
|
2022-05-21 00:31:26 +02:00
|
|
|
type ErrorFunc func(file string, err error) error
|
2018-03-30 22:43:18 +02:00
|
|
|
|
|
|
|
// ItemStats collects some statistics about a particular file or directory.
|
|
|
|
type ItemStats struct {
|
2022-05-01 14:41:36 +02:00
|
|
|
DataBlobs int // number of new data blobs added for this item
|
|
|
|
DataSize uint64 // sum of the sizes of all new data blobs
|
|
|
|
DataSizeInRepo uint64 // sum of the bytes added to the repo (including compression and crypto overhead)
|
|
|
|
TreeBlobs int // number of new tree blobs added for this item
|
|
|
|
TreeSize uint64 // sum of the sizes of all new tree blobs
|
|
|
|
TreeSizeInRepo uint64 // sum of the bytes added to the repo (including compression and crypto overhead)
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Add adds other to the current ItemStats.
|
|
|
|
func (s *ItemStats) Add(other ItemStats) {
|
|
|
|
s.DataBlobs += other.DataBlobs
|
|
|
|
s.DataSize += other.DataSize
|
2022-05-01 14:41:36 +02:00
|
|
|
s.DataSizeInRepo += other.DataSizeInRepo
|
2018-03-30 22:43:18 +02:00
|
|
|
s.TreeBlobs += other.TreeBlobs
|
|
|
|
s.TreeSize += other.TreeSize
|
2022-05-01 14:41:36 +02:00
|
|
|
s.TreeSizeInRepo += other.TreeSizeInRepo
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Archiver saves a directory structure to the repo.
|
|
|
|
type Archiver struct {
|
2018-07-31 17:25:25 +02:00
|
|
|
Repo restic.Repository
|
|
|
|
SelectByName SelectByNameFunc
|
|
|
|
Select SelectFunc
|
|
|
|
FS fs.FS
|
|
|
|
Options Options
|
2018-03-30 22:43:18 +02:00
|
|
|
|
|
|
|
blobSaver *BlobSaver
|
|
|
|
fileSaver *FileSaver
|
2018-04-30 15:13:03 +02:00
|
|
|
treeSaver *TreeSaver
|
2018-03-30 22:43:18 +02:00
|
|
|
|
|
|
|
// Error is called for all errors that occur during backup.
|
|
|
|
Error ErrorFunc
|
|
|
|
|
|
|
|
// CompleteItem is called for all files and dirs once they have been
|
|
|
|
// processed successfully. The parameter item contains the path as it will
|
|
|
|
// be in the snapshot after saving. s contains some statistics about this
|
|
|
|
// particular file/dir.
|
|
|
|
//
|
|
|
|
// CompleteItem may be called asynchronously from several different
|
|
|
|
// goroutines!
|
|
|
|
CompleteItem func(item string, previous, current *restic.Node, s ItemStats, d time.Duration)
|
|
|
|
|
|
|
|
// StartFile is called when a file is being processed by a worker.
|
|
|
|
StartFile func(filename string)
|
|
|
|
|
|
|
|
// CompleteBlob is called for all saved blobs for files.
|
2022-10-15 15:21:17 +02:00
|
|
|
CompleteBlob func(bytes uint64)
|
2018-03-30 22:43:18 +02:00
|
|
|
|
|
|
|
// WithAtime configures if the access time for files and directories should
|
|
|
|
// be saved. Enabling it may result in much metadata, so it's off by
|
|
|
|
// default.
|
2020-07-08 09:59:00 +02:00
|
|
|
WithAtime bool
|
|
|
|
|
|
|
|
// Flags controlling change detection. See doc/040_backup.rst for details.
|
|
|
|
ChangeIgnoreFlags uint
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
|
2020-07-08 09:59:00 +02:00
|
|
|
// Flags for the ChangeIgnoreFlags bitfield.
|
|
|
|
const (
|
|
|
|
ChangeIgnoreCtime = 1 << iota
|
|
|
|
ChangeIgnoreInode
|
|
|
|
)
|
|
|
|
|
2018-03-30 22:43:18 +02:00
|
|
|
// Options is used to configure the archiver.
|
|
|
|
type Options struct {
|
2022-09-24 11:57:16 +02:00
|
|
|
// ReadConcurrency sets how many files are read in concurrently. If
|
2018-03-30 22:43:18 +02:00
|
|
|
// it's set to zero, at most two files are read in concurrently (which
|
|
|
|
// turned out to be a good default for most situations).
|
2022-09-24 11:57:16 +02:00
|
|
|
ReadConcurrency uint
|
2018-03-30 22:43:18 +02:00
|
|
|
|
|
|
|
// SaveBlobConcurrency sets how many blobs are hashed and saved
|
|
|
|
// concurrently. If it's set to zero, the default is the number of CPUs
|
|
|
|
// available in the system.
|
|
|
|
SaveBlobConcurrency uint
|
2018-04-30 15:13:03 +02:00
|
|
|
|
|
|
|
// SaveTreeConcurrency sets how many trees are marshalled and saved to the
|
|
|
|
// repo concurrently.
|
|
|
|
SaveTreeConcurrency uint
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// ApplyDefaults returns a copy of o with the default options set for all unset
|
|
|
|
// fields.
|
|
|
|
func (o Options) ApplyDefaults() Options {
|
2022-09-24 11:57:16 +02:00
|
|
|
if o.ReadConcurrency == 0 {
|
2018-03-30 22:43:18 +02:00
|
|
|
// two is a sweet spot for almost all situations. We've done some
|
|
|
|
// experiments documented here:
|
|
|
|
// https://github.com/borgbackup/borg/issues/3500
|
2022-09-24 11:57:16 +02:00
|
|
|
o.ReadConcurrency = 2
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
if o.SaveBlobConcurrency == 0 {
|
2021-08-07 23:18:37 +02:00
|
|
|
// blob saving is CPU bound due to hash checking and encryption
|
|
|
|
// the actual upload is handled by the repository itself
|
|
|
|
o.SaveBlobConcurrency = uint(runtime.GOMAXPROCS(0))
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
|
2018-04-30 15:13:03 +02:00
|
|
|
if o.SaveTreeConcurrency == 0 {
|
2021-08-07 23:33:43 +02:00
|
|
|
// can either wait for a file, wait for a tree, serialize a tree or wait for saveblob
|
|
|
|
// the last two are cpu-bound and thus mutually exclusive.
|
|
|
|
// Also allow waiting for FileReadConcurrency files, this is the maximum of FutureFiles
|
|
|
|
// which currently can be in progress. The main backup loop blocks when trying to queue
|
|
|
|
// more files to read.
|
2022-09-24 11:57:16 +02:00
|
|
|
o.SaveTreeConcurrency = uint(runtime.GOMAXPROCS(0)) + o.ReadConcurrency
|
2018-04-30 15:13:03 +02:00
|
|
|
}
|
|
|
|
|
2018-03-30 22:43:18 +02:00
|
|
|
return o
|
|
|
|
}
|
|
|
|
|
|
|
|
// New initializes a new archiver.
|
|
|
|
func New(repo restic.Repository, fs fs.FS, opts Options) *Archiver {
|
|
|
|
arch := &Archiver{
|
2018-07-31 17:25:25 +02:00
|
|
|
Repo: repo,
|
|
|
|
SelectByName: func(item string) bool { return true },
|
|
|
|
Select: func(item string, fi os.FileInfo) bool { return true },
|
|
|
|
FS: fs,
|
|
|
|
Options: opts.ApplyDefaults(),
|
2018-03-30 22:43:18 +02:00
|
|
|
|
|
|
|
CompleteItem: func(string, *restic.Node, *restic.Node, ItemStats, time.Duration) {},
|
|
|
|
StartFile: func(string) {},
|
2022-10-15 15:21:17 +02:00
|
|
|
CompleteBlob: func(uint64) {},
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
return arch
|
|
|
|
}
|
|
|
|
|
2018-05-08 22:28:37 +02:00
|
|
|
// error calls arch.Error if it is set and the error is different from context.Canceled.
|
2022-05-21 00:31:26 +02:00
|
|
|
func (arch *Archiver) error(item string, err error) error {
|
2018-03-30 22:43:18 +02:00
|
|
|
if arch.Error == nil || err == nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2018-05-08 22:28:37 +02:00
|
|
|
if err == context.Canceled {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2022-05-21 00:31:26 +02:00
|
|
|
errf := arch.Error(item, err)
|
2018-03-30 22:43:18 +02:00
|
|
|
if err != errf {
|
|
|
|
debug.Log("item %v: error was filtered by handler, before: %q, after: %v", item, err, errf)
|
|
|
|
}
|
|
|
|
return errf
|
|
|
|
}
|
|
|
|
|
2020-05-16 08:05:26 +02:00
|
|
|
// nodeFromFileInfo returns the restic node from an os.FileInfo.
|
2022-08-19 23:08:13 +02:00
|
|
|
func (arch *Archiver) nodeFromFileInfo(snPath, filename string, fi os.FileInfo) (*restic.Node, error) {
|
2018-03-30 22:43:18 +02:00
|
|
|
node, err := restic.NodeFromFileInfo(filename, fi)
|
|
|
|
if !arch.WithAtime {
|
|
|
|
node.AccessTime = node.ModTime
|
|
|
|
}
|
2022-08-19 23:08:13 +02:00
|
|
|
// overwrite name to match that within the snapshot
|
|
|
|
node.Name = path.Base(snPath)
|
2018-03-30 22:43:18 +02:00
|
|
|
return node, errors.Wrap(err, "NodeFromFileInfo")
|
|
|
|
}
|
|
|
|
|
|
|
|
// loadSubtree tries to load the subtree referenced by node. In case of an error, nil is returned.
|
2020-08-08 00:05:07 +02:00
|
|
|
// If there is no node to load, then nil is returned without an error.
|
|
|
|
func (arch *Archiver) loadSubtree(ctx context.Context, node *restic.Node) (*restic.Tree, error) {
|
2018-03-30 22:43:18 +02:00
|
|
|
if node == nil || node.Type != "dir" || node.Subtree == nil {
|
2020-08-08 00:05:07 +02:00
|
|
|
return nil, nil
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
|
2022-06-12 14:38:19 +02:00
|
|
|
tree, err := restic.LoadTree(ctx, arch.Repo, *node.Subtree)
|
2018-03-30 22:43:18 +02:00
|
|
|
if err != nil {
|
|
|
|
debug.Log("unable to load tree %v: %v", node.Subtree.Str(), err)
|
2020-08-08 00:05:07 +02:00
|
|
|
// a tree in the repository is not readable -> warn the user
|
|
|
|
return nil, arch.wrapLoadTreeError(*node.Subtree, err)
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
|
2020-08-08 00:05:07 +02:00
|
|
|
return tree, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error {
|
2020-11-05 22:18:00 +01:00
|
|
|
if arch.Repo.Index().Has(restic.BlobHandle{ID: id, Type: restic.TreeBlob}) {
|
2020-08-08 00:05:07 +02:00
|
|
|
err = errors.Errorf("tree %v could not be loaded; the repository could be damaged: %v", id, err)
|
|
|
|
} else {
|
|
|
|
err = errors.Errorf("tree %v is not known; the repository could be damaged, run `rebuild-index` to try to repair it", id)
|
|
|
|
}
|
|
|
|
return err
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// SaveDir stores a directory in the repo and returns the node. snPath is the
|
|
|
|
// path within the current snapshot.
|
2022-05-29 11:57:10 +02:00
|
|
|
func (arch *Archiver) SaveDir(ctx context.Context, snPath string, dir string, fi os.FileInfo, previous *restic.Tree, complete CompleteFunc) (d FutureNode, err error) {
|
2018-03-30 22:43:18 +02:00
|
|
|
debug.Log("%v %v", snPath, dir)
|
|
|
|
|
2022-08-19 23:08:13 +02:00
|
|
|
treeNode, err := arch.nodeFromFileInfo(snPath, dir, fi)
|
2018-03-30 22:43:18 +02:00
|
|
|
if err != nil {
|
2022-05-29 11:57:10 +02:00
|
|
|
return FutureNode{}, err
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
|
2020-02-17 09:22:32 +01:00
|
|
|
names, err := readdirnames(arch.FS, dir, fs.O_NOFOLLOW)
|
2018-03-30 22:43:18 +02:00
|
|
|
if err != nil {
|
2022-05-29 11:57:10 +02:00
|
|
|
return FutureNode{}, err
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
2020-02-17 09:22:32 +01:00
|
|
|
sort.Strings(names)
|
2018-03-30 22:43:18 +02:00
|
|
|
|
2018-04-30 15:13:03 +02:00
|
|
|
nodes := make([]FutureNode, 0, len(names))
|
2018-03-30 22:43:18 +02:00
|
|
|
|
|
|
|
for _, name := range names {
|
2018-05-08 22:28:37 +02:00
|
|
|
// test if context has been cancelled
|
|
|
|
if ctx.Err() != nil {
|
2018-05-12 23:54:20 +02:00
|
|
|
debug.Log("context has been cancelled, aborting")
|
2022-05-29 11:57:10 +02:00
|
|
|
return FutureNode{}, ctx.Err()
|
2018-05-08 22:28:37 +02:00
|
|
|
}
|
|
|
|
|
2018-03-30 22:43:18 +02:00
|
|
|
pathname := arch.FS.Join(dir, name)
|
|
|
|
oldNode := previous.Find(name)
|
|
|
|
snItem := join(snPath, name)
|
|
|
|
fn, excluded, err := arch.Save(ctx, snItem, pathname, oldNode)
|
|
|
|
|
|
|
|
// return error early if possible
|
|
|
|
if err != nil {
|
2022-05-21 00:31:26 +02:00
|
|
|
err = arch.error(pathname, err)
|
2018-03-30 22:43:18 +02:00
|
|
|
if err == nil {
|
|
|
|
// ignore error
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2022-05-29 11:57:10 +02:00
|
|
|
return FutureNode{}, err
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
if excluded {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2018-04-30 15:13:03 +02:00
|
|
|
nodes = append(nodes, fn)
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
|
2022-05-29 11:57:10 +02:00
|
|
|
fn := arch.treeSaver.Save(ctx, snPath, dir, treeNode, nodes, complete)
|
2018-03-30 22:43:18 +02:00
|
|
|
|
2022-05-29 11:57:10 +02:00
|
|
|
return fn, nil
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
|
2022-05-29 11:57:10 +02:00
|
|
|
// FutureNode holds a reference to a channel that returns a FutureNodeResult
|
|
|
|
// or a reference to an already existing result. If the result is available
|
|
|
|
// immediatelly, then storing a reference directly requires less memory than
|
|
|
|
// using the indirection via a channel.
|
2018-03-30 22:43:18 +02:00
|
|
|
type FutureNode struct {
|
2022-05-29 11:57:10 +02:00
|
|
|
ch <-chan futureNodeResult
|
|
|
|
res *futureNodeResult
|
|
|
|
}
|
|
|
|
|
|
|
|
type futureNodeResult struct {
|
2018-03-30 22:43:18 +02:00
|
|
|
snPath, target string
|
|
|
|
|
|
|
|
node *restic.Node
|
|
|
|
stats ItemStats
|
|
|
|
err error
|
2022-05-29 11:57:10 +02:00
|
|
|
}
|
2018-03-30 22:43:18 +02:00
|
|
|
|
2022-05-29 11:57:10 +02:00
|
|
|
func newFutureNode() (FutureNode, chan<- futureNodeResult) {
|
|
|
|
ch := make(chan futureNodeResult, 1)
|
|
|
|
return FutureNode{ch: ch}, ch
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
|
2022-05-29 11:57:10 +02:00
|
|
|
func newFutureNodeWithResult(res futureNodeResult) FutureNode {
|
|
|
|
return FutureNode{
|
|
|
|
res: &res,
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-05-29 11:57:10 +02:00
|
|
|
func (fn *FutureNode) take(ctx context.Context) futureNodeResult {
|
|
|
|
if fn.res != nil {
|
|
|
|
res := fn.res
|
|
|
|
// free result
|
|
|
|
fn.res = nil
|
|
|
|
return *res
|
|
|
|
}
|
|
|
|
select {
|
|
|
|
case res, ok := <-fn.ch:
|
|
|
|
if ok {
|
|
|
|
// free channel
|
|
|
|
fn.ch = nil
|
|
|
|
return res
|
|
|
|
}
|
|
|
|
case <-ctx.Done():
|
|
|
|
}
|
2022-08-19 23:19:29 +02:00
|
|
|
return futureNodeResult{err: errors.Errorf("no result")}
|
2022-05-29 11:57:10 +02:00
|
|
|
}
|
|
|
|
|
2020-07-09 22:35:04 +02:00
|
|
|
// allBlobsPresent checks if all blobs (contents) of the given node are
|
|
|
|
// present in the index.
|
|
|
|
func (arch *Archiver) allBlobsPresent(previous *restic.Node) bool {
|
|
|
|
// check if all blobs are contained in index
|
|
|
|
for _, id := range previous.Content {
|
2020-11-05 22:18:00 +01:00
|
|
|
if !arch.Repo.Index().Has(restic.BlobHandle{ID: id, Type: restic.DataBlob}) {
|
2020-07-09 22:35:04 +02:00
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2018-03-30 22:43:18 +02:00
|
|
|
// Save saves a target (file or directory) to the repo. If the item is
|
2018-07-31 17:25:25 +02:00
|
|
|
// excluded, this function returns a nil node and error, with excluded set to
|
2018-05-12 21:59:38 +02:00
|
|
|
// true.
|
2018-03-30 22:43:18 +02:00
|
|
|
//
|
2018-07-31 17:25:25 +02:00
|
|
|
// Errors and completion needs to be handled by the caller.
|
2018-03-30 22:43:18 +02:00
|
|
|
//
|
|
|
|
// snPath is the path within the current snapshot.
|
|
|
|
func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous *restic.Node) (fn FutureNode, excluded bool, err error) {
|
2018-04-30 15:13:03 +02:00
|
|
|
start := time.Now()
|
|
|
|
|
2018-03-30 22:43:18 +02:00
|
|
|
debug.Log("%v target %q, previous %v", snPath, target, previous)
|
|
|
|
abstarget, err := arch.FS.Abs(target)
|
|
|
|
if err != nil {
|
|
|
|
return FutureNode{}, false, err
|
|
|
|
}
|
|
|
|
|
2018-07-31 17:25:25 +02:00
|
|
|
// exclude files by path before running Lstat to reduce number of lstat calls
|
|
|
|
if !arch.SelectByName(abstarget) {
|
|
|
|
debug.Log("%v is excluded by path", target)
|
|
|
|
return FutureNode{}, true, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// get file info and run remaining select functions that require file information
|
archiver: Use lstat before open/fstat
The previous code tried to be as efficient as possible and only do a
single open() on an item to save, and then fstat() on the fd to find out
what the item is (file, dir, other). For normal files, it would then
start reading the data without opening the file again, so it could not
be exchanged for e.g. a symlink.
This behavior starts the watchdog on my machine when /dev is saved
with restic, and after a few seconds, the machine reboots.
This commit reverts the behavior to the strategy the old archiver code
used: run lstat(), then decide what to do. For normal files, open the
file and then run fstat() on the fd to verify it's still a normal file,
then start reading the data.
The downside is that for normal files we now do two stat() calls
(lstat+fstat) instead of only one. On the upside, this does not start
the watchdog. :)
2018-05-01 23:05:50 +02:00
|
|
|
fi, err := arch.FS.Lstat(target)
|
|
|
|
if err != nil {
|
|
|
|
debug.Log("lstat() for %v returned error: %v", target, err)
|
2022-05-21 00:31:26 +02:00
|
|
|
err = arch.error(abstarget, err)
|
archiver: Use lstat before open/fstat
The previous code tried to be as efficient as possible and only do a
single open() on an item to save, and then fstat() on the fd to find out
what the item is (file, dir, other). For normal files, it would then
start reading the data without opening the file again, so it could not
be exchanged for e.g. a symlink.
This behavior starts the watchdog on my machine when /dev is saved
with restic, and after a few seconds, the machine reboots.
This commit reverts the behavior to the strategy the old archiver code
used: run lstat(), then decide what to do. For normal files, open the
file and then run fstat() on the fd to verify it's still a normal file,
then start reading the data.
The downside is that for normal files we now do two stat() calls
(lstat+fstat) instead of only one. On the upside, this does not start
the watchdog. :)
2018-05-01 23:05:50 +02:00
|
|
|
if err != nil {
|
|
|
|
return FutureNode{}, false, errors.Wrap(err, "Lstat")
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
archiver: Use lstat before open/fstat
The previous code tried to be as efficient as possible and only do a
single open() on an item to save, and then fstat() on the fd to find out
what the item is (file, dir, other). For normal files, it would then
start reading the data without opening the file again, so it could not
be exchanged for e.g. a symlink.
This behavior starts the watchdog on my machine when /dev is saved
with restic, and after a few seconds, the machine reboots.
This commit reverts the behavior to the strategy the old archiver code
used: run lstat(), then decide what to do. For normal files, open the
file and then run fstat() on the fd to verify it's still a normal file,
then start reading the data.
The downside is that for normal files we now do two stat() calls
(lstat+fstat) instead of only one. On the upside, this does not start
the watchdog. :)
2018-05-01 23:05:50 +02:00
|
|
|
return FutureNode{}, true, nil
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
2020-10-05 11:11:04 +02:00
|
|
|
if !arch.Select(abstarget, fi) {
|
|
|
|
debug.Log("%v is excluded", target)
|
|
|
|
return FutureNode{}, true, nil
|
|
|
|
}
|
2018-03-30 22:43:18 +02:00
|
|
|
|
|
|
|
switch {
|
|
|
|
case fs.IsRegularFile(fi):
|
|
|
|
debug.Log(" %v regular file", target)
|
|
|
|
start := time.Now()
|
|
|
|
|
2020-10-02 14:09:01 +02:00
|
|
|
// check if the file has not changed before performing a fopen operation (more expensive, specially
|
|
|
|
// in network filesystems)
|
2020-07-08 09:59:00 +02:00
|
|
|
if previous != nil && !fileChanged(fi, previous, arch.ChangeIgnoreFlags) {
|
2020-10-02 14:09:01 +02:00
|
|
|
if arch.allBlobsPresent(previous) {
|
|
|
|
debug.Log("%v hasn't changed, using old list of blobs", target)
|
|
|
|
arch.CompleteItem(snPath, previous, previous, ItemStats{}, time.Since(start))
|
2022-10-15 15:21:17 +02:00
|
|
|
arch.CompleteBlob(previous.Size)
|
2022-08-19 23:08:13 +02:00
|
|
|
node, err := arch.nodeFromFileInfo(snPath, target, fi)
|
2020-10-02 14:09:01 +02:00
|
|
|
if err != nil {
|
|
|
|
return FutureNode{}, false, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// copy list of blobs
|
2022-05-29 11:57:10 +02:00
|
|
|
node.Content = previous.Content
|
2020-10-02 14:09:01 +02:00
|
|
|
|
2022-05-29 11:57:10 +02:00
|
|
|
fn = newFutureNodeWithResult(futureNodeResult{
|
|
|
|
snPath: snPath,
|
|
|
|
target: target,
|
|
|
|
node: node,
|
|
|
|
})
|
2020-10-02 14:09:01 +02:00
|
|
|
return fn, false, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
debug.Log("%v hasn't changed, but contents are missing!", target)
|
|
|
|
// There are contents missing - inform user!
|
|
|
|
err := errors.Errorf("parts of %v not found in the repository index; storing the file again", target)
|
2022-05-21 00:31:26 +02:00
|
|
|
err = arch.error(abstarget, err)
|
2021-01-30 17:25:10 +01:00
|
|
|
if err != nil {
|
|
|
|
return FutureNode{}, false, err
|
|
|
|
}
|
2020-10-02 14:09:01 +02:00
|
|
|
}
|
|
|
|
|
archiver: Use lstat before open/fstat
The previous code tried to be as efficient as possible and only do a
single open() on an item to save, and then fstat() on the fd to find out
what the item is (file, dir, other). For normal files, it would then
start reading the data without opening the file again, so it could not
be exchanged for e.g. a symlink.
This behavior starts the watchdog on my machine when /dev is saved
with restic, and after a few seconds, the machine reboots.
This commit reverts the behavior to the strategy the old archiver code
used: run lstat(), then decide what to do. For normal files, open the
file and then run fstat() on the fd to verify it's still a normal file,
then start reading the data.
The downside is that for normal files we now do two stat() calls
(lstat+fstat) instead of only one. On the upside, this does not start
the watchdog. :)
2018-05-01 23:05:50 +02:00
|
|
|
// reopen file and do an fstat() on the open file to check it is still
|
|
|
|
// a file (and has not been exchanged for e.g. a symlink)
|
2018-05-20 16:05:53 +02:00
|
|
|
file, err := arch.FS.OpenFile(target, fs.O_RDONLY|fs.O_NOFOLLOW, 0)
|
archiver: Use lstat before open/fstat
The previous code tried to be as efficient as possible and only do a
single open() on an item to save, and then fstat() on the fd to find out
what the item is (file, dir, other). For normal files, it would then
start reading the data without opening the file again, so it could not
be exchanged for e.g. a symlink.
This behavior starts the watchdog on my machine when /dev is saved
with restic, and after a few seconds, the machine reboots.
This commit reverts the behavior to the strategy the old archiver code
used: run lstat(), then decide what to do. For normal files, open the
file and then run fstat() on the fd to verify it's still a normal file,
then start reading the data.
The downside is that for normal files we now do two stat() calls
(lstat+fstat) instead of only one. On the upside, this does not start
the watchdog. :)
2018-05-01 23:05:50 +02:00
|
|
|
if err != nil {
|
|
|
|
debug.Log("Openfile() for %v returned error: %v", target, err)
|
2022-05-21 00:31:26 +02:00
|
|
|
err = arch.error(abstarget, err)
|
archiver: Use lstat before open/fstat
The previous code tried to be as efficient as possible and only do a
single open() on an item to save, and then fstat() on the fd to find out
what the item is (file, dir, other). For normal files, it would then
start reading the data without opening the file again, so it could not
be exchanged for e.g. a symlink.
This behavior starts the watchdog on my machine when /dev is saved
with restic, and after a few seconds, the machine reboots.
This commit reverts the behavior to the strategy the old archiver code
used: run lstat(), then decide what to do. For normal files, open the
file and then run fstat() on the fd to verify it's still a normal file,
then start reading the data.
The downside is that for normal files we now do two stat() calls
(lstat+fstat) instead of only one. On the upside, this does not start
the watchdog. :)
2018-05-01 23:05:50 +02:00
|
|
|
if err != nil {
|
|
|
|
return FutureNode{}, false, errors.Wrap(err, "Lstat")
|
|
|
|
}
|
|
|
|
return FutureNode{}, true, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
fi, err = file.Stat()
|
|
|
|
if err != nil {
|
|
|
|
debug.Log("stat() on opened file %v returned error: %v", target, err)
|
|
|
|
_ = file.Close()
|
2022-05-21 00:31:26 +02:00
|
|
|
err = arch.error(abstarget, err)
|
archiver: Use lstat before open/fstat
The previous code tried to be as efficient as possible and only do a
single open() on an item to save, and then fstat() on the fd to find out
what the item is (file, dir, other). For normal files, it would then
start reading the data without opening the file again, so it could not
be exchanged for e.g. a symlink.
This behavior starts the watchdog on my machine when /dev is saved
with restic, and after a few seconds, the machine reboots.
This commit reverts the behavior to the strategy the old archiver code
used: run lstat(), then decide what to do. For normal files, open the
file and then run fstat() on the fd to verify it's still a normal file,
then start reading the data.
The downside is that for normal files we now do two stat() calls
(lstat+fstat) instead of only one. On the upside, this does not start
the watchdog. :)
2018-05-01 23:05:50 +02:00
|
|
|
if err != nil {
|
|
|
|
return FutureNode{}, false, errors.Wrap(err, "Lstat")
|
|
|
|
}
|
|
|
|
return FutureNode{}, true, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// make sure it's still a file
|
|
|
|
if !fs.IsRegularFile(fi) {
|
2022-10-14 14:18:52 +02:00
|
|
|
err = errors.Errorf("file %v changed type, refusing to archive", fi.Name())
|
2020-02-07 22:14:50 +01:00
|
|
|
_ = file.Close()
|
2022-05-21 00:31:26 +02:00
|
|
|
err = arch.error(abstarget, err)
|
archiver: Use lstat before open/fstat
The previous code tried to be as efficient as possible and only do a
single open() on an item to save, and then fstat() on the fd to find out
what the item is (file, dir, other). For normal files, it would then
start reading the data without opening the file again, so it could not
be exchanged for e.g. a symlink.
This behavior starts the watchdog on my machine when /dev is saved
with restic, and after a few seconds, the machine reboots.
This commit reverts the behavior to the strategy the old archiver code
used: run lstat(), then decide what to do. For normal files, open the
file and then run fstat() on the fd to verify it's still a normal file,
then start reading the data.
The downside is that for normal files we now do two stat() calls
(lstat+fstat) instead of only one. On the upside, this does not start
the watchdog. :)
2018-05-01 23:05:50 +02:00
|
|
|
if err != nil {
|
|
|
|
return FutureNode{}, false, err
|
|
|
|
}
|
|
|
|
return FutureNode{}, true, nil
|
|
|
|
}
|
|
|
|
|
2018-03-30 22:43:18 +02:00
|
|
|
// Save will close the file, we don't need to do that
|
2022-05-29 11:57:10 +02:00
|
|
|
fn = arch.fileSaver.Save(ctx, snPath, target, file, fi, func() {
|
2018-03-30 22:43:18 +02:00
|
|
|
arch.StartFile(snPath)
|
|
|
|
}, func(node *restic.Node, stats ItemStats) {
|
|
|
|
arch.CompleteItem(snPath, previous, node, stats, time.Since(start))
|
|
|
|
})
|
|
|
|
|
|
|
|
case fi.IsDir():
|
|
|
|
debug.Log(" %v dir", target)
|
|
|
|
|
|
|
|
snItem := snPath + "/"
|
|
|
|
start := time.Now()
|
2020-08-08 00:05:07 +02:00
|
|
|
oldSubtree, err := arch.loadSubtree(ctx, previous)
|
|
|
|
if err != nil {
|
2022-05-21 00:31:26 +02:00
|
|
|
err = arch.error(abstarget, err)
|
2021-01-30 17:25:10 +01:00
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
return FutureNode{}, false, err
|
2020-08-08 00:05:07 +02:00
|
|
|
}
|
2018-04-30 15:13:03 +02:00
|
|
|
|
2022-05-29 11:57:10 +02:00
|
|
|
fn, err = arch.SaveDir(ctx, snPath, target, fi, oldSubtree,
|
2020-04-22 22:23:02 +02:00
|
|
|
func(node *restic.Node, stats ItemStats) {
|
|
|
|
arch.CompleteItem(snItem, previous, node, stats, time.Since(start))
|
|
|
|
})
|
|
|
|
if err != nil {
|
2018-05-12 23:08:00 +02:00
|
|
|
debug.Log("SaveDir for %v returned error: %v", snPath, err)
|
2018-03-30 22:43:18 +02:00
|
|
|
return FutureNode{}, false, err
|
|
|
|
}
|
|
|
|
|
|
|
|
case fi.Mode()&os.ModeSocket > 0:
|
|
|
|
debug.Log(" %v is a socket, ignoring", target)
|
|
|
|
return FutureNode{}, true, nil
|
|
|
|
|
|
|
|
default:
|
|
|
|
debug.Log(" %v other", target)
|
|
|
|
|
2022-08-19 23:08:13 +02:00
|
|
|
node, err := arch.nodeFromFileInfo(snPath, target, fi)
|
2018-03-30 22:43:18 +02:00
|
|
|
if err != nil {
|
|
|
|
return FutureNode{}, false, err
|
|
|
|
}
|
2022-05-29 11:57:10 +02:00
|
|
|
fn = newFutureNodeWithResult(futureNodeResult{
|
|
|
|
snPath: snPath,
|
|
|
|
target: target,
|
|
|
|
node: node,
|
|
|
|
})
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
|
2018-04-30 15:13:03 +02:00
|
|
|
debug.Log("return after %.3f", time.Since(start).Seconds())
|
|
|
|
|
2018-03-30 22:43:18 +02:00
|
|
|
return fn, false, nil
|
|
|
|
}
|
|
|
|
|
2020-07-08 09:59:00 +02:00
|
|
|
// fileChanged tries to detect whether a file's content has changed compared
|
|
|
|
// to the contents of node, which describes the same path in the parent backup.
|
|
|
|
// It should only be run for regular files.
|
|
|
|
func fileChanged(fi os.FileInfo, node *restic.Node, ignoreFlags uint) bool {
|
|
|
|
switch {
|
|
|
|
case node == nil:
|
2018-03-30 22:43:18 +02:00
|
|
|
return true
|
2020-07-08 09:59:00 +02:00
|
|
|
case node.Type != "file":
|
|
|
|
// We're only called for regular files, so this is a type change.
|
2018-03-30 22:43:18 +02:00
|
|
|
return true
|
2020-07-08 09:59:00 +02:00
|
|
|
case uint64(fi.Size()) != node.Size:
|
2018-03-30 22:43:18 +02:00
|
|
|
return true
|
2020-07-08 09:59:00 +02:00
|
|
|
case !fi.ModTime().Equal(node.ModTime):
|
2019-03-20 02:27:37 +01:00
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2020-07-08 09:59:00 +02:00
|
|
|
checkCtime := ignoreFlags&ChangeIgnoreCtime == 0
|
|
|
|
checkInode := ignoreFlags&ChangeIgnoreInode == 0
|
2018-03-30 22:43:18 +02:00
|
|
|
|
2020-07-08 09:59:00 +02:00
|
|
|
extFI := fs.ExtendedStat(fi)
|
|
|
|
switch {
|
|
|
|
case checkCtime && !extFI.ChangeTime.Equal(node.ChangeTime):
|
|
|
|
return true
|
|
|
|
case checkInode && node.Inode != extFI.Inode:
|
2018-03-30 22:43:18 +02:00
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
// join returns all elements separated with a forward slash.
|
|
|
|
func join(elem ...string) string {
|
|
|
|
return path.Join(elem...)
|
|
|
|
}
|
|
|
|
|
|
|
|
// statDir returns the file info for the directory. Symbolic links are
|
|
|
|
// resolved. If the target directory is not a directory, an error is returned.
|
|
|
|
func (arch *Archiver) statDir(dir string) (os.FileInfo, error) {
|
|
|
|
fi, err := arch.FS.Stat(dir)
|
|
|
|
if err != nil {
|
|
|
|
return nil, errors.Wrap(err, "Lstat")
|
|
|
|
}
|
|
|
|
|
|
|
|
tpe := fi.Mode() & (os.ModeType | os.ModeCharDevice)
|
|
|
|
if tpe != os.ModeDir {
|
|
|
|
return fi, errors.Errorf("path is not a directory: %v", dir)
|
|
|
|
}
|
|
|
|
|
|
|
|
return fi, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// SaveTree stores a Tree in the repo, returned is the tree. snPath is the path
|
|
|
|
// within the current snapshot.
|
2022-08-19 23:08:13 +02:00
|
|
|
func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree, previous *restic.Tree, complete CompleteFunc) (FutureNode, int, error) {
|
2018-03-30 22:43:18 +02:00
|
|
|
|
2022-08-19 23:08:13 +02:00
|
|
|
var node *restic.Node
|
|
|
|
if snPath != "/" {
|
|
|
|
if atree.FileInfoPath == "" {
|
|
|
|
return FutureNode{}, 0, errors.Errorf("FileInfoPath for %v is empty", snPath)
|
|
|
|
}
|
2018-03-30 22:43:18 +02:00
|
|
|
|
2022-08-19 23:08:13 +02:00
|
|
|
fi, err := arch.statDir(atree.FileInfoPath)
|
|
|
|
if err != nil {
|
|
|
|
return FutureNode{}, 0, err
|
|
|
|
}
|
|
|
|
|
|
|
|
debug.Log("%v, dir node data loaded from %v", snPath, atree.FileInfoPath)
|
|
|
|
node, err = arch.nodeFromFileInfo(snPath, atree.FileInfoPath, fi)
|
|
|
|
if err != nil {
|
|
|
|
return FutureNode{}, 0, err
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
// fake root node
|
|
|
|
node = &restic.Node{}
|
|
|
|
}
|
|
|
|
|
|
|
|
debug.Log("%v (%v nodes), parent %v", snPath, len(atree.Nodes), previous)
|
|
|
|
nodeNames := atree.NodeNames()
|
|
|
|
nodes := make([]FutureNode, 0, len(nodeNames))
|
2018-03-30 22:43:18 +02:00
|
|
|
|
2018-05-12 23:07:16 +02:00
|
|
|
// iterate over the nodes of atree in lexicographic (=deterministic) order
|
2021-09-26 17:18:42 +02:00
|
|
|
for _, name := range nodeNames {
|
2018-05-12 23:07:16 +02:00
|
|
|
subatree := atree.Nodes[name]
|
|
|
|
|
2018-05-08 22:28:37 +02:00
|
|
|
// test if context has been cancelled
|
|
|
|
if ctx.Err() != nil {
|
2022-08-19 23:08:13 +02:00
|
|
|
return FutureNode{}, 0, ctx.Err()
|
2018-05-08 22:28:37 +02:00
|
|
|
}
|
2018-03-30 22:43:18 +02:00
|
|
|
|
|
|
|
// this is a leaf node
|
2021-01-29 11:10:28 +01:00
|
|
|
if subatree.Leaf() {
|
2018-03-30 22:43:18 +02:00
|
|
|
fn, excluded, err := arch.Save(ctx, join(snPath, name), subatree.Path, previous.Find(name))
|
|
|
|
|
|
|
|
if err != nil {
|
2022-05-21 00:31:26 +02:00
|
|
|
err = arch.error(subatree.Path, err)
|
2018-03-30 22:43:18 +02:00
|
|
|
if err == nil {
|
|
|
|
// ignore error
|
|
|
|
continue
|
|
|
|
}
|
2022-08-19 23:08:13 +02:00
|
|
|
return FutureNode{}, 0, err
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
if err != nil {
|
2022-08-19 23:08:13 +02:00
|
|
|
return FutureNode{}, 0, err
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
if !excluded {
|
2022-08-19 23:08:13 +02:00
|
|
|
nodes = append(nodes, fn)
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
snItem := join(snPath, name) + "/"
|
|
|
|
start := time.Now()
|
|
|
|
|
|
|
|
oldNode := previous.Find(name)
|
2020-08-08 00:05:07 +02:00
|
|
|
oldSubtree, err := arch.loadSubtree(ctx, oldNode)
|
|
|
|
if err != nil {
|
2022-05-21 00:31:26 +02:00
|
|
|
err = arch.error(join(snPath, name), err)
|
2021-01-30 17:25:10 +01:00
|
|
|
}
|
|
|
|
if err != nil {
|
2022-08-19 23:08:13 +02:00
|
|
|
return FutureNode{}, 0, err
|
2020-08-08 00:05:07 +02:00
|
|
|
}
|
2018-03-30 22:43:18 +02:00
|
|
|
|
|
|
|
// not a leaf node, archive subtree
|
2022-08-19 23:08:13 +02:00
|
|
|
fn, _, err := arch.SaveTree(ctx, join(snPath, name), &subatree, oldSubtree, func(n *restic.Node, is ItemStats) {
|
|
|
|
arch.CompleteItem(snItem, oldNode, n, is, time.Since(start))
|
|
|
|
})
|
2018-03-30 22:43:18 +02:00
|
|
|
if err != nil {
|
2022-08-19 23:08:13 +02:00
|
|
|
return FutureNode{}, 0, err
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
2022-08-19 23:08:13 +02:00
|
|
|
nodes = append(nodes, fn)
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
|
2022-08-19 23:08:13 +02:00
|
|
|
fn := arch.treeSaver.Save(ctx, snPath, atree.FileInfoPath, node, nodes, complete)
|
|
|
|
return fn, len(nodes), nil
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
|
2020-02-17 09:22:32 +01:00
|
|
|
// flags are passed to fs.OpenFile. O_RDONLY is implied.
|
|
|
|
func readdirnames(filesystem fs.FS, dir string, flags int) ([]string, error) {
|
|
|
|
f, err := filesystem.OpenFile(dir, fs.O_RDONLY|flags, 0)
|
2018-03-30 22:43:18 +02:00
|
|
|
if err != nil {
|
|
|
|
return nil, errors.Wrap(err, "Open")
|
|
|
|
}
|
|
|
|
|
|
|
|
entries, err := f.Readdirnames(-1)
|
|
|
|
if err != nil {
|
|
|
|
_ = f.Close()
|
2018-05-15 11:03:33 +02:00
|
|
|
return nil, errors.Wrapf(err, "Readdirnames %v failed", dir)
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
err = f.Close()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return entries, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// resolveRelativeTargets replaces targets that only contain relative
|
|
|
|
// directories ("." or "../../") with the contents of the directory. Each
|
|
|
|
// element of target is processed with fs.Clean().
|
2020-02-17 09:22:32 +01:00
|
|
|
func resolveRelativeTargets(filesys fs.FS, targets []string) ([]string, error) {
|
2018-03-30 22:43:18 +02:00
|
|
|
debug.Log("targets before resolving: %v", targets)
|
|
|
|
result := make([]string, 0, len(targets))
|
|
|
|
for _, target := range targets {
|
2020-02-17 09:22:32 +01:00
|
|
|
target = filesys.Clean(target)
|
|
|
|
pc, _ := pathComponents(filesys, target, false)
|
2018-03-30 22:43:18 +02:00
|
|
|
if len(pc) > 0 {
|
|
|
|
result = append(result, target)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
debug.Log("replacing %q with readdir(%q)", target, target)
|
2020-02-17 09:22:32 +01:00
|
|
|
entries, err := readdirnames(filesys, target, fs.O_NOFOLLOW)
|
2018-03-30 22:43:18 +02:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2020-02-17 09:22:32 +01:00
|
|
|
sort.Strings(entries)
|
2018-03-30 22:43:18 +02:00
|
|
|
|
|
|
|
for _, name := range entries {
|
2020-02-17 09:22:32 +01:00
|
|
|
result = append(result, filesys.Join(target, name))
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
debug.Log("targets after resolving: %v", result)
|
|
|
|
return result, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// SnapshotOptions collect attributes for a new snapshot.
|
|
|
|
type SnapshotOptions struct {
|
2020-04-13 04:29:13 +02:00
|
|
|
Tags restic.TagList
|
2018-03-30 22:43:18 +02:00
|
|
|
Hostname string
|
|
|
|
Excludes []string
|
|
|
|
Time time.Time
|
2022-10-03 14:48:14 +02:00
|
|
|
ParentSnapshot *restic.Snapshot
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// loadParentTree loads a tree referenced by snapshot id. If id is null, nil is returned.
|
2022-10-03 14:48:14 +02:00
|
|
|
func (arch *Archiver) loadParentTree(ctx context.Context, sn *restic.Snapshot) *restic.Tree {
|
|
|
|
if sn == nil {
|
2018-03-30 22:43:18 +02:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
if sn.Tree == nil {
|
2022-10-03 14:48:14 +02:00
|
|
|
debug.Log("snapshot %v has empty tree %v", *sn.ID())
|
2018-03-30 22:43:18 +02:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
debug.Log("load parent tree %v", *sn.Tree)
|
2022-06-12 14:38:19 +02:00
|
|
|
tree, err := restic.LoadTree(ctx, arch.Repo, *sn.Tree)
|
2018-03-30 22:43:18 +02:00
|
|
|
if err != nil {
|
|
|
|
debug.Log("unable to load tree %v: %v", *sn.Tree, err)
|
2022-05-21 00:31:26 +02:00
|
|
|
_ = arch.error("/", arch.wrapLoadTreeError(*sn.Tree, err))
|
2018-03-30 22:43:18 +02:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
return tree
|
|
|
|
}
|
|
|
|
|
|
|
|
// runWorkers starts the worker pools, which are stopped when the context is cancelled.
|
2022-05-27 19:08:50 +02:00
|
|
|
func (arch *Archiver) runWorkers(ctx context.Context, wg *errgroup.Group) {
|
|
|
|
arch.blobSaver = NewBlobSaver(ctx, wg, arch.Repo, arch.Options.SaveBlobConcurrency)
|
2018-04-30 15:13:03 +02:00
|
|
|
|
2022-05-27 19:08:50 +02:00
|
|
|
arch.fileSaver = NewFileSaver(ctx, wg,
|
2018-05-12 21:40:31 +02:00
|
|
|
arch.blobSaver.Save,
|
2018-04-29 13:20:12 +02:00
|
|
|
arch.Repo.Config().ChunkerPolynomial,
|
2022-09-24 11:57:16 +02:00
|
|
|
arch.Options.ReadConcurrency, arch.Options.SaveBlobConcurrency)
|
2018-03-30 22:43:18 +02:00
|
|
|
arch.fileSaver.CompleteBlob = arch.CompleteBlob
|
|
|
|
arch.fileSaver.NodeFromFileInfo = arch.nodeFromFileInfo
|
2018-04-30 15:13:03 +02:00
|
|
|
|
2022-08-19 23:08:13 +02:00
|
|
|
arch.treeSaver = NewTreeSaver(ctx, wg, arch.Options.SaveTreeConcurrency, arch.blobSaver.Save, arch.Error)
|
2022-05-27 19:08:50 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
func (arch *Archiver) stopWorkers() {
|
|
|
|
arch.blobSaver.TriggerShutdown()
|
|
|
|
arch.fileSaver.TriggerShutdown()
|
|
|
|
arch.treeSaver.TriggerShutdown()
|
2022-06-05 15:48:10 +02:00
|
|
|
arch.blobSaver = nil
|
|
|
|
arch.fileSaver = nil
|
|
|
|
arch.treeSaver = nil
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Snapshot saves several targets and returns a snapshot.
|
|
|
|
func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts SnapshotOptions) (*restic.Snapshot, restic.ID, error) {
|
|
|
|
cleanTargets, err := resolveRelativeTargets(arch.FS, targets)
|
|
|
|
if err != nil {
|
|
|
|
return nil, restic.ID{}, err
|
|
|
|
}
|
|
|
|
|
|
|
|
atree, err := NewTree(arch.FS, cleanTargets)
|
|
|
|
if err != nil {
|
|
|
|
return nil, restic.ID{}, err
|
|
|
|
}
|
|
|
|
|
2020-12-30 17:31:22 +01:00
|
|
|
var rootTreeID restic.ID
|
|
|
|
|
2021-08-07 22:52:05 +02:00
|
|
|
wgUp, wgUpCtx := errgroup.WithContext(ctx)
|
|
|
|
arch.Repo.StartPackUploader(wgUpCtx, wgUp)
|
2018-05-08 22:28:37 +02:00
|
|
|
|
2021-08-07 22:52:05 +02:00
|
|
|
wgUp.Go(func() error {
|
|
|
|
wg, wgCtx := errgroup.WithContext(wgUpCtx)
|
|
|
|
start := time.Now()
|
2018-05-20 15:58:55 +02:00
|
|
|
|
2021-08-07 22:52:05 +02:00
|
|
|
wg.Go(func() error {
|
|
|
|
arch.runWorkers(wgCtx, wg)
|
2018-05-08 22:28:37 +02:00
|
|
|
|
2021-08-07 22:52:05 +02:00
|
|
|
debug.Log("starting snapshot")
|
2022-08-19 23:08:13 +02:00
|
|
|
fn, nodeCount, err := arch.SaveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), func(n *restic.Node, is ItemStats) {
|
|
|
|
arch.CompleteItem("/", nil, nil, is, time.Since(start))
|
|
|
|
})
|
2021-08-07 22:52:05 +02:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2018-03-30 22:43:18 +02:00
|
|
|
|
2022-08-19 23:08:13 +02:00
|
|
|
fnr := fn.take(wgCtx)
|
|
|
|
if fnr.err != nil {
|
|
|
|
return fnr.err
|
2021-08-07 22:52:05 +02:00
|
|
|
}
|
|
|
|
|
2022-08-19 23:08:13 +02:00
|
|
|
if wgCtx.Err() != nil {
|
|
|
|
return wgCtx.Err()
|
2022-05-21 13:33:08 +02:00
|
|
|
}
|
2022-08-19 23:08:13 +02:00
|
|
|
|
|
|
|
if nodeCount == 0 {
|
|
|
|
return errors.New("snapshot is empty")
|
|
|
|
}
|
|
|
|
|
|
|
|
rootTreeID = *fnr.node.Subtree
|
2021-08-07 22:52:05 +02:00
|
|
|
arch.stopWorkers()
|
2022-08-19 23:08:13 +02:00
|
|
|
return nil
|
2021-08-07 22:52:05 +02:00
|
|
|
})
|
2018-03-30 22:43:18 +02:00
|
|
|
|
2021-08-07 22:52:05 +02:00
|
|
|
err = wg.Wait()
|
|
|
|
debug.Log("err is %v", err)
|
2018-03-30 22:43:18 +02:00
|
|
|
|
2021-08-07 22:52:05 +02:00
|
|
|
if err != nil {
|
|
|
|
debug.Log("error while saving tree: %v", err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return arch.Repo.Flush(ctx)
|
|
|
|
})
|
|
|
|
err = wgUp.Wait()
|
2018-03-30 22:43:18 +02:00
|
|
|
if err != nil {
|
|
|
|
return nil, restic.ID{}, err
|
|
|
|
}
|
|
|
|
|
|
|
|
sn, err := restic.NewSnapshot(targets, opts.Tags, opts.Hostname, opts.Time)
|
2020-02-12 22:37:37 +01:00
|
|
|
if err != nil {
|
|
|
|
return nil, restic.ID{}, err
|
|
|
|
}
|
|
|
|
|
2018-03-30 22:43:18 +02:00
|
|
|
sn.Excludes = opts.Excludes
|
2022-10-03 14:48:14 +02:00
|
|
|
if opts.ParentSnapshot != nil {
|
|
|
|
sn.Parent = opts.ParentSnapshot.ID()
|
2018-03-30 22:43:18 +02:00
|
|
|
}
|
|
|
|
sn.Tree = &rootTreeID
|
|
|
|
|
2022-06-12 14:38:19 +02:00
|
|
|
id, err := restic.SaveSnapshot(ctx, arch.Repo, sn)
|
2018-03-30 22:43:18 +02:00
|
|
|
if err != nil {
|
|
|
|
return nil, restic.ID{}, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return sn, id, nil
|
|
|
|
}
|