diff --git a/archiver.go b/archiver.go index 8632b82fd..ca8542b65 100644 --- a/archiver.go +++ b/archiver.go @@ -20,14 +20,14 @@ import ( ) const ( - maxConcurrentBlobs = 32 - maxConcurrency = 10 - maxConcurrencyPreload = 20 + maxConcurrentBlobs = 32 + maxConcurrency = 10 ) var archiverAbortOnAllErrors = func(str string, fi os.FileInfo, err error) error { return err } var archiverAllowAllFiles = func(string, os.FileInfo) bool { return true } +// Archiver is used to backup a set of directories. type Archiver struct { s *server.Server @@ -37,6 +37,7 @@ type Archiver struct { Filter func(item string, fi os.FileInfo) bool } +// NewArchiver returns a new archiver. func NewArchiver(s *server.Server) *Archiver { arch := &Archiver{ s: s, @@ -53,6 +54,7 @@ func NewArchiver(s *server.Server) *Archiver { return arch } +// Save stores a blob read from rd in the server. func (arch *Archiver) Save(t pack.BlobType, id backend.ID, length uint, rd io.Reader) error { debug.Log("Archiver.Save", "Save(%v, %v)\n", t, id.Str()) @@ -73,6 +75,7 @@ func (arch *Archiver) Save(t pack.BlobType, id backend.ID, length uint, rd io.Re return nil } +// SaveTreeJSON stores a tree in the server. func (arch *Archiver) SaveTreeJSON(item interface{}) (backend.ID, error) { data, err := json.Marshal(item) if err != nil { @@ -209,99 +212,6 @@ func (arch *Archiver) SaveFile(p *Progress, node *Node) error { return err } -func (arch *Archiver) saveTree(p *Progress, t *Tree) (backend.ID, error) { - debug.Log("Archiver.saveTree", "saveTree(%v)\n", t) - var wg sync.WaitGroup - - // TODO: do all this in parallel - for _, node := range t.Nodes { - if node.tree != nil { - id, err := arch.saveTree(p, node.tree) - if err != nil { - return nil, err - } - node.Subtree = id - p.Report(Stat{Dirs: 1}) - } else if node.Type == "file" { - if len(node.Content) > 0 { - removeContent := false - - // check content - for _, id := range node.Content { - packID, _, _, _, err := arch.s.Index().Lookup(id) - if err != nil { - debug.Log("Archiver.saveTree", "unable to find storage id for data blob %v: %v", id.Str(), err) - arch.Error(node.path, nil, fmt.Errorf("unable to find storage id for data blob %v: %v", id.Str(), err)) - removeContent = true - continue - } - - if ok, err := arch.s.Test(backend.Data, packID.String()); !ok || err != nil { - debug.Log("Archiver.saveTree", "pack %v of blob %v not in repository (error is %v)", packID, id, err) - arch.Error(node.path, nil, fmt.Errorf("pack %v of blob %v not in repository (error is %v)", packID, id, err)) - removeContent = true - } - } - - if removeContent { - debug.Log("Archiver.saveTree", "removing content for %s", node.path) - node.Content = node.Content[:0] - } - } - - if len(node.Content) == 0 { - // start goroutine - wg.Add(1) - go func(n *Node) { - defer wg.Done() - - n.err = arch.SaveFile(p, n) - p.Report(Stat{Files: 1}) - }(node) - } - } - } - - wg.Wait() - - usedIDs := backend.NewIDSet() - - // check for invalid file nodes - for _, node := range t.Nodes { - if node.Type == "file" && node.Content == nil && node.err == nil { - return nil, fmt.Errorf("node %v has empty content", node.Name) - } - - // remember used hashes - if node.Type == "file" && node.Content != nil { - for _, id := range node.Content { - usedIDs.Insert(id) - } - } - - if node.Type == "dir" && node.Subtree != nil { - usedIDs.Insert(node.Subtree) - } - - if node.err != nil { - err := arch.Error(node.path, nil, node.err) - if err != nil { - return nil, err - } - - // save error message in node - node.Error = node.err.Error() - } - } - - id, err := arch.SaveTreeJSON(t) - if err != nil { - return nil, err - } - - return id, nil -} - func (arch *Archiver) fileWorker(wg *sync.WaitGroup, p *Progress, done <-chan struct{}, entCh <-chan pipe.Entry) { defer func() { debug.Log("Archiver.fileWorker", "done") @@ -459,36 +369,36 @@ func (arch *Archiver) dirWorker(wg *sync.WaitGroup, p *Progress, done <-chan str } } -type ArchivePipe struct { +type archivePipe struct { Old <-chan WalkTreeJob New <-chan pipe.Job } func copyJobs(done <-chan struct{}, in <-chan pipe.Job, out chan<- pipe.Job) { - i := in - o := out - - o = nil - var ( - j pipe.Job - ok bool + // disable sending on the outCh until we received a job + outCh chan<- pipe.Job + // enable receiving from in + inCh = in + job pipe.Job + ok bool ) + for { select { case <-done: return - case j, ok = <-i: + case job, ok = <-inCh: if !ok { - // in ch closed, we're done - debug.Log("copyJobs", "in channel closed, we're done") + // input channel closed, we're done + debug.Log("copyJobs", "input channel closed, we're done") return } - i = nil - o = out - case o <- j: - o = nil - i = in + inCh = nil + outCh = out + case outCh <- job: + outCh = nil + inCh = in } } } @@ -499,7 +409,7 @@ type archiveJob struct { new pipe.Job } -func (a *ArchivePipe) compare(done <-chan struct{}, out chan<- pipe.Job) { +func (a *archivePipe) compare(done <-chan struct{}, out chan<- pipe.Job) { defer func() { close(out) debug.Log("ArchivePipe.compare", "done") @@ -619,7 +529,10 @@ func (j archiveJob) Copy() pipe.Job { return j.new } -func (arch *Archiver) Snapshot(p *Progress, paths []string, pid backend.ID) (*Snapshot, backend.ID, error) { +// Snapshot creates a snapshot of the given paths. If parentID is set, this is +// used to compare the files to the ones archived at the time this snapshot was +// taken. +func (arch *Archiver) Snapshot(p *Progress, paths []string, parentID backend.ID) (*Snapshot, backend.ID, error) { debug.Log("Archiver.Snapshot", "start for %v", paths) debug.Break("Archiver.Snapshot") @@ -638,14 +551,14 @@ func (arch *Archiver) Snapshot(p *Progress, paths []string, pid backend.ID) (*Sn return nil, nil, err } - jobs := ArchivePipe{} + jobs := archivePipe{} // use parent snapshot (if some was given) - if pid != nil { - sn.Parent = pid + if parentID != nil { + sn.Parent = parentID // load parent snapshot - parent, err := LoadSnapshot(arch.s, pid) + parent, err := LoadSnapshot(arch.s, parentID) if err != nil { return nil, nil, err } @@ -745,6 +658,8 @@ func isFile(fi os.FileInfo) bool { return fi.Mode()&(os.ModeType|os.ModeCharDevice) == 0 } +// Scan traverses the dirs to collect Stat information while emitting progress +// information with p. func Scan(dirs []string, p *Progress) (Stat, error) { p.Start() defer p.Done() diff --git a/archiver_int_test.go b/archiver_int_test.go index 948d44ba6..d26a9ce53 100644 --- a/archiver_int_test.go +++ b/archiver_int_test.go @@ -115,7 +115,7 @@ func TestArchivePipe(t *testing.T) { go testTreeWalker(done, treeCh) go testPipeWalker(done, pipeCh) - p := ArchivePipe{Old: treeCh, New: pipeCh} + p := archivePipe{Old: treeCh, New: pipeCh} ch := make(chan pipe.Job) diff --git a/backend/generic.go b/backend/generic.go index 0244cdab1..c7d569972 100644 --- a/backend/generic.go +++ b/backend/generic.go @@ -100,7 +100,7 @@ outer: // wrap around io.LimitedReader that implements io.ReadCloser type blobReader struct { - f io.Closer + cl io.Closer rd io.Reader closed bool } @@ -120,7 +120,7 @@ func (l *blobReader) Close() error { } if !l.closed { - err := l.f.Close() + err := l.cl.Close() l.closed = true return err } @@ -128,6 +128,8 @@ func (l *blobReader) Close() error { return nil } -func LimitReader(f io.ReadCloser, n int64) *blobReader { - return &blobReader{f: f, rd: io.LimitReader(f, n)} +// LimitReadCloser returns a new reader wraps r in an io.LimitReader, but also +// implements the Close() method. +func LimitReadCloser(r io.ReadCloser, n int64) *blobReader { + return &blobReader{cl: r, rd: io.LimitReader(r, n)} } diff --git a/backend/id.go b/backend/id.go index 261f4802d..e960b8eaf 100644 --- a/backend/id.go +++ b/backend/id.go @@ -10,7 +10,7 @@ import ( // IDSize contains the size of an ID, in bytes. const IDSize = hashSize -// References content within a repository. +// ID references content within a repository. type ID []byte // ParseID converts the given string to an ID. diff --git a/backend/local/local.go b/backend/local/local.go index 0fbd5fcf7..a50d2ef31 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -312,7 +312,7 @@ func (b *Local) GetReader(t backend.Type, name string, offset, length uint) (io. return f, nil } - return backend.LimitReader(f, int64(length)), nil + return backend.LimitReadCloser(f, int64(length)), nil } // Test returns true if a blob of the given type and name exists in the backend. diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index 09dad6242..5c850ce77 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -446,7 +446,7 @@ func (r *SFTP) GetReader(t backend.Type, name string, offset, length uint) (io.R return f, nil } - return backend.LimitReader(f, int64(length)), nil + return backend.LimitReadCloser(f, int64(length)), nil } // Test returns true if a blob of the given type and name exists in the backend. diff --git a/cache.go b/cache.go index 53e14873e..24f7a17fd 100644 --- a/cache.go +++ b/cache.go @@ -13,6 +13,7 @@ import ( "github.com/restic/restic/server" ) +// Cache is used to locally cache items from a server. type Cache struct { base string } @@ -29,6 +30,7 @@ func NewCache(be backend.Identifier) (*Cache, error) { return &Cache{base: basedir}, nil } +// Has checks if the local cache has the id. func (c *Cache) Has(t backend.Type, subtype string, id backend.ID) (bool, error) { filename, err := c.filename(t, subtype, id) if err != nil { @@ -51,6 +53,9 @@ func (c *Cache) Has(t backend.Type, subtype string, id backend.ID) (bool, error) return true, nil } +// Store returns an io.WriteCloser that is used to save new information to the +// cache. The returned io.WriteCloser must be closed by the caller after all +// data has been written. func (c *Cache) Store(t backend.Type, subtype string, id backend.ID) (io.WriteCloser, error) { filename, err := c.filename(t, subtype, id) if err != nil { @@ -73,6 +78,8 @@ func (c *Cache) Store(t backend.Type, subtype string, id backend.ID) (io.WriteCl return file, nil } +// Load returns information from the cache. The returned io.ReadCloser must be +// closed by the caller. func (c *Cache) Load(t backend.Type, subtype string, id backend.ID) (io.ReadCloser, error) { filename, err := c.filename(t, subtype, id) if err != nil { @@ -98,8 +105,9 @@ func (c *Cache) purge(t backend.Type, subtype string, id backend.ID) error { return err } +// Clear removes information from the cache that isn't present in the server any more. func (c *Cache) Clear(s *server.Server) error { - list, err := c.List(backend.Snapshot) + list, err := c.list(backend.Snapshot) if err != nil { return err } @@ -120,19 +128,19 @@ func (c *Cache) Clear(s *server.Server) error { return nil } -type CacheEntry struct { +type cacheEntry struct { ID backend.ID Subtype string } -func (c CacheEntry) String() string { +func (c cacheEntry) String() string { if c.Subtype != "" { return c.ID.Str() + "." + c.Subtype } return c.ID.Str() } -func (c *Cache) List(t backend.Type) ([]CacheEntry, error) { +func (c *Cache) list(t backend.Type) ([]cacheEntry, error) { var dir string switch t { @@ -145,7 +153,7 @@ func (c *Cache) List(t backend.Type) ([]CacheEntry, error) { fd, err := os.Open(dir) if err != nil { if os.IsNotExist(err) { - return []CacheEntry{}, nil + return []cacheEntry{}, nil } return nil, err } @@ -156,7 +164,7 @@ func (c *Cache) List(t backend.Type) ([]CacheEntry, error) { return nil, err } - entries := make([]CacheEntry, 0, len(fis)) + entries := make([]cacheEntry, 0, len(fis)) for _, fi := range fis { parts := strings.SplitN(fi.Name(), ".", 2) @@ -168,7 +176,7 @@ func (c *Cache) List(t backend.Type) ([]CacheEntry, error) { continue } - e := CacheEntry{ID: id} + e := cacheEntry{ID: id} if len(parts) == 2 { e.Subtype = parts[1] diff --git a/chunker/chunker.go b/chunker/chunker.go index eb92a50c7..9e016e4dd 100644 --- a/chunker/chunker.go +++ b/chunker/chunker.go @@ -12,16 +12,17 @@ const ( MiB = 1024 * KiB // WindowSize is the size of the sliding window. - WindowSize = 64 + windowSize = 64 // aim to create chunks of 20 bits or about 1MiB on average. - AverageBits = 20 + averageBits = 20 - // Chunks should be in the range of 512KiB to 8MiB. + // MinSize is the minimal size of a chunk. MinSize = 512 * KiB + // MaxSize is the maximal size of a chunk. MaxSize = 8 * MiB - splitmask = (1 << AverageBits) - 1 + splitmask = (1 << averageBits) - 1 ) type tables struct { @@ -39,7 +40,7 @@ func init() { cache.entries = make(map[Pol]*tables) } -// A chunk is one content-dependent chunk of bytes whose end was cut when the +// Chunk is one content-dependent chunk of bytes whose end was cut when the // Rabin Fingerprint had the value stored in Cut. type Chunk struct { Start uint @@ -52,7 +53,7 @@ func (c Chunk) Reader(r io.ReaderAt) io.Reader { return io.NewSectionReader(r, int64(c.Start), int64(c.Length)) } -// A chunker internally holds everything needed to split content. +// Chunker splits content with Rabin Fingerprints. type Chunker struct { pol Pol polShift uint @@ -61,7 +62,7 @@ type Chunker struct { rd io.Reader closed bool - window [WindowSize]byte + window [windowSize]byte wpos int buf []byte @@ -97,7 +98,7 @@ func (c *Chunker) Reset(rd io.Reader, p Pol) { c.fillTables() c.rd = rd - for i := 0; i < WindowSize; i++ { + for i := 0; i < windowSize; i++ { c.window[i] = 0 } c.closed = false @@ -116,7 +117,7 @@ func (c *Chunker) Reset(rd io.Reader, p Pol) { } // do not start a new chunk unless at least MinSize bytes have been read - c.pre = MinSize - WindowSize + c.pre = MinSize - windowSize } // Calculate out_table and mod_table for optimization. Must be called only @@ -154,7 +155,7 @@ func (c *Chunker) fillTables() { var h Pol h = appendByte(h, byte(b), c.pol) - for i := 0; i < WindowSize-1; i++ { + for i := 0; i < windowSize-1; i++ { h = appendByte(h, 0, c.pol) } c.tables.out[b] = h @@ -246,7 +247,7 @@ func (c *Chunker) Next() (*Chunk, error) { out := c.window[c.wpos] c.window[c.wpos] = b c.digest ^= uint64(c.tables.out[out]) - c.wpos = (c.wpos + 1) % WindowSize + c.wpos = (c.wpos + 1) % windowSize // c.append(b) index := c.digest >> c.polShift @@ -284,7 +285,7 @@ func (c *Chunker) Next() (*Chunk, error) { c.Reset(c.rd, c.pol) c.pos = pos c.start = pos - c.pre = MinSize - WindowSize + c.pre = MinSize - windowSize return chunk, nil } @@ -330,7 +331,7 @@ func (c *Chunker) slide(b byte) { out := c.window[c.wpos] c.window[c.wpos] = b c.digest ^= uint64(c.tables.out[out]) - c.wpos = (c.wpos + 1) % WindowSize + c.wpos = (c.wpos + 1) % windowSize c.append(b) } diff --git a/node.go b/node.go index 39f1da2cb..3060d54fb 100644 --- a/node.go +++ b/node.go @@ -17,6 +17,7 @@ import ( "github.com/restic/restic/server" ) +// Node is a file, directory or other item in a backup. type Node struct { Name string `json:"name"` Type string `json:"type"` @@ -62,6 +63,7 @@ func (node Node) Tree() *Tree { return node.tree } +// NodeFromFileInfo returns a new node from the given path and FileInfo. func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) { node := &Node{ path: path, @@ -100,6 +102,7 @@ func nodeTypeFromFileInfo(fi os.FileInfo) string { return "" } +// CreateAt creates the node at the given path and restores all the meta data. func (node *Node) CreateAt(path string, s *server.Server) error { switch node.Type { case "dir": diff --git a/restorer.go b/restorer.go index 740677473..cebd9c894 100644 --- a/restorer.go +++ b/restorer.go @@ -12,6 +12,7 @@ import ( "github.com/juju/errors" ) +// Restorer is used to restore a snapshot to a directory. type Restorer struct { s *server.Server sn *Snapshot @@ -103,6 +104,7 @@ func (res *Restorer) RestoreTo(dir string) error { return res.restoreTo(dir, "", res.sn.Tree) } +// Snapshot returns the snapshot this restorer is configured to use. func (res *Restorer) Snapshot() *Snapshot { return res.sn } diff --git a/server/key.go b/server/key.go index bcecb7719..68f00c481 100644 --- a/server/key.go +++ b/server/key.go @@ -258,13 +258,13 @@ func (k *Key) DecryptFrom(rd io.Reader) (io.ReadCloser, error) { return crypto.DecryptFrom(k.master, rd) } -// Master() returns the master keys for this repository. Only included for +// Master returns the master keys for this repository. Only included for // debug purposes. func (k *Key) Master() *crypto.Key { return k.master } -// User() returns the user keys for this key. Only included for debug purposes. +// User returns the user keys for this key. Only included for debug purposes. func (k *Key) User() *crypto.Key { return k.user } diff --git a/server/server.go b/server/server.go index da1974a99..e2a84650c 100644 --- a/server/server.go +++ b/server/server.go @@ -422,12 +422,10 @@ func (s *Server) Flush() error { return nil } -// Returns the backend used for this server. func (s *Server) Backend() backend.Backend { return s.be } -// Returns the index of this server. func (s *Server) Index() *Index { return s.idx } diff --git a/test/helpers.go b/test/helpers.go index a3e5a743c..b99f56dec 100644 --- a/test/helpers.go +++ b/test/helpers.go @@ -21,7 +21,7 @@ func Assert(tb testing.TB, condition bool, msg string, v ...interface{}) { } } -// ok fails the test if an err is not nil. +// OK fails the test if an err is not nil. func OK(tb testing.TB, err error) { if err != nil { _, file, line, _ := runtime.Caller(1) @@ -30,7 +30,7 @@ func OK(tb testing.TB, err error) { } } -// equals fails the test if exp is not equal to act. +// Equals fails the test if exp is not equal to act. func Equals(tb testing.TB, exp, act interface{}) { if !reflect.DeepEqual(exp, act) { _, file, line, _ := runtime.Caller(1) diff --git a/zerrors_linux.go b/zerrors_linux.go deleted file mode 100644 index c90a86293..000000000 --- a/zerrors_linux.go +++ /dev/null @@ -1,5 +0,0 @@ -package restic - -// Add constant O_PATH missing from Go1.3, will be added to Go1.4 according to -// https://code.google.com/p/go/issues/detail?id=7830 -const O_PATH = 010000000