Load dir info of all folders

model/request/request.go

@@ -90,3 +90,21 @@ func ReverseProxyIpFrom(ctx context.Context) (string, bool) {
 	v, ok := ctx.Value(ReverseProxyIp).(string)
 	return v, ok
 }
+
+func AddValues(ctx context.Context) context.Context {
+	keys := []contextKey{
+		User,
+		Username,
+		Client,
+		Version,
+		Player,
+		Transcoding,
+		ClientUniqueId,
+	}
+	for _, key := range keys {
+		if v, ok := ctx.Value(key).(string); ok {
+			ctx = context.WithValue(ctx, key, v)
+		}
+	}
+	return ctx
+}

scanner2/folder.go

@@ -0,0 +1,139 @@
+package scanner2
+
+import (
+	"context"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+	"time"
+
+	"github.com/charlievieth/fastwalk"
+	"github.com/navidrome/navidrome/log"
+	"github.com/navidrome/navidrome/model"
+)
+
+type folderEntry struct {
+	fastwalk.DirEntry
+	path            string
+	scanCtx         *scanContext
+	modTime         time.Time
+	images          []string
+	imagesUpdatedAt time.Time
+	hasPlaylists    bool
+	audioFilesCount uint32
+}
+
+func loadDir(ctx context.Context, scanCtx *scanContext, dirPath string, d fastwalk.DirEntry) (folder *folderEntry, children []string, err error) {
+	folder = &folderEntry{DirEntry: d, scanCtx: scanCtx, path: dirPath}
+
+	dirInfo, err := d.Stat()
+	if err != nil {
+		log.Error(ctx, "Error stating dir", "path", dirPath, err)
+		return nil, nil, err
+	}
+	folder.modTime = dirInfo.ModTime()
+
+	dir, err := os.Open(dirPath)
+	if err != nil {
+		log.Error(ctx, "Error in Opening directory", "path", dirPath, err)
+		return folder, children, err
+	}
+	defer dir.Close()
+
+	for _, entry := range fullReadDir(ctx, dir) {
+		isDir, err := isDirOrSymlinkToDir(dirPath, entry)
+		// Skip invalid symlinks
+		if err != nil {
+			log.Error(ctx, "Invalid symlink", "dir", filepath.Join(dirPath, entry.Name()), err)
+			continue
+		}
+		if isDir && isDirReadable(ctx, dirPath, entry) {
+			children = append(children, filepath.Join(dirPath, entry.Name()))
+		} else {
+			fileInfo, err := entry.Info()
+			if err != nil {
+				log.Error(ctx, "Error getting fileInfo", "name", entry.Name(), err)
+				return folder, children, err
+			}
+			if fileInfo.ModTime().After(folder.modTime) {
+				folder.modTime = fileInfo.ModTime()
+			}
+			switch {
+			case model.IsAudioFile(entry.Name()):
+				folder.audioFilesCount++
+			case model.IsValidPlaylist(entry.Name()):
+				folder.hasPlaylists = true
+			case model.IsImageFile(entry.Name()):
+				folder.images = append(folder.images, entry.Name())
+				if fileInfo.ModTime().After(folder.imagesUpdatedAt) {
+					folder.imagesUpdatedAt = fileInfo.ModTime()
+				}
+			}
+		}
+	}
+	return folder, children, nil
+}
+
+// fullReadDir reads all files in the folder, skipping the ones with errors.
+// It also detects when it is "stuck" with an error in the same directory over and over.
+// In this case, it stops and returns whatever it was able to read until it got stuck.
+// See discussion here: https://github.com/navidrome/navidrome/issues/1164#issuecomment-881922850
+func fullReadDir(ctx context.Context, dir fs.ReadDirFile) []os.DirEntry {
+	var allEntries []os.DirEntry
+	var prevErrStr = ""
+	for {
+		entries, err := dir.ReadDir(-1)
+		allEntries = append(allEntries, entries...)
+		if err == nil {
+			break
+		}
+		log.Warn(ctx, "Skipping DirEntry", err)
+		if prevErrStr == err.Error() {
+			log.Error(ctx, "Duplicate DirEntry failure, bailing", err)
+			break
+		}
+		prevErrStr = err.Error()
+	}
+	sort.Slice(allEntries, func(i, j int) bool { return allEntries[i].Name() < allEntries[j].Name() })
+	return allEntries
+}
+
+// isDirOrSymlinkToDir returns true if and only if the dirEnt represents a file
+// system directory, or a symbolic link to a directory. Note that if the dirEnt
+// is not a directory but is a symbolic link, this method will resolve by
+// sending a request to the operating system to follow the symbolic link.
+// originally copied from github.com/karrick/godirwalk, modified to use dirEntry for
+// efficiency for go 1.16 and beyond
+func isDirOrSymlinkToDir(baseDir string, dirEnt fs.DirEntry) (bool, error) {
+	if dirEnt.IsDir() {
+		return true, nil
+	}
+	if dirEnt.Type()&os.ModeSymlink == 0 {
+		return false, nil
+	}
+	// Does this symlink point to a directory?
+	fileInfo, err := os.Stat(filepath.Join(baseDir, dirEnt.Name()))
+	if err != nil {
+		return false, err
+	}
+	return fileInfo.IsDir(), nil
+}
+
+// isDirReadable returns true if the directory represented by dirEnt is readable
+func isDirReadable(ctx context.Context, baseDir string, dirEnt os.DirEntry) bool {
+	path := filepath.Join(baseDir, dirEnt.Name())
+
+	dir, err := os.Open(path)
+	if err != nil {
+		log.Warn("Skipping unreadable directory", "path", path, err)
+		return false
+	}
+
+	err = dir.Close()
+	if err != nil {
+		log.Warn(ctx, "Error closing directory", "path", path, err)
+	}
+
+	return true
+}

scanner2/scan_context.go

@@ -0,0 +1,19 @@
+package scanner2
+
+import (
+	"time"
+
+	"github.com/navidrome/navidrome/model"
+)
+
+type scanContext struct {
+	lib       model.Library
+	startTime time.Time
+}
+
+func newScannerContext(lib model.Library) *scanContext {
+	return &scanContext{
+		lib:       lib,
+		startTime: time.Now(),
+	}
+}

scanner2/scanner2.go

@@ -3,10 +3,12 @@ package scanner2
 import (
 	"context"
 	"io/fs"
+	"time"
 
 	"github.com/charlievieth/fastwalk"
 	"github.com/navidrome/navidrome/log"
 	"github.com/navidrome/navidrome/model"
+	"github.com/navidrome/navidrome/model/request"
 	"github.com/navidrome/navidrome/scanner"
 	"github.com/navidrome/navidrome/utils/pl"
 )
@@ -21,52 +23,89 @@ func New(ctx context.Context, ds model.DataStore) scanner.Scanner {
 }
 
 func (s *scanner2) RescanAll(ctx context.Context, fullRescan bool) error {
-	libs, err := s.ds.Library(s.ctx).GetAll()
+	ctx = request.AddValues(s.ctx)
+
+	libs, err := s.ds.Library(ctx).GetAll()
 	if err != nil {
 		return err
 	}
 
-	libsChan := pl.FromSlice(s.ctx, libs)
-	folderChan, folderErrChan := walkDirEntries(s.ctx, libsChan)
-	for folder := range folderChan {
-		log.Debug(s.ctx, "Scanner: Found folder", "folder", folder.Name())
-	}
+	startTime := time.Now()
+	log.Info(ctx, "Scanner: Starting scan", "fullRescan", fullRescan, "numLibraries", len(libs))
+	scanCtxChan := createScanContexts(ctx, libs)
+	folderChan, folderErrChan := walkDirEntries(ctx, scanCtxChan)
+	logErrChan := pl.Sink(ctx, 4, folderChan, func(ctx context.Context, folder *folderEntry) error {
+		log.Debug(ctx, "Scanner: Found folder", "folder", folder.Name(), "_path", folder.path, "audioCount", folder.audioFilesCount, "images", folder.images, "hasPlaylist", folder.hasPlaylists)
+		return nil
+	})
 
 	// Wait for pipeline to end, return first error found
-	for err := range pl.Merge(ctx, folderErrChan) {
+	for err := range pl.Merge(ctx, folderErrChan, logErrChan) {
 		return err
 	}
+
+	log.Info(ctx, "Scanner: Scan finished", "duration", time.Since(startTime))
 	return nil
 }
 
-func walkDirEntries(ctx context.Context, libsChan <-chan model.Library) (chan fastwalk.DirEntry, chan error) {
-	outputChannel := make(chan fastwalk.DirEntry)
+func createScanContexts(ctx context.Context, libs []model.Library) chan *scanContext {
+	outputChannel := make(chan *scanContext, len(libs))
+	go func() {
+		defer close(outputChannel)
+		for _, lib := range libs {
+			outputChannel <- newScannerContext(lib)
+		}
+	}()
+	return outputChannel
+}
+
+func walkDirEntries(ctx context.Context, libsChan <-chan *scanContext) (chan *folderEntry, chan error) {
+	outputChannel := make(chan *folderEntry)
 	errChannel := make(chan error)
 	go func() {
 		defer close(outputChannel)
 		defer close(errChannel)
-		errChan := pl.Sink(ctx, 1, libsChan, func(ctx context.Context, lib model.Library) error {
+		errChan := pl.Sink(ctx, 1, libsChan, func(ctx context.Context, scanCtx *scanContext) error {
 			conf := &fastwalk.Config{Follow: true}
-			return fastwalk.Walk(conf, lib.Path, func(path string, d fs.DirEntry, err error) error {
+			// lib.Path
+			err := fastwalk.Walk(conf, scanCtx.lib.Path, func(path string, d fs.DirEntry, err error) error {
 				if err != nil {
-					log.Error(ctx, "Scanner: Error walking path", "lib", lib.Name, "path", path, err)
+					log.Warn(ctx, "Scanner: Error walking path", "lib", scanCtx.lib.Name, "path", path, err)
+					return nil
 				}
-				if d.IsDir() {
-					outputChannel <- d.(fastwalk.DirEntry)
+
+				// Skip non-directories
+				if !d.IsDir() {
+					return nil
 				}
+
+				// Load all pertinent info from directory
+				folder, _, err := loadDir(ctx, scanCtx, path, d.(fastwalk.DirEntry))
+				if err != nil {
+					log.Warn(ctx, "Scanner: Error loading dir", "lib", scanCtx.lib.Name, "path", path, err)
+					return nil
+				}
+				outputChannel <- folder
 				return nil
 			})
+			if err != nil {
+				log.Warn(ctx, "Scanner: Error scanning library", "lib", scanCtx.lib.Name, err)
+			}
+			return nil
 		})
 
 		// Wait for pipeline to end, and forward any errors
 		for err := range pl.ReadOrDone(ctx, errChan) {
-			errChannel <- err
+			select {
+			case errChannel <- err:
+			default:
+			}
 		}
 	}()
 	return outputChannel, errChannel
 }
 func (s *scanner2) Status(context.Context) (*scanner.StatusInfo, error) {
-	return nil, nil
+	return &scanner.StatusInfo{}, nil
 }
 
 //nolint:unused