Load dir info of all folders
This commit is contained in:
parent
968df3c2a3
commit
7d73b9cdd4
|
@ -90,3 +90,21 @@ func ReverseProxyIpFrom(ctx context.Context) (string, bool) {
|
|||
v, ok := ctx.Value(ReverseProxyIp).(string)
|
||||
return v, ok
|
||||
}
|
||||
|
||||
func AddValues(ctx context.Context) context.Context {
|
||||
keys := []contextKey{
|
||||
User,
|
||||
Username,
|
||||
Client,
|
||||
Version,
|
||||
Player,
|
||||
Transcoding,
|
||||
ClientUniqueId,
|
||||
}
|
||||
for _, key := range keys {
|
||||
if v, ok := ctx.Value(key).(string); ok {
|
||||
ctx = context.WithValue(ctx, key, v)
|
||||
}
|
||||
}
|
||||
return ctx
|
||||
}
|
||||
|
|
|
@ -0,0 +1,139 @@
|
|||
package scanner2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/charlievieth/fastwalk"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
)
|
||||
|
||||
type folderEntry struct {
|
||||
fastwalk.DirEntry
|
||||
path string
|
||||
scanCtx *scanContext
|
||||
modTime time.Time
|
||||
images []string
|
||||
imagesUpdatedAt time.Time
|
||||
hasPlaylists bool
|
||||
audioFilesCount uint32
|
||||
}
|
||||
|
||||
func loadDir(ctx context.Context, scanCtx *scanContext, dirPath string, d fastwalk.DirEntry) (folder *folderEntry, children []string, err error) {
|
||||
folder = &folderEntry{DirEntry: d, scanCtx: scanCtx, path: dirPath}
|
||||
|
||||
dirInfo, err := d.Stat()
|
||||
if err != nil {
|
||||
log.Error(ctx, "Error stating dir", "path", dirPath, err)
|
||||
return nil, nil, err
|
||||
}
|
||||
folder.modTime = dirInfo.ModTime()
|
||||
|
||||
dir, err := os.Open(dirPath)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Error in Opening directory", "path", dirPath, err)
|
||||
return folder, children, err
|
||||
}
|
||||
defer dir.Close()
|
||||
|
||||
for _, entry := range fullReadDir(ctx, dir) {
|
||||
isDir, err := isDirOrSymlinkToDir(dirPath, entry)
|
||||
// Skip invalid symlinks
|
||||
if err != nil {
|
||||
log.Error(ctx, "Invalid symlink", "dir", filepath.Join(dirPath, entry.Name()), err)
|
||||
continue
|
||||
}
|
||||
if isDir && isDirReadable(ctx, dirPath, entry) {
|
||||
children = append(children, filepath.Join(dirPath, entry.Name()))
|
||||
} else {
|
||||
fileInfo, err := entry.Info()
|
||||
if err != nil {
|
||||
log.Error(ctx, "Error getting fileInfo", "name", entry.Name(), err)
|
||||
return folder, children, err
|
||||
}
|
||||
if fileInfo.ModTime().After(folder.modTime) {
|
||||
folder.modTime = fileInfo.ModTime()
|
||||
}
|
||||
switch {
|
||||
case model.IsAudioFile(entry.Name()):
|
||||
folder.audioFilesCount++
|
||||
case model.IsValidPlaylist(entry.Name()):
|
||||
folder.hasPlaylists = true
|
||||
case model.IsImageFile(entry.Name()):
|
||||
folder.images = append(folder.images, entry.Name())
|
||||
if fileInfo.ModTime().After(folder.imagesUpdatedAt) {
|
||||
folder.imagesUpdatedAt = fileInfo.ModTime()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return folder, children, nil
|
||||
}
|
||||
|
||||
// fullReadDir reads all files in the folder, skipping the ones with errors.
|
||||
// It also detects when it is "stuck" with an error in the same directory over and over.
|
||||
// In this case, it stops and returns whatever it was able to read until it got stuck.
|
||||
// See discussion here: https://github.com/navidrome/navidrome/issues/1164#issuecomment-881922850
|
||||
func fullReadDir(ctx context.Context, dir fs.ReadDirFile) []os.DirEntry {
|
||||
var allEntries []os.DirEntry
|
||||
var prevErrStr = ""
|
||||
for {
|
||||
entries, err := dir.ReadDir(-1)
|
||||
allEntries = append(allEntries, entries...)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
log.Warn(ctx, "Skipping DirEntry", err)
|
||||
if prevErrStr == err.Error() {
|
||||
log.Error(ctx, "Duplicate DirEntry failure, bailing", err)
|
||||
break
|
||||
}
|
||||
prevErrStr = err.Error()
|
||||
}
|
||||
sort.Slice(allEntries, func(i, j int) bool { return allEntries[i].Name() < allEntries[j].Name() })
|
||||
return allEntries
|
||||
}
|
||||
|
||||
// isDirOrSymlinkToDir returns true if and only if the dirEnt represents a file
|
||||
// system directory, or a symbolic link to a directory. Note that if the dirEnt
|
||||
// is not a directory but is a symbolic link, this method will resolve by
|
||||
// sending a request to the operating system to follow the symbolic link.
|
||||
// originally copied from github.com/karrick/godirwalk, modified to use dirEntry for
|
||||
// efficiency for go 1.16 and beyond
|
||||
func isDirOrSymlinkToDir(baseDir string, dirEnt fs.DirEntry) (bool, error) {
|
||||
if dirEnt.IsDir() {
|
||||
return true, nil
|
||||
}
|
||||
if dirEnt.Type()&os.ModeSymlink == 0 {
|
||||
return false, nil
|
||||
}
|
||||
// Does this symlink point to a directory?
|
||||
fileInfo, err := os.Stat(filepath.Join(baseDir, dirEnt.Name()))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return fileInfo.IsDir(), nil
|
||||
}
|
||||
|
||||
// isDirReadable returns true if the directory represented by dirEnt is readable
|
||||
func isDirReadable(ctx context.Context, baseDir string, dirEnt os.DirEntry) bool {
|
||||
path := filepath.Join(baseDir, dirEnt.Name())
|
||||
|
||||
dir, err := os.Open(path)
|
||||
if err != nil {
|
||||
log.Warn("Skipping unreadable directory", "path", path, err)
|
||||
return false
|
||||
}
|
||||
|
||||
err = dir.Close()
|
||||
if err != nil {
|
||||
log.Warn(ctx, "Error closing directory", "path", path, err)
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
|
@ -0,0 +1,19 @@
|
|||
package scanner2
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/model"
|
||||
)
|
||||
|
||||
type scanContext struct {
|
||||
lib model.Library
|
||||
startTime time.Time
|
||||
}
|
||||
|
||||
func newScannerContext(lib model.Library) *scanContext {
|
||||
return &scanContext{
|
||||
lib: lib,
|
||||
startTime: time.Now(),
|
||||
}
|
||||
}
|
|
@ -3,10 +3,12 @@ package scanner2
|
|||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
"time"
|
||||
|
||||
"github.com/charlievieth/fastwalk"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/model/request"
|
||||
"github.com/navidrome/navidrome/scanner"
|
||||
"github.com/navidrome/navidrome/utils/pl"
|
||||
)
|
||||
|
@ -21,52 +23,89 @@ func New(ctx context.Context, ds model.DataStore) scanner.Scanner {
|
|||
}
|
||||
|
||||
func (s *scanner2) RescanAll(ctx context.Context, fullRescan bool) error {
|
||||
libs, err := s.ds.Library(s.ctx).GetAll()
|
||||
ctx = request.AddValues(s.ctx)
|
||||
|
||||
libs, err := s.ds.Library(ctx).GetAll()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
libsChan := pl.FromSlice(s.ctx, libs)
|
||||
folderChan, folderErrChan := walkDirEntries(s.ctx, libsChan)
|
||||
for folder := range folderChan {
|
||||
log.Debug(s.ctx, "Scanner: Found folder", "folder", folder.Name())
|
||||
}
|
||||
startTime := time.Now()
|
||||
log.Info(ctx, "Scanner: Starting scan", "fullRescan", fullRescan, "numLibraries", len(libs))
|
||||
scanCtxChan := createScanContexts(ctx, libs)
|
||||
folderChan, folderErrChan := walkDirEntries(ctx, scanCtxChan)
|
||||
logErrChan := pl.Sink(ctx, 4, folderChan, func(ctx context.Context, folder *folderEntry) error {
|
||||
log.Debug(ctx, "Scanner: Found folder", "folder", folder.Name(), "_path", folder.path, "audioCount", folder.audioFilesCount, "images", folder.images, "hasPlaylist", folder.hasPlaylists)
|
||||
return nil
|
||||
})
|
||||
|
||||
// Wait for pipeline to end, return first error found
|
||||
for err := range pl.Merge(ctx, folderErrChan) {
|
||||
for err := range pl.Merge(ctx, folderErrChan, logErrChan) {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Info(ctx, "Scanner: Scan finished", "duration", time.Since(startTime))
|
||||
return nil
|
||||
}
|
||||
|
||||
func walkDirEntries(ctx context.Context, libsChan <-chan model.Library) (chan fastwalk.DirEntry, chan error) {
|
||||
outputChannel := make(chan fastwalk.DirEntry)
|
||||
func createScanContexts(ctx context.Context, libs []model.Library) chan *scanContext {
|
||||
outputChannel := make(chan *scanContext, len(libs))
|
||||
go func() {
|
||||
defer close(outputChannel)
|
||||
for _, lib := range libs {
|
||||
outputChannel <- newScannerContext(lib)
|
||||
}
|
||||
}()
|
||||
return outputChannel
|
||||
}
|
||||
|
||||
func walkDirEntries(ctx context.Context, libsChan <-chan *scanContext) (chan *folderEntry, chan error) {
|
||||
outputChannel := make(chan *folderEntry)
|
||||
errChannel := make(chan error)
|
||||
go func() {
|
||||
defer close(outputChannel)
|
||||
defer close(errChannel)
|
||||
errChan := pl.Sink(ctx, 1, libsChan, func(ctx context.Context, lib model.Library) error {
|
||||
errChan := pl.Sink(ctx, 1, libsChan, func(ctx context.Context, scanCtx *scanContext) error {
|
||||
conf := &fastwalk.Config{Follow: true}
|
||||
return fastwalk.Walk(conf, lib.Path, func(path string, d fs.DirEntry, err error) error {
|
||||
// lib.Path
|
||||
err := fastwalk.Walk(conf, scanCtx.lib.Path, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error walking path", "lib", lib.Name, "path", path, err)
|
||||
log.Warn(ctx, "Scanner: Error walking path", "lib", scanCtx.lib.Name, "path", path, err)
|
||||
return nil
|
||||
}
|
||||
if d.IsDir() {
|
||||
outputChannel <- d.(fastwalk.DirEntry)
|
||||
|
||||
// Skip non-directories
|
||||
if !d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Load all pertinent info from directory
|
||||
folder, _, err := loadDir(ctx, scanCtx, path, d.(fastwalk.DirEntry))
|
||||
if err != nil {
|
||||
log.Warn(ctx, "Scanner: Error loading dir", "lib", scanCtx.lib.Name, "path", path, err)
|
||||
return nil
|
||||
}
|
||||
outputChannel <- folder
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
log.Warn(ctx, "Scanner: Error scanning library", "lib", scanCtx.lib.Name, err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// Wait for pipeline to end, and forward any errors
|
||||
for err := range pl.ReadOrDone(ctx, errChan) {
|
||||
errChannel <- err
|
||||
select {
|
||||
case errChannel <- err:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}()
|
||||
return outputChannel, errChannel
|
||||
}
|
||||
func (s *scanner2) Status(context.Context) (*scanner.StatusInfo, error) {
|
||||
return nil, nil
|
||||
return &scanner.StatusInfo{}, nil
|
||||
}
|
||||
|
||||
//nolint:unused
|
||||
|
|
Loading…
Reference in New Issue