mirrors
/
navidrome
mirror of https://github.com/navidrome/navidrome.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

WIP

This commit is contained in:
Deluan 2020-10-29 11:31:45 -04:00
parent f1a24b971a
commit 2f394623c8
5 changed files with 117 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scanner/playlist_sync.go
View File

@ -24,8 +24,8 @@ func newPlaylistSync(ds model.DataStore) *playlistSync {
return &playlistSync{ds: ds}
}
func (s *playlistSync) processPlaylists(ctx context.Context, dir string) int {
count := 0
func (s *playlistSync) processPlaylists(ctx context.Context, dir string) int64 {
var count int64
files, err := ioutil.ReadDir(dir)
if err != nil {
log.Error(ctx, "Error reading files", "dir", dir, err)

112
scanner/tag_scanner.go
View File

@ -36,26 +36,30 @@ func NewTagScanner(rootFolder string, ds model.DataStore, cacheWarmer core.Cache
}
}
type counters struct {
added int64
updated int64
deleted int64
}
type (
counters struct {
added int64
updated int64
deleted int64
playlists int64
}
dirMap map[string]dirStats
)
const (
// filesBatchSize used for batching file metadata extraction
filesBatchSize = 100
)
// Scanner algorithm overview:
// Load all directories under the music folder, with their ModTime (self or any non-dir children, whichever is newer)
// TagScanner algorithm overview:
// Load all directories from the DB
// Compare both collections to find changed folders (based on lastModifiedSince) and deleted folders
// For each deleted folder: delete all files from DB whose path starts with the delete folder path (non-recursively)
// Traverse the music folder, collecting each subfolder's ModTime (self or any non-dir children, whichever is newer)
// For each changed folder: get all files from DB whose path starts with the changed folder (non-recursively), check each file:
// if file in folder is newer, update the one in DB
// if file in folder does not exists in DB, add it
// for each file in the DB that is not found in the folder, delete it from DB
// Compare directories in the fs with the ones in the DB to find deleted folders
// For each deleted folder: delete all files from DB whose path starts with the delete folder path (non-recursively)
// Create new albums/artists, update counters:
// collect all albumIDs and artistIDs from previous steps
// refresh the collected albums and artists with the metadata from the mediafiles
@ -65,60 +69,59 @@ const (
// Delete all empty albums, delete all empty artists, clean-up playlists
func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time) error {
ctx = s.withAdminUser(ctx)
start := time.Now()
allFSDirs, err := s.getDirTree(ctx)
if err != nil {
return err
}
allDBDirs, err := s.getDBDirTree(ctx)
if err != nil {
return err
}
changedDirs := s.getChangedDirs(ctx, allFSDirs, allDBDirs, lastModifiedSince)
deletedDirs := s.getDeletedDirs(ctx, allFSDirs, allDBDirs)
allFSDirs := dirMap{}
var changedDirs []string
s.cnt = &counters{}
if len(changedDirs)+len(deletedDirs) == 0 {
foldersFound := s.getRootFolderWalker(ctx)
for {
folderStats, more := <-foldersFound
if !more {
break
}
allFSDirs[folderStats.Path] = folderStats
if s.isChangedDirs(ctx, folderStats, allDBDirs, lastModifiedSince) {
changedDirs = append(changedDirs, folderStats.Path)
err := s.processChangedDir(ctx, folderStats.Path)
if err != nil {
log.Error("Error updating folder in the DB", "path", folderStats.Path, err)
}
}
}
deletedDirs := s.getDeletedDirs(ctx, allFSDirs, allDBDirs)
if len(deletedDirs)+len(changedDirs) == 0 {
log.Debug(ctx, "No changes found in Music Folder", "folder", s.rootFolder)
return nil
}
if log.CurrentLevel() >= log.LevelTrace {
log.Info(ctx, "Folder changes detected", "changedFolders", len(changedDirs), "deletedFolders", len(deletedDirs),
"changed", strings.Join(changedDirs, ";"), "deleted", strings.Join(deletedDirs, ";"))
} else {
log.Info(ctx, "Folder changes detected", "changedFolders", len(changedDirs), "deletedFolders", len(deletedDirs))
}
s.cnt = &counters{}
for _, dir := range deletedDirs {
err := s.processDeletedDir(ctx, dir)
if err != nil {
log.Error("Error removing deleted folder from DB", "path", dir, err)
}
}
for _, dir := range changedDirs {
err := s.processChangedDir(ctx, dir)
if err != nil {
log.Error("Error updating folder in the DB", "path", dir, err)
}
}
plsCount := 0
s.cnt.playlists = 0
if conf.Server.AutoImportPlaylists {
// Now that all mediafiles are imported/updated, search for and import playlists
// Now that all mediafiles are imported/updated, search for and import/update playlists
u, _ := request.UserFrom(ctx)
for _, dir := range changedDirs {
info := allFSDirs[dir]
if info.hasPlaylist {
if info.HasPlaylist {
if !u.IsAdmin {
log.Warn("Playlists will not be imported, as there are no admin users yet, "+
"Please create an admin user first, and then update the playlists for them to be imported", "dir", dir)
} else {
plsCount = s.plsSync.processPlaylists(ctx, dir)
s.cnt.playlists = s.plsSync.processPlaylists(ctx, dir)
}
}
}
@ -128,20 +131,22 @@ func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time) erro
err = s.ds.GC(log.NewContext(ctx), s.rootFolder)
log.Info("Finished processing Music Folder", "folder", s.rootFolder, "elapsed", time.Since(start),
"added", s.cnt.added, "updated", s.cnt.updated, "deleted", s.cnt.deleted, "playlistsImported", plsCount)
"added", s.cnt.added, "updated", s.cnt.updated, "deleted", s.cnt.deleted, "playlistsImported", s.cnt.playlists)
return err
}
func (s *TagScanner) getDirTree(ctx context.Context) (dirMap, error) {
func (s *TagScanner) getRootFolderWalker(ctx context.Context) walkResults {
start := time.Now()
log.Trace(ctx, "Loading directory tree from music folder", "folder", s.rootFolder)
dirs, err := loadDirTree(ctx, s.rootFolder)
if err != nil {
return nil, err
}
log.Debug("Directory tree loaded from music folder", "total", len(dirs), "elapsed", time.Since(start))
return dirs, nil
results := make(chan dirStats, 5000)
go func() {
if err := walkDirTree(ctx, s.rootFolder, results); err != nil {
log.Error("Scan was interrupted by error", err)
}
log.Debug("Finished reading directories from filesystem", "total", "elapsed", time.Since(start))
}()
return results
}
func (s *TagScanner) getDBDirTree(ctx context.Context) (map[string]struct{}, error) {
@ -162,21 +167,10 @@ func (s *TagScanner) getDBDirTree(ctx context.Context) (map[string]struct{}, err
return resp, nil
}
func (s *TagScanner) getChangedDirs(ctx context.Context, fsDirs dirMap, dbDirs map[string]struct{}, lastModified time.Time) []string {
start := time.Now()
log.Trace(ctx, "Checking for changed folders")
var changed []string
for d, info := range fsDirs {
_, inDB := dbDirs[d]
if (!inDB && (info.hasAudioFiles)) || info.modTime.After(lastModified) {
changed = append(changed, d)
}
}
sort.Strings(changed)
log.Debug(ctx, "Finished changed folders check", "total", len(changed), "elapsed", time.Since(start))
return changed
func (s *TagScanner) isChangedDirs(ctx context.Context, folder dirStats, dbDirs map[string]struct{}, lastModified time.Time) bool {
_, inDB := dbDirs[folder.Path]
// If is a new folder with at least one song OR it was modified after lastModified
return (!inDB && (folder.AudioFilesCount > 0)) || folder.ModTime.After(lastModified)
}
func (s *TagScanner) getDeletedDirs(ctx context.Context, fsDirs dirMap, dbDirs map[string]struct{}) []string {

50
scanner/load_tree.go → scanner/walk_dir_tree.go
View File

@ -14,49 +14,53 @@ import (
)
type (
dirMapValue struct {
modTime time.Time
hasImages bool
hasPlaylist bool
hasAudioFiles bool
dirStats struct {
Path string
ModTime time.Time
HasImages bool
HasPlaylist bool
AudioFilesCount int64
}
dirMap = map[string]dirMapValue
walkResults = chan dirStats
)
func loadDirTree(ctx context.Context, rootFolder string) (dirMap, error) {
newMap := make(dirMap)
err := loadMap(ctx, rootFolder, rootFolder, newMap)
func walkDirTree(ctx context.Context, rootFolder string, results walkResults) error {
err := walkFolder(ctx, rootFolder, rootFolder, results)
if err != nil {
log.Error(ctx, "Error loading directory tree", err)
}
return newMap, err
close(results)
return err
}
func loadMap(ctx context.Context, rootPath string, currentFolder string, dirMap dirMap) error {
children, dirMapValue, err := loadDir(ctx, currentFolder)
func walkFolder(ctx context.Context, rootPath string, currentFolder string, results walkResults) error {
children, stats, err := loadDir(ctx, currentFolder)
if err != nil {
return err
}
for _, c := range children {
err := loadMap(ctx, rootPath, c, dirMap)
err := walkFolder(ctx, rootPath, c, results)
if err != nil {
return err
}
}
dir := filepath.Clean(currentFolder)
dirMap[dir] = dirMapValue
log.Trace(ctx, "Found directory", "dir", dir, "audioCount", stats.AudioFilesCount,
"hasImages", stats.HasImages, "HasPlaylist", stats.HasPlaylist)
stats.Path = dir
results <- stats
return nil
}
func loadDir(ctx context.Context, dirPath string) (children []string, info dirMapValue, err error) {
func loadDir(ctx context.Context, dirPath string) (children []string, stats dirStats, err error) {
dirInfo, err := os.Stat(dirPath)
if err != nil {
log.Error(ctx, "Error stating dir", "path", dirPath, err)
return
}
info.modTime = dirInfo.ModTime()
stats.ModTime = dirInfo.ModTime()
files, err := ioutil.ReadDir(dirPath)
if err != nil {
@ -67,17 +71,21 @@ func loadDir(ctx context.Context, dirPath string) (children []string, info dirMa
isDir, err := isDirOrSymlinkToDir(dirPath, f)
// Skip invalid symlinks
if err != nil {
log.Error(ctx, "Invalid symlink", "dir", dirPath)
continue
}
if isDir && !isDirIgnored(dirPath, f) && isDirReadable(dirPath, f) {
children = append(children, filepath.Join(dirPath, f.Name()))
} else {
if f.ModTime().After(info.modTime) {
info.modTime = f.ModTime()
if f.ModTime().After(stats.ModTime) {
stats.ModTime = f.ModTime()
}
if utils.IsAudioFile(f.Name()) {
stats.AudioFilesCount++
} else {
stats.HasPlaylist = stats.HasPlaylist || utils.IsPlaylist(f.Name())
stats.HasImages = stats.HasImages || utils.IsImageFile(f.Name())
}
info.hasImages = info.hasImages || utils.IsImageFile(f.Name())
info.hasPlaylist = info.hasPlaylist || utils.IsPlaylist(f.Name())
info.hasAudioFiles = info.hasAudioFiles || utils.IsAudioFile(f.Name())
}
}
return

32
scanner/load_tree_test.go → scanner/walk_dir_tree_test.go
View File

@ -1,14 +1,46 @@
package scanner
import (
"context"
"os"
"path/filepath"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
. "github.com/onsi/gomega/gstruct"
)
var _ = Describe("load_tree", func() {
Describe("walkDirTree", func() {
It("reads all info correctly", func() {
var collected = dirMap{}
results := make(walkResults, 5000)
var err error
go func() {
err = walkDirTree(context.TODO(), "tests/fixtures", results)
}()
for {
stats, more := <-results
if !more {
break
}
collected[stats.Path] = stats
}
Expect(err).To(BeNil())
Expect(collected["tests/fixtures"]).To(MatchFields(IgnoreExtras, Fields{
"HasImages": BeTrue(),
"HasPlaylist": BeFalse(),
"AudioFilesCount": BeNumerically("==", 4),
}))
Expect(collected["tests/fixtures/playlists"].HasPlaylist).To(BeTrue())
Expect(collected).To(HaveKey("tests/fixtures/symlink2dir"))
Expect(collected).To(HaveKey("tests/fixtures/empty_folder"))
})
})
Describe("isDirOrSymlinkToDir", func() {
It("returns true for normal dirs", func() {
dir, _ := os.Stat("tests/fixtures")

2
tests/fixtures/symlink2dir vendored
View File

@ -1 +1 @@
../
empty_folder