Add new scanner algorithm, can be enabled with DevNewScanner config option

This commit is contained in:
Deluan 2020-07-17 10:27:30 -04:00
parent de0cc1f268
commit 51c295d1de
11 changed files with 750 additions and 160 deletions

View File

@ -38,6 +38,7 @@ type configOptions struct {
// DevFlags. These are used to enable/disable debugging and incomplete features
DevLogSourceLine bool
DevAutoCreateAdminPassword string
DevNewScanner bool
}
var Server = &configOptions{}
@ -94,6 +95,7 @@ func init() {
// DevFlags. These are used to enable/disable debugging and incomplete features
viper.SetDefault("devlogsourceline", false)
viper.SetDefault("devautocreateadminpassword", "")
viper.SetDefault("devnewscanner", false)
}
func InitConfig(cfgFile string) {

View File

@ -7,9 +7,7 @@ import (
"path/filepath"
"time"
"github.com/deluan/navidrome/consts"
"github.com/deluan/navidrome/log"
"github.com/deluan/navidrome/utils"
)
type dirInfo struct {
@ -18,19 +16,19 @@ type dirInfo struct {
}
type dirInfoMap map[string]dirInfo
type ChangeDetector struct {
type changeDetector struct {
rootFolder string
dirMap dirInfoMap
}
func NewChangeDetector(rootFolder string) *ChangeDetector {
return &ChangeDetector{
func newChangeDetector(rootFolder string) *changeDetector {
return &changeDetector{
rootFolder: rootFolder,
dirMap: dirInfoMap{},
}
}
func (s *ChangeDetector) Scan(ctx context.Context, lastModifiedSince time.Time) (changed []string, deleted []string, err error) {
func (s *changeDetector) Scan(ctx context.Context, lastModifiedSince time.Time) (changed []string, deleted []string, err error) {
start := time.Now()
newMap := make(dirInfoMap)
err = s.loadMap(ctx, newMap, s.rootFolder, lastModifiedSince, false)
@ -48,7 +46,7 @@ func (s *ChangeDetector) Scan(ctx context.Context, lastModifiedSince time.Time)
return
}
func (s *ChangeDetector) loadDir(ctx context.Context, dirPath string) (children []string, lastUpdated time.Time, err error) {
func (s *changeDetector) loadDir(ctx context.Context, dirPath string) (children []string, lastUpdated time.Time, err error) {
dirInfo, err := os.Stat(dirPath)
if err != nil {
log.Error(ctx, "Error stating dir", "path", dirPath, err)
@ -78,44 +76,7 @@ func (s *ChangeDetector) loadDir(ctx context.Context, dirPath string) (children
return
}
// isDirOrSymlinkToDir returns true if and only if the dirInfo represents a file
// system directory, or a symbolic link to a directory. Note that if the dirInfo
// is not a directory but is a symbolic link, this method will resolve by
// sending a request to the operating system to follow the symbolic link.
// Copied from github.com/karrick/godirwalk
func isDirOrSymlinkToDir(baseDir string, dirInfo os.FileInfo) (bool, error) {
if dirInfo.IsDir() {
return true, nil
}
if dirInfo.Mode()&os.ModeSymlink == 0 {
return false, nil
}
// Does this symlink point to a directory?
dirInfo, err := os.Stat(filepath.Join(baseDir, dirInfo.Name()))
if err != nil {
return false, err
}
return dirInfo.IsDir(), nil
}
// isDirIgnored returns true if the directory represented by dirInfo contains an
// `ignore` file (named after consts.SkipScanFile)
func isDirIgnored(baseDir string, dirInfo os.FileInfo) bool {
_, err := os.Stat(filepath.Join(baseDir, dirInfo.Name(), consts.SkipScanFile))
return err == nil
}
// isDirReadable returns true if the directory represented by dirInfo is readable
func isDirReadable(baseDir string, dirInfo os.FileInfo) bool {
path := filepath.Join(baseDir, dirInfo.Name())
res, err := utils.IsDirReadable(path)
if !res {
log.Debug("Warning: Skipping unreadable directory", "path", path, err)
}
return res
}
func (s *ChangeDetector) loadMap(ctx context.Context, dirMap dirInfoMap, path string, since time.Time, maybe bool) error {
func (s *changeDetector) loadMap(ctx context.Context, dirMap dirInfoMap, path string, since time.Time, maybe bool) error {
children, lastUpdated, err := s.loadDir(ctx, path)
if err != nil {
return err
@ -134,7 +95,7 @@ func (s *ChangeDetector) loadMap(ctx context.Context, dirMap dirInfoMap, path st
return nil
}
func (s *ChangeDetector) getRelativePath(subFolder string) string {
func (s *changeDetector) getRelativePath(subFolder string) string {
dir, _ := filepath.Rel(s.rootFolder, subFolder)
if dir == "" {
dir = "."
@ -142,7 +103,7 @@ func (s *ChangeDetector) getRelativePath(subFolder string) string {
return dir
}
func (s *ChangeDetector) checkForUpdates(lastModifiedSince time.Time, newMap dirInfoMap) (changed []string, deleted []string, err error) {
func (s *changeDetector) checkForUpdates(lastModifiedSince time.Time, newMap dirInfoMap) (changed []string, deleted []string, err error) {
for dir, newEntry := range newMap {
lastUpdated := newEntry.mdate
oldLastUpdated := lastModifiedSince

View File

@ -11,9 +11,9 @@ import (
. "github.com/onsi/gomega"
)
var _ = Describe("ChangeDetector", func() {
var _ = Describe("changeDetector", func() {
var testFolder string
var scanner *ChangeDetector
var scanner *changeDetector
lastModifiedSince := time.Time{}
@ -23,7 +23,7 @@ var _ = Describe("ChangeDetector", func() {
if err != nil {
panic(err)
}
scanner = NewChangeDetector(testFolder)
scanner = newChangeDetector(testFolder)
})
It("detects changes recursively", func() {
@ -97,7 +97,7 @@ var _ = Describe("ChangeDetector", func() {
// Only returns changes after lastModifiedSince
lastModifiedSince = nowWithDelay()
newScanner := NewChangeDetector(testFolder)
newScanner := newChangeDetector(testFolder)
changed, deleted, err = newScanner.Scan(context.TODO(), lastModifiedSince)
Expect(err).To(BeNil())
Expect(deleted).To(BeEmpty())

58
scanner/flushable_map.go Normal file
View File

@ -0,0 +1,58 @@
package scanner
import (
"context"
"fmt"
"github.com/deluan/navidrome/log"
)
const (
// batchSize used for albums/artists updates
batchSize = 5
)
type refreshCallbackFunc = func(ids ...string) error
type flushableMap struct {
ctx context.Context
flushFunc refreshCallbackFunc
entity string
m map[string]struct{}
}
func newFlushableMap(ctx context.Context, entity string, flushFunc refreshCallbackFunc) *flushableMap {
return &flushableMap{
ctx: ctx,
flushFunc: flushFunc,
entity: entity,
m: map[string]struct{}{},
}
}
func (f *flushableMap) update(id string) error {
f.m[id] = struct{}{}
if len(f.m) >= batchSize {
err := f.flush()
if err != nil {
return err
}
}
return nil
}
func (f *flushableMap) flush() error {
if len(f.m) == 0 {
return nil
}
var ids []string
for id := range f.m {
ids = append(ids, id)
delete(f.m, id)
}
if err := f.flushFunc(ids...); err != nil {
log.Error(f.ctx, fmt.Sprintf("Error writing %ss to the DB", f.entity), err)
return err
}
return nil
}

109
scanner/load_tree.go Normal file
View File

@ -0,0 +1,109 @@
package scanner
import (
"context"
"io/ioutil"
"os"
"path/filepath"
"time"
"github.com/deluan/navidrome/consts"
"github.com/deluan/navidrome/log"
"github.com/deluan/navidrome/utils"
)
type dirMap = map[string]time.Time
func loadDirTree(ctx context.Context, rootFolder string) (dirMap, error) {
newMap := make(map[string]time.Time)
err := loadMap(ctx, rootFolder, rootFolder, newMap)
if err != nil {
log.Error(ctx, "Error loading directory tree", err)
}
return newMap, err
}
func loadMap(ctx context.Context, rootPath string, currentFolder string, dirMap dirMap) error {
children, lastUpdated, err := loadDir(ctx, currentFolder)
if err != nil {
return err
}
for _, c := range children {
err := loadMap(ctx, rootPath, c, dirMap)
if err != nil {
return err
}
}
dir := filepath.Clean(currentFolder)
dirMap[dir] = lastUpdated
return nil
}
func loadDir(ctx context.Context, dirPath string) (children []string, lastUpdated time.Time, err error) {
dirInfo, err := os.Stat(dirPath)
if err != nil {
log.Error(ctx, "Error stating dir", "path", dirPath, err)
return
}
lastUpdated = dirInfo.ModTime()
files, err := ioutil.ReadDir(dirPath)
if err != nil {
log.Error(ctx, "Error reading dir", "path", dirPath, err)
return
}
for _, f := range files {
isDir, err := isDirOrSymlinkToDir(dirPath, f)
// Skip invalid symlinks
if err != nil {
continue
}
if isDir && !isDirIgnored(dirPath, f) && isDirReadable(dirPath, f) {
children = append(children, filepath.Join(dirPath, f.Name()))
} else {
if f.ModTime().After(lastUpdated) {
lastUpdated = f.ModTime()
}
}
}
return
}
// isDirOrSymlinkToDir returns true if and only if the dirInfo represents a file
// system directory, or a symbolic link to a directory. Note that if the dirInfo
// is not a directory but is a symbolic link, this method will resolve by
// sending a request to the operating system to follow the symbolic link.
// Copied from github.com/karrick/godirwalk
func isDirOrSymlinkToDir(baseDir string, dirInfo os.FileInfo) (bool, error) {
if dirInfo.IsDir() {
return true, nil
}
if dirInfo.Mode()&os.ModeSymlink == 0 {
return false, nil
}
// Does this symlink point to a directory?
dirInfo, err := os.Stat(filepath.Join(baseDir, dirInfo.Name()))
if err != nil {
return false, err
}
return dirInfo.IsDir(), nil
}
// isDirIgnored returns true if the directory represented by dirInfo contains an
// `ignore` file (named after consts.SkipScanFile)
func isDirIgnored(baseDir string, dirInfo os.FileInfo) bool {
_, err := os.Stat(filepath.Join(baseDir, dirInfo.Name(), consts.SkipScanFile))
return err == nil
}
// isDirReadable returns true if the directory represented by dirInfo is readable
func isDirReadable(baseDir string, dirInfo os.FileInfo) bool {
path := filepath.Join(baseDir, dirInfo.Name())
res, err := utils.IsDirReadable(path)
if !res {
log.Debug("Warning: Skipping unreadable directory", "path", path, err)
}
return res
}

120
scanner/mapping.go Normal file
View File

@ -0,0 +1,120 @@
package scanner
import (
"crypto/md5"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/deluan/navidrome/consts"
"github.com/deluan/navidrome/model"
"github.com/deluan/navidrome/utils"
"github.com/kennygrant/sanitize"
)
type mediaFileMapper struct {
rootFolder string
}
func newMediaFileMapper(rootFolder string) *mediaFileMapper {
return &mediaFileMapper{rootFolder: rootFolder}
}
func (s *mediaFileMapper) toMediaFile(md *Metadata) model.MediaFile {
mf := &model.MediaFile{}
mf.ID = s.trackID(md)
mf.Title = s.mapTrackTitle(md)
mf.Album = md.Album()
mf.AlbumID = s.albumID(md)
mf.Album = s.mapAlbumName(md)
mf.ArtistID = s.artistID(md)
mf.Artist = s.mapArtistName(md)
mf.AlbumArtistID = s.albumArtistID(md)
mf.AlbumArtist = s.mapAlbumArtistName(md)
mf.Genre = md.Genre()
mf.Compilation = md.Compilation()
mf.Year = md.Year()
mf.TrackNumber, _ = md.TrackNumber()
mf.DiscNumber, _ = md.DiscNumber()
mf.DiscSubtitle = md.DiscSubtitle()
mf.Duration = md.Duration()
mf.BitRate = md.BitRate()
mf.Path = md.FilePath()
mf.Suffix = md.Suffix()
mf.Size = md.Size()
mf.HasCoverArt = md.HasPicture()
mf.SortTitle = md.SortTitle()
mf.SortAlbumName = md.SortAlbum()
mf.SortArtistName = md.SortArtist()
mf.SortAlbumArtistName = md.SortAlbumArtist()
mf.OrderAlbumName = sanitizeFieldForSorting(mf.Album)
mf.OrderArtistName = sanitizeFieldForSorting(mf.Artist)
mf.OrderAlbumArtistName = sanitizeFieldForSorting(mf.AlbumArtist)
// TODO Get Creation time. https://github.com/djherbis/times ?
mf.CreatedAt = md.ModificationTime()
mf.UpdatedAt = md.ModificationTime()
return *mf
}
func sanitizeFieldForSorting(originalValue string) string {
v := utils.NoArticle(originalValue)
v = strings.TrimSpace(sanitize.Accents(v))
return utils.NoArticle(v)
}
func (s *mediaFileMapper) mapTrackTitle(md *Metadata) string {
if md.Title() == "" {
s := strings.TrimPrefix(md.FilePath(), s.rootFolder+string(os.PathSeparator))
e := filepath.Ext(s)
return strings.TrimSuffix(s, e)
}
return md.Title()
}
func (s *mediaFileMapper) mapAlbumArtistName(md *Metadata) string {
switch {
case md.Compilation():
return consts.VariousArtists
case md.AlbumArtist() != "":
return md.AlbumArtist()
case md.Artist() != "":
return md.Artist()
default:
return consts.UnknownArtist
}
}
func (s *mediaFileMapper) mapArtistName(md *Metadata) string {
if md.Artist() != "" {
return md.Artist()
}
return consts.UnknownArtist
}
func (s *mediaFileMapper) mapAlbumName(md *Metadata) string {
name := md.Album()
if name == "" {
return "[Unknown Album]"
}
return name
}
func (s *mediaFileMapper) trackID(md *Metadata) string {
return fmt.Sprintf("%x", md5.Sum([]byte(md.FilePath())))
}
func (s *mediaFileMapper) albumID(md *Metadata) string {
albumPath := strings.ToLower(fmt.Sprintf("%s\\%s", s.mapAlbumArtistName(md), s.mapAlbumName(md)))
return fmt.Sprintf("%x", md5.Sum([]byte(albumPath)))
}
func (s *mediaFileMapper) artistID(md *Metadata) string {
return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapArtistName(md)))))
}
func (s *mediaFileMapper) albumArtistID(md *Metadata) string {
return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapAlbumArtistName(md)))))
}

View File

@ -7,6 +7,7 @@ import (
"strconv"
"time"
"github.com/deluan/navidrome/conf"
"github.com/deluan/navidrome/log"
"github.com/deluan/navidrome/model"
)
@ -81,10 +82,17 @@ func (s *Scanner) loadFolders() {
fs, _ := s.ds.MediaFolder(context.TODO()).GetAll()
for _, f := range fs {
log.Info("Configuring Media Folder", "name", f.Name, "path", f.Path)
s.folders[f.Path] = NewTagScanner(f.Path, s.ds)
s.folders[f.Path] = s.newScanner(f)
}
}
func (s *Scanner) newScanner(f model.MediaFolder) FolderScanner {
if conf.Server.DevNewScanner {
return NewTagScanner2(f.Path, s.ds)
}
return NewTagScanner(f.Path, s.ds)
}
type Status int
type StatusInfo struct {

View File

@ -2,8 +2,6 @@ package scanner
import (
"context"
"crypto/md5"
"fmt"
"os"
"path/filepath"
"sort"
@ -11,17 +9,16 @@ import (
"sync"
"time"
"github.com/deluan/navidrome/consts"
"github.com/deluan/navidrome/log"
"github.com/deluan/navidrome/model"
"github.com/deluan/navidrome/utils"
"github.com/kennygrant/sanitize"
)
type TagScanner struct {
rootFolder string
ds model.DataStore
detector *ChangeDetector
detector *changeDetector
mapper *mediaFileMapper
firstRun sync.Once
}
@ -29,7 +26,8 @@ func NewTagScanner(rootFolder string, ds model.DataStore) *TagScanner {
return &TagScanner{
rootFolder: rootFolder,
ds: ds,
detector: NewChangeDetector(rootFolder),
detector: newChangeDetector(rootFolder),
mapper: newMediaFileMapper(rootFolder),
firstRun: sync.Once{},
}
}
@ -46,9 +44,6 @@ type (
)
const (
// batchSize used for albums/artists updates
batchSize = 5
// filesBatchSize used for extract file metadata
filesBatchSize = 100
)
@ -339,110 +334,12 @@ func (s *TagScanner) loadTracks(filePaths []string) (model.MediaFiles, error) {
var mfs model.MediaFiles
for _, md := range mds {
mf := s.toMediaFile(md)
mf := s.mapper.toMediaFile(md)
mfs = append(mfs, mf)
}
return mfs, nil
}
func (s *TagScanner) toMediaFile(md *Metadata) model.MediaFile {
mf := &model.MediaFile{}
mf.ID = s.trackID(md)
mf.Title = s.mapTrackTitle(md)
mf.Album = md.Album()
mf.AlbumID = s.albumID(md)
mf.Album = s.mapAlbumName(md)
mf.ArtistID = s.artistID(md)
mf.Artist = s.mapArtistName(md)
mf.AlbumArtistID = s.albumArtistID(md)
mf.AlbumArtist = s.mapAlbumArtistName(md)
mf.Genre = md.Genre()
mf.Compilation = md.Compilation()
mf.Year = md.Year()
mf.TrackNumber, _ = md.TrackNumber()
mf.DiscNumber, _ = md.DiscNumber()
mf.DiscSubtitle = md.DiscSubtitle()
mf.Duration = md.Duration()
mf.BitRate = md.BitRate()
mf.Path = md.FilePath()
mf.Suffix = md.Suffix()
mf.Size = md.Size()
mf.HasCoverArt = md.HasPicture()
mf.SortTitle = md.SortTitle()
mf.SortAlbumName = md.SortAlbum()
mf.SortArtistName = md.SortArtist()
mf.SortAlbumArtistName = md.SortAlbumArtist()
mf.OrderAlbumName = sanitizeFieldForSorting(mf.Album)
mf.OrderArtistName = sanitizeFieldForSorting(mf.Artist)
mf.OrderAlbumArtistName = sanitizeFieldForSorting(mf.AlbumArtist)
// TODO Get Creation time. https://github.com/djherbis/times ?
mf.CreatedAt = md.ModificationTime()
mf.UpdatedAt = md.ModificationTime()
return *mf
}
func sanitizeFieldForSorting(originalValue string) string {
v := utils.NoArticle(originalValue)
v = strings.TrimSpace(sanitize.Accents(v))
return utils.NoArticle(v)
}
func (s *TagScanner) mapTrackTitle(md *Metadata) string {
if md.Title() == "" {
s := strings.TrimPrefix(md.FilePath(), s.rootFolder+string(os.PathSeparator))
e := filepath.Ext(s)
return strings.TrimSuffix(s, e)
}
return md.Title()
}
func (s *TagScanner) mapAlbumArtistName(md *Metadata) string {
switch {
case md.Compilation():
return consts.VariousArtists
case md.AlbumArtist() != "":
return md.AlbumArtist()
case md.Artist() != "":
return md.Artist()
default:
return consts.UnknownArtist
}
}
func (s *TagScanner) mapArtistName(md *Metadata) string {
if md.Artist() != "" {
return md.Artist()
}
return consts.UnknownArtist
}
func (s *TagScanner) mapAlbumName(md *Metadata) string {
name := md.Album()
if name == "" {
return "[Unknown Album]"
}
return name
}
func (s *TagScanner) trackID(md *Metadata) string {
return fmt.Sprintf("%x", md5.Sum([]byte(md.FilePath())))
}
func (s *TagScanner) albumID(md *Metadata) string {
albumPath := strings.ToLower(fmt.Sprintf("%s\\%s", s.mapAlbumArtistName(md), s.mapAlbumName(md)))
return fmt.Sprintf("%x", md5.Sum([]byte(albumPath)))
}
func (s *TagScanner) artistID(md *Metadata) string {
return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapArtistName(md)))))
}
func (s *TagScanner) albumArtistID(md *Metadata) string {
return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapAlbumArtistName(md)))))
}
func LoadAllAudioFiles(dirPath string) (map[string]os.FileInfo, error) {
dir, err := os.Open(dirPath)
if err != nil {

297
scanner/tag_scanner_2.go Normal file
View File

@ -0,0 +1,297 @@
package scanner
import (
"context"
"path/filepath"
"sort"
"strings"
"time"
"github.com/deluan/navidrome/log"
"github.com/deluan/navidrome/model"
"github.com/deluan/navidrome/utils"
)
type TagScanner2 struct {
rootFolder string
ds model.DataStore
mapper *mediaFileMapper
albumMap *flushableMap
artistMap *flushableMap
cnt *counters
}
func NewTagScanner2(rootFolder string, ds model.DataStore) *TagScanner2 {
return &TagScanner2{
rootFolder: rootFolder,
mapper: newMediaFileMapper(rootFolder),
ds: ds,
}
}
// Scan algorithm overview:
// Load all directories under the music folder, with their ModTime (self or any non-dir children)
// Find changed folders (based on lastModifiedSince) and deletes folders (comparing to the DB)
// For each deleted folder: delete all files from DB whose path starts with the delete folder path
// For each changed folder: Get all files from DB whose path starts with the changed folder, scan each file:
// if file in folder is newer, update the one in DB
// if file in folder does not exists in DB, add
// for each file in the DB that is not found in the folder, delete from DB
// Create new albums/artists, update counters:
// collect all albumIDs and artistIDs from previous steps
// refresh the collected albums and artists with the metadata from the mediafiles
// Delete all empty albums, delete all empty Artists
func (s *TagScanner2) Scan(ctx context.Context, lastModifiedSince time.Time) error {
start := time.Now()
allDirs, err := s.getDirTree(ctx)
if err != nil {
return err
}
changedDirs := s.getChangedDirs(ctx, allDirs, lastModifiedSince)
if len(changedDirs) == 0 {
log.Debug(ctx, "No changes found in Music Folder", "folder", s.rootFolder)
return nil
}
deletedDirs, _ := s.getDeletedDirs(ctx, allDirs, changedDirs)
if log.CurrentLevel() >= log.LevelTrace {
log.Info(ctx, "Folder changes found", "changedFolders", len(changedDirs), "deletedFolders", len(deletedDirs),
"changed", strings.Join(changedDirs, ";"), "deleted", strings.Join(deletedDirs, ";"))
} else {
log.Info(ctx, "Folder changes found", "changedFolders", len(changedDirs), "deletedFolders", len(deletedDirs))
}
s.albumMap = newFlushableMap(ctx, "album", s.ds.Album(ctx).Refresh)
s.artistMap = newFlushableMap(ctx, "artist", s.ds.Artist(ctx).Refresh)
s.cnt = &counters{}
for _, dir := range deletedDirs {
err := s.processDeletedDir(ctx, dir)
if err != nil {
log.Error("Error removing deleted folder from DB", "path", dir, err)
continue
}
}
for _, dir := range changedDirs {
err := s.processChangedDir(ctx, dir)
if err != nil {
log.Error("Error updating folder in the DB", "path", dir, err)
continue
}
}
_ = s.albumMap.flush()
_ = s.artistMap.flush()
err = s.ds.GC(log.NewContext(ctx))
log.Info("Finished processing Music Folder", "folder", s.rootFolder, "elapsed", time.Since(start),
"added", s.cnt.added, "updated", s.cnt.updated, "deleted", s.cnt.deleted)
return err
}
func (s *TagScanner2) getDirTree(ctx context.Context) (dirMap, error) {
start := time.Now()
log.Trace(ctx, "Loading directory tree from music folder", "folder", s.rootFolder)
dirs, err := loadDirTree(ctx, s.rootFolder)
if err != nil {
return nil, err
}
log.Trace("Directory tree loaded", "total", len(dirs), "elapsed", time.Since(start))
return dirs, nil
}
func (s *TagScanner2) getChangedDirs(ctx context.Context, dirs dirMap, lastModified time.Time) []string {
start := time.Now()
log.Trace(ctx, "Checking for changed folders")
var changed []string
for d, t := range dirs {
if t.After(lastModified) {
changed = append(changed, d)
}
}
sort.Strings(changed)
log.Trace(ctx, "Finished changed folders check", "total", len(changed), "elapsed", time.Since(start))
return changed
}
func (s *TagScanner2) getDeletedDirs(ctx context.Context, allDirs dirMap, changedDirs []string) ([]string, error) {
start := time.Now()
log.Trace(ctx, "Checking for deleted folders")
var deleted []string
repo := s.ds.MediaFile(ctx)
// If rootFolder is in the list of changedDirs, optimize and only do one query to the DB
var foldersToCheck []string
if utils.StringInSlice(s.rootFolder, changedDirs) {
foldersToCheck = []string{s.rootFolder}
} else {
foldersToCheck = changedDirs
}
for _, changedDir := range foldersToCheck {
dirs, err := repo.FindPathsRecursively(changedDir)
if err != nil {
log.Error("Error getting subfolders from DB", "path", changedDir, err)
continue
}
for _, d := range dirs {
d := filepath.Clean(d)
if _, ok := allDirs[d]; !ok {
deleted = append(deleted, d)
}
}
}
sort.Strings(deleted)
log.Trace(ctx, "Finished deleted folders check", "total", len(deleted), "elapsed", time.Since(start))
return deleted, nil
}
func (s *TagScanner2) processDeletedDir(ctx context.Context, dir string) error {
start := time.Now()
mfs, err := s.ds.MediaFile(ctx).FindByPath(dir)
if err != nil {
return err
}
for _, t := range mfs {
err = s.albumMap.update(t.AlbumID)
if err != nil {
return err
}
err = s.artistMap.update(t.AlbumArtistID)
if err != nil {
return err
}
}
log.Info(ctx, "Finished processing deleted folder", "path", dir, "purged", len(mfs), "elapsed", time.Since(start))
c, err := s.ds.MediaFile(ctx).DeleteByPath(dir)
s.cnt.deleted += c
return err
}
func (s *TagScanner2) processChangedDir(ctx context.Context, dir string) error {
start := time.Now()
// Load folder's current tracks from DB into a map
currentTracks := map[string]model.MediaFile{}
ct, err := s.ds.MediaFile(ctx).FindByPath(dir)
if err != nil {
return err
}
for _, t := range ct {
currentTracks[t.Path] = t
}
// Load tracks FileInfo from the folder
files, err := LoadAllAudioFiles(dir)
if err != nil {
return err
}
// If no files to process, return
if len(files)+len(currentTracks) == 0 {
return nil
}
// If track from folder is newer than the one in DB, select for update/insert in DB and delete from the current tracks
log.Trace(ctx, "Processing changed folder", "dir", dir, "tracksInDB", len(currentTracks), "tracksInFolder", len(files))
var filesToUpdate []string
for filePath, info := range files {
c, ok := currentTracks[filePath]
if !ok {
filesToUpdate = append(filesToUpdate, filePath)
s.cnt.added++
}
if ok && info.ModTime().After(c.UpdatedAt) {
filesToUpdate = append(filesToUpdate, filePath)
s.cnt.updated++
}
delete(currentTracks, filePath)
// Force a refresh of the album and artist, to cater for cover art files
err = s.albumMap.update(c.AlbumID)
if err != nil {
return err
}
err = s.artistMap.update(c.AlbumArtistID)
if err != nil {
return err
}
}
numUpdatedTracks := 0
numPurgedTracks := 0
if len(filesToUpdate) > 0 {
// Break the file list in chunks to avoid calling ffmpeg with too many parameters
chunks := utils.BreakUpStringSlice(filesToUpdate, filesBatchSize)
for _, chunk := range chunks {
// Load tracks Metadata from the folder
newTracks, err := s.loadTracks(chunk)
if err != nil {
return err
}
// If track from folder is newer than the one in DB, update/insert in DB
log.Trace(ctx, "Updating mediaFiles in DB", "dir", dir, "files", chunk, "numFiles", len(chunk))
for i := range newTracks {
n := newTracks[i]
err := s.ds.MediaFile(ctx).Put(&n)
if err != nil {
return err
}
err = s.albumMap.update(n.AlbumID)
if err != nil {
return err
}
err = s.artistMap.update(n.AlbumArtistID)
if err != nil {
return err
}
numUpdatedTracks++
}
}
}
if len(currentTracks) > 0 {
log.Trace(ctx, "Deleting dangling tracks from DB", "dir", dir, "numTracks", len(currentTracks))
// Remaining tracks from DB that are not in the folder are deleted
for _, ct := range currentTracks {
numPurgedTracks++
err = s.albumMap.update(ct.AlbumID)
if err != nil {
return err
}
err = s.artistMap.update(ct.AlbumArtistID)
if err != nil {
return err
}
if err := s.ds.MediaFile(ctx).Delete(ct.ID); err != nil {
return err
}
s.cnt.deleted++
}
}
log.Info(ctx, "Finished processing changed folder", "dir", dir, "updated", numUpdatedTracks, "purged", numPurgedTracks, "elapsed", time.Since(start))
return nil
}
func (s *TagScanner2) loadTracks(filePaths []string) (model.MediaFiles, error) {
mds, err := ExtractAllMetadata(filePaths)
if err != nil {
return nil, err
}
var mfs model.MediaFiles
for _, md := range mds {
mf := s.mapper.toMediaFile(md)
mfs = append(mfs, mf)
}
return mfs, nil
}

View File

@ -52,3 +52,19 @@ func BreakUpStringSlice(mediaFileIds []string, chunkSize int) [][]string {
}
return chunks
}
func LongestCommonPrefix(list []string) string {
if len(list) == 0 {
return ""
}
for l := 0; l < len(list[0]); l++ {
c := list[0][l]
for i := 1; i < len(list); i++ {
if l >= len(list[i]) || list[i][l] != c {
return list[i][0:l]
}
}
}
return list[0]
}

View File

@ -81,4 +81,126 @@ var _ = Describe("Strings", func() {
Expect(chunks[1]).To(ConsistOf("d", "e"))
})
})
Describe("LongestCommonPrefix", func() {
It("finds the longest common prefix", func() {
Expect(LongestCommonPrefix(testPaths)).To(Equal("/Music/iTunes 1/iTunes Media/Music/"))
})
})
})
var testPaths = []string{
"/Music/iTunes 1/iTunes Media/Music/ABBA/Gold_ Greatest Hits/Dancing Queen.m4a",
"/Music/iTunes 1/iTunes Media/Music/ABBA/Gold_ Greatest Hits/Mamma Mia.m4a",
"/Music/iTunes 1/iTunes Media/Music/Art Blakey/A Night At Birdland, Vol. 1/01 Annoucement By Pee Wee Marquette.m4a",
"/Music/iTunes 1/iTunes Media/Music/Art Blakey/A Night At Birdland, Vol. 1/02 Split Kick.m4a",
"/Music/iTunes 1/iTunes Media/Music/As Frenéticas/As Frenéticas/Perigosa.m4a",
"/Music/iTunes 1/iTunes Media/Music/Bachman-Turner Overdrive/Gold/Down Down.m4a",
"/Music/iTunes 1/iTunes Media/Music/Bachman-Turner Overdrive/Gold/Hey You.m4a",
"/Music/iTunes 1/iTunes Media/Music/Bachman-Turner Overdrive/Gold/Hold Back The Water.m4a",
"/Music/iTunes 1/iTunes Media/Music/Belle And Sebastian/Write About Love/01 I Didn't See It Coming.m4a",
"/Music/iTunes 1/iTunes Media/Music/Belle And Sebastian/Write About Love/02 Come On Sister.m4a",
"/Music/iTunes 1/iTunes Media/Music/Black Eyed Peas/Elephunk/03 Let's Get Retarded.m4a",
"/Music/iTunes 1/iTunes Media/Music/Black Eyed Peas/Elephunk/04 Hey Mama.m4a",
"/Music/iTunes 1/iTunes Media/Music/Black Eyed Peas/Monkey Business/10 They Don't Want Music (Feat. James Brown).m4a",
"/Music/iTunes 1/iTunes Media/Music/Black Eyed Peas/The E.N.D/1-01 Boom Boom Pow.m4a",
"/Music/iTunes 1/iTunes Media/Music/Black Eyed Peas/Timeless/01 Mas Que Nada.m4a",
"/Music/iTunes 1/iTunes Media/Music/Blondie/Heart Of Glass/Heart Of Glass.m4a",
"/Music/iTunes 1/iTunes Media/Music/Bob Dylan/Nashville Skyline/06 Lay Lady Lay.m4a",
"/Music/iTunes 1/iTunes Media/Music/Botany/Feeling Today - EP/03 Waterparker.m4a",
"/Music/iTunes 1/iTunes Media/Music/Céu/CéU/06 10 Contados.m4a",
"/Music/iTunes 1/iTunes Media/Music/Chance/Six Through Ten/03 Forgive+Forget.m4a",
"/Music/iTunes 1/iTunes Media/Music/Clive Tanaka Y Su Orquesta/Jet Set Siempre 1°/03 Neu Chicago (Side A) [For Dance].m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Absolute Rock Classics/1-02 Smoke on the water.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Almost Famous Soundtrack/10 Simple Man.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Audio News - Rock'n' Roll Forever/01 Rock Around The Clock.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Austin Powers_ International Man Of Mystery/01 The Magic Piper (Of Love).m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Austin Powers_ The Spy Who Shagged Me/04 American Woman.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Back To Dance/03 Long Cool Woman In A Black Dress.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Back To The 70's - O Album Da Década/03 American Pie.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Bambolê/09 In The Mood.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Bambolê - Volume II/03 Blue Moon.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Big Brother Brasil 2004/04 I Will Survive.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Collateral Soundtrack/03 Hands Of Time.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Estúpido Cupido (Internacional)/08 The Twist.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Forrest Gump - The Soundtrack/1-12 California Dreamin'.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Forrest Gump - The Soundtrack/1-16 Mrs. Robinson.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Ghost World - Original Motion Picture Soundtrack/01 Jaan Pechechaan Ho.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Grease [Original Soundtrack]/01 Grease.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/La Bamba/09 Summertime Blues.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Pretty Woman/10 Oh Pretty Woman.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents African Groove/01 Saye Mogo Bana.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Arabic Groove/02 Galbi.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Asian Groove/03 Remember Tomorrow.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Blues Lounge/01 Midnight Dream.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Blues Lounge/03 Banal Reality.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Blues Lounge/04 Parchman Blues.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Blues Lounge/06 Run On.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Brazilian Groove/01 Maria Moita.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Brazilian Lounge/08 E Depois....m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Brazilian Lounge/11 Os Grilos.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Euro Lounge/01 Un Simple Histoire.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Euro Lounge/02 Limbe.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Euro Lounge/05 Sempre Di Domenica.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Euro Lounge/12 Voulez-Vous_.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents World Lounge/03 Santa Maria.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents_ A New Groove/02 Dirty Laundry.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents_ Blues Around the World/02 Canceriano Sem Lar (Clinica Tobias Blues).m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents_ Euro Groove/03 Check In.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents_ World Groove/01 Attention.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Saturday Night Fever/01 Stayin' Alive.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/Saturday Night Fever/03 Night Fever.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/The Best Air Guitar Album In The World... Ever!/2-06 Johnny B. Goode.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/The Full Monty - Soundtrack/02 You Sexy Thing.m4a",
"/Music/iTunes 1/iTunes Media/Music/Compilations/The Full Monty - Soundtrack/11 We Are Family.m4a",
"/Music/iTunes 1/iTunes Media/Music/Cut Copy/Zonoscope (Bonus Version)/10 Corner of the Sky.m4a",
"/Music/iTunes 1/iTunes Media/Music/David Bowie/Changesbowie/07 Diamond Dogs.m4a",
"/Music/iTunes 1/iTunes Media/Music/Douster & Savage Skulls/Get Rich or High Tryin' - EP/01 Bad Gal.m4a",
"/Music/iTunes 1/iTunes Media/Music/Elton John/Greatest Hits 1970-2002/1-04 Rocket Man (I Think It's Going to Be a Long, Long Time).m4a",
"/Music/iTunes 1/iTunes Media/Music/Elvis Presley/ELV1S 30 #1 Hits/02 Don't Be Cruel.m4a",
"/Music/iTunes 1/iTunes Media/Music/Eric Clapton/The Cream Of Clapton/03 I Feel Free.m4a",
"/Music/iTunes 1/iTunes Media/Music/Fleetwood Mac/The Very Best Of Fleetwood Mac/02 Don't Stop.m4a",
"/Music/iTunes 1/iTunes Media/Music/Françoise Hardy/Comment te dire adieu/Comment te dire adieu.m4a",
"/Music/iTunes 1/iTunes Media/Music/Games/That We Can Play - EP/01 Strawberry Skies.m4a",
"/Music/iTunes 1/iTunes Media/Music/Grand Funk Railroad/Collectors Series/The Loco-Motion.m4a",
"/Music/iTunes 1/iTunes Media/Music/Henry Mancini/The Pink Panther (Music from the Film Score)/The Pink Panther Theme.m4a",
"/Music/iTunes 1/iTunes Media/Music/Holy Ghost!/Do It Again - Single/01 Do It Again.m4a",
"/Music/iTunes 1/iTunes Media/Music/K.C. & The Sunshine Band/The Best of/03 I'm Your Boogie Man.m4a",
"/Music/iTunes 1/iTunes Media/Music/K.C. & The Sunshine Band/Unknown Album/Megamix (Thats The Way, Shake Your Booty, Get Down Tonight, Give It Up).m4a",
"/Music/iTunes 1/iTunes Media/Music/Kim Ann Foxman & Andy Butler/Creature - EP/01 Creature.m4a",
"/Music/iTunes 1/iTunes Media/Music/Nico/Chelsea Girl/01 The Fairest Of The Seasons.m4a",
"/Music/iTunes 1/iTunes Media/Music/oOoOO/oOoOO - EP/02 Burnout Eyess.m4a",
"/Music/iTunes 1/iTunes Media/Music/Peter Frampton/The Very Best of Peter Frampton/Baby, I Love Your Way.m4a",
"/Music/iTunes 1/iTunes Media/Music/Peter Frampton/The Very Best of Peter Frampton/Show Me The Way.m4a",
"/Music/iTunes 1/iTunes Media/Music/Raul Seixas/A Arte De Raul Seixas/03 Metamorfose Ambulante.m4a",
"/Music/iTunes 1/iTunes Media/Music/Raul Seixas/A Arte De Raul Seixas/18 Eu Nasci há 10 Mil Anos Atrás.m4a",
"/Music/iTunes 1/iTunes Media/Music/Rick James/Street Songs/Super Freak.m4a",
"/Music/iTunes 1/iTunes Media/Music/Rita Lee/Fruto Proibido/Agora Só Falta Você.m4a",
"/Music/iTunes 1/iTunes Media/Music/Rita Lee/Fruto Proibido/Esse Tal De Roque Enrow.m4a",
"/Music/iTunes 1/iTunes Media/Music/Roberto Carlos/Roberto Carlos 1966/05 Negro Gato.m4a",
"/Music/iTunes 1/iTunes Media/Music/SOHO/Goddess/02 Hippychick.m4a",
"/Music/iTunes 1/iTunes Media/Music/Stan Getz/Getz_Gilberto/05 Corcovado (Quiet Nights of Quiet Stars).m4a",
"/Music/iTunes 1/iTunes Media/Music/Steely Dan/Pretzel Logic/Rikki Don't Loose That Number.m4a",
"/Music/iTunes 1/iTunes Media/Music/Stevie Wonder/For Once In My Life/I Don't Know Why.m4a",
"/Music/iTunes 1/iTunes Media/Music/Teebs/Ardour/While You Doooo.m4a",
"/Music/iTunes 1/iTunes Media/Music/The Beatles/Magical Mystery Tour/08 Strawberry Fields Forever.m4a",
"/Music/iTunes 1/iTunes Media/Music/The Beatles/Past Masters, Vol. 1/10 Long Tall Sally.m4a",
"/Music/iTunes 1/iTunes Media/Music/The Beatles/Please Please Me/14 Twist And Shout.m4a",
"/Music/iTunes 1/iTunes Media/Music/The Beatles/Sgt. Pepper's Lonely Hearts Club Band/03 Lucy In The Sky With Diamonds.m4a",
"/Music/iTunes 1/iTunes Media/Music/The Black Crowes/Amorica/09 Wiser Time.m4a",
"/Music/iTunes 1/iTunes Media/Music/The Black Crowes/By Your Side/05 Only A Fool.m4a",
"/Music/iTunes 1/iTunes Media/Music/The Black Crowes/Shake Your Money Maker/04 Could I''ve Been So Blind.m4a",
"/Music/iTunes 1/iTunes Media/Music/The Black Crowes/The Southern Harmony And Musical Companion/01 Sting Me.m4a",
"/Music/iTunes 1/iTunes Media/Music/The Black Crowes/Three Snakes And One Charm/02 Good Friday.m4a",
"/Music/iTunes 1/iTunes Media/Music/The Doors/Strange Days (40th Anniversary Mixes)/01 Strange Days.m4a",
"/Music/iTunes 1/iTunes Media/Music/The Rolling Stones/Forty Licks/1-03 (I Can't Get No) Satisfaction.m4a",
"/Music/iTunes 1/iTunes Media/Music/The Velvet Underground/The Velvet Underground & Nico/02 I'm Waiting For The Man.m4a",
"/Music/iTunes 1/iTunes Media/Music/The Velvet Underground/The Velvet Underground & Nico/03 Femme Fatale.m4a",
"/Music/iTunes 1/iTunes Media/Music/The Velvet Underground/White Light_White Heat/04 Here She Comes Now.m4a",
"/Music/iTunes 1/iTunes Media/Music/The Who/Sings My Generation/My Generation.m4a",
"/Music/iTunes 1/iTunes Media/Music/Village People/The Very Best Of Village People/Macho Man.m4a",
"/Music/iTunes 1/iTunes Media/Music/Vondelpark/Sauna - EP/01 California Analog Dream.m4a",
"/Music/iTunes 1/iTunes Media/Music/War/Why Can't We Be Friends/Low Rider.m4a",
"/Music/iTunes 1/iTunes Media/Music/Yes/Fragile/01 Roundabout.m4a",
}