445 lines
12 KiB
Go
445 lines
12 KiB
Go
package scanner
|
|
|
|
import (
|
|
"context"
|
|
"crypto/md5"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/deluan/navidrome/consts"
|
|
"github.com/deluan/navidrome/log"
|
|
"github.com/deluan/navidrome/model"
|
|
"github.com/deluan/navidrome/utils"
|
|
"github.com/kennygrant/sanitize"
|
|
)
|
|
|
|
type TagScanner struct {
|
|
rootFolder string
|
|
ds model.DataStore
|
|
detector *ChangeDetector
|
|
firstRun sync.Once
|
|
}
|
|
|
|
func NewTagScanner(rootFolder string, ds model.DataStore) *TagScanner {
|
|
return &TagScanner{
|
|
rootFolder: rootFolder,
|
|
ds: ds,
|
|
detector: NewChangeDetector(rootFolder),
|
|
firstRun: sync.Once{},
|
|
}
|
|
}
|
|
|
|
type (
|
|
artistMap map[string]struct{}
|
|
albumMap map[string]struct{}
|
|
|
|
counters struct {
|
|
added int64
|
|
updated int64
|
|
deleted int64
|
|
}
|
|
)
|
|
|
|
const (
|
|
// batchSize used for albums/artists updates
|
|
batchSize = 5
|
|
|
|
// filesBatchSize used for extract file metadata
|
|
filesBatchSize = 100
|
|
)
|
|
|
|
// Scan algorithm overview:
|
|
// For each changed folder: Get all files from DB that starts with the folder, scan each file:
|
|
// if file in folder is newer, update the one in DB
|
|
// if file in folder does not exists in DB, add
|
|
// for each file in the DB that is not found in the folder, delete from DB
|
|
// For each deleted folder: delete all files from DB that starts with the folder path
|
|
// Only on first run, check if any folder under each changed folder is missing.
|
|
// if it is, delete everything under it
|
|
// Create new albums/artists, update counters:
|
|
// collect all albumIDs and artistIDs from previous steps
|
|
// refresh the collected albums and artists with the metadata from the mediafiles
|
|
// Delete all empty albums, delete all empty Artists
|
|
func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time) error {
|
|
start := time.Now()
|
|
log.Trace(ctx, "Looking for changes in music folder", "folder", s.rootFolder)
|
|
|
|
changed, deleted, err := s.detector.Scan(ctx, lastModifiedSince)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if len(changed)+len(deleted) == 0 {
|
|
log.Debug(ctx, "No changes found in Music Folder", "folder", s.rootFolder)
|
|
return nil
|
|
}
|
|
|
|
if log.CurrentLevel() >= log.LevelTrace {
|
|
log.Info(ctx, "Folder changes found", "numChanged", len(changed), "numDeleted", len(deleted),
|
|
"changed", strings.Join(changed, ";"), "deleted", strings.Join(deleted, ";"))
|
|
} else {
|
|
log.Info(ctx, "Folder changes found", "numChanged", len(changed), "numDeleted", len(deleted))
|
|
}
|
|
|
|
sort.Strings(changed)
|
|
sort.Strings(deleted)
|
|
|
|
updatedArtists := artistMap{}
|
|
updatedAlbums := albumMap{}
|
|
cnt := &counters{}
|
|
|
|
for _, c := range changed {
|
|
err := s.processChangedDir(ctx, c, updatedArtists, updatedAlbums, cnt)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
for _, c := range deleted {
|
|
err := s.processDeletedDir(ctx, c, updatedArtists, updatedAlbums, cnt)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
err = s.flushAlbums(ctx, updatedAlbums)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = s.flushArtists(ctx, updatedArtists)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
s.firstRun.Do(func() {
|
|
s.removeDeletedFolders(context.TODO(), changed, cnt)
|
|
})
|
|
|
|
err = s.ds.GC(log.NewContext(context.TODO()))
|
|
log.Info("Finished Music Folder", "folder", s.rootFolder, "elapsed", time.Since(start),
|
|
"added", cnt.added, "updated", cnt.updated, "deleted", cnt.deleted)
|
|
|
|
return err
|
|
}
|
|
|
|
func (s *TagScanner) flushAlbums(ctx context.Context, updatedAlbums albumMap) error {
|
|
if len(updatedAlbums) == 0 {
|
|
return nil
|
|
}
|
|
var ids []string
|
|
for id := range updatedAlbums {
|
|
ids = append(ids, id)
|
|
delete(updatedAlbums, id)
|
|
}
|
|
return s.ds.Album(ctx).Refresh(ids...)
|
|
}
|
|
|
|
func (s *TagScanner) flushArtists(ctx context.Context, updatedArtists artistMap) error {
|
|
if len(updatedArtists) == 0 {
|
|
return nil
|
|
}
|
|
var ids []string
|
|
for id := range updatedArtists {
|
|
ids = append(ids, id)
|
|
delete(updatedArtists, id)
|
|
}
|
|
return s.ds.Artist(ctx).Refresh(ids...)
|
|
}
|
|
|
|
func (s *TagScanner) processChangedDir(ctx context.Context, dir string, updatedArtists artistMap, updatedAlbums albumMap, cnt *counters) error {
|
|
dir = filepath.Join(s.rootFolder, dir)
|
|
start := time.Now()
|
|
|
|
// Load folder's current tracks from DB into a map
|
|
currentTracks := map[string]model.MediaFile{}
|
|
ct, err := s.ds.MediaFile(ctx).FindByPath(dir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, t := range ct {
|
|
currentTracks[t.Path] = t
|
|
}
|
|
|
|
// Load tracks FileInfo from the folder
|
|
files, err := LoadAllAudioFiles(dir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// If no files to process, return
|
|
if len(files)+len(currentTracks) == 0 {
|
|
return nil
|
|
}
|
|
|
|
// If track from folder is newer than the one in DB, select for update/insert in DB and delete from the current tracks
|
|
log.Trace("Processing changed folder", "dir", dir, "tracksInDB", len(currentTracks), "tracksInFolder", len(files))
|
|
var filesToUpdate []string
|
|
for filePath, info := range files {
|
|
c, ok := currentTracks[filePath]
|
|
if !ok {
|
|
filesToUpdate = append(filesToUpdate, filePath)
|
|
cnt.added++
|
|
}
|
|
if ok && info.ModTime().After(c.UpdatedAt) {
|
|
filesToUpdate = append(filesToUpdate, filePath)
|
|
cnt.updated++
|
|
}
|
|
delete(currentTracks, filePath)
|
|
|
|
// Force a refresh of the album and artist, to cater for cover art files. Ideally we would only do this
|
|
// if there are any image file in the folder (TODO)
|
|
err = s.updateAlbum(ctx, c.AlbumID, updatedAlbums)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = s.updateArtist(ctx, c.AlbumArtistID, updatedArtists)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
numUpdatedTracks := 0
|
|
numPurgedTracks := 0
|
|
|
|
if len(filesToUpdate) > 0 {
|
|
// Break the file list in chunks to avoid calling ffmpeg with too many parameters
|
|
chunks := utils.BreakUpStringSlice(filesToUpdate, filesBatchSize)
|
|
for _, chunk := range chunks {
|
|
// Load tracks Metadata from the folder
|
|
newTracks, err := s.loadTracks(chunk)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// If track from folder is newer than the one in DB, update/insert in DB
|
|
log.Trace("Updating mediaFiles in DB", "dir", dir, "files", chunk, "numFiles", len(chunk))
|
|
for i := range newTracks {
|
|
n := newTracks[i]
|
|
err := s.ds.MediaFile(ctx).Put(&n)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = s.updateAlbum(ctx, n.AlbumID, updatedAlbums)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = s.updateArtist(ctx, n.AlbumArtistID, updatedArtists)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
numUpdatedTracks++
|
|
}
|
|
}
|
|
}
|
|
|
|
if len(currentTracks) > 0 {
|
|
log.Trace("Deleting dangling tracks from DB", "dir", dir, "numTracks", len(currentTracks))
|
|
// Remaining tracks from DB that are not in the folder are deleted
|
|
for _, ct := range currentTracks {
|
|
numPurgedTracks++
|
|
err = s.updateAlbum(ctx, ct.AlbumID, updatedAlbums)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = s.updateArtist(ctx, ct.AlbumArtistID, updatedArtists)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := s.ds.MediaFile(ctx).Delete(ct.ID); err != nil {
|
|
return err
|
|
}
|
|
cnt.deleted++
|
|
}
|
|
}
|
|
|
|
log.Info("Finished processing changed folder", "dir", dir, "updated", numUpdatedTracks, "purged", numPurgedTracks, "elapsed", time.Since(start))
|
|
return nil
|
|
}
|
|
|
|
func (s *TagScanner) updateAlbum(ctx context.Context, albumId string, updatedAlbums albumMap) error {
|
|
updatedAlbums[albumId] = struct{}{}
|
|
if len(updatedAlbums) >= batchSize {
|
|
err := s.flushAlbums(ctx, updatedAlbums)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *TagScanner) updateArtist(ctx context.Context, artistId string, updatedArtists artistMap) error {
|
|
updatedArtists[artistId] = struct{}{}
|
|
if len(updatedArtists) >= batchSize {
|
|
err := s.flushArtists(ctx, updatedArtists)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *TagScanner) processDeletedDir(ctx context.Context, dir string, updatedArtists artistMap, updatedAlbums albumMap, cnt *counters) error {
|
|
dir = filepath.Join(s.rootFolder, dir)
|
|
start := time.Now()
|
|
|
|
mfs, err := s.ds.MediaFile(ctx).FindByPath(dir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, t := range mfs {
|
|
err = s.updateAlbum(ctx, t.AlbumID, updatedAlbums)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = s.updateArtist(ctx, t.AlbumArtistID, updatedArtists)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
log.Info("Finished processing deleted folder", "dir", dir, "purged", len(mfs), "elapsed", time.Since(start))
|
|
c, err := s.ds.MediaFile(ctx).DeleteByPath(dir)
|
|
cnt.deleted += c
|
|
return err
|
|
}
|
|
|
|
func (s *TagScanner) removeDeletedFolders(ctx context.Context, changed []string, cnt *counters) {
|
|
for _, dir := range changed {
|
|
fullPath := filepath.Join(s.rootFolder, dir)
|
|
paths, err := s.ds.MediaFile(ctx).FindPathsRecursively(fullPath)
|
|
if err != nil {
|
|
log.Error(ctx, "Error reading paths from DB", "path", dir, err)
|
|
return
|
|
}
|
|
|
|
// If a path is unreadable, remove from the DB
|
|
for _, path := range paths {
|
|
if readable, err := utils.IsDirReadable(path); !readable {
|
|
log.Info(ctx, "Path unavailable. Removing tracks from DB", "path", path, err)
|
|
c, err := s.ds.MediaFile(ctx).DeleteByPath(path)
|
|
if err != nil {
|
|
log.Error(ctx, "Error removing MediaFiles from DB", "path", path, err)
|
|
}
|
|
cnt.deleted += c
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *TagScanner) loadTracks(filePaths []string) (model.MediaFiles, error) {
|
|
mds, err := ExtractAllMetadata(filePaths)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var mfs model.MediaFiles
|
|
for _, md := range mds {
|
|
mf := s.toMediaFile(md)
|
|
mfs = append(mfs, mf)
|
|
}
|
|
return mfs, nil
|
|
}
|
|
|
|
func (s *TagScanner) toMediaFile(md *Metadata) model.MediaFile {
|
|
mf := &model.MediaFile{}
|
|
mf.ID = s.trackID(md)
|
|
mf.Title = s.mapTrackTitle(md)
|
|
mf.Album = md.Album()
|
|
mf.AlbumID = s.albumID(md)
|
|
mf.Album = s.mapAlbumName(md)
|
|
mf.ArtistID = s.artistID(md)
|
|
mf.Artist = s.mapArtistName(md)
|
|
mf.AlbumArtistID = s.albumArtistID(md)
|
|
mf.AlbumArtist = s.mapAlbumArtistName(md)
|
|
mf.Genre = md.Genre()
|
|
mf.Compilation = md.Compilation()
|
|
mf.Year = md.Year()
|
|
mf.TrackNumber, _ = md.TrackNumber()
|
|
mf.DiscNumber, _ = md.DiscNumber()
|
|
mf.DiscSubtitle = md.DiscSubtitle()
|
|
mf.Duration = md.Duration()
|
|
mf.BitRate = md.BitRate()
|
|
mf.Path = md.FilePath()
|
|
mf.Suffix = md.Suffix()
|
|
mf.Size = md.Size()
|
|
mf.HasCoverArt = md.HasPicture()
|
|
mf.SortTitle = md.SortTitle()
|
|
mf.SortAlbumName = md.SortAlbum()
|
|
mf.SortArtistName = md.SortArtist()
|
|
mf.SortAlbumArtistName = md.SortAlbumArtist()
|
|
mf.OrderAlbumName = sanitizeFieldForSorting(mf.Album)
|
|
mf.OrderArtistName = sanitizeFieldForSorting(mf.Artist)
|
|
mf.OrderAlbumArtistName = sanitizeFieldForSorting(mf.AlbumArtist)
|
|
|
|
// TODO Get Creation time. https://github.com/djherbis/times ?
|
|
mf.CreatedAt = md.ModificationTime()
|
|
mf.UpdatedAt = md.ModificationTime()
|
|
|
|
return *mf
|
|
}
|
|
|
|
func sanitizeFieldForSorting(originalValue string) string {
|
|
v := utils.NoArticle(originalValue)
|
|
v = strings.TrimSpace(sanitize.Accents(v))
|
|
return utils.NoArticle(v)
|
|
}
|
|
|
|
func (s *TagScanner) mapTrackTitle(md *Metadata) string {
|
|
if md.Title() == "" {
|
|
s := strings.TrimPrefix(md.FilePath(), s.rootFolder+string(os.PathSeparator))
|
|
e := filepath.Ext(s)
|
|
return strings.TrimSuffix(s, e)
|
|
}
|
|
return md.Title()
|
|
}
|
|
|
|
func (s *TagScanner) mapAlbumArtistName(md *Metadata) string {
|
|
switch {
|
|
case md.Compilation():
|
|
return consts.VariousArtists
|
|
case md.AlbumArtist() != "":
|
|
return md.AlbumArtist()
|
|
case md.Artist() != "":
|
|
return md.Artist()
|
|
default:
|
|
return consts.UnknownArtist
|
|
}
|
|
}
|
|
|
|
func (s *TagScanner) mapArtistName(md *Metadata) string {
|
|
if md.Artist() != "" {
|
|
return md.Artist()
|
|
}
|
|
return consts.UnknownArtist
|
|
}
|
|
|
|
func (s *TagScanner) mapAlbumName(md *Metadata) string {
|
|
name := md.Album()
|
|
if name == "" {
|
|
return "[Unknown Album]"
|
|
}
|
|
return name
|
|
}
|
|
|
|
func (s *TagScanner) trackID(md *Metadata) string {
|
|
return fmt.Sprintf("%x", md5.Sum([]byte(md.FilePath())))
|
|
}
|
|
|
|
func (s *TagScanner) albumID(md *Metadata) string {
|
|
albumPath := strings.ToLower(fmt.Sprintf("%s\\%s", s.mapAlbumArtistName(md), s.mapAlbumName(md)))
|
|
return fmt.Sprintf("%x", md5.Sum([]byte(albumPath)))
|
|
}
|
|
|
|
func (s *TagScanner) artistID(md *Metadata) string {
|
|
return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapArtistName(md)))))
|
|
}
|
|
|
|
func (s *TagScanner) albumArtistID(md *Metadata) string {
|
|
return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapAlbumArtistName(md)))))
|
|
}
|