mirrors
/
navidrome
mirror of https://github.com/navidrome/navidrome.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Flush albums and artists after each folder added/updated/deleted

This commit is contained in:
Deluan 2020-07-22 12:48:24 -04:00
parent 1b7f628759
commit 036f9d6730
4 changed files with 79 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
scanner/flushable_map.go
View File

@ -1,58 +0,0 @@
package scanner
import (
"context"
"fmt"
"github.com/deluan/navidrome/log"
)
const (
// batchSize used for albums/artists updates
batchSize = 5
)
type refreshCallbackFunc = func(ids ...string) error
type flushableMap struct {
ctx context.Context
flushFunc refreshCallbackFunc
entity string
m map[string]struct{}
}
func newFlushableMap(ctx context.Context, entity string, flushFunc refreshCallbackFunc) *flushableMap {
return &flushableMap{
ctx: ctx,
flushFunc: flushFunc,
entity: entity,
m: map[string]struct{}{},
}
}
func (f *flushableMap) update(id string) error {
f.m[id] = struct{}{}
if len(f.m) >= batchSize {
err := f.flush()
if err != nil {
return err
}
}
return nil
}
func (f *flushableMap) flush() error {
if len(f.m) == 0 {
return nil
}
var ids []string
for id := range f.m {
ids = append(ids, id)
delete(f.m, id)
}
if err := f.flushFunc(ids...); err != nil {
log.Error(f.ctx, fmt.Sprintf("Error writing %ss to the DB", f.entity), err)
return err
}
return nil
}

59
scanner/refresh_buffer.go Normal file
View File

@ -0,0 +1,59 @@
package scanner
import (
"context"
"fmt"
"github.com/deluan/navidrome/log"
"github.com/deluan/navidrome/model"
)
type refreshBuffer struct {
ctx context.Context
ds model.DataStore
album map[string]struct{}
artist map[string]struct{}
}
func newRefreshBuffer(ctx context.Context, ds model.DataStore) *refreshBuffer {
return &refreshBuffer{
ctx: ctx,
ds: ds,
album: map[string]struct{}{},
}
}
func (f *refreshBuffer) accumulate(mf model.MediaFile) {
f.album[mf.AlbumID] = struct{}{}
f.album[mf.AlbumArtistID] = struct{}{}
}
type refreshCallbackFunc = func(ids ...string) error
func (f *refreshBuffer) flushMap(m map[string]struct{}, entity string, refresh refreshCallbackFunc) error {
if len(m) == 0 {
return nil
}
var ids []string
for id := range m {
ids = append(ids, id)
delete(m, id)
}
if err := refresh(ids...); err != nil {
log.Error(f.ctx, fmt.Sprintf("Error writing %ss to the DB", entity), err)
return err
}
return nil
}
func (f *refreshBuffer) flush() error {
err := f.flushMap(f.album, "album", f.ds.Album(f.ctx).Refresh)
if err != nil {
return err
}
err = f.flushMap(f.artist, "artist", f.ds.Artist(f.ctx).Refresh)
if err != nil {
return err
}
return nil
}

2
scanner/tag_scanner.go
View File

@ -32,6 +32,8 @@ func NewTagScanner(rootFolder string, ds model.DataStore) *TagScanner {
}
}
const batchSize = 5
type (
artistMap map[string]struct{}
albumMap map[string]struct{}

64
scanner/tag_scanner_2.go
View File

@ -18,8 +18,6 @@ type TagScanner2 struct {
ds model.DataStore
mapper *mediaFileMapper
plsSync *playlistSync
albumMap *flushableMap
artistMap *flushableMap
cnt *counters
}
@ -77,8 +75,6 @@ func (s *TagScanner2) Scan(ctx context.Context, lastModifiedSince time.Time) err
log.Info(ctx, "Folder changes detected", "changedFolders", len(changedDirs), "deletedFolders", len(deletedDirs))
}
s.albumMap = newFlushableMap(ctx, "album", s.ds.Album(ctx).Refresh)
s.artistMap = newFlushableMap(ctx, "artist", s.ds.Artist(ctx).Refresh)
s.cnt = &counters{}
for _, dir := range deletedDirs {
@ -94,9 +90,6 @@ func (s *TagScanner2) Scan(ctx context.Context, lastModifiedSince time.Time) err
}
}
_ = s.albumMap.flush()
_ = s.artistMap.flush()
// Now that all mediafiles are imported/updated, search for and import playlists
u, _ := request.UserFrom(ctx)
plsCount := 0
@ -126,13 +119,15 @@ func (s *TagScanner2) getDirTree(ctx context.Context) (dirMap, error) {
if err != nil {
return nil, err
}
log.Debug("Directory tree loaded", "total", len(dirs), "elapsed", time.Since(start))
log.Debug("Directory tree loaded from music folder", "total", len(dirs), "elapsed", time.Since(start))
return dirs, nil
}
func (s *TagScanner2) getDBDirTree(ctx context.Context) (map[string]struct{}, error) {
repo := s.ds.MediaFile(ctx)
start := time.Now()
log.Trace(ctx, "Loading directory tree from database", "folder", s.rootFolder)
repo := s.ds.MediaFile(ctx)
dirs, err := repo.FindPathsRecursively(s.rootFolder)
if err != nil {
return nil, err
@ -142,6 +137,7 @@ func (s *TagScanner2) getDBDirTree(ctx context.Context) (map[string]struct{}, er
resp[filepath.Clean(d)] = struct{}{}
}
log.Debug("Directory tree loaded from DB", "total", len(resp), "elapsed", time.Since(start))
return resp, nil
}
@ -184,6 +180,7 @@ func (s *TagScanner2) getDeletedDirs(ctx context.Context, allDirs dirMap, dbDirs
func (s *TagScanner2) processDeletedDir(ctx context.Context, dir string) error {
start := time.Now()
buffer := newRefreshBuffer(ctx, s.ds)
mfs, err := s.ds.MediaFile(ctx).FindAllByPath(dir)
if err != nil {
@ -197,22 +194,17 @@ func (s *TagScanner2) processDeletedDir(ctx context.Context, dir string) error {
s.cnt.deleted += c
for _, t := range mfs {
err = s.albumMap.update(t.AlbumID)
if err != nil {
return err
}
err = s.artistMap.update(t.AlbumArtistID)
if err != nil {
return err
}
buffer.accumulate(t)
}
err = buffer.flush()
log.Info(ctx, "Finished processing deleted folder", "path", dir, "purged", len(mfs), "elapsed", time.Since(start))
return err
}
func (s *TagScanner2) processChangedDir(ctx context.Context, dir string) error {
start := time.Now()
buffer := newRefreshBuffer(ctx, s.ds)
// Load folder's current tracks from DB into a map
currentTracks := map[string]model.MediaFile{}
@ -250,14 +242,7 @@ func (s *TagScanner2) processChangedDir(ctx context.Context, dir string) error {
}
// Force a refresh of the album and artist, to cater for cover art files
err = s.albumMap.update(c.AlbumID)
if err != nil {
return err
}
err = s.artistMap.update(c.AlbumArtistID)
if err != nil {
return err
}
buffer.accumulate(c)
// Remove it from currentTracks (the ones found in DB). After this loop any currentTracks remaining
// are considered gone from the music folder and will be deleted from DB
@ -268,39 +253,33 @@ func (s *TagScanner2) processChangedDir(ctx context.Context, dir string) error {
numPurgedTracks := 0
if len(filesToUpdate) > 0 {
numUpdatedTracks, err = s.addOrUpdateTracksInDB(ctx, dir, filesToUpdate)
numUpdatedTracks, err = s.addOrUpdateTracksInDB(ctx, dir, filesToUpdate, buffer)
if err != nil {
return err
}
}
if len(currentTracks) > 0 {
numPurgedTracks, err = s.deleteOrphanSongs(ctx, dir, currentTracks)
numPurgedTracks, err = s.deleteOrphanSongs(ctx, dir, currentTracks, buffer)
if err != nil {
return err
}
}
err = buffer.flush()
log.Info(ctx, "Finished processing changed folder", "dir", dir, "updated", numUpdatedTracks,
"purged", numPurgedTracks, "elapsed", time.Since(start))
return nil
return err
}
func (s *TagScanner2) deleteOrphanSongs(ctx context.Context, dir string, tracksToDelete map[string]model.MediaFile) (int, error) {
func (s *TagScanner2) deleteOrphanSongs(ctx context.Context, dir string, tracksToDelete map[string]model.MediaFile, buffer *refreshBuffer) (int, error) {
numPurgedTracks := 0
log.Debug(ctx, "Deleting orphan tracks from DB", "dir", dir, "numTracks", len(tracksToDelete))
// Remaining tracks from DB that are not in the folder are deleted
for _, ct := range tracksToDelete {
numPurgedTracks++
err := s.albumMap.update(ct.AlbumID)
if err != nil {
return 0, err
}
err = s.artistMap.update(ct.AlbumArtistID)
if err != nil {
return 0, err
}
buffer.accumulate(ct)
if err := s.ds.MediaFile(ctx).Delete(ct.ID); err != nil {
return 0, err
}
@ -309,7 +288,7 @@ func (s *TagScanner2) deleteOrphanSongs(ctx context.Context, dir string, tracksT
return numPurgedTracks, nil
}
func (s *TagScanner2) addOrUpdateTracksInDB(ctx context.Context, dir string, filesToUpdate []string) (int, error) {
func (s *TagScanner2) addOrUpdateTracksInDB(ctx context.Context, dir string, filesToUpdate []string, buffer *refreshBuffer) (int, error) {
numUpdatedTracks := 0
log.Trace(ctx, "Updating mediaFiles in DB", "dir", dir, "numFiles", len(filesToUpdate))
@ -330,14 +309,7 @@ func (s *TagScanner2) addOrUpdateTracksInDB(ctx context.Context, dir string, fil
if err != nil {
return 0, err
}
err = s.albumMap.update(n.AlbumID)
if err != nil {
return 0, err
}
err = s.artistMap.update(n.AlbumArtistID)
if err != nil {
return 0, err
}
buffer.accumulate(n)
numUpdatedTracks++
}
}