From 2f90fc9bd40e24afd900b5ee85426f32684b960b Mon Sep 17 00:00:00 2001 From: Deluan Date: Mon, 19 Dec 2022 12:08:39 -0500 Subject: [PATCH] Move album refresh to scanner --- model/album.go | 1 - persistence/album_repository.go | 34 ----------- scanner/refresh_buffer.go | 64 --------------------- scanner/refresher.go | 99 +++++++++++++++++++++++++++++++++ scanner/tag_scanner.go | 8 +-- 5 files changed, 103 insertions(+), 103 deletions(-) delete mode 100644 scanner/refresh_buffer.go create mode 100644 scanner/refresher.go diff --git a/model/album.go b/model/album.go index 94b5ff9f..ec5a6842 100644 --- a/model/album.go +++ b/model/album.go @@ -54,6 +54,5 @@ type AlbumRepository interface { GetAll(...QueryOptions) (Albums, error) GetAllWithoutGenres(...QueryOptions) (Albums, error) Search(q string, offset int, size int) (Albums, error) - Refresh(ids ...string) error AnnotatedRepository } diff --git a/persistence/album_repository.go b/persistence/album_repository.go index 8a47a283..f04f12dd 100644 --- a/persistence/album_repository.go +++ b/persistence/album_repository.go @@ -11,8 +11,6 @@ import ( "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" - "github.com/navidrome/navidrome/utils" - "github.com/navidrome/navidrome/utils/slice" ) type albumRepository struct { @@ -142,38 +140,6 @@ func (r *albumRepository) GetAllWithoutGenres(options ...model.QueryOptions) (mo return res, err } -func (r *albumRepository) Refresh(ids ...string) error { - chunks := utils.BreakUpStringSlice(ids, 100) - for _, chunk := range chunks { - err := r.refresh(chunk...) - if err != nil { - return err - } - } - return nil -} - -func (r *albumRepository) refresh(ids ...string) error { - mfRepo := NewMediaFileRepository(r.ctx, r.ormer) - mfs, err := mfRepo.GetAll(model.QueryOptions{Filters: Eq{"album_id": ids}}) - if err != nil { - return err - } - if len(mfs) == 0 { - return nil - } - - grouped := slice.Group(mfs, func(m model.MediaFile) string { return m.AlbumID }) - for _, songs := range grouped { - a := model.MediaFiles(songs).ToAlbum() - err := r.Put(&a) - if err != nil { - return err - } - } - return nil -} - func (r *albumRepository) purgeEmpty() error { del := Delete(r.tableName).Where("id not in (select distinct(album_id) from media_file)") c, err := r.executeSQL(del) diff --git a/scanner/refresh_buffer.go b/scanner/refresh_buffer.go deleted file mode 100644 index e6a1f221..00000000 --- a/scanner/refresh_buffer.go +++ /dev/null @@ -1,64 +0,0 @@ -package scanner - -import ( - "context" - "fmt" - - "github.com/navidrome/navidrome/log" - "github.com/navidrome/navidrome/model" -) - -type refreshBuffer struct { - ctx context.Context - ds model.DataStore - album map[string]struct{} - artist map[string]struct{} -} - -func newRefreshBuffer(ctx context.Context, ds model.DataStore) *refreshBuffer { - return &refreshBuffer{ - ctx: ctx, - ds: ds, - album: map[string]struct{}{}, - artist: map[string]struct{}{}, - } -} - -func (f *refreshBuffer) accumulate(mf model.MediaFile) { - if mf.AlbumID != "" { - f.album[mf.AlbumID] = struct{}{} - } - if mf.AlbumArtistID != "" { - f.artist[mf.AlbumArtistID] = struct{}{} - } -} - -type refreshCallbackFunc = func(ids ...string) error - -func (f *refreshBuffer) flushMap(m map[string]struct{}, entity string, refresh refreshCallbackFunc) error { - if len(m) == 0 { - return nil - } - var ids []string - for id := range m { - ids = append(ids, id) - delete(m, id) - } - if err := refresh(ids...); err != nil { - log.Error(f.ctx, fmt.Sprintf("Error writing %ss to the DB", entity), err) - return err - } - return nil -} - -func (f *refreshBuffer) flush() error { - err := f.flushMap(f.album, "album", f.ds.Album(f.ctx).Refresh) - if err != nil { - return err - } - err = f.flushMap(f.artist, "artist", f.ds.Artist(f.ctx).Refresh) - if err != nil { - return err - } - return nil -} diff --git a/scanner/refresher.go b/scanner/refresher.go new file mode 100644 index 00000000..605c17cf --- /dev/null +++ b/scanner/refresher.go @@ -0,0 +1,99 @@ +package scanner + +import ( + "context" + "fmt" + + "github.com/Masterminds/squirrel" + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/utils" + "github.com/navidrome/navidrome/utils/slice" +) + +type refresher struct { + ctx context.Context + ds model.DataStore + album map[string]struct{} + artist map[string]struct{} +} + +func newRefresher(ctx context.Context, ds model.DataStore) *refresher { + return &refresher{ + ctx: ctx, + ds: ds, + album: map[string]struct{}{}, + artist: map[string]struct{}{}, + } +} + +func (f *refresher) accumulate(mf model.MediaFile) { + if mf.AlbumID != "" { + f.album[mf.AlbumID] = struct{}{} + } + if mf.AlbumArtistID != "" { + f.artist[mf.AlbumArtistID] = struct{}{} + } +} + +type refreshCallbackFunc = func(ids ...string) error + +func (f *refresher) flushMap(m map[string]struct{}, entity string, refresh refreshCallbackFunc) error { + if len(m) == 0 { + return nil + } + var ids []string + for id := range m { + ids = append(ids, id) + delete(m, id) + } + if err := refresh(ids...); err != nil { + log.Error(f.ctx, fmt.Sprintf("Error writing %ss to the DB", entity), err) + return err + } + return nil +} + +func (f *refresher) chunkRefreshAlbums(ids ...string) error { + chunks := utils.BreakUpStringSlice(ids, 100) + for _, chunk := range chunks { + err := f.refreshAlbums(chunk...) + if err != nil { + return err + } + } + return nil +} + +func (f *refresher) refreshAlbums(ids ...string) error { + mfs, err := f.ds.MediaFile(f.ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"album_id": ids}}) + if err != nil { + return err + } + if len(mfs) == 0 { + return nil + } + + repo := f.ds.Album(f.ctx) + grouped := slice.Group(mfs, func(m model.MediaFile) string { return m.AlbumID }) + for _, songs := range grouped { + a := model.MediaFiles(songs).ToAlbum() + err := repo.Put(&a) + if err != nil { + return err + } + } + return nil +} + +func (f *refresher) flush() error { + err := f.flushMap(f.album, "album", f.chunkRefreshAlbums) + if err != nil { + return err + } + err = f.flushMap(f.artist, "artist", f.ds.Artist(f.ctx).Refresh) // TODO Move Artist Refresh out of persistence + if err != nil { + return err + } + return nil +} diff --git a/scanner/tag_scanner.go b/scanner/tag_scanner.go index 278e3d65..bc256c10 100644 --- a/scanner/tag_scanner.go +++ b/scanner/tag_scanner.go @@ -225,7 +225,7 @@ func (s *TagScanner) getDeletedDirs(ctx context.Context, fsDirs dirMap, dbDirs m func (s *TagScanner) processDeletedDir(ctx context.Context, dir string) error { start := time.Now() - buffer := newRefreshBuffer(ctx, s.ds) + buffer := newRefresher(ctx, s.ds) mfs, err := s.ds.MediaFile(ctx).FindAllByPath(dir) if err != nil { @@ -250,7 +250,7 @@ func (s *TagScanner) processDeletedDir(ctx context.Context, dir string) error { func (s *TagScanner) processChangedDir(ctx context.Context, dir string, fullScan bool) error { start := time.Now() - buffer := newRefreshBuffer(ctx, s.ds) + buffer := newRefresher(ctx, s.ds) // Load folder's current tracks from DB into a map currentTracks := map[string]model.MediaFile{} @@ -334,7 +334,7 @@ func (s *TagScanner) processChangedDir(ctx context.Context, dir string, fullScan return err } -func (s *TagScanner) deleteOrphanSongs(ctx context.Context, dir string, tracksToDelete map[string]model.MediaFile, buffer *refreshBuffer) (int, error) { +func (s *TagScanner) deleteOrphanSongs(ctx context.Context, dir string, tracksToDelete map[string]model.MediaFile, buffer *refresher) (int, error) { numPurgedTracks := 0 log.Debug(ctx, "Deleting orphan tracks from DB", "dir", dir, "numTracks", len(tracksToDelete)) @@ -350,7 +350,7 @@ func (s *TagScanner) deleteOrphanSongs(ctx context.Context, dir string, tracksTo return numPurgedTracks, nil } -func (s *TagScanner) addOrUpdateTracksInDB(ctx context.Context, dir string, currentTracks map[string]model.MediaFile, filesToUpdate []string, buffer *refreshBuffer) (int, error) { +func (s *TagScanner) addOrUpdateTracksInDB(ctx context.Context, dir string, currentTracks map[string]model.MediaFile, filesToUpdate []string, buffer *refresher) (int, error) { numUpdatedTracks := 0 log.Trace(ctx, "Updating mediaFiles in DB", "dir", dir, "numFiles", len(filesToUpdate))