Merge 16f39b8b41 into ca005f6457

cmd/inspect.go

@@ -75,7 +75,7 @@ func runInspector(args []string) {
 	if err != nil {
 		log.Fatal("Error extracting tags", err)
 	}
-	mapper := scanner.NewMediaFileMapper(conf.Server.MusicFolder, &tests.MockedGenreRepo{})
+	mapper := scanner.NewMediaFileMapper(conf.Server.MusicFolder, &tests.MockedGenreRepo{}, true)
 	marshal := marshalers[format]
 	if marshal == nil {
 		log.Fatal("Invalid format", "format", format)

cmd/mbzid_migration.go

@@ -0,0 +1,379 @@
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/signal"
+	"time"
+
+	"github.com/navidrome/navidrome/db"
+	"github.com/navidrome/navidrome/log"
+	"github.com/navidrome/navidrome/model"
+	"github.com/navidrome/navidrome/model/request"
+	"github.com/navidrome/navidrome/persistence"
+	"github.com/navidrome/navidrome/utils/slice"
+	"github.com/spf13/cobra"
+)
+
+var mbzidNoScan bool
+var mbzidNoConfirm bool
+
+var mbzIdCmd = &cobra.Command{
+	Use:   "use_mbzid",
+	Short: "Use MusicBrainz IDs",
+	Long:  "Convert Navidrome's database to use MusicBrainz IDs",
+	Run: func(cmd *cobra.Command, args []string) {
+		db.Init()
+		if err := convertToMbzIDs(cmd.Context()); err != nil {
+			log.Error("Error handling MusicBrainz cataloging. Aborting", err)
+			os.Exit(1)
+			return
+		}
+	},
+}
+
+func init() {
+	mbzIdCmd.Flags().BoolVar(&mbzidNoScan, "no-scan", false, `don't re-scan afterwards.
+WARNING: Your database will be in an inconsistent state unless a full rescan is completed.`)
+	mbzIdCmd.Flags().BoolVar(&mbzidNoConfirm, "no-confirm", false, "don't ask for confirmation")
+	rootCmd.AddCommand(mbzIdCmd)
+}
+
+func warnMbzMigration(dur time.Duration) bool {
+	log.Warn("About to convert database to use MusicBrainz metadata. This CANNOT be undone.")
+	log.Warn(fmt.Sprintf("If this isn't intentional, press ^C NOW. Will begin in %s...", dur))
+
+	sc := make(chan os.Signal, 1)
+	signal.Notify(sc, os.Interrupt)
+
+	defer signal.Stop(sc)
+
+	select {
+	case <-sc:
+		return false
+	case <-time.After(dur):
+		return true
+	}
+}
+
+type deleteManyable interface {
+	DeleteMany(ids ...string) error
+}
+
+func deleteManyIDs(repo deleteManyable, ids map[string]bool) error {
+	s := make([]string, 0, len(ids))
+	for id := range ids {
+		s = append(s, id)
+	}
+
+	return slice.RangeByChunks(s, 100, func(s []string) error {
+		return repo.DeleteMany(s...)
+	})
+}
+
+func migrateUserPlaylists(ctx context.Context, ds model.DataStore, user model.User, ndIdToMbz map[string]*model.MediaFile) error {
+	var err error
+
+	repo := ds.Playlist(request.WithUser(ctx, user))
+	playlists, err := repo.GetAll()
+	if err != nil {
+		return err
+	}
+
+	for _, playlist := range playlists {
+		newPlaylist, err2 := repo.GetWithTracks(playlist.ID, false)
+		if err2 != nil {
+			return err2
+		}
+
+		for i, track := range newPlaylist.Tracks {
+			if newTrack, found := ndIdToMbz[track.MediaFileID]; found {
+				newPlaylist.Tracks[i].MediaFileID = newTrack.ID
+				newPlaylist.Tracks[i].MediaFile.ID = newTrack.ID
+			}
+		}
+
+		if err2 = repo.Put(newPlaylist); err2 != nil {
+			return err2
+		}
+	}
+	return nil
+}
+
+func migrateUserPlayQueue(ctx context.Context, ds model.DataStore, user model.User, ndIdToMbz map[string]*model.MediaFile) error {
+	repo := ds.PlayQueue(request.WithUser(ctx, user))
+	playQueue, err := repo.Retrieve(user.ID)
+	if err != nil {
+		if errors.Is(err, model.ErrNotFound) {
+			return nil
+		}
+
+		return err
+	}
+
+	if newTrack, found := ndIdToMbz[playQueue.Current]; found {
+		playQueue.Current = newTrack.ID
+	}
+
+	for i, item := range playQueue.Items {
+		if newTrack, found := ndIdToMbz[item.ID]; found {
+			playQueue.Items[i].ID = newTrack.ID
+		}
+	}
+
+	return repo.Store(playQueue)
+}
+
+func fillArtists(repo model.ArtistRepository, newArtists map[string]*model.Artist) error {
+	artists, err := repo.GetAll()
+	if err != nil {
+		return err
+	}
+
+	for _, artist := range artists {
+		if newArtist, ok := newArtists[artist.MbzArtistID]; ok {
+			tmp := *newArtist
+			*newArtist = artist
+			newArtist.ID = tmp.ID
+			newArtist.MbzArtistID = tmp.MbzArtistID
+		}
+	}
+
+	return nil
+}
+
+func fillAlbums(repo model.AlbumRepository, newAlbums map[string]*model.Album) error {
+	albums, err := repo.GetAll()
+	if err != nil {
+		return err
+	}
+
+	for _, album := range albums {
+		if newAlbum, ok := newAlbums[album.MbzAlbumID]; ok {
+			tmp := *newAlbum
+			*newAlbum = album
+			newAlbum.ID = tmp.ID
+			newAlbum.ArtistID = tmp.ArtistID
+			newAlbum.AlbumArtistID = tmp.AlbumArtistID
+			newAlbum.MbzAlbumID = tmp.MbzAlbumID
+			newAlbum.MbzAlbumArtistID = tmp.MbzAlbumArtistID
+			newAlbum.AllArtistIDs = "" // Nuking this, the rescan will fix it
+		}
+	}
+
+	return nil
+}
+
+// Migrate all the database entities to use MusicBrainz IDs.
+// Uses the Mbz* fields in model.MediaFile to define the relationships, ignoring
+// the Navidrome ones.
+func migrateEverything(ctx context.Context, ds model.DataStore) error {
+	artistRepo := ds.Artist(ctx)
+	albumRepo := ds.Album(ctx)
+	mfRepo := ds.MediaFile(ctx)
+
+	log.Info("Pass 1: Rebuild hierarchy")
+
+	mediaFiles, err := mfRepo.GetAll()
+	if err != nil {
+		return err
+	}
+
+	newMediaFiles := make(map[string]*model.MediaFile, len(mediaFiles))
+	newArtists := map[string]*model.Artist{}
+	newAlbums := map[string]*model.Album{}
+
+	oldMediaFiles := make(map[string]bool, len(mediaFiles))
+	oldArtists := map[string]bool{}
+	oldAlbums := map[string]bool{}
+
+	oldToNewMF := make(map[string]*model.MediaFile, len(mediaFiles)) // For play queue/playlist remapping
+	newToOldMF := make(map[string]string, len(mediaFiles))           // For mediafile annotations
+	newToOldAlbum := map[string]string{}                             // For album annotations
+	newToOldArtist := map[string]string{}                            // For artist annotations
+
+	for _, mf := range mediaFiles {
+		// Don't touch partial files. The final rescan should take care of them.
+		if mf.MbzReleaseTrackID == "" || mf.MbzAlbumID == "" || mf.MbzArtistID == "" || mf.MbzAlbumArtistID == "" {
+			continue
+		}
+
+		newID := mf.MbzReleaseTrackID
+
+		if _, ok := newMediaFiles[newID]; !ok {
+			newMediaFile := &model.MediaFile{}
+			*newMediaFile = mf
+
+			newMediaFile.ID = newID
+			newMediaFile.AlbumID = mf.MbzAlbumID
+			newMediaFile.ArtistID = mf.MbzArtistID
+			newMediaFile.AlbumArtistID = mf.MbzAlbumArtistID
+			newMediaFiles[newID] = newMediaFile
+
+			oldToNewMF[mf.ID] = newMediaFile
+			newToOldMF[newID] = mf.ID
+			oldMediaFiles[mf.ID] = true
+		}
+
+		if _, ok := newArtists[mf.MbzArtistID]; !ok {
+			newArtists[mf.MbzArtistID] = &model.Artist{ID: mf.MbzArtistID, MbzArtistID: mf.MbzArtistID}
+			newToOldArtist[mf.MbzArtistID] = mf.ArtistID
+			oldArtists[mf.ArtistID] = true
+		}
+
+		if _, ok := newArtists[mf.MbzAlbumArtistID]; !ok {
+			newArtists[mf.MbzAlbumArtistID] = &model.Artist{ID: mf.MbzAlbumArtistID, MbzArtistID: mf.MbzAlbumArtistID}
+			newToOldArtist[mf.MbzAlbumArtistID] = mf.AlbumArtistID
+			oldArtists[mf.AlbumArtistID] = true
+		}
+
+		if _, ok := newAlbums[mf.MbzAlbumID]; !ok {
+			newAlbums[mf.MbzAlbumID] = &model.Album{
+				ID:               mf.MbzAlbumID,
+				ArtistID:         mf.MbzArtistID,
+				AlbumArtistID:    mf.MbzAlbumArtistID,
+				MbzAlbumID:       mf.MbzAlbumID,
+				MbzAlbumArtistID: mf.MbzAlbumArtistID,
+			}
+			newToOldAlbum[mf.MbzAlbumID] = mf.AlbumID
+			oldAlbums[mf.AlbumID] = true
+		}
+	}
+
+	// Attempt to salvage some artist/album information.
+	// These parts are completely optional, as all the information will be recovered by the final rescan.
+	if err = fillAlbums(albumRepo, newAlbums); err != nil {
+		return err
+	}
+
+	if err = fillArtists(artistRepo, newArtists); err != nil {
+		return err
+	}
+
+	log.Info("Pass 2: Add new artists", "count", len(newArtists))
+	for _, artist := range newArtists {
+		if err = artistRepo.Put(artist); err != nil {
+			return err
+		}
+
+		if err = artistRepo.CopyAnnotation(newToOldArtist[artist.ID], artist.ID); err != nil {
+			return err
+		}
+	}
+
+	log.Info("Pass 3: Add new albums", "count", len(newAlbums))
+	for _, album := range newAlbums {
+		if err = albumRepo.Put(album); err != nil {
+			return err
+		}
+
+		if err = albumRepo.CopyAnnotation(newToOldAlbum[album.ID], album.ID); err != nil {
+			return err
+		}
+	}
+
+	log.Info("Pass 4: Add new tracks", "count", len(newMediaFiles))
+	for _, mf := range newMediaFiles {
+		if err = mfRepo.Put(mf); err != nil {
+			return err
+		}
+
+		if err = mfRepo.CopyAnnotation(newToOldMF[mf.ID], mf.ID); err != nil {
+			return err
+		}
+	}
+
+	// Playlists and Play queues require a user in the context
+	users, err := ds.User(ctx).GetAll()
+	if err != nil {
+		return err
+	}
+
+	log.Info("Pass 5: Update playlist references")
+	for _, user := range users {
+		if err = migrateUserPlaylists(ctx, ds, user, oldToNewMF); err != nil {
+			return err
+		}
+	}
+
+	log.Info("Pass 6: Update play queue references", "count", len(users))
+	for _, user := range users {
+		if err = migrateUserPlayQueue(ctx, ds, user, oldToNewMF); err != nil {
+			return err
+		}
+	}
+
+	log.Info("Pass 7: Cleanup leftover tracks", "count", len(oldMediaFiles))
+	if err = deleteManyIDs(mfRepo, oldMediaFiles); err != nil {
+		return err
+	}
+
+	log.Info("Pass 8: Cleanup leftover albums", "count", len(oldAlbums))
+	if err = deleteManyIDs(albumRepo, oldAlbums); err != nil {
+		return err
+	}
+
+	log.Info("Pass 9: Cleanup leftover artists", "count", len(oldArtists))
+	if err = deleteManyIDs(artistRepo, oldArtists); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func convertToMbzIDs(ctx context.Context) error {
+	var err error
+
+	ds := persistence.New(db.Db())
+
+	alreadyDone := false
+
+	err = ds.WithTx(func(tx model.DataStore) error {
+		props := tx.Property(ctx)
+
+		useMbzIDs, err := props.DefaultGetBool(model.PropUsingMbzIDs, false)
+		if err != nil {
+			return err
+		}
+
+		// Nothing to do
+		if useMbzIDs {
+			alreadyDone = true
+			return nil
+		}
+
+		if !mbzidNoConfirm && !warnMbzMigration(10*time.Second) {
+			return errors.New("user aborted")
+		}
+
+		if err := migrateEverything(ctx, tx); err != nil {
+			return err
+		}
+
+		if err = props.Put(model.PropUsingMbzIDs, "true"); err != nil {
+			return err
+		}
+
+		return props.DeletePrefixed(model.PropLastScan)
+	})
+
+	if err != nil {
+		return err
+	}
+
+	if alreadyDone {
+		log.Info("Migration already done.")
+		return nil
+	}
+
+	if mbzidNoScan {
+		log.Info("Skipping post-migration scan by request.")
+		return nil
+	}
+
+	fullRescan = true
+	runScanner()
+	return nil
+}

model/album.go

@@ -111,5 +111,6 @@ type AlbumRepository interface {
 	GetAll(...QueryOptions) (Albums, error)
 	GetAllWithoutGenres(...QueryOptions) (Albums, error)
 	Search(q string, offset int, size int) (Albums, error)
+	DeleteMany(ids ...string) error
 	AnnotatedRepository
 }

model/annotation.go

@@ -14,4 +14,5 @@ type AnnotatedRepository interface {
 	IncPlayCount(itemID string, ts time.Time) error
 	SetStar(starred bool, itemIDs ...string) error
 	SetRating(rating int, itemID string) error
+	CopyAnnotation(fromID string, toID string) error
 }

model/artist.go

@@ -54,5 +54,6 @@ type ArtistRepository interface {
 	GetAll(options ...QueryOptions) (Artists, error)
 	Search(q string, offset int, size int) (Artists, error)
 	GetIndex() (ArtistIndexes, error)
+	DeleteMany(ids ...string) error
 	AnnotatedRepository
 }

model/mediafile.go

@@ -262,6 +262,7 @@ type MediaFileRepository interface {
 	GetAll(options ...QueryOptions) (MediaFiles, error)
 	Search(q string, offset int, size int) (MediaFiles, error)
 	Delete(id string) error
+	DeleteMany(ids ...string) error
 
 	// Queries by path to support the scanner, no Annotations or Bookmarks required in the response
 	FindAllByPath(path string) (MediaFiles, error)

model/properties.go

@@ -2,12 +2,15 @@ package model
 
 const (
 	// TODO Move other prop keys to here
-	PropLastScan = "LastScan"
+	PropLastScan    = "LastScan"
+	PropUsingMbzIDs = "UsingMbzIDs"
 )
 
 type PropertyRepository interface {
 	Put(id string, value string) error
 	Get(id string) (string, error)
 	Delete(id string) error
+	DeletePrefixed(prefix string) error
 	DefaultGet(id string, defaultValue string) (string, error)
+	DefaultGetBool(id string, defaultValue bool) (bool, error)
 }

model/user.go

@@ -27,6 +27,7 @@ type Users []User
 type UserRepository interface {
 	CountAll(...QueryOptions) (int64, error)
 	Get(id string) (*User, error)
+	GetAll(options ...QueryOptions) (Users, error)
 	Put(*User) error
 	UpdateLastLoginAt(id string) error
 	UpdateLastAccessAt(id string) error

persistence/album_repository.go

@@ -229,5 +229,9 @@ func (r *albumRepository) NewInstance() interface{} {
 	return &model.Album{}
 }
 
+func (r *albumRepository) DeleteMany(ids ...string) error {
+	return r.delete(Eq{"album.id": ids})
+}
+
 var _ model.AlbumRepository = (*albumRepository)(nil)
 var _ model.ResourceRepository = (*albumRepository)(nil)

persistence/artist_repository.go

@@ -218,5 +218,9 @@ func (r *artistRepository) NewInstance() interface{} {
 	return &model.Artist{}
 }
 
+func (r *artistRepository) DeleteMany(ids ...string) error {
+	return r.delete(Eq{"artist.id": ids})
+}
+
 var _ model.ArtistRepository = (*artistRepository)(nil)
 var _ model.ResourceRepository = (*artistRepository)(nil)

persistence/mediafile_repository.go

@@ -173,7 +173,11 @@ func (r *mediaFileRepository) deleteNotInPath(basePath string) error {
 }
 
 func (r *mediaFileRepository) Delete(id string) error {
-	return r.delete(Eq{"id": id})
+	return r.DeleteMany(id)
+}
+
+func (r *mediaFileRepository) DeleteMany(ids ...string) error {
+	return r.delete(Eq{"id": ids})
 }
 
 // DeleteByPath delete from the DB all mediafiles that are direct children of path

persistence/property_repository.go

@@ -3,6 +3,8 @@ package persistence
 import (
 	"context"
 	"errors"
+	"strconv"
+	"strings"
 
 	. "github.com/Masterminds/squirrel"
 	"github.com/navidrome/navidrome/model"
@@ -58,6 +60,19 @@ func (r propertyRepository) DefaultGet(id string, defaultValue string) (string,
 	return value, nil
 }
 
+func (r propertyRepository) DefaultGetBool(id string, defaultValue bool) (bool, error) {
+	val, err := r.DefaultGet(id, strconv.FormatBool(defaultValue))
+	if err != nil {
+		return false, err
+	}
+
+	return strconv.ParseBool(val)
+}
+
 func (r propertyRepository) Delete(id string) error {
 	return r.delete(Eq{"id": id})
 }
+
+func (r propertyRepository) DeletePrefixed(prefix string) error {
+	return r.delete(Like{"id": strings.Replace(prefix, "%", "%%", -1) + "%"})
+}

persistence/sql_annotations.go

@@ -101,3 +101,30 @@ func (r sqlRepository) cleanAnnotations() error {
 	}
 	return nil
 }
+
+func (r sqlRepository) CopyAnnotation(fromID string, toID string) error {
+	if fromID == toID {
+		return nil
+	}
+
+	/* Delete existing ones so we don't get conflicts. */
+	del := Delete(annotationTable).Where(And{
+		Eq{annotationTable + ".item_type": r.tableName},
+		Eq{annotationTable + ".item_id": toID},
+	})
+	_, err := r.executeSQL(del)
+	if err != nil && !errors.Is(err, sql.ErrNoRows) {
+		return err
+	}
+
+	upd := Update(annotationTable).Where(And{
+		Eq{annotationTable + ".item_type": r.tableName},
+		Eq{annotationTable + ".item_id": fromID},
+	}).Set("item_id", toID)
+
+	c, err := r.executeSQL(upd)
+	if c == 0 || errors.Is(err, sql.ErrNoRows) {
+		return nil
+	}
+	return err
+}

scanner/mapping.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/deluan/sanitize"
+	"github.com/google/uuid"
 	"github.com/navidrome/navidrome/conf"
 	"github.com/navidrome/navidrome/consts"
 	"github.com/navidrome/navidrome/model"
@@ -18,12 +19,14 @@ import (
 type MediaFileMapper struct {
 	rootFolder string
 	genres     model.GenreRepository
+	useMbzIDs  bool
 }
 
-func NewMediaFileMapper(rootFolder string, genres model.GenreRepository) *MediaFileMapper {
+func NewMediaFileMapper(rootFolder string, genres model.GenreRepository, useMbzIDs bool) *MediaFileMapper {
 	return &MediaFileMapper{
 		rootFolder: rootFolder,
 		genres:     genres,
+		useMbzIDs:  useMbzIDs,
 	}
 }
 
@@ -123,11 +126,30 @@ func (s MediaFileMapper) mapAlbumName(md metadata.Tags) string {
 	return name
 }
 
+func (s MediaFileMapper) mapMbzID(id string) string {
+	if !s.useMbzIDs {
+		return ""
+	}
+
+	if _, err := uuid.Parse(id); err != nil {
+		return ""
+	}
+
+	return id
+}
+
 func (s MediaFileMapper) trackID(md metadata.Tags) string {
+	if mbzID := s.mapMbzID(md.MbzReleaseTrackID()); mbzID != "" {
+		return mbzID
+	}
 	return fmt.Sprintf("%x", md5.Sum([]byte(md.FilePath())))
 }
 
 func (s MediaFileMapper) albumID(md metadata.Tags, releaseDate string) string {
+	if id := s.mapMbzID(md.MbzAlbumID()); id != "" {
+		return id
+	}
+
 	albumPath := strings.ToLower(fmt.Sprintf("%s\\%s", s.mapAlbumArtistName(md), s.mapAlbumName(md)))
 	if !conf.Server.Scanner.GroupAlbumReleases {
 		if len(releaseDate) != 0 {
@@ -138,10 +160,16 @@ func (s MediaFileMapper) albumID(md metadata.Tags, releaseDate string) string {
 }
 
 func (s MediaFileMapper) artistID(md metadata.Tags) string {
+	if id := s.mapMbzID(md.MbzArtistID()); id != "" {
+		return id
+	}
 	return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapArtistName(md)))))
 }
 
 func (s MediaFileMapper) albumArtistID(md metadata.Tags) string {
+	if id := s.mapMbzID(md.MbzAlbumArtistID()); id != "" {
+		return id
+	}
 	return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapAlbumArtistName(md)))))
 }
 

scanner/mapping_internal_test.go

@@ -16,7 +16,7 @@ var _ = Describe("mapping", func() {
 		var mapper *MediaFileMapper
 		Describe("mapTrackTitle", func() {
 			BeforeEach(func() {
-				mapper = NewMediaFileMapper("/music", nil)
+				mapper = NewMediaFileMapper("/music", nil, false)
 			})
 			It("returns the Title when it is available", func() {
 				md := metadata.NewTag("/music/artist/album01/Song.mp3", nil, metadata.ParsedTags{"title": []string{"This is not a love song"}})
@@ -37,7 +37,7 @@ var _ = Describe("mapping", func() {
 				ds := &tests.MockDataStore{}
 				gr = ds.Genre(ctx)
 				gr = newCachedGenreRepository(ctx, gr)
-				mapper = NewMediaFileMapper("/", gr)
+				mapper = NewMediaFileMapper("/", gr, false)
 			})
 
 			It("returns empty if no genres are available", func() {
@@ -79,7 +79,7 @@ var _ = Describe("mapping", func() {
 		Describe("mapDates", func() {
 			var md metadata.Tags
 			BeforeEach(func() {
-				mapper = NewMediaFileMapper("/", nil)
+				mapper = NewMediaFileMapper("/", nil, false)
 			})
 			Context("when all date fields are provided", func() {
 				BeforeEach(func() {

scanner/tag_scanner.go

@@ -101,7 +101,13 @@ func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time, prog
 	var changedDirs []string
 	s.cnt = &counters{}
 	genres := newCachedGenreRepository(ctx, s.ds.Genre(ctx))
-	s.mapper = NewMediaFileMapper(s.rootFolder, genres)
+
+	useMbzIds, err := s.ds.Property(ctx).DefaultGetBool(model.PropUsingMbzIDs, false)
+	if err != nil {
+		return 0, err
+	}
+
+	s.mapper = NewMediaFileMapper(s.rootFolder, genres, useMbzIds)
 	refresher := newRefresher(s.ds, s.cacheWarmer, allFSDirs)
 
 	log.Trace(ctx, "Loading directory tree from music folder", "folder", s.rootFolder)