mirrors
/
navidrome
mirror of https://github.com/navidrome/navidrome.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

WIP

This commit is contained in:
Deluan 2023-12-22 20:13:11 -05:00
parent 8614a20f7e
commit e47ddd740a
10 changed files with 180 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
db/migration/20231219003407_add_folder_table.go
View File

@ -27,6 +27,22 @@ create table if not exists folder(
references folder (id)
on delete cascade
);
alter table media_file
add column folder_id varchar default "" not null;
alter table media_file
add column pid varchar default id not null;
alter table media_file
add column album_pid varchar default album_id not null;
create index if not exists media_file_folder_id_index
on media_file (folder_id);
create index if not exists media_file_pid_index
on media_file (pid);
create index if not exists media_file_album_pid_index
on media_file (album_pid);
-- FIXME Needs to process current media_file.paths, creating folders as needed
`)
return err

92
model/album.go
View File

@ -11,51 +11,53 @@ import (
type Album struct {
Annotations `structs:"-"`
ID string `structs:"id" json:"id"`
Name string `structs:"name" json:"name"`
EmbedArtPath string `structs:"embed_art_path" json:"embedArtPath"`
ArtistID string `structs:"artist_id" json:"artistId"`
Artist string `structs:"artist" json:"artist"`
AlbumArtistID string `structs:"album_artist_id" json:"albumArtistId"`
AlbumArtist string `structs:"album_artist" json:"albumArtist"`
AllArtistIDs string `structs:"all_artist_ids" json:"allArtistIds"`
MaxYear int `structs:"max_year" json:"maxYear"`
MinYear int `structs:"min_year" json:"minYear"`
Date string `structs:"date" json:"date,omitempty"`
MaxOriginalYear int `structs:"max_original_year" json:"maxOriginalYear"`
MinOriginalYear int `structs:"min_original_year" json:"minOriginalYear"`
OriginalDate string `structs:"original_date" json:"originalDate,omitempty"`
ReleaseDate string `structs:"release_date" json:"releaseDate,omitempty"`
Releases int `structs:"releases" json:"releases"`
Compilation bool `structs:"compilation" json:"compilation"`
Comment string `structs:"comment" json:"comment,omitempty"`
SongCount int `structs:"song_count" json:"songCount"`
Duration float32 `structs:"duration" json:"duration"`
Size int64 `structs:"size" json:"size"`
Genre string `structs:"genre" json:"genre"`
Genres Genres `structs:"-" json:"genres"`
Discs Discs `structs:"discs" json:"discs,omitempty"`
FullText string `structs:"full_text" json:"fullText"`
SortAlbumName string `structs:"sort_album_name" json:"sortAlbumName,omitempty"`
SortArtistName string `structs:"sort_artist_name" json:"sortArtistName,omitempty"`
SortAlbumArtistName string `structs:"sort_album_artist_name" json:"sortAlbumArtistName,omitempty"`
OrderAlbumName string `structs:"order_album_name" json:"orderAlbumName"`
OrderAlbumArtistName string `structs:"order_album_artist_name" json:"orderAlbumArtistName"`
CatalogNum string `structs:"catalog_num" json:"catalogNum,omitempty"`
MbzAlbumID string `structs:"mbz_album_id" json:"mbzAlbumId,omitempty"`
MbzAlbumArtistID string `structs:"mbz_album_artist_id" json:"mbzAlbumArtistId,omitempty"`
MbzAlbumType string `structs:"mbz_album_type" json:"mbzAlbumType,omitempty"`
MbzAlbumComment string `structs:"mbz_album_comment" json:"mbzAlbumComment,omitempty"`
ImageFiles string `structs:"image_files" json:"imageFiles,omitempty"`
Paths string `structs:"paths" json:"paths,omitempty"`
Description string `structs:"description" json:"description,omitempty"`
SmallImageUrl string `structs:"small_image_url" json:"smallImageUrl,omitempty"`
MediumImageUrl string `structs:"medium_image_url" json:"mediumImageUrl,omitempty"`
LargeImageUrl string `structs:"large_image_url" json:"largeImageUrl,omitempty"`
ExternalUrl string `structs:"external_url" json:"externalUrl,omitempty"`
ExternalInfoUpdatedAt *time.Time `structs:"external_info_updated_at" json:"externalInfoUpdatedAt"`
CreatedAt time.Time `structs:"created_at" json:"createdAt"`
UpdatedAt time.Time `structs:"updated_at" json:"updatedAt"`
ID string `structs:"id" json:"id"`
PID string `structs:"pid" json:"pid"`
LibraryID string `structs:"library_id" json:"libraryId"`
Name string `structs:"name" json:"name"`
EmbedArtPath string `structs:"embed_art_path" json:"embedArtPath"`
ArtistID string `structs:"artist_id" json:"artistId"`
Artist string `structs:"artist" json:"artist"`
AlbumArtistID string `structs:"album_artist_id" json:"albumArtistId"`
AlbumArtist string `structs:"album_artist" json:"albumArtist"`
AllArtistIDs string `structs:"all_artist_ids" json:"allArtistIds"`
MaxYear int `structs:"max_year" json:"maxYear"`
MinYear int `structs:"min_year" json:"minYear"`
Date string `structs:"date" json:"date,omitempty"`
MaxOriginalYear int `structs:"max_original_year" json:"maxOriginalYear"`
MinOriginalYear int `structs:"min_original_year" json:"minOriginalYear"`
OriginalDate string `structs:"original_date" json:"originalDate,omitempty"`
ReleaseDate string `structs:"release_date" json:"releaseDate,omitempty"`
Releases int `structs:"releases" json:"releases"`
Compilation bool `structs:"compilation" json:"compilation"`
Comment string `structs:"comment" json:"comment,omitempty"`
SongCount int `structs:"song_count" json:"songCount"`
Duration float32 `structs:"duration" json:"duration"`
Size int64 `structs:"size" json:"size"`
Genre string `structs:"genre" json:"genre"`
Genres Genres `structs:"-" json:"genres"`
Discs Discs `structs:"discs" json:"discs,omitempty"`
FullText string `structs:"full_text" json:"fullText"`
SortAlbumName string `structs:"sort_album_name" json:"sortAlbumName,omitempty"`
SortArtistName string `structs:"sort_artist_name" json:"sortArtistName,omitempty"`
SortAlbumArtistName string `structs:"sort_album_artist_name" json:"sortAlbumArtistName,omitempty"`
OrderAlbumName string `structs:"order_album_name" json:"orderAlbumName"`
OrderAlbumArtistName string `structs:"order_album_artist_name" json:"orderAlbumArtistName"`
CatalogNum string `structs:"catalog_num" json:"catalogNum,omitempty"`
MbzAlbumID string `structs:"mbz_album_id" json:"mbzAlbumId,omitempty"`
MbzAlbumArtistID string `structs:"mbz_album_artist_id" json:"mbzAlbumArtistId,omitempty"`
MbzAlbumType string `structs:"mbz_album_type" json:"mbzAlbumType,omitempty"`
MbzAlbumComment string `structs:"mbz_album_comment" json:"mbzAlbumComment,omitempty"`
ImageFiles string `structs:"image_files" json:"imageFiles,omitempty"`
Paths string `structs:"paths" json:"paths,omitempty"`
Description string `structs:"description" json:"description,omitempty"`
SmallImageUrl string `structs:"small_image_url" json:"smallImageUrl,omitempty"`
MediumImageUrl string `structs:"medium_image_url" json:"mediumImageUrl,omitempty"`
LargeImageUrl string `structs:"large_image_url" json:"largeImageUrl,omitempty"`
ExternalUrl string `structs:"external_url" json:"externalUrl,omitempty"`
ExternalInfoUpdatedAt time.Time `structs:"external_info_updated_at" json:"externalInfoUpdatedAt"`
CreatedAt time.Time `structs:"created_at" json:"createdAt"`
UpdatedAt time.Time `structs:"updated_at" json:"updatedAt"`
}
func (a Album) CoverArtID() ArtworkID {

5
model/mediafile.go
View File

@ -21,6 +21,9 @@ type MediaFile struct {
Bookmarkable `structs:"-"`
ID string `structs:"id" json:"id"`
PID string `structs:"pid" json:"pid"`
LibraryID string `structs:"library_id" json:"libraryId"`
FolderID string `structs:"folder_id" json:"folderId"`
Path string `structs:"path" json:"path"`
Title string `structs:"title" json:"title"`
Album string `structs:"album" json:"album"`
@ -29,6 +32,7 @@ type MediaFile struct {
AlbumArtistID string `structs:"album_artist_id" json:"albumArtistId"`
AlbumArtist string `structs:"album_artist" json:"albumArtist"`
AlbumID string `structs:"album_id" json:"albumId"`
AlbumPID string `structs:"album_pid" json:"albumPid"`
HasCoverArt bool `structs:"has_cover_art" json:"hasCoverArt"`
TrackNumber int `structs:"track_number" json:"trackNumber"`
DiscNumber int `structs:"disc_number" json:"discNumber"`
@ -271,4 +275,5 @@ type MediaFileRepository interface {
AnnotatedRepository
BookmarkableRepository
GetByFolder(folderID string) (MediaFiles, error)
}

7
persistence/mediafile_repository.go
View File

@ -109,6 +109,13 @@ func (r *mediaFileRepository) GetAll(options ...model.QueryOptions) (model.Media
return res, err
}
func (r *mediaFileRepository) GetByFolder(folderID string) (model.MediaFiles, error) {
sq := r.newSelect().Columns("*").Where(Eq{"folder_id": folderID})
res := model.MediaFiles{}
err := r.queryAll(sq, &res)
return res, err
}
func (r *mediaFileRepository) FindByPath(path string) (*model.MediaFile, error) {
sel := r.newSelect().Columns("*").Where(Like{"path": path})
var res model.MediaFiles

1
scanner/metadata/metadata.go
View File

@ -222,6 +222,7 @@ func (t Tags) Channels() int { return t.getInt("channels") }
func (t Tags) ModificationTime() time.Time { return t.fileInfo.ModTime() }
func (t Tags) Size() int64 { return t.fileInfo.Size() }
func (t Tags) FilePath() string { return t.filePath }
func (t Tags) Folder() string { return path.Dir(t.filePath) }
func (t Tags) Suffix() string { return strings.ToLower(strings.TrimPrefix(path.Ext(t.filePath), ".")) }
func (t Tags) BirthTime() time.Time {
if ts := times.Get(t.fileInfo); ts.HasBirthTime() {

15
scanner2/folder.go
View File

@ -11,7 +11,6 @@ import (
"github.com/charlievieth/fastwalk"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"golang.org/x/exp/slices"
)
type folderEntry struct {
@ -21,8 +20,8 @@ type folderEntry struct {
id string // DB ID
updTime time.Time // From DB
modTime time.Time // From FS
audioFiles []fs.DirEntry
imageFiles []fs.DirEntry
audioFiles map[string]fs.DirEntry
imageFiles map[string]fs.DirEntry
playlists []fs.DirEntry
imagesUpdatedAt time.Time
}
@ -35,6 +34,8 @@ func loadDir(ctx context.Context, scanCtx *scanContext, dirPath string, d fastwa
folder = &folderEntry{DirEntry: d, scanCtx: scanCtx, path: dirPath}
folder.id = model.FolderID(scanCtx.lib, dirPath)
folder.updTime = scanCtx.getLastUpdatedInDB(folder.id)
folder.audioFiles = make(map[string]fs.DirEntry)
folder.imageFiles = make(map[string]fs.DirEntry)
dirInfo, err := d.Stat()
if err != nil {
@ -68,22 +69,20 @@ func loadDir(ctx context.Context, scanCtx *scanContext, dirPath string, d fastwa
if fileInfo.ModTime().After(folder.modTime) {
folder.modTime = fileInfo.ModTime()
}
filePath := filepath.Join(dirPath, entry.Name())
switch {
case model.IsAudioFile(entry.Name()):
folder.audioFiles = append(folder.audioFiles, entry)
folder.audioFiles[filePath] = entry
case model.IsValidPlaylist(entry.Name()):
folder.playlists = append(folder.playlists, entry)
case model.IsImageFile(entry.Name()):
folder.imageFiles = append(folder.imageFiles, entry)
folder.imageFiles[filePath] = entry
if fileInfo.ModTime().After(folder.imagesUpdatedAt) {
folder.imagesUpdatedAt = fileInfo.ModTime()
}
}
}
}
slices.SortFunc(folder.audioFiles, func(i, j fs.DirEntry) bool { return i.Name() < j.Name() })
slices.SortFunc(folder.imageFiles, func(i, j fs.DirEntry) bool { return i.Name() < j.Name() })
slices.SortFunc(folder.playlists, func(i, j fs.DirEntry) bool { return i.Name() < j.Name() })
return folder, children, nil
}

33
scanner2/persistent_ids.go Normal file
View File

@ -0,0 +1,33 @@
package scanner2
import (
"crypto/md5"
"fmt"
"github.com/navidrome/navidrome/scanner/metadata"
. "github.com/navidrome/navidrome/utils/gg"
)
func artistPID(md metadata.Tags) string {
key := FirstOr(md.Artist(), "M"+md.MbzArtistID())
return fmt.Sprintf("%x", md5.Sum([]byte(key)))
}
func albumArtistPID(md metadata.Tags) string {
key := FirstOr(md.AlbumArtist(), "M"+md.MbzAlbumArtistID())
return fmt.Sprintf("%x", md5.Sum([]byte(key)))
}
func albumPID(md metadata.Tags) string {
var key string
if md.MbzAlbumID() != "" {
key = "M" + md.MbzAlbumID()
} else {
key = fmt.Sprintf("%s%s%t", albumArtistPID(md), md.Album(), md.Compilation())
}
return fmt.Sprintf("%x", md5.Sum([]byte(key)))
}
func trackPID(md metadata.Tags) string {
return fmt.Sprintf("%s%x", albumPID(md), md5.Sum([]byte(md.FilePath())))
}

4
scanner2/scan_context.go
View File

@ -15,9 +15,10 @@ type scanContext struct {
startTime time.Time
lastUpdates map[string]time.Time
lock sync.RWMutex
fullRescan bool
}
func newScannerContext(ctx context.Context, ds model.DataStore, lib model.Library) (*scanContext, error) {
func newScannerContext(ctx context.Context, ds model.DataStore, lib model.Library, fullRescan bool) (*scanContext, error) {
lastUpdates, err := ds.Folder(ctx).GetLastUpdates(lib)
if err != nil {
return nil, fmt.Errorf("error getting last updates: %w", err)
@ -27,6 +28,7 @@ func newScannerContext(ctx context.Context, ds model.DataStore, lib model.Librar
ds: ds,
startTime: time.Now(),
lastUpdates: lastUpdates,
fullRescan: fullRescan,
}, nil
}

68
scanner2/scanner2.go
View File

@ -11,6 +11,7 @@ import (
"github.com/navidrome/navidrome/model/request"
"github.com/navidrome/navidrome/scanner"
"github.com/navidrome/navidrome/utils/pl"
"github.com/navidrome/navidrome/utils/slice"
)
type scanner2 struct {
@ -33,20 +34,19 @@ func (s *scanner2) RescanAll(requestCtx context.Context, fullRescan bool) error
startTime := time.Now()
log.Info(ctx, "Scanner: Starting scan", "fullRescan", fullRescan, "numLibraries", len(libs))
scanCtxChan := createScanContexts(ctx, s.ds, libs)
scanCtxChan := createScanContexts(ctx, s.ds, libs, fullRescan)
folderChan, folderErrChan := walkDirEntries(ctx, scanCtxChan)
changedFolderChan, changedFolderErrChan := pl.Filter(ctx, 4, folderChan, onlyOutdated(fullRescan))
changedFolderChan, changedFolderErrChan := pl.Filter(ctx, 4, folderChan, onlyOutdated)
processedFolderChan, processedFolderErrChan := pl.Stage(ctx, 4, changedFolderChan, processFolder)
// TODO Next: load tags from all files that are newer than or not in DB
logErrChan := pl.Sink(ctx, 4, changedFolderChan, func(ctx context.Context, folder *folderEntry) error {
logErrChan := pl.Sink(ctx, 4, processedFolderChan, func(ctx context.Context, folder *folderEntry) error {
log.Debug(ctx, "Scanner: Found folder", "folder", folder.Name(), "_path", folder.path,
"audioCount", len(folder.audioFiles), "imageCount", len(folder.imageFiles), "plsCount", len(folder.playlists))
return nil
})
// Wait for pipeline to end, return first error found
for err := range pl.Merge(ctx, folderErrChan, logErrChan, changedFolderErrChan) {
for err := range pl.Merge(ctx, folderErrChan, logErrChan, changedFolderErrChan, processedFolderErrChan) {
return err
}
@ -54,19 +54,12 @@ func (s *scanner2) RescanAll(requestCtx context.Context, fullRescan bool) error
return nil
}
// onlyOutdated returns a filter function that returns true if the folder is outdated (needs to be scanned)
func onlyOutdated(fullScan bool) func(ctx context.Context, entry *folderEntry) (bool, error) {
return func(ctx context.Context, entry *folderEntry) (bool, error) {
return fullScan || entry.isExpired(), nil
}
}
func createScanContexts(ctx context.Context, ds model.DataStore, libs []model.Library) chan *scanContext {
func createScanContexts(ctx context.Context, ds model.DataStore, libs []model.Library, fullRescan bool) chan *scanContext {
outputChannel := make(chan *scanContext, len(libs))
go func() {
defer close(outputChannel)
for _, lib := range libs {
scanCtx, err := newScannerContext(ctx, ds, lib)
scanCtx, err := newScannerContext(ctx, ds, lib, fullRescan)
if err != nil {
log.Error(ctx, "Scanner: Error creating scan context", "lib", lib.Name, err)
continue
@ -122,12 +115,55 @@ func walkDirEntries(ctx context.Context, libsChan <-chan *scanContext) (chan *fo
}()
return outputChannel, errChannel
}
// onlyOutdated returns a filter function that returns true if the folder is outdated (needs to be scanned)
func onlyOutdated(_ context.Context, entry *folderEntry) (bool, error) {
return entry.scanCtx.fullRescan || entry.isExpired(), nil
}
func processFolder(ctx context.Context, entry *folderEntry) (*folderEntry, error) {
// Load children mediafiles from DB
mfs, err := entry.scanCtx.ds.MediaFile(ctx).GetByFolder(entry.id)
if err != nil {
log.Warn(ctx, "Scanner: Error loading mediafiles from DB. Skipping", "folder", entry.path, err)
return entry, nil
}
dbTracks := slice.ToMap(mfs, func(mf model.MediaFile) (string, model.MediaFile) { return mf.Path, mf })
// Get list of files to import, leave dbTracks with tracks to be removed
var filesToImport []string
for afPath, af := range entry.audioFiles {
dbTrack, foundInDB := dbTracks[afPath]
if !foundInDB || entry.scanCtx.fullRescan {
filesToImport = append(filesToImport, afPath)
} else {
info, err := af.Info()
if err != nil {
log.Warn(ctx, "Scanner: Error getting file info", "folder", entry.path, "file", af.Name(), err)
return nil, err
}
if info.ModTime().After(dbTrack.UpdatedAt) {
filesToImport = append(filesToImport, afPath)
}
}
delete(dbTracks, afPath)
}
//tracksToRemove := dbTracks // Just to name things properly
// Load tags from files to import
// Add new/updated files to DB
// Remove deleted mediafiles from DB
// Update folder info in DB
return entry, nil
}
func (s *scanner2) Status(context.Context) (*scanner.StatusInfo, error) {
return &scanner.StatusInfo{}, nil
}
//nolint:unused
func (s *scanner2) doScan(ctx context.Context, lib model.Library, fullRescan bool, folders <-chan string) error {
func (s *scanner2) doScan(ctx context.Context, fullRescan bool, folders <-chan string) error {
return nil
}

9
utils/slice/slice.go
View File

@ -17,6 +17,15 @@ func Group[T any, K comparable](s []T, keyFunc func(T) K) map[K][]T {
return m
}
func ToMap[T any, K comparable, V any](s []T, transformFunc func(T) (K, V)) map[K]V {
m := map[K]V{}
for _, item := range s {
k, v := transformFunc(item)
m[k] = v
}
return m
}
func MostFrequent[T comparable](list []T) T {
if len(list) == 0 {
var zero T