mirrors
/
navidrome
mirror of https://github.com/navidrome/navidrome.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

WIP: Persisting tracks and tags to DB

This commit is contained in:
Deluan 2023-12-26 17:11:02 -05:00
parent bbfe8727ca
commit d6b82154c9
25 changed files with 404 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
db/migration/20231219003407_add_folder_table.go
View File

@ -34,15 +34,36 @@ alter table media_file
add column pid varchar default id not null;
alter table media_file
add column album_pid varchar default album_id not null;
alter table media_file
add column tags JSONB default '{}' not null;
create index if not exists media_file_folder_id_index
create index if not exists media_file_folder_id_ix
on media_file (folder_id);
create index if not exists media_file_pid_index
create unique index if not exists media_file_pid_ix
on media_file (pid);
create index if not exists media_file_album_pid_index
create index if not exists media_file_album_pid_ix
on media_file (album_pid);
-- FIXME Needs to process current media_file.paths, creating folders as needed
create table if not exists tag(
id varchar not null primary key,
name varchar default '' not null,
value varchar default '' not null,
constraint tags_name_value_ux
unique (name, value)
);
create table if not exists item_tags(
item_id varchar not null,
item_type varchar not null,
tag_name varchar not null,
tag_id varchar not null,
constraint item_tags_ux
unique (item_id, item_type, tag_id)
);
create index if not exists item_tag_name_ix on item_tags(item_id, tag_name)
`)
return err

1
model/datastore.go
View File

@ -26,6 +26,7 @@ type DataStore interface {
Artist(ctx context.Context) ArtistRepository
MediaFile(ctx context.Context) MediaFileRepository
Genre(ctx context.Context) GenreRepository
Tag(ctx context.Context) TagRepository
Playlist(ctx context.Context) PlaylistRepository
PlayQueue(ctx context.Context) PlayQueueRepository
Transcoding(ctx context.Context) TranscodingRepository

4
model/mediafile.go
View File

@ -22,7 +22,7 @@ type MediaFile struct {
ID string `structs:"id" json:"id"`
PID string `structs:"pid" json:"pid"`
LibraryID string `structs:"library_id" json:"libraryId"`
LibraryID int `structs:"library_id" json:"libraryId"`
FolderID string `structs:"folder_id" json:"folderId"`
Path string `structs:"path" json:"path"`
Title string `structs:"title" json:"title"`
@ -76,6 +76,8 @@ type MediaFile struct {
RgTrackGain float64 `structs:"rg_track_gain" json:"rgTrackGain"`
RgTrackPeak float64 `structs:"rg_track_peak" json:"rgTrackPeak"`
Tags Tags `structs:"tags" json:"tags,omitempty"` // All tags from the original file
CreatedAt time.Time `structs:"created_at" json:"createdAt"` // Time this entry was created in the DB
UpdatedAt time.Time `structs:"updated_at" json:"updatedAt"` // Time of file last update (mtime)
}

59
model/tag.go Normal file
View File

@ -0,0 +1,59 @@
package model
import (
"crypto/md5"
"fmt"
"strings"
"github.com/navidrome/navidrome/consts"
)
type Tag struct {
ID string
Name string
Value string
}
type FlattenedTags []Tag
func (t Tag) String() string {
return fmt.Sprintf("%s=%s", t.Name, t.Value)
}
type Tags map[string][]string
func (t Tags) Values(name string) []string {
return t[name]
}
func (t Tags) Flatten(name string) FlattenedTags {
var tags FlattenedTags
for _, v := range t[name] {
tags = append(tags, NewTag(name, v))
}
return tags
}
func (t Tags) FlattenAll() FlattenedTags {
var tags FlattenedTags
for name, values := range t {
for _, v := range values {
tags = append(tags, NewTag(name, v))
}
}
return tags
}
func NewTag(name, value string) Tag {
name = strings.ToLower(name)
id := fmt.Sprintf("%x", md5.Sum([]byte(name+consts.Zwsp+strings.ToLower(value))))
return Tag{
ID: id,
Name: name,
Value: value,
}
}
type TagRepository interface {
Add(...Tag) error
}

2
persistence/album_repository.go
View File

@ -155,7 +155,7 @@ func (r *albumRepository) Get(id string) (*model.Album, error) {
}
func (r *albumRepository) Put(m *model.Album) error {
_, err := r.put(m.ID, &dbAlbum{Album: m})
_, err := r.put("pid", m.PID, &dbAlbum{Album: m})
if err != nil {
return err
}

2
persistence/artist_repository.go
View File

@ -95,7 +95,7 @@ func (r *artistRepository) Exists(id string) (bool, error) {
func (r *artistRepository) Put(a *model.Artist, colsToUpdate ...string) error {
a.FullText = getFullText(a.Name, a.SortArtistName)
dba := &dbArtist{Artist: a}
_, err := r.put(dba.ID, dba, colsToUpdate...)
_, err := r.put("id", dba.ID, dba, colsToUpdate...)
if err != nil {
return err
}

2
persistence/folder_repository.go
View File

@ -55,7 +55,7 @@ func (r folderRepository) GetLastUpdates(lib model.Library) (map[string]time.Tim
func (r folderRepository) Put(lib model.Library, path string) error {
folder := model.NewFolder(lib, path)
_, err := r.put(folder.ID, folder)
_, err := r.put("id", folder.ID, folder)
return err
}

124
persistence/mediafile_repository.go
View File

@ -2,6 +2,8 @@ package persistence
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
@ -21,7 +23,51 @@ type mediaFileRepository struct {
sqlRestful
}
func NewMediaFileRepository(ctx context.Context, db dbx.Builder) *mediaFileRepository {
type dbMediaFile struct {
*model.MediaFile `structs:",flatten"`
Tags string `structs:"-" json:"tags"`
}
func (m *dbMediaFile) PostScan() error {
if m.Tags == "" {
m.MediaFile.Tags = make(map[string][]string)
return nil
}
err := json.Unmarshal([]byte(m.Tags), &m.MediaFile.Tags)
if err != nil {
return err
}
// Map genres from tags
for _, g := range m.MediaFile.Tags.Flatten("genre") {
m.MediaFile.Genres = append(m.MediaFile.Genres, model.Genre{Name: g.Value, ID: g.ID})
}
return nil
}
func (m *dbMediaFile) PostMapArgs(args map[string]any) error {
if len(m.MediaFile.Tags) == 0 {
args["tags"] = "{}"
return nil
}
b, err := json.Marshal(m.MediaFile.Tags)
if err != nil {
return err
}
args["tags"] = string(b)
return nil
}
type dbMediaFiles []dbMediaFile
func (m *dbMediaFiles) toModels() model.MediaFiles {
res := make(model.MediaFiles, len(*m))
for i, mf := range *m {
res[i] = *mf.MediaFile
}
return res
}
func NewMediaFileRepository(ctx context.Context, db dbx.Builder) model.MediaFileRepository {
r := &mediaFileRepository{}
r.ctx = ctx
r.db = db
@ -50,7 +96,8 @@ func NewMediaFileRepository(ctx context.Context, db dbx.Builder) *mediaFileRepos
func (r *mediaFileRepository) CountAll(options ...model.QueryOptions) (int64, error) {
sql := r.newSelectWithAnnotation("media_file.id")
sql = r.withGenres(sql) // Required for filtering by genre
// FIXME Genres
//sql = r.withGenres(sql) // Required for filtering by genre
return r.count(sql, options...)
}
@ -59,73 +106,89 @@ func (r *mediaFileRepository) Exists(id string) (bool, error) {
}
func (r *mediaFileRepository) Put(m *model.MediaFile) error {
if m.ID == "" || m.PID == "" {
return errors.New("id and pid are required")
}
m.FullText = getFullText(m.Title, m.Album, m.Artist, m.AlbumArtist,
m.SortTitle, m.SortAlbumName, m.SortArtistName, m.SortAlbumArtistName, m.DiscSubtitle)
_, err := r.put(m.ID, m)
_, err := r.put("pid", m.PID, &dbMediaFile{MediaFile: m})
if err != nil {
return err
}
return r.updateGenres(m.ID, r.tableName, m.Genres)
return r.updateTags(m.ID, m.Tags)
// FIXME Genres
//if err != nil {
// return err
//}
//return r.updateGenres(m.ID, r.tableName, m.Genres)
}
func (r *mediaFileRepository) selectMediaFile(options ...model.QueryOptions) SelectBuilder {
sql := r.newSelectWithAnnotation("media_file.id", options...).Columns("media_file.*")
sql = r.withBookmark(sql, "media_file.id")
if len(options) > 0 && options[0].Filters != nil {
s, _, _ := options[0].Filters.ToSql()
// If there's any reference of genre in the filter, joins with genre
if strings.Contains(s, "genre") {
sql = r.withGenres(sql)
// If there's no filter on genre_id, group the results by media_file.id
if !strings.Contains(s, "genre_id") {
sql = sql.GroupBy("media_file.id")
}
}
}
// FIXME Genres
//if len(options) > 0 && options[0].Filters != nil {
// s, _, _ := options[0].Filters.ToSql()
// // If there's any reference of genre in the filter, joins with genre
// if strings.Contains(s, "genre") {
// sql = r.withGenres(sql)
// // If there's no filter on genre_id, group the results by media_file.id
// if !strings.Contains(s, "genre_id") {
// sql = sql.GroupBy("media_file.id")
// }
// }
//}
return sql
}
func (r *mediaFileRepository) Get(id string) (*model.MediaFile, error) {
sel := r.selectMediaFile().Where(Eq{"media_file.id": id})
var res model.MediaFiles
var res dbMediaFiles
if err := r.queryAll(sel, &res); err != nil {
return nil, err
}
if len(res) == 0 {
return nil, model.ErrNotFound
}
err := r.loadMediaFileGenres(&res)
return &res[0], err
// FIXME Genres
//err := r.loadMediaFileGenres(&res)
return res[0].MediaFile, nil
}
func (r *mediaFileRepository) GetAll(options ...model.QueryOptions) (model.MediaFiles, error) {
sq := r.selectMediaFile(options...)
res := model.MediaFiles{}
var res dbMediaFiles
err := r.queryAll(sq, &res, options...)
if err != nil {
return nil, err
}
err = r.loadMediaFileGenres(&res)
return res, err
// FIXME Genres
//err = r.loadMediaFileGenres(&rows)
return res.toModels(), nil
}
func (r *mediaFileRepository) GetByFolder(folderID string) (model.MediaFiles, error) {
sq := r.newSelect().Columns("*").Where(Eq{"folder_id": folderID})
res := model.MediaFiles{}
var res dbMediaFiles
err := r.queryAll(sq, &res)
return res, err
if err != nil {
return nil, err
}
return res.toModels(), nil
}
func (r *mediaFileRepository) FindByPath(path string) (*model.MediaFile, error) {
sel := r.newSelect().Columns("*").Where(Like{"path": path})
var res model.MediaFiles
var res dbMediaFiles
if err := r.queryAll(sel, &res); err != nil {
return nil, err
}
if len(res) == 0 {
return nil, model.ErrNotFound
}
return &res[0], nil
return res[0].MediaFile, nil
}
func cleanPath(path string) string {
@ -151,9 +214,9 @@ func (r *mediaFileRepository) FindAllByPath(path string) (model.MediaFiles, erro
sel := r.newSelect().Columns("*", "item NOT GLOB '*"+string(os.PathSeparator)+"*' AS isLast").
Where(Eq{"isLast": 1}).FromSelect(sel0, "sel0")
res := model.MediaFiles{}
res := dbMediaFiles{}
err := r.queryAll(sel, &res)
return res, err
return res.toModels(), err
}
// FindPathsRecursively returns a list of all subfolders of basePath, recursively
@ -202,13 +265,14 @@ func (r *mediaFileRepository) removeNonAlbumArtistIds() error {
}
func (r *mediaFileRepository) Search(q string, offset int, size int) (model.MediaFiles, error) {
results := model.MediaFiles{}
results := dbMediaFiles{}
err := r.doSearch(q, offset, size, &results, "title")
if err != nil {
return nil, err
}
err = r.loadMediaFileGenres(&results)
return results, err
// FIXME Genres
//err = r.loadMediaFileGenres(&results)
return results.toModels(), err
}
func (r *mediaFileRepository) Count(options ...rest.QueryOptions) (int64, error) {

4
persistence/persistence.go
View File

@ -43,6 +43,10 @@ func (s *SQLStore) Genre(ctx context.Context) model.GenreRepository {
return NewGenreRepository(ctx, s.getDBXBuilder())
}
func (s *SQLStore) Tag(ctx context.Context) model.TagRepository {
return NewTagRepository(ctx, s.getDBXBuilder())
}
func (s *SQLStore) PlayQueue(ctx context.Context) model.PlayQueueRepository {
return NewPlayQueueRepository(ctx, s.getDBXBuilder())
}

6
persistence/player_repository.go
View File

@ -27,7 +27,7 @@ func NewPlayerRepository(ctx context.Context, db dbx.Builder) model.PlayerReposi
}
func (r *playerRepository) Put(p *model.Player) error {
_, err := r.put(p.ID, p)
_, err := r.put("id", p.ID, p)
return err
}
@ -102,7 +102,7 @@ func (r *playerRepository) Save(entity interface{}) (string, error) {
if !r.isPermitted(t) {
return "", rest.ErrPermissionDenied
}
id, err := r.put(t.ID, t)
id, err := r.put("id", t.ID, t)
if errors.Is(err, model.ErrNotFound) {
return "", rest.ErrNotFound
}
@ -115,7 +115,7 @@ func (r *playerRepository) Update(id string, entity interface{}, cols ...string)
if !r.isPermitted(t) {
return rest.ErrPermissionDenied
}
_, err := r.put(id, t, cols...)
_, err := r.put("id", id, t, cols...)
if errors.Is(err, model.ErrNotFound) {
return rest.ErrNotFound
}

4
persistence/playlist_repository.go
View File

@ -118,7 +118,7 @@ func (r *playlistRepository) Put(p *model.Playlist) error {
}
pls.UpdatedAt = time.Now()
id, err := r.put(pls.ID, pls)
id, err := r.put("id", pls.ID, pls)
if err != nil {
return err
}
@ -417,7 +417,7 @@ func (r *playlistRepository) Update(id string, entity interface{}, cols ...strin
}
pls.ID = id
pls.UpdatedAt = time.Now()
_, err = r.put(id, pls, append(cols, "updatedAt")...)
_, err = r.put("id", id, pls, append(cols, "updatedAt")...)
if errors.Is(err, model.ErrNotFound) {
return rest.ErrNotFound
}

2
persistence/playqueue_repository.go
View File

@ -47,7 +47,7 @@ func (r *playQueueRepository) Store(q *model.PlayQueue) error {
pq.CreatedAt = time.Now()
}
pq.UpdatedAt = time.Now()
_, err = r.put(pq.ID, pq)
_, err = r.put("id", pq.ID, pq)
if err != nil {
log.Error(r.ctx, "Error saving playqueue", "user", u.UserName, err)
return err

4
persistence/share_repository.go
View File

@ -138,7 +138,7 @@ func (r *shareRepository) Update(id string, entity interface{}, cols ...string)
s.ID = id
s.UpdatedAt = time.Now()
cols = append(cols, "updated_at")
_, err := r.put(id, s, cols...)
_, err := r.put("id", id, s, cols...)
if errors.Is(err, model.ErrNotFound) {
return rest.ErrNotFound
}
@ -154,7 +154,7 @@ func (r *shareRepository) Save(entity interface{}) (string, error) {
}
s.CreatedAt = time.Now()
s.UpdatedAt = time.Now()
id, err := r.put(s.ID, s)
id, err := r.put("id", s.ID, s)
if errors.Is(err, model.ErrNotFound) {
return "", rest.ErrNotFound
}

16
persistence/sql_base_repository.go
View File

@ -237,10 +237,10 @@ func (r sqlRepository) count(countQuery SelectBuilder, options ...model.QueryOpt
return res.Count, err
}
func (r sqlRepository) put(id string, m interface{}, colsToUpdate ...string) (newId string, err error) {
func (r sqlRepository) put(idCol string, idVal string, m interface{}, colsToUpdate ...string) (newId string, err error) {
values, _ := toSQLArgs(m)
// If there's an ID, try to update first
if id != "" {
if idVal != "" {
updateValues := map[string]interface{}{}
// This is a map of the columns that need to be updated, if specified
@ -255,23 +255,23 @@ func (r sqlRepository) put(id string, m interface{}, colsToUpdate ...string) (ne
}
delete(updateValues, "created_at")
update := Update(r.tableName).Where(Eq{"id": id}).SetMap(updateValues)
update := Update(r.tableName).Where(Eq{idCol: idVal}).SetMap(updateValues)
count, err := r.executeSQL(update)
if err != nil {
return "", err
}
if count > 0 {
return id, nil
return idVal, nil
}
}
// If it does not have an ID OR the ID was not found (when it is a new record with predefined id)
if id == "" {
id = uuid.NewString()
values["id"] = id
if idVal == "" {
idVal = uuid.NewString()
values[idCol] = idVal
}
insert := Insert(r.tableName).SetMap(values)
_, err = r.executeSQL(insert)
return id, err
return idVal, err
}
func (r sqlRepository) delete(cond Sqlizer) error {

52
persistence/tag_repository.go Normal file
View File

@ -0,0 +1,52 @@
package persistence
import (
"context"
. "github.com/Masterminds/squirrel"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/utils/slice"
"github.com/pocketbase/dbx"
)
type tagRepository struct {
sqlRepository
}
func NewTagRepository(ctx context.Context, db dbx.Builder) model.TagRepository {
r := &tagRepository{}
r.ctx = ctx
r.db = db
r.tableName = "tag"
return r
}
func (r *tagRepository) Add(tags ...model.Tag) error {
return slice.RangeByChunks(tags, 200, func(chunk []model.Tag) error {
sq := Insert(r.tableName).Columns("id", "name", "value").
Suffix("on conflict (id) do nothing")
for _, t := range chunk {
sq = sq.Values(t.ID, t.Name, t.Value)
}
_, err := r.executeSQL(sq)
return err
})
}
func (r *sqlRepository) updateTags(itemID string, tags model.Tags) error {
sqd := Delete("item_tags").Where(Eq{"item_id": itemID, "item_type": r.tableName})
_, err := r.executeSQL(sqd)
if err != nil {
return err
}
sqi := Insert("item_tags").Columns("item_id", "item_type", "tag_name", "tag_id").
Suffix("on conflict (item_id, item_type, tag_id) do nothing")
for name, values := range tags {
for _, value := range values {
tag := model.NewTag(name, value)
sqi = sqi.Values(itemID, r.tableName, tag.Name, tag.ID)
}
}
_, err = r.executeSQL(sqi)
return err
}

6
persistence/transcoding_repository.go
View File

@ -42,7 +42,7 @@ func (r *transcodingRepository) FindByFormat(format string) (*model.Transcoding,
}
func (r *transcodingRepository) Put(t *model.Transcoding) error {
_, err := r.put(t.ID, t)
_, err := r.put("id", t.ID, t)
return err
}
@ -71,7 +71,7 @@ func (r *transcodingRepository) NewInstance() interface{} {
func (r *transcodingRepository) Save(entity interface{}) (string, error) {
t := entity.(*model.Transcoding)
id, err := r.put(t.ID, t)
id, err := r.put("id", t.ID, t)
if errors.Is(err, model.ErrNotFound) {
return "", rest.ErrNotFound
}
@ -81,7 +81,7 @@ func (r *transcodingRepository) Save(entity interface{}) (string, error) {
func (r *transcodingRepository) Update(id string, entity interface{}, cols ...string) error {
t := entity.(*model.Transcoding)
t.ID = id
_, err := r.put(id, t)
_, err := r.put("id", id, t)
if errors.Is(err, model.ErrNotFound) {
return rest.ErrNotFound
}

67
scanner/metadata/metadata.go
View File

@ -231,7 +231,7 @@ func (t Tags) BirthTime() time.Time {
return time.Now()
}
// ReplayGain Properties
// ReplayGain Properties TODO: check rg_* tags
func (t Tags) RGAlbumGain() float64 { return t.getGainValue("replaygain_album_gain") }
func (t Tags) RGAlbumPeak() float64 { return t.getPeakValue("replaygain_album_peak") }
@ -385,3 +385,68 @@ func (t Tags) getFloat(tagNames ...string) float64 {
}
return value
}
// We exclude all tags that are already first-class citizens in the model
var excludedTags = map[string]struct{}{
"duration": {},
"bitrate": {},
"channels": {},
"bpm": {},
"has_picture": {},
"title": {},
"album": {},
"artist": {},
"artists": {},
"albumartist": {},
"albumartists": {},
"track": {},
"tracknumber": {},
"tracktotal": {},
"totaltracks": {},
"disc": {},
"discnumber": {},
"disctotal": {},
"totaldiscs": {},
"lyrics": {},
"year": {},
"date": {},
"originaldate": {},
"releasedate": {},
"comment": {},
}
// Also exclude any tag that starts with one of these prefixes
var excludedPrefixes = []string{
"musicbrainz",
"replaygain",
"sort",
}
func isExcludedTag(tagName string) bool {
if _, ok := excludedTags[tagName]; ok {
return true
}
for _, prefix := range excludedPrefixes {
if strings.HasPrefix(tagName, prefix) {
return true
}
}
return false
}
func (t Tags) ModelTags() model.Tags {
models := model.Tags{}
for tagName, values := range t.tags {
if isExcludedTag(tagName) {
continue
}
for _, value := range values {
if value == "" {
continue
}
models[tagName] = append(models[tagName], value)
}
}
return models
}

1
scanner/metadata/taglib/taglib.go
View File

@ -50,6 +50,7 @@ func (e *Extractor) extractMetadata(filePath string) (metadata.ParsedTags, error
if duration := float64(millis) / 1000.0; duration > 0 {
tags["duration"] = []string{strconv.FormatFloat(duration, 'f', 2, 32)}
}
delete(tags, "lengthinmilliseconds")
}
// Adjust some ID3 tags
parseTIPL(tags)

1
scanner2/folder.go
View File

@ -20,6 +20,7 @@ type folderEntry struct {
tracks model.MediaFiles
albums model.Albums
artists model.Artists
tags model.FlattenedTags
missingTracks model.MediaFiles
}

10
scanner2/mapping.go
View File

@ -16,12 +16,12 @@ import (
)
type mediaFileMapper struct {
rootFolder string
entry *folderEntry
}
func newMediaFileMapper(entry *folderEntry) *mediaFileMapper {
return &mediaFileMapper{
rootFolder: entry.path,
entry: entry,
}
}
@ -74,6 +74,10 @@ func (s mediaFileMapper) toMediaFile(md metadata.Tags) model.MediaFile {
mf.Bpm = md.Bpm()
mf.CreatedAt = md.BirthTime()
mf.UpdatedAt = md.ModificationTime()
mf.Tags = md.ModelTags()
mf.FolderID = s.entry.id
mf.LibraryID = s.entry.scanCtx.lib.ID
mf.PID = mf.ID
return *mf
}
@ -85,7 +89,7 @@ func sanitizeFieldForSorting(originalValue string) string {
func (s mediaFileMapper) mapTrackTitle(md metadata.Tags) string {
if md.Title() == "" {
s := strings.TrimPrefix(md.FilePath(), s.rootFolder+string(os.PathSeparator))
s := strings.TrimPrefix(md.FilePath(), s.entry.path+string(os.PathSeparator))
e := filepath.Ext(s)
return strings.TrimSuffix(s, e)
}

47
scanner2/persiste_changes.go Normal file
View File

@ -0,0 +1,47 @@
package scanner2
import (
"context"
"github.com/google/go-pipeline/pkg/pipeline"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/utils/slice"
)
func persistChanges(ctx context.Context) pipeline.StageFn[*folderEntry] {
return func(entry *folderEntry) (*folderEntry, error) {
err := entry.scanCtx.ds.WithTx(func(tx model.DataStore) error {
// Save all tags to DB
err := slice.RangeByChunks(entry.tags, 100, func(chunk []model.Tag) error {
err := tx.Tag(ctx).Add(chunk...)
if err != nil {
log.Error(ctx, "Scanner: Error adding tags to DB", "folder", entry.path, err)
return err
}
return nil
})
if err != nil {
return err
}
// Save all tracks to DB
err = slice.RangeByChunks(entry.tracks, 100, func(chunk []model.MediaFile) error {
for i := range chunk {
track := chunk[i]
err = tx.MediaFile(ctx).Put(&track)
if err != nil {
log.Error(ctx, "Scanner: Error adding mediafile to DB", "folder", entry.path, "track", track, err)
return err
}
}
return nil
})
return err
})
if err != nil {
log.Error(ctx, "Scanner: Error persisting changes to DB", "folder", entry.path, err)
}
return entry, err
}
}

32
scanner2/persistent_ids.go
View File

@ -1,33 +1 @@
package scanner2
import (
"crypto/md5"
"fmt"
"github.com/navidrome/navidrome/scanner/metadata"
. "github.com/navidrome/navidrome/utils/gg"
)
func artistPID(md metadata.Tags) string {
key := FirstOr(md.Artist(), "M"+md.MbzArtistID())
return fmt.Sprintf("%x", md5.Sum([]byte(key)))
}
func albumArtistPID(md metadata.Tags) string {
key := FirstOr(md.AlbumArtist(), "M"+md.MbzAlbumArtistID())
return fmt.Sprintf("%x", md5.Sum([]byte(key)))
}
func albumPID(md metadata.Tags) string {
var key string
if md.MbzAlbumID() != "" {
key = "M" + md.MbzAlbumID()
} else {
key = fmt.Sprintf("%s%s%t", albumArtistPID(md), md.Album(), md.Compilation())
}
return fmt.Sprintf("%x", md5.Sum([]byte(key)))
}
func trackPID(md metadata.Tags) string {
return fmt.Sprintf("%s%x", albumPID(md), md5.Sum([]byte(md.FilePath())))
}

30
scanner2/process_folders.go
View File

@ -50,35 +50,41 @@ func processFolder(ctx context.Context) pipeline.StageFn[*folderEntry] {
// Remaining dbTracks are tracks that were not found in the folder, so they should be marked as missing
entry.missingTracks = maps.Values(dbTracks)
entry.tracks, err = loadTagsFromFiles(ctx, entry, filesToImport)
if err != nil {
log.Warn(ctx, "Scanner: Error loading tags from files. Skipping", "folder", entry.path, err)
return entry, nil
}
if len(filesToImport) > 0 {
entry.tracks, entry.tags, err = loadTagsFromFiles(ctx, entry, filesToImport)
if err != nil {
log.Warn(ctx, "Scanner: Error loading tags from files. Skipping", "folder", entry.path, err)
return entry, nil
}
entry.albums = loadAlbumsFromTags(ctx, entry)
entry.artists = loadArtistsFromTags(ctx, entry)
entry.albums = loadAlbumsFromTags(ctx, entry)
entry.artists = loadArtistsFromTags(ctx, entry)
}
return entry, nil
}
}
func loadTagsFromFiles(ctx context.Context, entry *folderEntry, toImport []string) (model.MediaFiles, error) {
func loadTagsFromFiles(ctx context.Context, entry *folderEntry, toImport []string) (model.MediaFiles, model.FlattenedTags, error) {
tracks := model.MediaFiles{}
uniqueTags := make(map[string]model.Tag)
mapper := newMediaFileMapper(entry)
err := slice.RangeByChunks(toImport, filesBatchSize, func(chunk []string) error {
allTags, err := metadata.Extract(toImport...)
allFileTags, err := metadata.Extract(toImport...)
if err != nil {
log.Warn(ctx, "Scanner: Error extracting tags from files. Skipping", "folder", entry.path, err)
return err
}
for _, tags := range allTags {
track := mapper.toMediaFile(tags)
for _, fileTags := range allFileTags {
track := mapper.toMediaFile(fileTags)
tracks = append(tracks, track)
for _, t := range track.Tags.FlattenAll() {
uniqueTags[t.ID] = t
}
}
return nil
})
return tracks, err
return tracks, maps.Values(uniqueTags), err
}
func loadAlbumsFromTags(ctx context.Context, entry *folderEntry) model.Albums {

1
scanner2/scanner2.go
View File

@ -34,6 +34,7 @@ func (s *scanner2) RescanAll(requestCtx context.Context, fullRescan bool) error
err = s.runPipeline(
pipeline.NewProducer(produceFolders(ctx, s.ds, libs, fullRescan), pipeline.Name("read folders from disk")),
pipeline.NewStage(processFolder(ctx), pipeline.Name("process folder")),
pipeline.NewStage(persistChanges(ctx), pipeline.Name("persist changes")),
pipeline.NewStage(logFolder(ctx), pipeline.Name("log results")),
)

4
tests/mock_persistence.go
View File

@ -51,6 +51,10 @@ func (db *MockDataStore) Folder(context.Context) model.FolderRepository {
return struct{ model.FolderRepository }{}
}
func (db *MockDataStore) Tag(context.Context) model.TagRepository {
return struct{ model.TagRepository }{}
}
func (db *MockDataStore) Genre(context.Context) model.GenreRepository {
if db.MockedGenre == nil {
db.MockedGenre = &MockedGenreRepo{}