diff --git a/persistence/playlist_repository.go b/persistence/playlist_repository.go index fe1ecd5e..f2addc15 100644 --- a/persistence/playlist_repository.go +++ b/persistence/playlist_repository.go @@ -13,7 +13,7 @@ import ( "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/model/criteria" - "github.com/navidrome/navidrome/utils" + "github.com/navidrome/navidrome/utils/slice" ) type playlistRepository struct { @@ -297,7 +297,7 @@ func (r *playlistRepository) updatePlaylist(playlistId string, mediaFileIds []st func (r *playlistRepository) addTracks(playlistId string, startingPos int, mediaFileIds []string) error { // Break the track list in chunks to avoid hitting SQLITE_MAX_FUNCTION_ARG limit - chunks := utils.BreakUpStringSlice(mediaFileIds, 200) + chunks := slice.BreakUp(mediaFileIds, 200) // Add new tracks, chunk by chunk pos := startingPos diff --git a/persistence/playqueue_repository.go b/persistence/playqueue_repository.go index 14bd1b9b..beda2a83 100644 --- a/persistence/playqueue_repository.go +++ b/persistence/playqueue_repository.go @@ -9,7 +9,7 @@ import ( "github.com/beego/beego/v2/client/orm" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" - "github.com/navidrome/navidrome/utils" + "github.com/navidrome/navidrome/utils/slice" ) type playQueueRepository struct { @@ -113,7 +113,7 @@ func (r *playQueueRepository) loadTracks(tracks model.MediaFiles) model.MediaFil } // Break the list in chunks, up to 50 items, to avoid hitting SQLITE_MAX_FUNCTION_ARG limit - chunks := utils.BreakUpStringSlice(ids, 50) + chunks := slice.BreakUp(ids, 50) // Query each chunk of media_file ids and store results in a map mfRepo := NewMediaFileRepository(r.ctx, r.ormer) diff --git a/persistence/sql_genres.go b/persistence/sql_genres.go index 397e4336..be245721 100644 --- a/persistence/sql_genres.go +++ b/persistence/sql_genres.go @@ -3,7 +3,7 @@ package persistence import ( . "github.com/Masterminds/squirrel" "github.com/navidrome/navidrome/model" - "github.com/navidrome/navidrome/utils" + "github.com/navidrome/navidrome/utils/slice" ) func (r sqlRepository) withGenres(sql SelectBuilder) SelectBuilder { @@ -25,7 +25,7 @@ func (r *sqlRepository) updateGenres(id string, tableName string, genres model.G for _, g := range genres { genreIds = append(genreIds, g.ID) } - err = utils.RangeByChunks(genreIds, 100, func(ids []string) error { + err = slice.RangeByChunks(genreIds, 100, func(ids []string) error { ins := Insert(tableName+"_genres").Columns("genre_id", tableName+"_id") for _, gid := range ids { ins = ins.Values(gid, id) @@ -45,7 +45,7 @@ func (r *sqlRepository) loadMediaFileGenres(mfs *model.MediaFiles) error { m[mf.ID] = mf } - return utils.RangeByChunks(ids, 900, func(ids []string) error { + return slice.RangeByChunks(ids, 900, func(ids []string) error { sql := Select("g.*", "mg.media_file_id").From("genre g").Join("media_file_genres mg on mg.genre_id = g.id"). Where(Eq{"mg.media_file_id": ids}).OrderBy("mg.media_file_id", "mg.rowid") var genres []struct { @@ -74,7 +74,7 @@ func (r *sqlRepository) loadAlbumGenres(mfs *model.Albums) error { m[mf.ID] = mf } - return utils.RangeByChunks(ids, 900, func(ids []string) error { + return slice.RangeByChunks(ids, 900, func(ids []string) error { sql := Select("g.*", "ag.album_id").From("genre g").Join("album_genres ag on ag.genre_id = g.id"). Where(Eq{"ag.album_id": ids}).OrderBy("ag.album_id", "ag.rowid") var genres []struct { @@ -103,7 +103,7 @@ func (r *sqlRepository) loadArtistGenres(mfs *model.Artists) error { m[mf.ID] = mf } - return utils.RangeByChunks(ids, 900, func(ids []string) error { + return slice.RangeByChunks(ids, 900, func(ids []string) error { sql := Select("g.*", "ag.artist_id").From("genre g").Join("artist_genres ag on ag.genre_id = g.id"). Where(Eq{"ag.artist_id": ids}).OrderBy("ag.artist_id", "ag.rowid") var genres []struct { diff --git a/scanner/refresher.go b/scanner/refresher.go index ec17fc1e..3a35e8fe 100644 --- a/scanner/refresher.go +++ b/scanner/refresher.go @@ -12,7 +12,6 @@ import ( "github.com/navidrome/navidrome/core/artwork" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" - "github.com/navidrome/navidrome/utils" "github.com/navidrome/navidrome/utils/slice" "golang.org/x/exp/maps" ) @@ -71,7 +70,7 @@ func (r *refresher) flushMap(ctx context.Context, m map[string]struct{}, entity } ids := maps.Keys(m) - chunks := utils.BreakUpStringSlice(ids, 100) + chunks := slice.BreakUp(ids, 100) for _, chunk := range chunks { err := refresh(ctx, chunk...) if err != nil { diff --git a/scanner/tag_scanner.go b/scanner/tag_scanner.go index 1317581b..5a719acd 100644 --- a/scanner/tag_scanner.go +++ b/scanner/tag_scanner.go @@ -19,7 +19,7 @@ import ( "github.com/navidrome/navidrome/scanner/metadata" _ "github.com/navidrome/navidrome/scanner/metadata/ffmpeg" _ "github.com/navidrome/navidrome/scanner/metadata/taglib" - "github.com/navidrome/navidrome/utils" + "github.com/navidrome/navidrome/utils/slice" ) type TagScanner struct { @@ -351,7 +351,7 @@ func (s *TagScanner) addOrUpdateTracksInDB( log.Trace(ctx, "Updating mediaFiles in DB", "dir", dir, "numFiles", len(filesToUpdate)) // Break the file list in chunks to avoid calling ffmpeg with too many parameters - chunks := utils.BreakUpStringSlice(filesToUpdate, filesBatchSize) + chunks := slice.BreakUp(filesToUpdate, filesBatchSize) for _, chunk := range chunks { // Load tracks Metadata from the folder newTracks, err := s.loadTracks(chunk) diff --git a/utils/slice/slice.go b/utils/slice/slice.go index cd012330..78169685 100644 --- a/utils/slice/slice.go +++ b/utils/slice/slice.go @@ -54,3 +54,28 @@ func Move[T any](slice []T, srcIndex int, dstIndex int) []T { value := slice[srcIndex] return Insert(Remove(slice, srcIndex), value, dstIndex) } + +func BreakUp[T any](items []T, chunkSize int) [][]T { + numTracks := len(items) + var chunks [][]T + for i := 0; i < numTracks; i += chunkSize { + end := i + chunkSize + if end > numTracks { + end = numTracks + } + + chunks = append(chunks, items[i:end]) + } + return chunks +} + +func RangeByChunks[T any](items []T, chunkSize int, cb func([]T) error) error { + chunks := BreakUp(items, chunkSize) + for _, chunk := range chunks { + err := cb(chunk) + if err != nil { + return err + } + } + return nil +} diff --git a/utils/slice/slice_test.go b/utils/slice/slice_test.go index a27416f3..7436a34e 100644 --- a/utils/slice/slice_test.go +++ b/utils/slice/slice_test.go @@ -69,4 +69,25 @@ var _ = Describe("Slice Utils", func() { Expect(slice.Move([]string{"1", "2", "3"}, 1, 1)).To(ConsistOf("1", "2", "3")) }) }) + + Describe("BreakUp", func() { + It("returns no chunks if slice is empty", func() { + var s []string + chunks := slice.BreakUp(s, 10) + Expect(chunks).To(HaveLen(0)) + }) + It("returns the slice in one chunk if len < chunkSize", func() { + s := []string{"a", "b", "c"} + chunks := slice.BreakUp(s, 10) + Expect(chunks).To(HaveLen(1)) + Expect(chunks[0]).To(ConsistOf("a", "b", "c")) + }) + It("breaks up the slice if len > chunkSize", func() { + s := []string{"a", "b", "c", "d", "e"} + chunks := slice.BreakUp(s, 3) + Expect(chunks).To(HaveLen(2)) + Expect(chunks[0]).To(ConsistOf("a", "b", "c")) + Expect(chunks[1]).To(ConsistOf("d", "e")) + }) + }) }) diff --git a/utils/strings.go b/utils/strings.go index 9fbe0d44..1bd26b10 100644 --- a/utils/strings.go +++ b/utils/strings.go @@ -17,31 +17,6 @@ func NoArticle(name string) string { return name } -func BreakUpStringSlice(items []string, chunkSize int) [][]string { - numTracks := len(items) - var chunks [][]string - for i := 0; i < numTracks; i += chunkSize { - end := i + chunkSize - if end > numTracks { - end = numTracks - } - - chunks = append(chunks, items[i:end]) - } - return chunks -} - -func RangeByChunks(items []string, chunkSize int, cb func([]string) error) error { - chunks := BreakUpStringSlice(items, chunkSize) - for _, chunk := range chunks { - err := cb(chunk) - if err != nil { - return err - } - } - return nil -} - func LongestCommonPrefix(list []string) string { if len(list) == 0 { return "" diff --git a/utils/strings_test.go b/utils/strings_test.go index 0983faa2..4b24dab0 100644 --- a/utils/strings_test.go +++ b/utils/strings_test.go @@ -35,27 +35,6 @@ var _ = Describe("Strings", func() { }) }) - Describe("BreakUpStringSlice", func() { - It("returns no chunks if slice is empty", func() { - var slice []string - chunks := BreakUpStringSlice(slice, 10) - Expect(chunks).To(HaveLen(0)) - }) - It("returns the slice in one chunk if len < chunkSize", func() { - slice := []string{"a", "b", "c"} - chunks := BreakUpStringSlice(slice, 10) - Expect(chunks).To(HaveLen(1)) - Expect(chunks[0]).To(ConsistOf("a", "b", "c")) - }) - It("breaks up the slice if len > chunkSize", func() { - slice := []string{"a", "b", "c", "d", "e"} - chunks := BreakUpStringSlice(slice, 3) - Expect(chunks).To(HaveLen(2)) - Expect(chunks[0]).To(ConsistOf("a", "b", "c")) - Expect(chunks[1]).To(ConsistOf("d", "e")) - }) - }) - Describe("LongestCommonPrefix", func() { It("finds the longest common prefix", func() { Expect(LongestCommonPrefix(testPaths)).To(Equal("/Music/iTunes 1/iTunes Media/Music/"))