mirrors
/
navidrome
mirror of https://github.com/navidrome/navidrome.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Generalize BreakUp/RangByChunks functions

This commit is contained in:
Deluan 2023-11-25 12:13:36 -05:00
parent c3efc57259
commit 9aa7b80d0d
9 changed files with 58 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
persistence/playlist_repository.go
View File

@ -13,7 +13,7 @@ import (
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/model/criteria"
"github.com/navidrome/navidrome/utils"
"github.com/navidrome/navidrome/utils/slice"
)
type playlistRepository struct {
@ -297,7 +297,7 @@ func (r *playlistRepository) updatePlaylist(playlistId string, mediaFileIds []st
func (r *playlistRepository) addTracks(playlistId string, startingPos int, mediaFileIds []string) error {
// Break the track list in chunks to avoid hitting SQLITE_MAX_FUNCTION_ARG limit
chunks := utils.BreakUpStringSlice(mediaFileIds, 200)
chunks := slice.BreakUp(mediaFileIds, 200)
// Add new tracks, chunk by chunk
pos := startingPos

4
persistence/playqueue_repository.go
View File

@ -9,7 +9,7 @@ import (
"github.com/beego/beego/v2/client/orm"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/utils"
"github.com/navidrome/navidrome/utils/slice"
)
type playQueueRepository struct {
@ -113,7 +113,7 @@ func (r *playQueueRepository) loadTracks(tracks model.MediaFiles) model.MediaFil
}
// Break the list in chunks, up to 50 items, to avoid hitting SQLITE_MAX_FUNCTION_ARG limit
chunks := utils.BreakUpStringSlice(ids, 50)
chunks := slice.BreakUp(ids, 50)
// Query each chunk of media_file ids and store results in a map
mfRepo := NewMediaFileRepository(r.ctx, r.ormer)

10
persistence/sql_genres.go
View File

@ -3,7 +3,7 @@ package persistence
import (
. "github.com/Masterminds/squirrel"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/utils"
"github.com/navidrome/navidrome/utils/slice"
)
func (r sqlRepository) withGenres(sql SelectBuilder) SelectBuilder {
@ -25,7 +25,7 @@ func (r *sqlRepository) updateGenres(id string, tableName string, genres model.G
for _, g := range genres {
genreIds = append(genreIds, g.ID)
}
err = utils.RangeByChunks(genreIds, 100, func(ids []string) error {
err = slice.RangeByChunks(genreIds, 100, func(ids []string) error {
ins := Insert(tableName+"_genres").Columns("genre_id", tableName+"_id")
for _, gid := range ids {
ins = ins.Values(gid, id)
@ -45,7 +45,7 @@ func (r *sqlRepository) loadMediaFileGenres(mfs *model.MediaFiles) error {
m[mf.ID] = mf
}
return utils.RangeByChunks(ids, 900, func(ids []string) error {
return slice.RangeByChunks(ids, 900, func(ids []string) error {
sql := Select("g.*", "mg.media_file_id").From("genre g").Join("media_file_genres mg on mg.genre_id = g.id").
Where(Eq{"mg.media_file_id": ids}).OrderBy("mg.media_file_id", "mg.rowid")
var genres []struct {
@ -74,7 +74,7 @@ func (r *sqlRepository) loadAlbumGenres(mfs *model.Albums) error {
m[mf.ID] = mf
}
return utils.RangeByChunks(ids, 900, func(ids []string) error {
return slice.RangeByChunks(ids, 900, func(ids []string) error {
sql := Select("g.*", "ag.album_id").From("genre g").Join("album_genres ag on ag.genre_id = g.id").
Where(Eq{"ag.album_id": ids}).OrderBy("ag.album_id", "ag.rowid")
var genres []struct {
@ -103,7 +103,7 @@ func (r *sqlRepository) loadArtistGenres(mfs *model.Artists) error {
m[mf.ID] = mf
}
return utils.RangeByChunks(ids, 900, func(ids []string) error {
return slice.RangeByChunks(ids, 900, func(ids []string) error {
sql := Select("g.*", "ag.artist_id").From("genre g").Join("artist_genres ag on ag.genre_id = g.id").
Where(Eq{"ag.artist_id": ids}).OrderBy("ag.artist_id", "ag.rowid")
var genres []struct {

3
scanner/refresher.go
View File

@ -12,7 +12,6 @@ import (
"github.com/navidrome/navidrome/core/artwork"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/utils"
"github.com/navidrome/navidrome/utils/slice"
"golang.org/x/exp/maps"
)
@ -71,7 +70,7 @@ func (r *refresher) flushMap(ctx context.Context, m map[string]struct{}, entity
}
ids := maps.Keys(m)
chunks := utils.BreakUpStringSlice(ids, 100)
chunks := slice.BreakUp(ids, 100)
for _, chunk := range chunks {
err := refresh(ctx, chunk...)
if err != nil {

4
scanner/tag_scanner.go
View File

@ -19,7 +19,7 @@ import (
"github.com/navidrome/navidrome/scanner/metadata"
_ "github.com/navidrome/navidrome/scanner/metadata/ffmpeg"
_ "github.com/navidrome/navidrome/scanner/metadata/taglib"
"github.com/navidrome/navidrome/utils"
"github.com/navidrome/navidrome/utils/slice"
)
type TagScanner struct {
@ -351,7 +351,7 @@ func (s *TagScanner) addOrUpdateTracksInDB(
log.Trace(ctx, "Updating mediaFiles in DB", "dir", dir, "numFiles", len(filesToUpdate))
// Break the file list in chunks to avoid calling ffmpeg with too many parameters
chunks := utils.BreakUpStringSlice(filesToUpdate, filesBatchSize)
chunks := slice.BreakUp(filesToUpdate, filesBatchSize)
for _, chunk := range chunks {
// Load tracks Metadata from the folder
newTracks, err := s.loadTracks(chunk)

25
utils/slice/slice.go
View File

@ -54,3 +54,28 @@ func Move[T any](slice []T, srcIndex int, dstIndex int) []T {
value := slice[srcIndex]
return Insert(Remove(slice, srcIndex), value, dstIndex)
}
func BreakUp[T any](items []T, chunkSize int) [][]T {
numTracks := len(items)
var chunks [][]T
for i := 0; i < numTracks; i += chunkSize {
end := i + chunkSize
if end > numTracks {
end = numTracks
}
chunks = append(chunks, items[i:end])
}
return chunks
}
func RangeByChunks[T any](items []T, chunkSize int, cb func([]T) error) error {
chunks := BreakUp(items, chunkSize)
for _, chunk := range chunks {
err := cb(chunk)
if err != nil {
return err
}
}
return nil
}

21
utils/slice/slice_test.go
View File

@ -69,4 +69,25 @@ var _ = Describe("Slice Utils", func() {
Expect(slice.Move([]string{"1", "2", "3"}, 1, 1)).To(ConsistOf("1", "2", "3"))
})
})
Describe("BreakUp", func() {
It("returns no chunks if slice is empty", func() {
var s []string
chunks := slice.BreakUp(s, 10)
Expect(chunks).To(HaveLen(0))
})
It("returns the slice in one chunk if len < chunkSize", func() {
s := []string{"a", "b", "c"}
chunks := slice.BreakUp(s, 10)
Expect(chunks).To(HaveLen(1))
Expect(chunks[0]).To(ConsistOf("a", "b", "c"))
})
It("breaks up the slice if len > chunkSize", func() {
s := []string{"a", "b", "c", "d", "e"}
chunks := slice.BreakUp(s, 3)
Expect(chunks).To(HaveLen(2))
Expect(chunks[0]).To(ConsistOf("a", "b", "c"))
Expect(chunks[1]).To(ConsistOf("d", "e"))
})
})
})

25
utils/strings.go
View File

@ -17,31 +17,6 @@ func NoArticle(name string) string {
return name
}
func BreakUpStringSlice(items []string, chunkSize int) [][]string {
numTracks := len(items)
var chunks [][]string
for i := 0; i < numTracks; i += chunkSize {
end := i + chunkSize
if end > numTracks {
end = numTracks
}
chunks = append(chunks, items[i:end])
}
return chunks
}
func RangeByChunks(items []string, chunkSize int, cb func([]string) error) error {
chunks := BreakUpStringSlice(items, chunkSize)
for _, chunk := range chunks {
err := cb(chunk)
if err != nil {
return err
}
}
return nil
}
func LongestCommonPrefix(list []string) string {
if len(list) == 0 {
return ""

21
utils/strings_test.go
View File

@ -35,27 +35,6 @@ var _ = Describe("Strings", func() {
})
})
Describe("BreakUpStringSlice", func() {
It("returns no chunks if slice is empty", func() {
var slice []string
chunks := BreakUpStringSlice(slice, 10)
Expect(chunks).To(HaveLen(0))
})
It("returns the slice in one chunk if len < chunkSize", func() {
slice := []string{"a", "b", "c"}
chunks := BreakUpStringSlice(slice, 10)
Expect(chunks).To(HaveLen(1))
Expect(chunks[0]).To(ConsistOf("a", "b", "c"))
})
It("breaks up the slice if len > chunkSize", func() {
slice := []string{"a", "b", "c", "d", "e"}
chunks := BreakUpStringSlice(slice, 3)
Expect(chunks).To(HaveLen(2))
Expect(chunks[0]).To(ConsistOf("a", "b", "c"))
Expect(chunks[1]).To(ConsistOf("d", "e"))
})
})
Describe("LongestCommonPrefix", func() {
It("finds the longest common prefix", func() {
Expect(LongestCommonPrefix(testPaths)).To(Equal("/Music/iTunes 1/iTunes Media/Music/"))