Deterministic pagination in random albums sort (#1841)

* Deterministic pagination in random albums sort

* Reseed on first random page

* Add unit tests

* Use rand in Subsonic API

* Use different seeds per user on SEEDEDRAND() SQLite3 function

* Small refactor

* Fix id mismatch

* Add seeded random to media_file (subsonic endpoint `getRandomSongs`)

* Refactor

* Remove unneeded import

---------

Co-authored-by: Deluan <deluan@navidrome.org>

db/db.go

@@ -5,10 +5,11 @@ import (
 	"embed"
 	"fmt"
 
-	_ "github.com/mattn/go-sqlite3"
+	"github.com/mattn/go-sqlite3"
 	"github.com/navidrome/navidrome/conf"
 	_ "github.com/navidrome/navidrome/db/migrations"
 	"github.com/navidrome/navidrome/log"
+	"github.com/navidrome/navidrome/utils/hasher"
 	"github.com/navidrome/navidrome/utils/singleton"
 	"github.com/pressly/goose/v3"
 )
@@ -25,13 +26,19 @@ const migrationsFolder = "migrations"
 
 func Db() *sql.DB {
 	return singleton.GetInstance(func() *sql.DB {
+		sql.Register(Driver+"_custom", &sqlite3.SQLiteDriver{
+			ConnectHook: func(conn *sqlite3.SQLiteConn) error {
+				return conn.RegisterFunc("SEEDEDRAND", hasher.HashFunc(), false)
+			},
+		})
+
 		Path = conf.Server.DbPath
 		if Path == ":memory:" {
 			Path = "file::memory:?cache=shared&_foreign_keys=on"
 			conf.Server.DbPath = Path
 		}
 		log.Debug("Opening DataBase", "dbPath", Path, "driver", Driver)
-		instance, err := sql.Open(Driver, Path)
+		instance, err := sql.Open(Driver+"_custom", Path)
 		if err != nil {
 			panic(err)
 		}

persistence/album_repository.go

@@ -75,7 +75,7 @@ func NewAlbumRepository(ctx context.Context, db dbx.Builder) model.AlbumReposito
 			"artist":         "compilation asc, COALESCE(NULLIF(sort_album_artist_name,''),order_album_artist_name) asc, COALESCE(NULLIF(sort_album_name,''),order_album_name) asc",
 			"albumArtist":    "compilation asc, COALESCE(NULLIF(sort_album_artist_name,''),order_album_artist_name) asc, COALESCE(NULLIF(sort_album_name,''),order_album_name) asc",
 			"max_year":       "coalesce(nullif(original_date,''), cast(max_year as text)), release_date, name, COALESCE(NULLIF(sort_album_name,''),order_album_name) asc",
-			"random":         "RANDOM()",
+			"random":         r.seededRandomSort(),
 			"recently_added": recentlyAddedSort(),
 		}
 	} else {
@@ -84,7 +84,7 @@ func NewAlbumRepository(ctx context.Context, db dbx.Builder) model.AlbumReposito
 			"artist":         "compilation asc, order_album_artist_name asc, order_album_name asc",
 			"albumArtist":    "compilation asc, order_album_artist_name asc, order_album_name asc",
 			"max_year":       "coalesce(nullif(original_date,''), cast(max_year as text)), release_date, name, order_album_name asc",
-			"random":         "RANDOM()",
+			"random":         r.seededRandomSort(),
 			"recently_added": recentlyAddedSort(),
 		}
 	}
@@ -180,6 +180,7 @@ func (r *albumRepository) GetAll(options ...model.QueryOptions) (model.Albums, e
 }
 
 func (r *albumRepository) GetAllWithoutGenres(options ...model.QueryOptions) (model.Albums, error) {
+	r.resetSeededRandom(options)
 	sq := r.selectAlbum(options...)
 	var dba dbAlbums
 	err := r.queryAll(sq, &dba)

persistence/mediafile_repository.go

@@ -36,7 +36,7 @@ func NewMediaFileRepository(ctx context.Context, db dbx.Builder) *mediaFileRepos
 			"title":     "COALESCE(NULLIF(sort_title,''),title)",
 			"artist":    "COALESCE(NULLIF(sort_artist_name,''),order_artist_name) asc, COALESCE(NULLIF(sort_album_name,''),order_album_name) asc, release_date asc, disc_number asc, track_number asc",
 			"album":     "COALESCE(NULLIF(sort_album_name,''),order_album_name) asc, release_date asc, disc_number asc, track_number asc, COALESCE(NULLIF(sort_artist_name,''),order_artist_name) asc, COALESCE(NULLIF(sort_title,''),title) asc",
-			"random":    "RANDOM()",
+			"random":    r.seededRandomSort(),
 			"createdAt": "media_file.created_at",
 		}
 	} else {
@@ -44,7 +44,7 @@ func NewMediaFileRepository(ctx context.Context, db dbx.Builder) *mediaFileRepos
 			"title":     "order_title",
 			"artist":    "order_artist_name asc, order_album_name asc, release_date asc, disc_number asc, track_number asc",
 			"album":     "order_album_name asc, release_date asc, disc_number asc, track_number asc, order_artist_name asc, title asc",
-			"random":    "RANDOM()",
+			"random":    r.seededRandomSort(),
 			"createdAt": "media_file.created_at",
 		}
 	}
@@ -102,6 +102,7 @@ func (r *mediaFileRepository) Get(id string) (*model.MediaFile, error) {
 }
 
 func (r *mediaFileRepository) GetAll(options ...model.QueryOptions) (model.MediaFiles, error) {
+	r.resetSeededRandom(options)
 	sq := r.selectMediaFile(options...)
 	res := model.MediaFiles{}
 	err := r.queryAll(sq, &res, options...)

persistence/sql_base_repository.go

@@ -14,6 +14,7 @@ import (
 	"github.com/navidrome/navidrome/log"
 	"github.com/navidrome/navidrome/model"
 	"github.com/navidrome/navidrome/model/request"
+	"github.com/navidrome/navidrome/utils/hasher"
 	"github.com/pocketbase/dbx"
 )
 
@@ -137,6 +138,18 @@ func (r sqlRepository) applyFilters(sq SelectBuilder, options ...model.QueryOpti
 	return sq
 }
 
+func (r sqlRepository) seededRandomSort() string {
+	u, _ := request.UserFrom(r.ctx)
+	return fmt.Sprintf("SEEDEDRAND('%s', id)", r.tableName+u.ID)
+}
+
+func (r sqlRepository) resetSeededRandom(options []model.QueryOptions) {
+	if len(options) > 0 && options[0].Offset == 0 && options[0].Sort == "random" {
+		u, _ := request.UserFrom(r.ctx)
+		hasher.Reseed(r.tableName + u.ID)
+	}
+}
+
 func (r sqlRepository) executeSQL(sq Sqlizer) (int64, error) {
 	query, args, err := r.toSQL(sq)
 	if err != nil {

server/subsonic/filter/filters.go

@@ -24,7 +24,7 @@ func AlbumsByFrequent() Options {
 }
 
 func AlbumsByRandom() Options {
-	return Options{Sort: "random()"}
+	return Options{Sort: "random"}
 }
 
 func AlbumsByName() Options {
@@ -100,7 +100,7 @@ func SongsByAlbum(albumId string) Options {
 
 func SongsByRandom(genre string, fromYear, toYear int) Options {
 	options := Options{
-		Sort: "random()",
+		Sort: "random",
 	}
 	ff := squirrel.And{}
 	if genre != "" {

utils/hasher/hasher.go

@@ -0,0 +1,44 @@
+package hasher
+
+import "hash/maphash"
+
+var instance = NewHasher()
+
+func Reseed(id string) {
+	instance.Reseed(id)
+}
+
+func HashFunc() func(id, str string) uint64 {
+	return instance.HashFunc()
+}
+
+type hasher struct {
+	seeds map[string]maphash.Seed
+}
+
+func NewHasher() *hasher {
+	h := new(hasher)
+	h.seeds = make(map[string]maphash.Seed)
+	return h
+}
+
+// Reseed generates a new seed for the given id
+func (h *hasher) Reseed(id string) {
+	h.seeds[id] = maphash.MakeSeed()
+}
+
+// HashFunc returns a function that hashes a string using the seed for the given id
+func (h *hasher) HashFunc() func(id, str string) uint64 {
+	return func(id, str string) uint64 {
+		var hash maphash.Hash
+		var seed maphash.Seed
+		var ok bool
+		if seed, ok = h.seeds[id]; !ok {
+			seed = maphash.MakeSeed()
+			h.seeds[id] = seed
+		}
+		hash.SetSeed(seed)
+		_, _ = hash.WriteString(str)
+		return hash.Sum64()
+	}
+}

utils/hasher/hasher_test.go

@@ -0,0 +1,36 @@
+package hasher_test
+
+import (
+	"github.com/navidrome/navidrome/utils/hasher"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("HashFunc", func() {
+	const input = "123e4567e89b12d3a456426614174000"
+
+	It("hashes the input and returns the sum", func() {
+		hashFunc := hasher.HashFunc()
+		sum := hashFunc("1", input)
+		Expect(sum > 0).To(BeTrue())
+	})
+
+	It("hashes the input, reseeds and returns a different sum", func() {
+		hashFunc := hasher.HashFunc()
+		sum := hashFunc("1", input)
+		hasher.Reseed("1")
+		sum2 := hashFunc("1", input)
+		Expect(sum).NotTo(Equal(sum2))
+	})
+
+	It("keeps different hashes for different ids", func() {
+		hashFunc := hasher.HashFunc()
+		sum := hashFunc("1", input)
+		sum2 := hashFunc("2", input)
+
+		Expect(sum).NotTo(Equal(sum2))
+
+		Expect(sum).To(Equal(hashFunc("1", input)))
+		Expect(sum2).To(Equal(hashFunc("2", input)))
+	})
+})