Album downloads working

This commit is contained in:
Nathan Thomas 2023-10-31 12:51:44 -07:00
parent 837e934476
commit 89f76b7f58
20 changed files with 338 additions and 125 deletions

View File

@ -1,20 +1,42 @@
import asyncio
import logging
import os
from dataclasses import dataclass
from .artwork import download_artwork
from .client import Client
from .config import Config
from .console import console
from .media import Media, Pending
from .metadata import AlbumMetadata, get_album_track_ids
from .track import PendingTrack, Track
logger = logging.getLogger("streamrip")
@dataclass(slots=True)
class Album(Media):
meta: AlbumMetadata
tracks: list[Track]
tracks: list[PendingTrack]
config: Config
directory: str
# folder where the tracks will be downloaded
folder: str
async def preprocess(self):
if self.config.session.cli.text_output:
console.print(
f"[cyan]Downloading {self.meta.album} by {self.meta.albumartist}"
)
async def download(self):
async def _resolve_and_download(pending):
track = await pending.resolve()
await track.rip()
await asyncio.gather(*[_resolve_and_download(p) for p in self.tracks])
async def postprocess(self):
pass
@dataclass(slots=True)
@ -28,7 +50,8 @@ class PendingAlbum(Pending):
meta = AlbumMetadata.from_resp(resp, self.client.source)
tracklist = get_album_track_ids(self.client.source, resp)
folder = self.config.session.downloads.folder
album_folder = self._album_folder(folder, meta.album)
album_folder = self._album_folder(folder, meta)
os.makedirs(album_folder, exist_ok=True)
embed_cover, _ = await download_artwork(
self.client.session, album_folder, meta.covers, self.config.session.artwork
)
@ -43,12 +66,10 @@ class PendingAlbum(Pending):
)
for id in tracklist
]
tracks: list[Track] = await asyncio.gather(
*(track.resolve() for track in pending_tracks)
)
return Album(meta, tracks, self.config, album_folder)
logger.debug("Pending tracks: %s", pending_tracks)
return Album(meta, pending_tracks, self.config, album_folder)
def _album_folder(self, parent: str, album_name: str) -> str:
# find name of album folder
# create album folder if it doesnt exist
raise NotImplementedError
def _album_folder(self, parent: str, meta: AlbumMetadata) -> str:
formatter = self.config.session.filepaths.folder_format
folder = meta.format_folder_path(formatter)
return os.path.join(parent, folder)

View File

@ -1,6 +1,12 @@
from .album import Album, PendingAlbum
from .client import Client
from .config import Config
from .media import Media, Pending
class Artist(Media):
name: str
albums: list[Album]
albums: list[PendingAlbum]
config: Config

View File

@ -12,15 +12,10 @@ from rich.logging import RichHandler
from rich.traceback import install
from .config import Config, set_user_defaults
from .console import console
from .main import Main
from .user_paths import BLANK_CONFIG_PATH, CONFIG_PATH
logging.basicConfig(
level="DEBUG", format="%(message)s", datefmt="[%X]", handlers=[RichHandler()]
)
logger = logging.getLogger("streamrip")
def echo_i(msg, **kwargs):
secho(msg, fg="green", **kwargs)
@ -59,12 +54,23 @@ def rip(ctx, config_path, verbose):
"""
Streamrip: the all in one music downloader.
"""
global logger
FORMAT = "%(message)s"
logging.basicConfig(
level="WARNING", format=FORMAT, datefmt="[%X]", handlers=[RichHandler()]
)
logger = logging.getLogger("streamrip")
if verbose:
install(suppress=[click], show_locals=True, locals_hide_sunder=False)
install(
console=console,
suppress=[click],
show_locals=True,
locals_hide_sunder=False,
)
logger.setLevel(logging.DEBUG)
logger.debug("Showing all debug logs")
else:
install(suppress=[click, asyncio], max_frames=1)
install(console=console, suppress=[click, asyncio], max_frames=1)
logger.setLevel(logging.WARNING)
ctx.ensure_object(dict)
@ -112,8 +118,7 @@ async def file(ctx, path):
with Config(config_path) as cfg:
main = Main(cfg)
with open(path) as f:
for u in f:
await main.add(u)
await asyncio.gather(*[main.add(url) for url in f])
await main.resolve()
await main.rip()

View File

@ -205,9 +205,11 @@ class LastFmConfig:
@dataclass(slots=True)
class ThemeConfig:
# Options: "dainty" or "plain"
progress_bar: str
class CliConfig:
# Print "Downloading {Album name}" etc. to screen
text_output: bool
# Show resolve, download progress bars
progress_bars: bool
@dataclass(slots=True)
@ -232,7 +234,7 @@ class ConfigData:
metadata: MetadataConfig
qobuz_filters: QobuzDiscographyFilterConfig
theme: ThemeConfig
cli: CliConfig
database: DatabaseConfig
conversion: ConversionConfig
@ -260,7 +262,7 @@ class ConfigData:
filepaths = FilepathsConfig(**toml["filepaths"]) # type: ignore
metadata = MetadataConfig(**toml["metadata"]) # type: ignore
qobuz_filters = QobuzDiscographyFilterConfig(**toml["qobuz_filters"]) # type: ignore
theme = ThemeConfig(**toml["theme"]) # type: ignore
cli = CliConfig(**toml["cli"]) # type: ignore
database = DatabaseConfig(**toml["database"]) # type: ignore
conversion = ConversionConfig(**toml["conversion"]) # type: ignore
misc = MiscConfig(**toml["misc"]) # type: ignore
@ -278,7 +280,7 @@ class ConfigData:
filepaths=filepaths,
metadata=metadata,
qobuz_filters=qobuz_filters,
theme=theme,
cli=cli,
database=database,
conversion=conversion,
misc=misc,
@ -308,7 +310,7 @@ class ConfigData:
update_toml_section_from_config(self.toml["filepaths"], self.filepaths)
update_toml_section_from_config(self.toml["metadata"], self.metadata)
update_toml_section_from_config(self.toml["qobuz_filters"], self.qobuz_filters)
update_toml_section_from_config(self.toml["theme"], self.theme)
update_toml_section_from_config(self.toml["cli"], self.cli)
update_toml_section_from_config(self.toml["database"], self.database)
update_toml_section_from_config(self.toml["conversion"], self.conversion)

View File

@ -15,7 +15,7 @@ concurrency = true
max_connections = 3
# Max number of API requests to handle per minute
# Set to -1 for no limit
requests_per_minute = -1
requests_per_minute = 60
[qobuz]
# 1: 320kbps MP3, 2: 16/44.1, 3: 24/<=96, 4: 24/>=96
@ -172,9 +172,11 @@ source = "qobuz"
# on this one.
fallback_source = "deezer"
[theme]
# Options: "dainty" or "plain"
progress_bar = "dainty"
[cli]
# Print "Downloading {Album name}" etc. to screen
text_output = true
# Show resolve, download progress bars
progress_bars = true
[misc]
# Metadata to identify this config file. Do not change.

3
streamrip/console.py Normal file
View File

@ -0,0 +1,3 @@
from rich.console import Console
console = Console()

View File

@ -1,9 +1,9 @@
"""Wrapper classes over FFMPEG."""
import asyncio
import logging
import os
import shutil
import subprocess
from tempfile import gettempdir
from typing import Optional
@ -68,7 +68,7 @@ class Converter:
logger.debug("FFmpeg codec extra argument: %s", self.ffmpeg_arg)
def convert(self, custom_fn: Optional[str] = None):
async def convert(self, custom_fn: Optional[str] = None):
"""Convert the file.
:param custom_fn: Custom output filename (defaults to the original
@ -81,8 +81,10 @@ class Converter:
self.command = self._gen_command()
logger.debug("Generated conversion command: %s", self.command)
process = subprocess.Popen(self.command, stderr=subprocess.PIPE)
process.wait()
process = await asyncio.create_subprocess_exec(
*self.command, stderr=asyncio.subprocess.PIPE
)
out, err = await process.communicate()
if process.returncode == 0 and os.path.isfile(self.tempfile):
if self.remove_source:
os.remove(self.filename)
@ -91,7 +93,7 @@ class Converter:
shutil.move(self.tempfile, self.final_fn)
logger.debug("Moved: %s -> %s", self.tempfile, self.final_fn)
else:
raise ConversionError(f"FFmpeg output:\n{process.communicate()[1]}")
raise ConversionError(f"FFmpeg output:\n{out, err}")
def _gen_command(self):
command = [
@ -172,7 +174,7 @@ class LAME(Converter):
https://trac.ffmpeg.org/wiki/Encode/MP3
"""
__bitrate_map = {
_bitrate_map = {
320: "-b:a 320k",
245: "-q:a 0",
225: "-q:a 1",
@ -192,7 +194,7 @@ class LAME(Converter):
default_ffmpeg_arg = "-q:a 0" # V0
def get_quality_arg(self, rate):
return self.__bitrate_map[rate]
return self._bitrate_map[rate]
class ALAC(Converter):

View File

@ -1,14 +1,13 @@
import asyncio
import logging
from click import secho
from .client import Client
from .config import Config
from .console import console
from .media import Media, Pending
from .progress import clear_progress
from .prompter import get_prompter
from .qobuz_client import QobuzClient
from .thread_pool import AsyncThreadPool
from .universal_url import parse_url
logger = logging.getLogger("streamrip")
@ -26,7 +25,8 @@ class Main:
def __init__(self, config: Config):
# Pipeline:
# input URL -> (URL) -> (Pending) -> (Media) -> (Downloadable) -> downloaded audio file
# input URL -> (URL) -> (Pending) -> (Media) -> (Downloadable)
# -> downloaded audio file
self.pending: list[Pending] = []
self.media: list[Media] = []
@ -42,11 +42,11 @@ class Main:
async def add(self, url: str):
parsed = parse_url(url)
if parsed is None:
secho(f"Unable to parse url {url}", fg="red")
raise Exception
raise Exception(f"Unable to parse url {url}")
client = await self.get_logged_in_client(parsed.source)
self.pending.append(await parsed.into_pending(client, self.config))
logger.debug("Added url=%s", url)
async def get_logged_in_client(self, source: str):
client = self.clients[source]
@ -57,30 +57,25 @@ class Main:
await prompter.prompt_and_login()
prompter.save()
else:
# Log into client using credentials from config
await client.login()
with console.status(f"[cyan]Logging into {source}", spinner="dots"):
# Log into client using credentials from config
await client.login()
assert client.logged_in
return client
async def resolve(self):
logger.info(f"Resolving {len(self.pending)} items")
assert len(self.pending) != 0
coros = [p.resolve() for p in self.pending]
new_media: list[Media] = await asyncio.gather(*coros)
with console.status("Resolving URLs...", spinner="dots"):
coros = [p.resolve() for p in self.pending]
new_media: list[Media] = await asyncio.gather(*coros)
self.media.extend(new_media)
self.pending.clear()
assert len(self.pending) == 0
async def rip(self):
c = self.config.session.downloads
if c.concurrency:
max_connections = c.max_connections if c.max_connections > 0 else 9999
else:
max_connections = 1
async with AsyncThreadPool(max_connections) as pool:
await pool.gather([item.rip() for item in self.media])
await asyncio.gather(*[item.rip() for item in self.media])
for client in self.clients.values():
await client.session.close()
clear_progress()

View File

@ -98,8 +98,8 @@ class Covers:
return f"Covers({covers})"
COPYRIGHT = "\u2117"
PHON_COPYRIGHT = "\u00a9"
PHON_COPYRIGHT = "\u2117"
COPYRIGHT = "\u00a9"
@dataclass(slots=True)
@ -201,10 +201,13 @@ class TrackInfo:
bit_depth: Optional[int] = None
explicit: bool = False
sampling_rate: Optional[int] = None
sampling_rate: Optional[int | float] = None
work: Optional[str] = None
genre_clean = re.compile(r"([^\u2192\/]+)")
@dataclass(slots=True)
class AlbumMetadata:
info: AlbumInfo
@ -214,25 +217,36 @@ class AlbumMetadata:
year: str
genre: list[str]
covers: Covers
tracktotal: int
disctotal: int = 1
albumcomposer: Optional[str] = None
comment: Optional[str] = None
compilation: Optional[str] = None
copyright: Optional[str] = None
date: Optional[str] = None
description: Optional[str] = None
disctotal: Optional[int] = None
encoder: Optional[str] = None
grouping: Optional[str] = None
lyrics: Optional[str] = None
purchase_date: Optional[str] = None
tracktotal: Optional[int] = None
def get_genres(self) -> str:
return ", ".join(self.genre)
def get_copyright(self) -> str | None:
if self.copyright is None:
return None
# Add special chars
_copyright = re.sub(r"(?i)\(P\)", PHON_COPYRIGHT, self.copyright)
_copyright = re.sub(r"(?i)\(C\)", COPYRIGHT, _copyright)
return _copyright
def format_folder_path(self, formatter: str) -> str:
# Available keys: "albumartist", "title", "year", "bit_depth", "sampling_rate",
# "id", and "albumcomposer",
none_str = "Unknown"
info: dict[str, str | int] = {
info: dict[str, str | int | float] = {
"albumartist": self.albumartist,
"albumcomposer": self.albumcomposer or none_str,
"bit_depth": self.info.bit_depth or none_str,
@ -249,13 +263,11 @@ class AlbumMetadata:
album = resp.get("title", "Unknown Album")
tracktotal = resp.get("tracks_count", 1)
genre = resp.get("genres_list") or resp.get("genre") or []
genres = list(set(re.findall(r"([^\u2192\/]+)", "/".join(genre))))
genres = list(set(genre_clean.findall("/".join(genre))))
date = resp.get("release_date_original") or resp.get("release_date")
year = date[:4] if date is not None else "Unknown"
_copyright = resp.get("copyright", "")
_copyright = re.sub(r"(?i)\(P\)", PHON_COPYRIGHT, _copyright)
_copyright = re.sub(r"(?i)\(C\)", COPYRIGHT, _copyright)
if artists := resp.get("artists"):
albumartist = ", ".join(a["name"] for a in artists)
@ -358,7 +370,7 @@ class AlbumInfo:
container: str
label: Optional[str] = None
explicit: bool = False
sampling_rate: Optional[int] = None
sampling_rate: Optional[int | float] = None
bit_depth: Optional[int] = None
booklets: list[dict] | None = None

13
streamrip/playlist.py Normal file
View File

@ -0,0 +1,13 @@
from dataclasses import dataclass
from .media import Media, Pending
@dataclass(slots=True)
class Playlist(Media):
pass
@dataclass(slots=True)
class PendingPlaylist(Pending):
pass

View File

@ -1,9 +1,9 @@
from typing import Optional
from typing import Callable
from click import style
from tqdm.asyncio import tqdm
from rich.progress import Progress
from .config import Config
from .console import console
THEMES = {
"plain": None,
@ -16,14 +16,39 @@ THEMES = {
}
def get_progress_bar(config: Config, total: int, desc: Optional[str], unit="B"):
theme = THEMES[config.session.theme.progress_bar]
return tqdm(
total=total,
unit=unit,
unit_scale=True,
unit_divisor=1024,
desc=desc,
dynamic_ncols=True,
bar_format=theme,
)
class ProgressManager:
def __init__(self):
self.started = False
self.progress = Progress(console=console)
def get_callback(self, total: int, desc: str):
if not self.started:
self.progress.start()
self.started = True
task = self.progress.add_task(f"[cyan]{desc}", total=total)
def _callback(x: int):
self.progress.update(task, advance=x)
return _callback
def cleanup(self):
if self.started:
self.progress.stop()
# global instance
_p = ProgressManager()
def get_progress_callback(
enabled: bool, total: int, desc: str
) -> Callable[[int], None]:
if not enabled:
return lambda _: None
return _p.get_callback(total, desc)
def clear_progress():
_p.cleanup()

View File

@ -61,8 +61,6 @@ class QobuzPrompter(CredentialPrompter):
except MissingCredentials:
self._prompt_creds_and_set_session_config()
secho("Successfully logged in to Qobuz", fg="green")
def _prompt_creds_and_set_session_config(self):
secho("Enter Qobuz email: ", fg="green", nl=False)
email = input()

View File

@ -276,9 +276,7 @@ class QobuzClient(Client):
logger.debug("api_request: endpoint=%s, params=%s", epoint, params)
if self.rate_limiter is not None:
async with self.rate_limiter:
async with self.session.get(
url, params=params, encoding="utf-8"
) as response:
async with self.session.get(url, params=params) as response:
return response.status, await response.json()
# return await self.session.get(url, params=params)
async with self.session.get(url, params=params) as response:

41
streamrip/semaphore.py Normal file
View File

@ -0,0 +1,41 @@
import asyncio
from .config import DownloadsConfig
INF = 9999
class UnlimitedSemaphore:
async def __aenter__(self):
return self
async def __aexit__(self, *_):
pass
_unlimited = UnlimitedSemaphore()
_global_semaphore: None | tuple[int, asyncio.Semaphore] = None
def global_download_semaphore(
c: DownloadsConfig,
) -> UnlimitedSemaphore | asyncio.Semaphore:
global _unlimited, _global_semaphore
if c.concurrency:
max_connections = c.max_connections if c.max_connections > 0 else INF
else:
max_connections = 1
assert max_connections > 0
if max_connections == INF:
return _unlimited
if _global_semaphore is None:
_global_semaphore = (max_connections, asyncio.Semaphore(max_connections))
assert (
max_connections == _global_semaphore[0]
), f"Already have other global semaphore {_global_semaphore}"
return _global_semaphore[1]

View File

@ -159,7 +159,7 @@ class Container(Enum):
out.append((v, text))
return out
def _attr_from_meta(self, meta: TrackMetadata, attr: str) -> str:
def _attr_from_meta(self, meta: TrackMetadata, attr: str) -> str | None:
# TODO: verify this works
in_trackmetadata = {
"title",
@ -170,9 +170,21 @@ class Container(Enum):
"composer",
}
if attr in in_trackmetadata:
return str(getattr(meta, attr))
if attr == "album":
return meta.album.album
val = getattr(meta, attr)
if val is None:
return None
return str(val)
else:
return str(getattr(meta.album, attr))
if attr == "genre":
return meta.album.get_genres()
elif attr == "copyright":
return meta.album.get_copyright()
val = getattr(meta.album, attr)
if val is None:
return None
return str(val)
def tag_audio(self, audio, tags: list[tuple]):
for k, v in tags:

View File

@ -1,21 +0,0 @@
import asyncio
class AsyncThreadPool:
"""Allows a maximum of `max_workers` coroutines to be running at once."""
def __init__(self, max_workers: int):
self.s = asyncio.Semaphore(max_workers)
async def gather(self, coros: list):
async def _wrapper(coro):
async with self.s:
await coro
return await asyncio.gather(*(_wrapper(c) for c in coros))
async def __aenter__(self):
return self
async def __aexit__(self, *_):
pass

View File

@ -10,7 +10,8 @@ from .downloadable import Downloadable
from .filepath_utils import clean_filename
from .media import Media, Pending
from .metadata import AlbumMetadata, Covers, TrackMetadata
from .progress import get_progress_bar
from .progress import get_progress_callback
from .semaphore import global_download_semaphore
from .tagger import tag_file
@ -31,14 +32,13 @@ class Track(Media):
async def download(self):
# TODO: progress bar description
with get_progress_bar(
self.config,
await self.downloadable.size(),
f"Track {self.meta.tracknumber}",
) as bar:
await self.downloadable.download(
self.download_path, lambda x: bar.update(x)
async with global_download_semaphore(self.config.session.downloads):
callback = get_progress_callback(
self.config.session.cli.progress_bars,
await self.downloadable.size(),
f"Track {self.meta.tracknumber}",
)
await self.downloadable.download(self.download_path, callback)
async def postprocess(self):
await self._tag()
@ -52,7 +52,7 @@ class Track(Media):
await tag_file(self.download_path, self.meta, self.cover_path)
async def _convert(self):
CONV_CLASS = {
CONV_CLASS: dict[str, type[converter.Converter]] = {
"FLAC": converter.FLAC,
"ALAC": converter.ALAC,
"MP3": converter.LAME,
@ -67,9 +67,10 @@ class Track(Media):
engine = CONV_CLASS[codec.upper()](
filename=self.download_path,
sampling_rate=c.sampling_rate,
bit_depth=c.bit_depth,
remove_source=True, # always going to delete the old file
)
engine.convert()
await engine.convert()
self.download_path = engine.final_fn # because the extension changed
def _set_download_path(self):
@ -93,6 +94,7 @@ class PendingTrack(Pending):
client: Client
config: Config
folder: str
# cover_path is None <==> Artwork for this track doesn't exist in API
cover_path: str | None
async def resolve(self) -> Track:

BIN
tests/1x1_pixel.jpg Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 631 B

File diff suppressed because one or more lines are too long

97
tests/test_tagger.py Normal file
View File

@ -0,0 +1,97 @@
import pytest
from mutagen.flac import FLAC
from util import arun
from streamrip.metadata import *
from streamrip.tagger import tag_file
test_flac = "tests/silence.flac"
test_cover = "tests/1x1_pixel.jpg"
def wipe_test_flac():
audio = FLAC(test_flac)
# Remove all tags
audio.delete()
audio.save()
@pytest.fixture
def sample_metadata() -> TrackMetadata:
return TrackMetadata(
TrackInfo(
id="12345",
quality=3,
bit_depth=24,
explicit=True,
sampling_rate=96,
work=None,
),
"testtitle",
AlbumMetadata(
AlbumInfo("5678", 4, "flac"),
"testalbum",
"testalbumartist",
"1999",
["rock", "pop"],
Covers(),
14,
3,
"testalbumcomposer",
"testcomment",
compilation="testcompilation",
copyright="(c) stuff (p) other stuff",
date="1998-02-13",
description="testdesc",
encoder="ffmpeg",
grouping="testgroup",
lyrics="ye ye ye",
purchase_date=None,
),
"testartist",
3,
1,
"testcomposer",
)
def test_tag_flac_no_cover(sample_metadata):
wipe_test_flac()
arun(tag_file(test_flac, sample_metadata, None))
file = FLAC(test_flac)
assert file["title"][0] == "testtitle"
assert file["album"][0] == "testalbum"
assert file["composer"][0] == "testcomposer"
assert file["comment"][0] == "testcomment"
assert file["artist"][0] == "testartist"
assert file["albumartist"][0] == "testalbumartist"
assert file["year"][0] == "1999"
assert file["genre"][0] == "rock, pop"
assert file["tracknumber"][0] == "03"
assert file["discnumber"][0] == "01"
assert file["copyright"][0] == "© stuff ℗ other stuff"
assert file["tracktotal"][0] == "14"
assert file["date"][0] == "1998-02-13"
assert "purchase_date" not in file, file["purchase_date"]
def test_tag_flac_cover(sample_metadata):
wipe_test_flac()
arun(tag_file(test_flac, sample_metadata, test_cover))
file = FLAC(test_flac)
assert file["title"][0] == "testtitle"
assert file["album"][0] == "testalbum"
assert file["composer"][0] == "testcomposer"
assert file["comment"][0] == "testcomment"
assert file["artist"][0] == "testartist"
assert file["albumartist"][0] == "testalbumartist"
assert file["year"][0] == "1999"
assert file["genre"][0] == "rock, pop"
assert file["tracknumber"][0] == "03"
assert file["discnumber"][0] == "01"
assert file["copyright"][0] == "© stuff ℗ other stuff"
assert file["tracktotal"][0] == "14"
assert file["date"][0] == "1998-02-13"
with open(test_cover, "rb") as img:
assert file.pictures[0].data == img.read()
assert "purchase_date" not in file, file["purchase_date"]