Formatting

This commit is contained in:
Nathan Thomas 2023-12-20 22:21:58 -08:00
parent cf770892f1
commit abb37f17fd
25 changed files with 181 additions and 78 deletions

View File

@ -52,7 +52,8 @@ class Client(ABC):
if headers is None:
headers = {}
return aiohttp.ClientSession(
headers={"User-Agent": DEFAULT_USER_AGENT}, **headers,
headers={"User-Agent": DEFAULT_USER_AGENT},
**headers,
)
def __del__(self):

View File

@ -114,7 +114,9 @@ class DeezerClient(Client):
return response
async def get_downloadable(
self, item_id: str, quality: int = 2,
self,
item_id: str,
quality: int = 2,
) -> DeezerDownloadable:
# TODO: optimize such that all of the ids are requested at once
dl_info: dict = {"quality": quality, "id": item_id}
@ -168,14 +170,19 @@ class DeezerClient(Client):
if url is None:
url = self._get_encrypted_file_url(
item_id, track_info["MD5_ORIGIN"], track_info["MEDIA_VERSION"],
item_id,
track_info["MD5_ORIGIN"],
track_info["MEDIA_VERSION"],
)
dl_info["url"] = url
return DeezerDownloadable(self.session, dl_info)
def _get_encrypted_file_url(
self, meta_id: str, track_hash: str, media_version: str,
self,
meta_id: str,
track_hash: str,
media_version: str,
):
logger.debug("Unable to fetch URL. Trying encryption method.")
format_number = 1

View File

@ -280,11 +280,16 @@ class QobuzClient(Client):
raise NonStreamable
return BasicDownloadable(
self.session, stream_url, "flac" if quality > 1 else "mp3",
self.session,
stream_url,
"flac" if quality > 1 else "mp3",
)
async def _paginate(
self, epoint: str, params: dict, limit: Optional[int] = None,
self,
epoint: str,
params: dict,
limit: Optional[int] = None,
) -> list[dict]:
"""Paginate search results.
@ -359,7 +364,10 @@ class QobuzClient(Client):
return None
async def _request_file_url(
self, track_id: str, quality: int, secret: str,
self,
track_id: str,
quality: int,
secret: str,
) -> tuple[int, dict]:
quality = self.get_quality(quality)
unix_ts = time.time()

View File

@ -159,7 +159,8 @@ class SoundcloudClient(Client):
resp_json, status = await self._api_request(f"tracks/{item_id}/download")
assert status == 200
return SoundcloudDownloadable(
self.session, {"url": resp_json["redirectUri"], "type": "original"},
self.session,
{"url": resp_json["redirectUri"], "type": "original"},
)
if download_info == self.NOT_RESOLVED:
@ -168,11 +169,16 @@ class SoundcloudClient(Client):
# download_info contains mp3 stream url
resp_json, status = await self._request(download_info)
return SoundcloudDownloadable(
self.session, {"url": resp_json["url"], "type": "mp3"},
self.session,
{"url": resp_json["url"], "type": "mp3"},
)
async def search(
self, media_type: str, query: str, limit: int = 50, offset: int = 0,
self,
media_type: str,
query: str,
limit: int = 50,
offset: int = 0,
) -> list[dict]:
# TODO: implement pagination
assert media_type in ("track", "playlist")
@ -236,7 +242,8 @@ class SoundcloudClient(Client):
page_text = await resp.text(encoding="utf-8")
*_, client_id_url_match = re.finditer(
r"<script\s+crossorigin\s+src=\"([^\"]+)\"", page_text,
r"<script\s+crossorigin\s+src=\"([^\"]+)\"",
page_text,
)
if client_id_url_match is None:
@ -245,7 +252,8 @@ class SoundcloudClient(Client):
client_id_url = client_id_url_match.group(1)
app_version_match = re.search(
r'<script>window\.__sc_version="(\d+)"</script>', page_text,
r'<script>window\.__sc_version="(\d+)"</script>',
page_text,
)
if app_version_match is None:
raise Exception("Could not find app version in %s" % client_id_url_match)

View File

@ -220,7 +220,8 @@ DEFAULT_DOWNLOADS_FOLDER = os.path.join(HOME, "StreamripDownloads")
DEFAULT_DOWNLOADS_DB_PATH = os.path.join(APP_DIR, "downloads.db")
DEFAULT_FAILED_DOWNLOADS_DB_PATH = os.path.join(APP_DIR, "failed_downloads.db")
DEFAULT_YOUTUBE_VIDEO_DOWNLOADS_FOLDER = os.path.join(
DEFAULT_DOWNLOADS_FOLDER, "YouTubeVideos",
DEFAULT_DOWNLOADS_FOLDER,
"YouTubeVideos",
)
BLANK_CONFIG_PATH = os.path.join(os.path.dirname(__file__), "config.toml")
assert os.path.isfile(BLANK_CONFIG_PATH), "Template config not found"
@ -324,7 +325,8 @@ class ConfigData:
update_toml_section_from_config(self.toml["conversion"], self.conversion)
def get_source(
self, source: str,
self,
source: str,
) -> QobuzConfig | DeezerConfig | SoundcloudConfig | TidalConfig:
d = {
"qobuz": self.qobuz,

View File

@ -85,7 +85,8 @@ class Converter:
logger.debug("Generated conversion command: %s", self.command)
process = await asyncio.create_subprocess_exec(
*self.command, stderr=asyncio.subprocess.PIPE,
*self.command,
stderr=asyncio.subprocess.PIPE,
)
out, err = await process.communicate()
if process.returncode == 0 and os.path.isfile(self.tempfile):

View File

@ -43,6 +43,6 @@ class AlbumList(Media):
@staticmethod
def batch(iterable, n=1):
l = len(iterable)
for ndx in range(0, l, n):
yield iterable[ndx : min(ndx + n, l)]
total = len(iterable)
for ndx in range(0, total, n):
yield iterable[ndx : min(ndx + n, total)]

View File

@ -69,7 +69,8 @@ async def download_artwork(
assert l_url is not None
downloadables.append(
BasicDownloadable(session, l_url, "jpg").download(
saved_cover_path, lambda _: None,
saved_cover_path,
lambda _: None,
),
)
@ -82,7 +83,8 @@ async def download_artwork(
embed_cover_path = os.path.join(embed_dir, f"cover{hash(embed_url)}.jpg")
downloadables.append(
BasicDownloadable(session, embed_url, "jpg").download(
embed_cover_path, lambda _: None,
embed_cover_path,
lambda _: None,
),
)

View File

@ -80,7 +80,12 @@ class PendingPlaylistTrack(Pending):
return None
return Track(
meta, downloadable, self.config, self.folder, embedded_cover_path, self.db,
meta,
downloadable,
self.config,
self.folder,
embedded_cover_path,
self.db,
)
async def _download_cover(self, covers: Covers, folder: str) -> str | None:
@ -125,9 +130,9 @@ class Playlist(Media):
@staticmethod
def batch(iterable, n=1):
l = len(iterable)
for ndx in range(0, l, n):
yield iterable[ndx : min(ndx + n, l)]
total = len(iterable)
for ndx in range(0, total, n):
yield iterable[ndx : min(ndx + n, total)]
@dataclass(slots=True)
@ -145,7 +150,13 @@ class PendingPlaylist(Pending):
folder = os.path.join(parent, clean_filename(name))
tracks = [
PendingPlaylistTrack(
id, self.client, self.config, folder, name, position + 1, self.db,
id,
self.client,
self.config,
folder,
name,
position + 1,
self.db,
)
for position, id in enumerate(meta.ids())
]
@ -191,12 +202,18 @@ class PendingLastfmPlaylist(Pending):
s = self.Status(0, 0, len(titles_artists))
if self.config.session.cli.progress_bars:
with console.status(s.text(), spinner="moon") as status:
callback = lambda: status.update(s.text())
def callback():
status.update(s.text())
for title, artist in titles_artists:
requests.append(self._make_query(f"{title} {artist}", s, callback))
results: list[tuple[str | None, bool]] = await asyncio.gather(*requests)
else:
callback = lambda: None
def callback():
pass
for title, artist in titles_artists:
requests.append(self._make_query(f"{title} {artist}", s, callback))
results: list[tuple[str | None, bool]] = await asyncio.gather(*requests)
@ -231,7 +248,10 @@ class PendingLastfmPlaylist(Pending):
return Playlist(playlist_title, self.config, self.client, pending_tracks)
async def _make_query(
self, query: str, s: Status, callback,
self,
query: str,
s: Status,
callback,
) -> tuple[str | None, bool]:
"""Try searching for `query` with main source. If that fails, try with next source.
@ -261,7 +281,9 @@ class PendingLastfmPlaylist(Pending):
s.found += 1
return (
SearchResults.from_pages(
self.fallback_client.source, "track", pages,
self.fallback_client.source,
"track",
pages,
)
.results[0]
.id
@ -272,7 +294,8 @@ class PendingLastfmPlaylist(Pending):
return None, True
async def _parse_lastfm_playlist(
self, playlist_url: str,
self,
playlist_url: str,
) -> tuple[str, list[tuple[str, str]]]:
"""From a last.fm url, return the playlist title, and a list of
track titles and artist names.
@ -337,7 +360,10 @@ class PendingLastfmPlaylist(Pending):
return playlist_title, title_artist_pairs
async def _make_query_mock(
self, _: str, s: Status, callback,
self,
_: str,
s: Status,
callback,
) -> tuple[str | None, bool]:
await asyncio.sleep(random.uniform(1, 20))
if random.randint(0, 4) >= 1:

View File

@ -73,13 +73,15 @@ class Track(Media):
c = self.config.session.filepaths
formatter = c.track_format
track_path = clean_filename(
self.meta.format_track_path(formatter), restrict=c.restrict_characters,
self.meta.format_track_path(formatter),
restrict=c.restrict_characters,
)
if c.truncate_to > 0 and len(track_path) > c.truncate_to:
track_path = track_path[: c.truncate_to]
self.download_path = os.path.join(
self.folder, f"{track_path}.{self.downloadable.extension}",
self.folder,
f"{track_path}.{self.downloadable.extension}",
)
@ -112,7 +114,12 @@ class PendingTrack(Pending):
quality = self.config.session.get_source(source).quality
downloadable = await self.client.get_downloadable(self.id, quality)
return Track(
meta, downloadable, self.config, self.folder, self.cover_path, self.db,
meta,
downloadable,
self.config,
self.folder,
self.cover_path,
self.db,
)
@ -163,7 +170,8 @@ class PendingSingle(Pending):
quality = getattr(self.config.session, self.client.source).quality
assert isinstance(quality, int)
folder = os.path.join(
self.config.session.downloads.folder, self._format_folder(album),
self.config.session.downloads.folder,
self._format_folder(album),
)
os.makedirs(folder, exist_ok=True)

View File

@ -227,7 +227,8 @@ class AlbumMetadata:
track_id = track["id"]
bit_depth, sampling_rate = None, None
explicit = typed(
safe_get(track, "publisher_metadata", "explicit", default=False), bool,
safe_get(track, "publisher_metadata", "explicit", default=False),
bool,
)
genre = typed(track["genre"], str)
artist = typed(safe_get(track, "publisher_metadata", "artist"), str | None)
@ -238,7 +239,8 @@ class AlbumMetadata:
label = typed(track["label_name"], str | None)
description = typed(track.get("description"), str | None)
album_title = typed(
safe_get(track, "publisher_metadata", "album_title"), str | None,
safe_get(track, "publisher_metadata", "album_title"),
str | None,
)
album_title = album_title or "Unknown album"
copyright = typed(safe_get(track, "publisher_metadata", "p_line"), str | None)

View File

@ -1,3 +1,6 @@
TIDAL_COVER_URL = "https://resources.tidal.com/images/{uuid}/{width}x{height}.jpg"
class Covers:
COVER_SIZES = ("thumbnail", "small", "large", "original")
CoverEntry = tuple[str, str | None, str | None]
@ -78,7 +81,8 @@ class Covers:
def from_soundcloud(cls, resp):
c = cls()
cover_url = (resp["artwork_url"] or resp["user"].get("avatar_url")).replace(
"large", "t500x500",
"large",
"t500x500",
)
c.set_cover_url("large", cover_url)
return c
@ -112,13 +116,12 @@ class Covers:
:param uuid: VALID uuid string
:param size:
"""
TIDAL_COVER_URL = (
"https://resources.tidal.com/images/{uuid}/{width}x{height}.jpg"
)
possibles = (80, 160, 320, 640, 1280)
assert size in possibles, f"size must be in {possibles}"
return TIDAL_COVER_URL.format(
uuid=uuid.replace("-", "/"), height=size, width=size,
uuid=uuid.replace("-", "/"),
height=size,
width=size,
)
def __repr__(self):

View File

@ -53,7 +53,8 @@ class PlaylistMetadata:
for i, track in enumerate(resp["tracks"]["items"]):
meta = TrackMetadata.from_qobuz(
AlbumMetadata.from_qobuz(track["album"]), track,
AlbumMetadata.from_qobuz(track["album"]),
track,
)
if meta is None:
logger.error(f"Track {i+1} in playlist {name} not available for stream")

View File

@ -123,7 +123,8 @@ class TrackMetadata:
track_id = track["id"]
bit_depth, sampling_rate = None, None
explicit = typed(
safe_get(track, "publisher_metadata", "explicit", default=False), bool,
safe_get(track, "publisher_metadata", "explicit", default=False),
bool,
)
title = typed(track["title"].strip(), str)

View File

@ -26,7 +26,8 @@ def typed(thing, expected_type: Type[T]) -> T:
def get_quality_id(
bit_depth: Optional[int], sampling_rate: Optional[int | float],
bit_depth: Optional[int],
sampling_rate: Optional[int | float],
) -> int:
"""Get the universal quality id from bit depth and sampling rate.

View File

@ -5,6 +5,7 @@ import shutil
import subprocess
from functools import wraps
import aiofiles
import click
from click_help_colors import HelpColorsGroup # type: ignore
from rich.logging import RichHandler
@ -15,7 +16,6 @@ from .. import db
from ..config import DEFAULT_CONFIG_PATH, Config, set_user_defaults
from ..console import console
from .main import Main
from .user_paths import DEFAULT_CONFIG_PATH
def coro(f):
@ -33,7 +33,9 @@ def coro(f):
)
@click.version_option(version="2.0")
@click.option(
"--config-path", default=DEFAULT_CONFIG_PATH, help="Path to the configuration file",
"--config-path",
default=DEFAULT_CONFIG_PATH,
help="Path to the configuration file",
)
@click.option("-f", "--folder", help="The folder to download items into.")
@click.option(
@ -50,18 +52,26 @@ def coro(f):
help="Convert the downloaded files to an audio codec (ALAC, FLAC, MP3, AAC, or OGG)",
)
@click.option(
"--no-progress", help="Do not show progress bars", is_flag=True, default=False,
"--no-progress",
help="Do not show progress bars",
is_flag=True,
default=False,
)
@click.option(
"-v", "--verbose", help="Enable verbose output (debug mode)", is_flag=True,
"-v",
"--verbose",
help="Enable verbose output (debug mode)",
is_flag=True,
)
@click.pass_context
def rip(ctx, config_path, folder, no_db, quality, convert, no_progress, verbose):
"""Streamrip: the all in one music downloader.
"""
"""Streamrip: the all in one music downloader."""
global logger
logging.basicConfig(
level="INFO", format="%(message)s", datefmt="[%X]", handlers=[RichHandler()],
level="INFO",
format="%(message)s",
datefmt="[%X]",
handlers=[RichHandler()],
)
logger = logging.getLogger("streamrip")
if verbose:
@ -147,8 +157,8 @@ async def file(ctx, path):
"""
with ctx.obj["config"] as cfg:
async with Main(cfg) as main:
with open(path) as f:
await main.add_all([line for line in f])
async with aiofiles.open(path) as f:
await main.add_all([line async for line in f])
await main.resolve()
await main.rip()

View File

@ -3,7 +3,7 @@ import logging
import os
from .. import db
from ..client import Client, DeezerClient, QobuzClient, SoundcloudClient
from ..client import Client, DeezerClient, QobuzClient, SoundcloudClient, TidalClient
from ..config import Config
from ..console import console
from ..media import Media, Pending, PendingLastfmPlaylist, remove_artwork_tempdirs
@ -33,7 +33,7 @@ class Main:
self.config = config
self.clients: dict[str, Client] = {
"qobuz": QobuzClient(config),
# "tidal": TidalClient(config),
"tidal": TidalClient(config),
"deezer": DeezerClient(config),
"soundcloud": SoundcloudClient(config),
}
@ -203,7 +203,11 @@ class Main:
fallback_client = None
pending_playlist = PendingLastfmPlaylist(
playlist_url, client, fallback_client, self.config, self.database,
playlist_url,
client,
fallback_client,
self.config,
self.database,
)
playlist = await pending_playlist.resolve()

View File

@ -35,7 +35,10 @@ class URL(ABC):
@abstractmethod
async def into_pending(
self, client: Client, config: Config, db: Database,
self,
client: Client,
config: Config,
db: Database,
) -> Pending:
raise NotImplementedError
@ -50,7 +53,10 @@ class GenericURL(URL):
return cls(generic_url, source)
async def into_pending(
self, client: Client, config: Config, db: Database,
self,
client: Client,
config: Config,
db: Database,
) -> Pending:
source, media_type, item_id = self.match.groups()
assert client.source == source
@ -80,7 +86,10 @@ class QobuzInterpreterURL(URL):
return cls(qobuz_interpreter_url, "qobuz")
async def into_pending(
self, client: Client, config: Config, db: Database,
self,
client: Client,
config: Config,
db: Database,
) -> Pending:
url = self.match.group(0)
artist_id = await self.extract_interpreter_url(url, client)
@ -119,7 +128,10 @@ class SoundcloudURL(URL):
self.url = url
async def into_pending(
self, client: SoundcloudClient, config: Config, db: Database,
self,
client: SoundcloudClient,
config: Config,
db: Database,
) -> Pending:
resolved = await client._resolve_url(self.url)
media_type = resolved["kind"]

View File

@ -13,7 +13,7 @@ def qobuz_client():
config = Config.defaults()
config.session.qobuz.email_or_userid = os.environ["QOBUZ_EMAIL"]
config.session.qobuz.password_or_token = hashlib.md5(
os.environ["QOBUZ_PASSWORD"].encode("utf-8")
os.environ["QOBUZ_PASSWORD"].encode("utf-8"),
).hexdigest()
if "QOBUZ_APP_ID" in os.environ and "QOBUZ_SECRETS" in os.environ:
config.session.qobuz.app_id = os.environ["QOBUZ_APP_ID"]

View File

@ -6,11 +6,11 @@ import pytest
from streamrip.config import Config
@pytest.fixture
@pytest.fixture()
def config():
c = Config.defaults()
c.session.qobuz.email_or_userid = os.environ["QOBUZ_EMAIL"]
c.session.qobuz.password_or_token = hashlib.md5(
os.environ["QOBUZ_PASSWORD"].encode("utf-8")
os.environ["QOBUZ_PASSWORD"].encode("utf-8"),
).hexdigest()
return c

View File

@ -8,7 +8,7 @@ SAMPLE_CONFIG = "tests/test_config.toml"
# Define a fixture to create a sample ConfigData instance for testing
@pytest.fixture
@pytest.fixture()
def sample_config_data() -> ConfigData:
# Create a sample ConfigData instance here
# You can customize this to your specific needs for testing
@ -18,7 +18,7 @@ def sample_config_data() -> ConfigData:
# Define a fixture to create a sample Config instance for testing
@pytest.fixture
@pytest.fixture()
def sample_config() -> Config:
# Create a sample Config instance here
# You can customize this to your specific needs for testing
@ -66,10 +66,15 @@ def test_sample_config_data_fields(sample_config_data):
download_videos=True,
),
deezer=DeezerConfig(
arl="testarl", quality=2, use_deezloader=True, deezloader_warnings=True
arl="testarl",
quality=2,
use_deezloader=True,
deezloader_warnings=True,
),
soundcloud=SoundcloudConfig(
client_id="clientid", app_version="appversion", quality=0
client_id="clientid",
app_version="appversion",
quality=0,
),
youtube=YoutubeConfig(
video_downloads_folder="videodownloadsfolder",
@ -92,7 +97,9 @@ def test_sample_config_data_fields(sample_config_data):
saved_max_width=-1,
),
metadata=MetadataConfig(
set_playlist_to_album=True, renumber_playlist_tracks=True, exclude=[]
set_playlist_to_album=True,
renumber_playlist_tracks=True,
exclude=[],
),
qobuz_filters=QobuzDiscographyFilterConfig(
extras=False,

View File

@ -4,14 +4,14 @@ import tomlkit
from streamrip.config import *
@pytest.fixture
@pytest.fixture()
def toml():
with open("streamrip/config.toml") as f:
t = tomlkit.parse(f.read()) # type: ignore
return t
@pytest.fixture
@pytest.fixture()
def config():
return ConfigData.defaults()

View File

@ -3,7 +3,7 @@ import pytest
from streamrip.metadata import Covers
@pytest.fixture
@pytest.fixture()
def covers_all():
c = Covers()
c.set_cover("original", "ourl", None)
@ -14,19 +14,19 @@ def covers_all():
return c
@pytest.fixture
@pytest.fixture()
def covers_none():
return Covers()
@pytest.fixture
@pytest.fixture()
def covers_one():
c = Covers()
c.set_cover("small", "surl", None)
return c
@pytest.fixture
@pytest.fixture()
def covers_some():
c = Covers()
c.set_cover("large", "lurl", None)

View File

@ -11,8 +11,7 @@ from streamrip.qobuz_client import QobuzClient
logger = logging.getLogger("streamrip")
@pytest.mark.usefixtures("qobuz_client")
@pytest.fixture
@pytest.fixture()
def client(qobuz_client):
return qobuz_client

View File

@ -16,7 +16,7 @@ def wipe_test_flac():
audio.save()
@pytest.fixture
@pytest.fixture()
def sample_metadata() -> TrackMetadata:
return TrackMetadata(
TrackInfo(