[scrobbles] Improve query efficency
This commit is contained in:
@ -62,9 +62,7 @@ logger = logging.getLogger(__name__)
|
||||
User = get_user_model()
|
||||
BNULL = {"blank": True, "null": True}
|
||||
|
||||
POINTS_FOR_MOVEMENT_HISTORY = int(
|
||||
getattr(settings, "POINTS_FOR_MOVEMENT_HISTORY", 3)
|
||||
)
|
||||
POINTS_FOR_MOVEMENT_HISTORY = int(getattr(settings, "POINTS_FOR_MOVEMENT_HISTORY", 3))
|
||||
|
||||
|
||||
class BaseFileImportMixin(TimeStampedModel):
|
||||
@ -107,9 +105,7 @@ class BaseFileImportMixin(TimeStampedModel):
|
||||
scrobble_id = line.split("\t")[0]
|
||||
scrobble = Scrobble.objects.filter(id=scrobble_id).first()
|
||||
if not scrobble:
|
||||
logger.warning(
|
||||
f"Could not find scrobble {scrobble_id} to undo"
|
||||
)
|
||||
logger.warning(f"Could not find scrobble {scrobble_id} to undo")
|
||||
continue
|
||||
logger.info(f"Removing scrobble {scrobble_id}")
|
||||
if not dryrun:
|
||||
@ -152,9 +148,7 @@ class BaseFileImportMixin(TimeStampedModel):
|
||||
return
|
||||
|
||||
for count, scrobble in enumerate(scrobbles):
|
||||
scrobble_str = (
|
||||
f"{scrobble.id}\t{scrobble.timestamp}\t{scrobble.media_obj}"
|
||||
)
|
||||
scrobble_str = f"{scrobble.id}\t{scrobble.timestamp}\t{scrobble.media_obj}"
|
||||
log_line = f"{scrobble_str}"
|
||||
if count > 0:
|
||||
log_line = "\n" + log_line
|
||||
@ -176,9 +170,7 @@ class KoReaderImport(BaseFileImportMixin):
|
||||
return "KOReader"
|
||||
|
||||
def get_absolute_url(self):
|
||||
return reverse(
|
||||
"scrobbles:koreader-import-detail", kwargs={"slug": self.uuid}
|
||||
)
|
||||
return reverse("scrobbles:koreader-import-detail", kwargs={"slug": self.uuid})
|
||||
|
||||
def get_path(instance, filename):
|
||||
extension = filename.split(".")[-1]
|
||||
@ -214,15 +206,11 @@ class KoReaderImport(BaseFileImportMixin):
|
||||
fix_profile_historic_timezones(self.user.profile)
|
||||
|
||||
if self.processed_finished and not force:
|
||||
logger.info(
|
||||
f"{self} already processed on {self.processed_finished}"
|
||||
)
|
||||
logger.info(f"{self} already processed on {self.processed_finished}")
|
||||
return
|
||||
|
||||
self.mark_started()
|
||||
scrobbles = process_koreader_sqlite_file(
|
||||
self.upload_file_path, self.user.id
|
||||
)
|
||||
scrobbles = process_koreader_sqlite_file(self.upload_file_path, self.user.id)
|
||||
self.record_log(scrobbles)
|
||||
self.mark_finished()
|
||||
|
||||
@ -236,9 +224,7 @@ class AudioScrobblerTSVImport(BaseFileImportMixin):
|
||||
return "AudiosScrobbler"
|
||||
|
||||
def get_absolute_url(self):
|
||||
return reverse(
|
||||
"scrobbles:tsv-import-detail", kwargs={"slug": self.uuid}
|
||||
)
|
||||
return reverse("scrobbles:tsv-import-detail", kwargs={"slug": self.uuid})
|
||||
|
||||
def get_path(instance, filename):
|
||||
extension = filename.split(".")[-1]
|
||||
@ -262,16 +248,12 @@ class AudioScrobblerTSVImport(BaseFileImportMixin):
|
||||
fix_profile_historic_timezones(self.user.profile)
|
||||
|
||||
if self.processed_finished and not force:
|
||||
logger.info(
|
||||
f"{self} already processed on {self.processed_finished}"
|
||||
)
|
||||
logger.info(f"{self} already processed on {self.processed_finished}")
|
||||
return
|
||||
|
||||
self.mark_started()
|
||||
|
||||
scrobbles = import_audioscrobbler_tsv_file(
|
||||
self.upload_file_path, self.user.id
|
||||
)
|
||||
scrobbles = import_audioscrobbler_tsv_file(self.upload_file_path, self.user.id)
|
||||
self.record_log(scrobbles)
|
||||
self.mark_finished()
|
||||
|
||||
@ -285,9 +267,7 @@ class LastFmImport(BaseFileImportMixin):
|
||||
return "LastFM"
|
||||
|
||||
def get_absolute_url(self):
|
||||
return reverse(
|
||||
"scrobbles:lastfm-import-detail", kwargs={"slug": self.uuid}
|
||||
)
|
||||
return reverse("scrobbles:lastfm-import-detail", kwargs={"slug": self.uuid})
|
||||
|
||||
def process(self, import_all=False):
|
||||
"""Import scrobbles found on LastFM"""
|
||||
@ -296,9 +276,7 @@ class LastFmImport(BaseFileImportMixin):
|
||||
fix_profile_historic_timezones(self.user.profile)
|
||||
|
||||
if self.processed_finished:
|
||||
logger.info(
|
||||
f"{self} already processed on {self.processed_finished}"
|
||||
)
|
||||
logger.info(f"{self} already processed on {self.processed_finished}")
|
||||
return
|
||||
|
||||
last_import = None
|
||||
@ -336,9 +314,7 @@ class RetroarchImport(BaseFileImportMixin):
|
||||
return "Retroarch"
|
||||
|
||||
def get_absolute_url(self):
|
||||
return reverse(
|
||||
"scrobbles:retroarch-import-detail", kwargs={"slug": self.uuid}
|
||||
)
|
||||
return reverse("scrobbles:retroarch-import-detail", kwargs={"slug": self.uuid})
|
||||
|
||||
def process(self, import_all=False, force=False):
|
||||
"""Import scrobbles found on Retroarch"""
|
||||
@ -346,9 +322,7 @@ class RetroarchImport(BaseFileImportMixin):
|
||||
fix_profile_historic_timezones(self.user.profile)
|
||||
|
||||
if self.processed_finished and not force:
|
||||
logger.info(
|
||||
f"{self} already processed on {self.processed_finished}"
|
||||
)
|
||||
logger.info(f"{self} already processed on {self.processed_finished}")
|
||||
return
|
||||
|
||||
if force:
|
||||
@ -504,9 +478,37 @@ class ChartRecord(TimeStampedModel):
|
||||
return cls.objects.filter(year=year, week=week, user=user)
|
||||
|
||||
|
||||
class ScrobbleQuerySet(models.QuerySet):
|
||||
def with_related(self):
|
||||
return self.select_related("user").prefetch_related(
|
||||
"video",
|
||||
"track",
|
||||
"track__artist",
|
||||
"podcast_episode",
|
||||
"podcast_episode__podcast",
|
||||
"sport_event",
|
||||
"book",
|
||||
"paper",
|
||||
"video_game",
|
||||
"board_game",
|
||||
"geo_location",
|
||||
"beer",
|
||||
"puzzle",
|
||||
"food",
|
||||
"trail",
|
||||
"task",
|
||||
"web_page",
|
||||
"life_event",
|
||||
"mood",
|
||||
"brick_set",
|
||||
)
|
||||
|
||||
|
||||
class Scrobble(TimeStampedModel):
|
||||
"""A scrobble tracks played media items by a user."""
|
||||
|
||||
objects = ScrobbleQuerySet.as_manager()
|
||||
|
||||
class MediaType(models.TextChoices):
|
||||
"""Enum mapping a media model type to a string"""
|
||||
|
||||
@ -539,39 +541,25 @@ class Scrobble(TimeStampedModel):
|
||||
podcast_episode = models.ForeignKey(
|
||||
PodcastEpisode, on_delete=models.DO_NOTHING, **BNULL
|
||||
)
|
||||
sport_event = models.ForeignKey(
|
||||
SportEvent, on_delete=models.DO_NOTHING, **BNULL
|
||||
)
|
||||
sport_event = models.ForeignKey(SportEvent, on_delete=models.DO_NOTHING, **BNULL)
|
||||
book = models.ForeignKey(Book, on_delete=models.DO_NOTHING, **BNULL)
|
||||
paper = models.ForeignKey(Paper, on_delete=models.DO_NOTHING, **BNULL)
|
||||
video_game = models.ForeignKey(
|
||||
VideoGame, on_delete=models.DO_NOTHING, **BNULL
|
||||
)
|
||||
board_game = models.ForeignKey(
|
||||
BoardGame, on_delete=models.DO_NOTHING, **BNULL
|
||||
)
|
||||
geo_location = models.ForeignKey(
|
||||
GeoLocation, on_delete=models.DO_NOTHING, **BNULL
|
||||
)
|
||||
video_game = models.ForeignKey(VideoGame, on_delete=models.DO_NOTHING, **BNULL)
|
||||
board_game = models.ForeignKey(BoardGame, on_delete=models.DO_NOTHING, **BNULL)
|
||||
geo_location = models.ForeignKey(GeoLocation, on_delete=models.DO_NOTHING, **BNULL)
|
||||
beer = models.ForeignKey(Beer, on_delete=models.DO_NOTHING, **BNULL)
|
||||
puzzle = models.ForeignKey(Puzzle, on_delete=models.DO_NOTHING, **BNULL)
|
||||
food = models.ForeignKey(Food, on_delete=models.DO_NOTHING, **BNULL)
|
||||
trail = models.ForeignKey(Trail, on_delete=models.DO_NOTHING, **BNULL)
|
||||
task = models.ForeignKey(Task, on_delete=models.DO_NOTHING, **BNULL)
|
||||
web_page = models.ForeignKey(WebPage, on_delete=models.DO_NOTHING, **BNULL)
|
||||
life_event = models.ForeignKey(
|
||||
LifeEvent, on_delete=models.DO_NOTHING, **BNULL
|
||||
)
|
||||
life_event = models.ForeignKey(LifeEvent, on_delete=models.DO_NOTHING, **BNULL)
|
||||
mood = models.ForeignKey(Mood, on_delete=models.DO_NOTHING, **BNULL)
|
||||
brick_set = models.ForeignKey(
|
||||
BrickSet, on_delete=models.DO_NOTHING, **BNULL
|
||||
)
|
||||
brick_set = models.ForeignKey(BrickSet, on_delete=models.DO_NOTHING, **BNULL)
|
||||
media_type = models.CharField(
|
||||
max_length=14, choices=MediaType.choices, default=MediaType.VIDEO
|
||||
)
|
||||
user = models.ForeignKey(
|
||||
User, blank=True, null=True, on_delete=models.DO_NOTHING
|
||||
)
|
||||
user = models.ForeignKey(User, blank=True, null=True, on_delete=models.DO_NOTHING)
|
||||
|
||||
# Time keeping
|
||||
timestamp = models.DateTimeField(**BNULL)
|
||||
@ -599,9 +587,7 @@ class Scrobble(TimeStampedModel):
|
||||
upload_to="scrobbles/videogame_save_data/", **BNULL
|
||||
)
|
||||
gpx_file = models.FileField(upload_to="scrobbles/gpx_file/", **BNULL)
|
||||
screenshot = models.ImageField(
|
||||
upload_to="scrobbles/videogame_screenshot/", **BNULL
|
||||
)
|
||||
screenshot = models.ImageField(upload_to="scrobbles/videogame_screenshot/", **BNULL)
|
||||
screenshot_small = ImageSpecField(
|
||||
source="screenshot",
|
||||
processors=[ResizeToFit(100, 100)],
|
||||
@ -662,8 +648,13 @@ class Scrobble(TimeStampedModel):
|
||||
@classmethod
|
||||
def as_dict_by_type(cls, scrobble_qs: models.QuerySet) -> dict:
|
||||
scrobbles_by_type = defaultdict(list)
|
||||
scrobbles = (
|
||||
scrobble_qs.with_related()
|
||||
if hasattr(scrobble_qs, "with_related")
|
||||
else scrobble_qs
|
||||
)
|
||||
|
||||
for scrobble in scrobble_qs:
|
||||
for scrobble in scrobbles:
|
||||
scrobbles_by_type[scrobble.media_type].append(scrobble)
|
||||
if not scrobbles_by_type.get(scrobble.media_type + "_count"):
|
||||
scrobbles_by_type[scrobble.media_type + "_count"] = 0
|
||||
@ -697,9 +688,7 @@ class Scrobble(TimeStampedModel):
|
||||
from scrobbles.models import Scrobble
|
||||
|
||||
if self.logdata and self.logdata.serial_scrobble_id:
|
||||
return Scrobble.objects.filter(
|
||||
id=self.logdata.serial_scrobble_id
|
||||
).first()
|
||||
return Scrobble.objects.filter(id=self.logdata.serial_scrobble_id).first()
|
||||
|
||||
@property
|
||||
def finish_url(self) -> str:
|
||||
@ -722,8 +711,7 @@ class Scrobble(TimeStampedModel):
|
||||
self.media_type = self.MediaType(self.media_obj.__class__.__name__)
|
||||
|
||||
if (self.timestamp and self.stop_timestamp) and (
|
||||
not self.playback_position_seconds
|
||||
or self.playback_position_seconds <= 0
|
||||
not self.playback_position_seconds or self.playback_position_seconds <= 0
|
||||
):
|
||||
self.playback_position_seconds = (
|
||||
self.stop_timestamp - self.timestamp
|
||||
@ -738,9 +726,9 @@ class Scrobble(TimeStampedModel):
|
||||
return reverse("scrobbles:detail", kwargs={"uuid": self.uuid})
|
||||
|
||||
def push_to_archivebox(self):
|
||||
pushable_media = hasattr(
|
||||
self.media_obj, "push_to_archivebox"
|
||||
) and callable(self.media_obj.push_to_archivebox)
|
||||
pushable_media = hasattr(self.media_obj, "push_to_archivebox") and callable(
|
||||
self.media_obj.push_to_archivebox
|
||||
)
|
||||
|
||||
if pushable_media and self.user.profile.archivebox_url:
|
||||
try:
|
||||
@ -810,10 +798,7 @@ class Scrobble(TimeStampedModel):
|
||||
logger.info(f"Redirecting to {self.media_obj.url}")
|
||||
redirect_url = self.media_obj.url
|
||||
|
||||
if (
|
||||
self.media_type == self.MediaType.VIDEO
|
||||
and self.media_obj.youtube_id
|
||||
):
|
||||
if self.media_type == self.MediaType.VIDEO and self.media_obj.youtube_id:
|
||||
redirect_url = self.media_obj.youtube_link
|
||||
|
||||
return redirect_url
|
||||
@ -829,9 +814,7 @@ class Scrobble(TimeStampedModel):
|
||||
@property
|
||||
def local_stop_timestamp(self):
|
||||
if self.stop_timestamp:
|
||||
return timezone.localtime(
|
||||
self.stop_timestamp, timezone=self.tzinfo
|
||||
)
|
||||
return timezone.localtime(self.stop_timestamp, timezone=self.tzinfo)
|
||||
|
||||
@property
|
||||
def scrobble_media_key(self) -> str:
|
||||
@ -959,9 +942,7 @@ class Scrobble(TimeStampedModel):
|
||||
long_play_secs = 0
|
||||
if self.previous and not self.previous.long_play_complete:
|
||||
long_play_secs = self.previous.long_play_seconds or 0
|
||||
percent = int(
|
||||
((playback_seconds + long_play_secs) / run_time_secs) * 100
|
||||
)
|
||||
percent = int(((playback_seconds + long_play_secs) / run_time_secs) * 100)
|
||||
|
||||
return percent
|
||||
|
||||
@ -1006,9 +987,7 @@ class Scrobble(TimeStampedModel):
|
||||
expected_end = self.timestamp + datetime.timedelta(
|
||||
seconds=self.media_obj.run_time_seconds
|
||||
)
|
||||
expected_end_padded = expected_end + datetime.timedelta(
|
||||
seconds=padding_seconds
|
||||
)
|
||||
expected_end_padded = expected_end + datetime.timedelta(seconds=padding_seconds)
|
||||
# Take our start time, add our media length and an extra 30 min (1800s) is it still in the future? keep going
|
||||
is_in_progress = expected_end_padded > pendulum.now()
|
||||
logger.info(
|
||||
@ -1071,11 +1050,9 @@ class Scrobble(TimeStampedModel):
|
||||
|
||||
@classmethod
|
||||
def by_date(cls, media_type: str = "Track"):
|
||||
cls.objects.filter(media_type=media_type).values(
|
||||
"timestamp__date"
|
||||
).annotate(count=models.Count("id")).values(
|
||||
"timestamp__date", "count"
|
||||
).order_by(
|
||||
cls.objects.filter(media_type=media_type).values("timestamp__date").annotate(
|
||||
count=models.Count("id")
|
||||
).values("timestamp__date", "count").order_by(
|
||||
"-count",
|
||||
)
|
||||
|
||||
@ -1209,9 +1186,7 @@ class Scrobble(TimeStampedModel):
|
||||
logger.warning(
|
||||
f"[create_or_update] geoloc requires create_or_update_location"
|
||||
)
|
||||
scrobble = cls.create_or_update_location(
|
||||
media, scrobble_data, user_id
|
||||
)
|
||||
scrobble = cls.create_or_update_location(media, scrobble_data, user_id)
|
||||
return scrobble
|
||||
|
||||
if not skip_in_progress_check or read_log_page:
|
||||
@ -1224,9 +1199,7 @@ class Scrobble(TimeStampedModel):
|
||||
"scrobble_data": scrobble_data,
|
||||
},
|
||||
)
|
||||
scrobble_data["playback_status"] = scrobble_data.pop(
|
||||
"status", None
|
||||
)
|
||||
scrobble_data["playback_status"] = scrobble_data.pop("status", None)
|
||||
# If it's marked as stopped, send it through our update mechanism, which will complete it
|
||||
if scrobble and (
|
||||
scrobble.can_be_updated
|
||||
@ -1238,12 +1211,8 @@ class Scrobble(TimeStampedModel):
|
||||
if page_list:
|
||||
for page in page_list:
|
||||
if not page.get("end_ts", None):
|
||||
page["end_ts"] = int(
|
||||
timezone.now().timestamp()
|
||||
)
|
||||
page["duration"] = page["end_ts"] - page.get(
|
||||
"start_ts"
|
||||
)
|
||||
page["end_ts"] = int(timezone.now().timestamp())
|
||||
page["duration"] = page["end_ts"] - page.get("start_ts")
|
||||
|
||||
page_list.append(
|
||||
BookPageLogData(
|
||||
@ -1279,9 +1248,9 @@ class Scrobble(TimeStampedModel):
|
||||
"source": source,
|
||||
},
|
||||
)
|
||||
if mtype == cls.MediaType.FOOD and not scrobble_data.get(
|
||||
"log", {}
|
||||
).get("calories", None):
|
||||
if mtype == cls.MediaType.FOOD and not scrobble_data.get("log", {}).get(
|
||||
"calories", None
|
||||
):
|
||||
if media.calories:
|
||||
scrobble_data["log"] = FoodLogData(calories=media.calories)
|
||||
|
||||
@ -1368,9 +1337,7 @@ class Scrobble(TimeStampedModel):
|
||||
if existing_locations := location.in_proximity(named=True):
|
||||
existing_location = existing_locations.first()
|
||||
ts = int(pendulum.now().timestamp())
|
||||
scrobble.log[
|
||||
ts
|
||||
] = f"Location {location.id} too close to this scrobble"
|
||||
scrobble.log[ts] = f"Location {location.id} too close to this scrobble"
|
||||
scrobble.save(update_fields=["log"])
|
||||
logger.info(
|
||||
f"[scrobbling] finished - found existing named location",
|
||||
@ -1549,9 +1516,7 @@ class Scrobble(TimeStampedModel):
|
||||
"""Returns true if our media is beyond our completion percent, unless
|
||||
our type is geolocation in which case we always return false
|
||||
"""
|
||||
beyond_completion = (
|
||||
self.percent_played >= self.media_obj.COMPLETION_PERCENT
|
||||
)
|
||||
beyond_completion = self.percent_played >= self.media_obj.COMPLETION_PERCENT
|
||||
|
||||
if self.media_type == "GeoLocation":
|
||||
logger.info(
|
||||
|
||||
Reference in New Issue
Block a user