From 93de6d1556ea9299d8185dcb8d6b7b30a058f84a Mon Sep 17 00:00:00 2001 From: Colin Powell Date: Wed, 24 May 2023 22:38:20 -0400 Subject: [PATCH] Add retroarch import tasks and models --- ...rprofile_retroarch_auto_import_and_more.py | 23 +++++++ vrobbler/apps/profiles/models.py | 3 + vrobbler/apps/scrobbles/admin.py | 12 +++- .../commands/import_from_retroarch.py | 18 +++++ .../migrations/0041_retroarchimport.py | 67 +++++++++++++++++++ vrobbler/apps/scrobbles/models.py | 52 +++++++++++--- vrobbler/apps/scrobbles/tasks.py | 10 +++ vrobbler/apps/scrobbles/urls.py | 5 ++ vrobbler/apps/scrobbles/utils.py | 26 ++++++- vrobbler/apps/scrobbles/views.py | 8 ++- vrobbler/apps/videogames/retroarch.py | 31 +++++---- 11 files changed, 227 insertions(+), 28 deletions(-) create mode 100644 vrobbler/apps/profiles/migrations/0009_userprofile_retroarch_auto_import_and_more.py create mode 100644 vrobbler/apps/scrobbles/management/commands/import_from_retroarch.py create mode 100644 vrobbler/apps/scrobbles/migrations/0041_retroarchimport.py diff --git a/vrobbler/apps/profiles/migrations/0009_userprofile_retroarch_auto_import_and_more.py b/vrobbler/apps/profiles/migrations/0009_userprofile_retroarch_auto_import_and_more.py new file mode 100644 index 0000000..86d33f0 --- /dev/null +++ b/vrobbler/apps/profiles/migrations/0009_userprofile_retroarch_auto_import_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 4.1.7 on 2023-05-25 02:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("profiles", "0008_userprofile_lastfm_auto_import"), + ] + + operations = [ + migrations.AddField( + model_name="userprofile", + name="retroarch_auto_import", + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name="userprofile", + name="retroarch_path", + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/vrobbler/apps/profiles/models.py b/vrobbler/apps/profiles/models.py index 1c56463..c68b47b 100644 --- a/vrobbler/apps/profiles/models.py +++ b/vrobbler/apps/profiles/models.py @@ -25,6 +25,9 @@ class UserProfile(TimeStampedModel): lastfm_password = EncryptedField(**BNULL) lastfm_auto_import = models.BooleanField(default=False) + retroarch_path = models.CharField(max_length=255, **BNULL) + retroarch_auto_import = models.BooleanField(default=False) + def __str__(self): return f"User profile for {self.user}" diff --git a/vrobbler/apps/scrobbles/admin.py b/vrobbler/apps/scrobbles/admin.py index 1d88a01..8651b8d 100644 --- a/vrobbler/apps/scrobbles/admin.py +++ b/vrobbler/apps/scrobbles/admin.py @@ -5,6 +5,7 @@ from scrobbles.models import ( ChartRecord, KoReaderImport, LastFmImport, + RetroarchImport, Scrobble, ) from scrobbles.mixins import Genre @@ -38,17 +39,22 @@ class ImportBaseAdmin(admin.ModelAdmin): @admin.register(AudioScrobblerTSVImport) class AudioScrobblerTSVImportAdmin(ImportBaseAdmin): - """""" + ... @admin.register(LastFmImport) class LastFmImportAdmin(ImportBaseAdmin): - """""" + ... @admin.register(KoReaderImport) class KoReaderImportAdmin(ImportBaseAdmin): - """""" + ... + + +@admin.register(RetroarchImport) +class RetroarchImportAdmin(ImportBaseAdmin): + ... @admin.register(Genre) diff --git a/vrobbler/apps/scrobbles/management/commands/import_from_retroarch.py b/vrobbler/apps/scrobbles/management/commands/import_from_retroarch.py new file mode 100644 index 0000000..09b4497 --- /dev/null +++ b/vrobbler/apps/scrobbles/management/commands/import_from_retroarch.py @@ -0,0 +1,18 @@ +from django.core.management.base import BaseCommand +from vrobbler.apps.scrobbles.utils import import_retroarch_for_all_users + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument( + "--restart", + action="store_true", + help="Restart failed imports", + ) + + def handle(self, *args, **options): + restart = False + if options["restart"]: + restart = True + count = import_retroarch_for_all_users(restart=restart) + print(f"Started {count} Retroarch imports") diff --git a/vrobbler/apps/scrobbles/migrations/0041_retroarchimport.py b/vrobbler/apps/scrobbles/migrations/0041_retroarchimport.py new file mode 100644 index 0000000..da6bb3f --- /dev/null +++ b/vrobbler/apps/scrobbles/migrations/0041_retroarchimport.py @@ -0,0 +1,67 @@ +# Generated by Django 4.1.7 on 2023-05-25 02:17 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import django_extensions.db.fields +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("scrobbles", "0040_alter_scrobble_media_type"), + ] + + operations = [ + migrations.CreateModel( + name="RetroarchImport", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name="created" + ), + ), + ( + "modified", + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name="modified" + ), + ), + ("uuid", models.UUIDField(default=uuid.uuid4, editable=False)), + ( + "processing_started", + models.DateTimeField(blank=True, null=True), + ), + ( + "processed_finished", + models.DateTimeField(blank=True, null=True), + ), + ("process_log", models.TextField(blank=True, null=True)), + ("process_count", models.IntegerField(blank=True, null=True)), + ( + "user", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "verbose_name": "Retroarch Import", + }, + ), + ] diff --git a/vrobbler/apps/scrobbles/models.py b/vrobbler/apps/scrobbles/models.py index 3bd9b2d..11dccb8 100644 --- a/vrobbler/apps/scrobbles/models.py +++ b/vrobbler/apps/scrobbles/models.py @@ -30,6 +30,7 @@ from scrobbles.utils import ( check_scrobble_for_finish, ) from sports.models import SportEvent +from videogames import retroarch from videogames.models import VideoGame from videos.models import Series, Video @@ -50,7 +51,7 @@ class BaseFileImportMixin(TimeStampedModel): abstract = True def __str__(self): - return f"Scrobble import {self.id}" + return f"{self.import_type} import on {self.human_start}" @property def human_start(self): @@ -144,9 +145,6 @@ class KoReaderImport(BaseFileImportMixin): def import_type(self) -> str: return "KOReader" - def __str__(self): - return f"KoReader import on {self.human_start}" - def get_absolute_url(self): return reverse( "scrobbles:koreader-import-detail", kwargs={"slug": self.uuid} @@ -192,9 +190,6 @@ class AudioScrobblerTSVImport(BaseFileImportMixin): def import_type(self) -> str: return "AudiosScrobbler" - def __str__(self): - return f"Audioscrobbler import on {self.human_start}" - def get_absolute_url(self): return reverse( "scrobbles:tsv-import-detail", kwargs={"slug": self.uuid} @@ -246,9 +241,6 @@ class LastFmImport(BaseFileImportMixin): def import_type(self) -> str: return "LastFM" - def __str__(self): - return f"LastFM import on {self.human_start}" - def get_absolute_url(self): return reverse( "scrobbles:lastfm-import-detail", kwargs={"slug": self.uuid} @@ -288,6 +280,46 @@ class LastFmImport(BaseFileImportMixin): self.mark_finished() +class RetroarchImport(BaseFileImportMixin): + class Meta: + verbose_name = "Retroarch Import" + + @property + def import_type(self) -> str: + return "Retroarch" + + def get_absolute_url(self): + return reverse( + "scrobbles:retroarch-import-detail", kwargs={"slug": self.uuid} + ) + + def process(self, import_all=False, force=False): + """Import scrobbles found on Retroarch""" + if self.processed_finished and not force: + logger.info( + f"{self} already processed on {self.processed_finished}" + ) + return + + if force: + logger.info(f"You told me to force import from Retroarch") + + if not self.user.profile.retroarch_path: + logger.info( + "Tying to import Retroarch logs, but user has no retroarch_path configured" + ) + + self.mark_started() + + scrobbles = retroarch.import_retroarch_lrtl_files( + self.user.profile.retroarch_path, + self.user.id, + ) + + self.record_log(scrobbles) + self.mark_finished() + + class ChartRecord(TimeStampedModel): """Sort of like a materialized view for what we could dynamically generate, but would kill the DB as it gets larger. Collects time-based records diff --git a/vrobbler/apps/scrobbles/tasks.py b/vrobbler/apps/scrobbles/tasks.py index 50abfcb..1523915 100644 --- a/vrobbler/apps/scrobbles/tasks.py +++ b/vrobbler/apps/scrobbles/tasks.py @@ -9,6 +9,16 @@ logger = logging.getLogger(__name__) User = get_user_model() +@shared_task +def process_retroarch_import(import_id): + RetroarchImport = apps.get_model("scrobbles", "RetroarchImport") + retroarch_import = RetroarchImport.objects.filter(id=import_id).first() + if not retroarch_import: + logger.warn(f"RetroarchImport not found with id {import_id}") + + retroarch_import.process() + + @shared_task def process_lastfm_import(import_id): LastFmImport = apps.get_model("scrobbles", "LastFMImport") diff --git a/vrobbler/apps/scrobbles/urls.py b/vrobbler/apps/scrobbles/urls.py index 36a7a40..1cd5485 100644 --- a/vrobbler/apps/scrobbles/urls.py +++ b/vrobbler/apps/scrobbles/urls.py @@ -65,6 +65,11 @@ urlpatterns = [ views.ScrobbleKoReaderImportDetailView.as_view(), name="koreader-import-detail", ), + path( + "imports/retroarch//", + views.ScrobbleRetroarchImportDetailView.as_view(), + name="retroarch-import-detail", + ), path( "charts/", views.ChartRecordView.as_view(), diff --git a/vrobbler/apps/scrobbles/utils.py b/vrobbler/apps/scrobbles/utils.py index e924ab2..a12a45f 100644 --- a/vrobbler/apps/scrobbles/utils.py +++ b/vrobbler/apps/scrobbles/utils.py @@ -11,7 +11,7 @@ from django.utils import timezone from profiles.models import UserProfile from profiles.utils import now_user_timezone from scrobbles.constants import LONG_PLAY_MEDIA -from scrobbles.tasks import process_lastfm_import +from scrobbles.tasks import process_lastfm_import, process_retroarch_import logger = logging.getLogger(__name__) User = get_user_model() @@ -208,6 +208,30 @@ def import_lastfm_for_all_users(restart=False): return lastfm_import_count +def import_retroarch_for_all_users(restart=False): + """Grab a list of all users with Retroarch enabled and kickoff imports for them""" + RetroarchImport = apps.get_model("scrobbles", "RetroarchImport") + retroarch_enabled_user_ids = UserProfile.objects.filter( + retroarch_path__isnull=False, + retroarch_auto_import=True, + ).values_list("user_id", flat=True) + + retroarch_import_count = 0 + + for user_id in retroarch_enabled_user_ids: + retroarch_import, created = RetroarchImport.objects.get_or_create( + user_id=user_id, processed_finished__isnull=True + ) + if not created and not restart: + logger.info( + f"Not resuming failed LastFM import {retroarch_import.id} for user {user_id}, use restart=True to restart" + ) + continue + process_retroarch_import.delay(retroarch_import.id) + retroarch_import_count += 1 + return retroarch_import_count + + def delete_zombie_scrobbles(dry_run=True): """Look for any scrobble over a day old that is not paused and still in progress and delete it""" Scrobble = apps.get_model("scrobbles", "Scrobble") diff --git a/vrobbler/apps/scrobbles/views.py b/vrobbler/apps/scrobbles/views.py index a9f52be..3d208a7 100644 --- a/vrobbler/apps/scrobbles/views.py +++ b/vrobbler/apps/scrobbles/views.py @@ -9,7 +9,6 @@ from django.conf import settings from django.contrib import messages from django.contrib.auth.mixins import LoginRequiredMixin from django.db.models import Q -from django.db.models.fields import timezone from django.db.models.query import QuerySet from django.http import FileResponse, HttpResponseRedirect, JsonResponse from django.urls import reverse, reverse_lazy @@ -42,16 +41,17 @@ from scrobbles.models import ( ChartRecord, KoReaderImport, LastFmImport, + RetroarchImport, Scrobble, ) from scrobbles.scrobblers import ( jellyfin_scrobble_track, jellyfin_scrobble_video, + manual_scrobble_board_game, manual_scrobble_book, manual_scrobble_event, manual_scrobble_video, manual_scrobble_video_game, - manual_scrobble_board_game, mopidy_scrobble_podcast, mopidy_scrobble_track, ) @@ -202,6 +202,10 @@ class ScrobbleLastFMImportDetailView(BaseScrobbleImportDetailView): model = LastFmImport +class ScrobbleRetroarchImportDetailView(BaseScrobbleImportDetailView): + model = RetroarchImport + + class ManualScrobbleView(FormView): form_class = ScrobbleForm template_name = "scrobbles/manual_form.html" diff --git a/vrobbler/apps/videogames/retroarch.py b/vrobbler/apps/videogames/retroarch.py index 41209ce..2410a5c 100644 --- a/vrobbler/apps/videogames/retroarch.py +++ b/vrobbler/apps/videogames/retroarch.py @@ -6,6 +6,7 @@ from typing import List import pytz from dateutil.parser import ParserError, parse +from django.apps import apps from vrobbler.apps.scrobbles.utils import convert_to_seconds from vrobbler.apps.videogames.utils import get_or_create_videogame @@ -71,6 +72,7 @@ def import_retroarch_lrtl_files(playlog_path: str, user_id: int) -> List[dict]: 3. Create new scrobble if last_played != last_scrobble.timestamp 4. Calculate scrobble time from runtime - last_scrobble.long_play_time """ + Scrobble = apps.get_model("scrobbles", "Scrobble") game_logs = load_game_data(playlog_path) found_game = None @@ -104,18 +106,23 @@ def import_retroarch_lrtl_files(playlog_path: str, user_id: int) -> List[dict]: stop_timestamp = game_data["last_played"] + timedelta( seconds=playback_position_seconds ) + if playback_position_seconds < 30: + logger.info( + f"Video game {found_game.id} played for less than 30 seconds, skipping" + ) new_scrobbles.append( - { - "video_game_id": found_game.id, - "timestamp": game_data["last_played"], - "stop_timestamp": stop_timestamp, - "playback_position_seconds": playback_position_seconds, - "played_to_completion": True, - "in_progress": False, - "long_play_seconds": game_data["runtime"], - "user_id": user_id, - "source_id": "Retroarch", - "source": "Imported from Retroarch play log file", - } + Scrobble( + video_game_id=found_game.id, + timestamp=game_data["last_played"], + stop_timestamp=stop_timestamp, + playback_position_seconds=playback_position_seconds, + played_to_completion=True, + in_progress=False, + long_play_seconds=game_data["runtime"], + user_id=user_id, + source="Retroarch", + source_id="Imported from Retroarch play log file", + ) ) + created_scrobbles = Scrobble.objects.bulk_create(new_scrobbles) return new_scrobbles