secstate/vrobbler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add retroarch import tasks and models

Browse Source
This commit is contained in:
Colin Powell
2023-05-24 22:38:20 -04:00
parent dbbb2b43a8
commit 93de6d1556
11 changed files with 227 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
vrobbler/apps/profiles/migrations/0009_userprofile_retroarch_auto_import_and_more.py Normal file
View File

@ -0,0 +1,23 @@
# Generated by Django 4.1.7 on 2023-05-25 02:14
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("profiles", "0008_userprofile_lastfm_auto_import"),
]
operations = [
migrations.AddField(
model_name="userprofile",
name="retroarch_auto_import",
field=models.BooleanField(default=False),
),
migrations.AddField(
model_name="userprofile",
name="retroarch_path",
field=models.CharField(blank=True, max_length=255, null=True),
),
]

3
vrobbler/apps/profiles/models.py
View File

@ -25,6 +25,9 @@ class UserProfile(TimeStampedModel):
lastfm_password = EncryptedField(**BNULL)
lastfm_auto_import = models.BooleanField(default=False)
retroarch_path = models.CharField(max_length=255, **BNULL)
retroarch_auto_import = models.BooleanField(default=False)
def __str__(self):
return f"User profile for {self.user}"

12
vrobbler/apps/scrobbles/admin.py
View File

@ -5,6 +5,7 @@ from scrobbles.models import (
ChartRecord,
KoReaderImport,
LastFmImport,
RetroarchImport,
Scrobble,
)
from scrobbles.mixins import Genre
@ -38,17 +39,22 @@ class ImportBaseAdmin(admin.ModelAdmin):
@admin.register(AudioScrobblerTSVImport)
class AudioScrobblerTSVImportAdmin(ImportBaseAdmin):
""""""
...
@admin.register(LastFmImport)
class LastFmImportAdmin(ImportBaseAdmin):
""""""
...
@admin.register(KoReaderImport)
class KoReaderImportAdmin(ImportBaseAdmin):
""""""
...
@admin.register(RetroarchImport)
class RetroarchImportAdmin(ImportBaseAdmin):
...
@admin.register(Genre)

18
vrobbler/apps/scrobbles/management/commands/import_from_retroarch.py Normal file
View File

@ -0,0 +1,18 @@
from django.core.management.base import BaseCommand
from vrobbler.apps.scrobbles.utils import import_retroarch_for_all_users
class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument(
"--restart",
action="store_true",
help="Restart failed imports",
)
def handle(self, *args, **options):
restart = False
if options["restart"]:
restart = True
count = import_retroarch_for_all_users(restart=restart)
print(f"Started {count} Retroarch imports")

67
vrobbler/apps/scrobbles/migrations/0041_retroarchimport.py Normal file
View File

@ -0,0 +1,67 @@
# Generated by Django 4.1.7 on 2023-05-25 02:17
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
import django_extensions.db.fields
import uuid
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
("scrobbles", "0040_alter_scrobble_media_type"),
]
operations = [
migrations.CreateModel(
name="RetroarchImport",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"created",
django_extensions.db.fields.CreationDateTimeField(
auto_now_add=True, verbose_name="created"
),
),
(
"modified",
django_extensions.db.fields.ModificationDateTimeField(
auto_now=True, verbose_name="modified"
),
),
("uuid", models.UUIDField(default=uuid.uuid4, editable=False)),
(
"processing_started",
models.DateTimeField(blank=True, null=True),
),
(
"processed_finished",
models.DateTimeField(blank=True, null=True),
),
("process_log", models.TextField(blank=True, null=True)),
("process_count", models.IntegerField(blank=True, null=True)),
(
"user",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.DO_NOTHING,
to=settings.AUTH_USER_MODEL,
),
),
],
options={
"verbose_name": "Retroarch Import",
},
),
]

52
vrobbler/apps/scrobbles/models.py
View File

@ -30,6 +30,7 @@ from scrobbles.utils import (
check_scrobble_for_finish,
)
from sports.models import SportEvent
from videogames import retroarch
from videogames.models import VideoGame
from videos.models import Series, Video
@ -50,7 +51,7 @@ class BaseFileImportMixin(TimeStampedModel):
abstract = True
def __str__(self):
return f"Scrobble import {self.id}"
return f"{self.import_type} import on {self.human_start}"
@property
def human_start(self):
@ -144,9 +145,6 @@ class KoReaderImport(BaseFileImportMixin):
def import_type(self) -> str:
return "KOReader"
def __str__(self):
return f"KoReader import on {self.human_start}"
def get_absolute_url(self):
return reverse(
"scrobbles:koreader-import-detail", kwargs={"slug": self.uuid}
@ -192,9 +190,6 @@ class AudioScrobblerTSVImport(BaseFileImportMixin):
def import_type(self) -> str:
return "AudiosScrobbler"
def __str__(self):
return f"Audioscrobbler import on {self.human_start}"
def get_absolute_url(self):
return reverse(
"scrobbles:tsv-import-detail", kwargs={"slug": self.uuid}
@ -246,9 +241,6 @@ class LastFmImport(BaseFileImportMixin):
def import_type(self) -> str:
return "LastFM"
def __str__(self):
return f"LastFM import on {self.human_start}"
def get_absolute_url(self):
return reverse(
"scrobbles:lastfm-import-detail", kwargs={"slug": self.uuid}
@ -288,6 +280,46 @@ class LastFmImport(BaseFileImportMixin):
self.mark_finished()
class RetroarchImport(BaseFileImportMixin):
class Meta:
verbose_name = "Retroarch Import"
@property
def import_type(self) -> str:
return "Retroarch"
def get_absolute_url(self):
return reverse(
"scrobbles:retroarch-import-detail", kwargs={"slug": self.uuid}
)
def process(self, import_all=False, force=False):
"""Import scrobbles found on Retroarch"""
if self.processed_finished and not force:
logger.info(
f"{self} already processed on {self.processed_finished}"
)
return
if force:
logger.info(f"You told me to force import from Retroarch")
if not self.user.profile.retroarch_path:
logger.info(
"Tying to import Retroarch logs, but user has no retroarch_path configured"
)
self.mark_started()
scrobbles = retroarch.import_retroarch_lrtl_files(
self.user.profile.retroarch_path,
self.user.id,
)
self.record_log(scrobbles)
self.mark_finished()
class ChartRecord(TimeStampedModel):
"""Sort of like a materialized view for what we could dynamically generate,
but would kill the DB as it gets larger. Collects time-based records

10
vrobbler/apps/scrobbles/tasks.py
View File

@ -9,6 +9,16 @@ logger = logging.getLogger(__name__)
User = get_user_model()
@shared_task
def process_retroarch_import(import_id):
RetroarchImport = apps.get_model("scrobbles", "RetroarchImport")
retroarch_import = RetroarchImport.objects.filter(id=import_id).first()
if not retroarch_import:
logger.warn(f"RetroarchImport not found with id {import_id}")
retroarch_import.process()
@shared_task
def process_lastfm_import(import_id):
LastFmImport = apps.get_model("scrobbles", "LastFMImport")

5
vrobbler/apps/scrobbles/urls.py
View File

@ -65,6 +65,11 @@ urlpatterns = [
views.ScrobbleKoReaderImportDetailView.as_view(),
name="koreader-import-detail",
),
path(
"imports/retroarch/<slug:slug>/",
views.ScrobbleRetroarchImportDetailView.as_view(),
name="retroarch-import-detail",
),
path(
"charts/",
views.ChartRecordView.as_view(),

26
vrobbler/apps/scrobbles/utils.py
View File

@ -11,7 +11,7 @@ from django.utils import timezone
from profiles.models import UserProfile
from profiles.utils import now_user_timezone
from scrobbles.constants import LONG_PLAY_MEDIA
from scrobbles.tasks import process_lastfm_import
from scrobbles.tasks import process_lastfm_import, process_retroarch_import
logger = logging.getLogger(__name__)
User = get_user_model()
@ -208,6 +208,30 @@ def import_lastfm_for_all_users(restart=False):
return lastfm_import_count
def import_retroarch_for_all_users(restart=False):
"""Grab a list of all users with Retroarch enabled and kickoff imports for them"""
RetroarchImport = apps.get_model("scrobbles", "RetroarchImport")
retroarch_enabled_user_ids = UserProfile.objects.filter(
retroarch_path__isnull=False,
retroarch_auto_import=True,
).values_list("user_id", flat=True)
retroarch_import_count = 0
for user_id in retroarch_enabled_user_ids:
retroarch_import, created = RetroarchImport.objects.get_or_create(
user_id=user_id, processed_finished__isnull=True
)
if not created and not restart:
logger.info(
f"Not resuming failed LastFM import {retroarch_import.id} for user {user_id}, use restart=True to restart"
)
continue
process_retroarch_import.delay(retroarch_import.id)
retroarch_import_count += 1
return retroarch_import_count
def delete_zombie_scrobbles(dry_run=True):
"""Look for any scrobble over a day old that is not paused and still in progress and delete it"""
Scrobble = apps.get_model("scrobbles", "Scrobble")

8
vrobbler/apps/scrobbles/views.py
View File

@ -9,7 +9,6 @@ from django.conf import settings
from django.contrib import messages
from django.contrib.auth.mixins import LoginRequiredMixin
from django.db.models import Q
from django.db.models.fields import timezone
from django.db.models.query import QuerySet
from django.http import FileResponse, HttpResponseRedirect, JsonResponse
from django.urls import reverse, reverse_lazy
@ -42,16 +41,17 @@ from scrobbles.models import (
ChartRecord,
KoReaderImport,
LastFmImport,
RetroarchImport,
Scrobble,
)
from scrobbles.scrobblers import (
jellyfin_scrobble_track,
jellyfin_scrobble_video,
manual_scrobble_board_game,
manual_scrobble_book,
manual_scrobble_event,
manual_scrobble_video,
manual_scrobble_video_game,
manual_scrobble_board_game,
mopidy_scrobble_podcast,
mopidy_scrobble_track,
)
@ -202,6 +202,10 @@ class ScrobbleLastFMImportDetailView(BaseScrobbleImportDetailView):
model = LastFmImport
class ScrobbleRetroarchImportDetailView(BaseScrobbleImportDetailView):
model = RetroarchImport
class ManualScrobbleView(FormView):
form_class = ScrobbleForm
template_name = "scrobbles/manual_form.html"

33
vrobbler/apps/videogames/retroarch.py
View File

@ -6,6 +6,7 @@ from typing import List
import pytz
from dateutil.parser import ParserError, parse
from django.apps import apps
from vrobbler.apps.scrobbles.utils import convert_to_seconds
from vrobbler.apps.videogames.utils import get_or_create_videogame
@ -71,6 +72,7 @@ def import_retroarch_lrtl_files(playlog_path: str, user_id: int) -> List[dict]:
3. Create new scrobble if last_played != last_scrobble.timestamp
4. Calculate scrobble time from runtime - last_scrobble.long_play_time
"""
Scrobble = apps.get_model("scrobbles", "Scrobble")
game_logs = load_game_data(playlog_path)
found_game = None
@ -104,18 +106,23 @@ def import_retroarch_lrtl_files(playlog_path: str, user_id: int) -> List[dict]:
stop_timestamp = game_data["last_played"] + timedelta(
seconds=playback_position_seconds
)
new_scrobbles.append(
{
"video_game_id": found_game.id,
"timestamp": game_data["last_played"],
"stop_timestamp": stop_timestamp,
"playback_position_seconds": playback_position_seconds,
"played_to_completion": True,
"in_progress": False,
"long_play_seconds": game_data["runtime"],
"user_id": user_id,
"source_id": "Retroarch",
"source": "Imported from Retroarch play log file",
}
if playback_position_seconds < 30:
logger.info(
f"Video game {found_game.id} played for less than 30 seconds, skipping"
)
new_scrobbles.append(
Scrobble(
video_game_id=found_game.id,
timestamp=game_data["last_played"],
stop_timestamp=stop_timestamp,
playback_position_seconds=playback_position_seconds,
played_to_completion=True,
in_progress=False,
long_play_seconds=game_data["runtime"],
user_id=user_id,
source="Retroarch",
source_id="Imported from Retroarch play log file",
)
)
created_scrobbles = Scrobble.objects.bulk_create(new_scrobbles)
return new_scrobbles