From a171517f92c456dca92677678d9427a30f256105 Mon Sep 17 00:00:00 2001 From: Colin Powell Date: Sat, 23 May 2026 17:30:25 -0400 Subject: [PATCH] [birds] Move importer to scrobbles and webdav --- vrobbler/apps/birds/views.py | 8 +- vrobbler/apps/scrobbles/admin.py | 6 ++ vrobbler/apps/scrobbles/importers/webdav.py | 63 +++++++++++++++- .../management/commands/import_from_webdav.py | 5 +- .../migrations/0082_ebird_csv_import_model.py | 75 +++++++++++++++++++ .../migrations/0083_migrate_birding_data.py | 36 +++++++++ vrobbler/apps/scrobbles/models.py | 47 ++++++++++++ vrobbler/apps/scrobbles/tasks.py | 10 +++ vrobbler/apps/scrobbles/urls.py | 5 ++ vrobbler/apps/scrobbles/views.py | 11 ++- vrobbler/apps/videogames/retroarch.py | 1 + vrobbler/templates/scrobbles/import_list.html | 25 +++++++ 12 files changed, 284 insertions(+), 8 deletions(-) create mode 100644 vrobbler/apps/scrobbles/migrations/0082_ebird_csv_import_model.py create mode 100644 vrobbler/apps/scrobbles/migrations/0083_migrate_birding_data.py diff --git a/vrobbler/apps/birds/views.py b/vrobbler/apps/birds/views.py index 0bc1333..c06ac7b 100644 --- a/vrobbler/apps/birds/views.py +++ b/vrobbler/apps/birds/views.py @@ -1,8 +1,9 @@ -from birds.models import Bird, BirdingCSVImport, BirdingLocation +from birds.models import Bird, BirdingLocation from django.contrib.auth.mixins import LoginRequiredMixin from django.http import HttpResponseRedirect from django.urls import reverse_lazy from django.views import generic +from scrobbles.models import EBirdCSVImport as BirdingCSVImport from scrobbles.views import ( ScrobbleableDetailView, ScrobbleableListView, @@ -40,6 +41,9 @@ class BirdingCSVImportCreateView( def form_valid(self, form): self.object = form.save(commit=False) self.object.user = self.request.user + self.object.original_filename = ( + form.cleaned_data["csv_file"].name + ) self.object.save() self.object.process() return HttpResponseRedirect(self.request.META.get("HTTP_REFERER")) @@ -55,5 +59,5 @@ class BirdingCSVImportDetailView(LoginRequiredMixin, generic.DetailView): def get_context_data(self, **kwargs): context_data = super().get_context_data(**kwargs) - context_data["title"] = "Birding CSV Import" + context_data["title"] = "eBird CSV Import" return context_data diff --git a/vrobbler/apps/scrobbles/admin.py b/vrobbler/apps/scrobbles/admin.py index 363500d..9c5ba9d 100644 --- a/vrobbler/apps/scrobbles/admin.py +++ b/vrobbler/apps/scrobbles/admin.py @@ -3,6 +3,7 @@ from django.contrib import admin from scrobbles.models import ( AudioScrobblerTSVImport, BGStatsImport, + EBirdCSVImport, KoReaderImport, LastFmImport, RetroarchImport, @@ -85,6 +86,11 @@ class BGStatsImportAdmin(ImportBaseAdmin): ... +@admin.register(EBirdCSVImport) +class EBirdCSVImportAdmin(ImportBaseAdmin): + ... + + @admin.register(ScaleCSVImport) class ScaleCSVImportAdmin(ImportBaseAdmin): ... diff --git a/vrobbler/apps/scrobbles/importers/webdav.py b/vrobbler/apps/scrobbles/importers/webdav.py index f6fa95f..736d997 100644 --- a/vrobbler/apps/scrobbles/importers/webdav.py +++ b/vrobbler/apps/scrobbles/importers/webdav.py @@ -17,6 +17,7 @@ DEFAULT_GPX_PATH = "var/gpx/" DEFAULT_KOREADER_PATH = "var/koreader/" DEFAULT_RETROARCH_PATH = "var/retroarch/" DEFAULT_BGSTATS_PATH = "var/bgstats/" +DEFAULT_EBIRD_PATH = "var/ebird/" def import_from_webdav_for_all_users(restart=False): @@ -33,6 +34,7 @@ def import_from_webdav_for_all_users(restart=False): gpx_count = 0 retro_count = 0 bgstats_count = 0 + ebird_count = 0 for user_id in webdav_enabled_user_ids: client = get_webdav_client(user_id) @@ -40,6 +42,7 @@ def import_from_webdav_for_all_users(restart=False): gpx_count += scan_webdav_for_gpx(client, user_id) retro_count += scan_webdav_for_retroarch(client, user_id) bgstats_count += scan_webdav_for_bgstats(client, user_id) + ebird_count += scan_webdav_for_ebird(client, user_id) logger.info( "WebDAV import complete", @@ -48,9 +51,10 @@ def import_from_webdav_for_all_users(restart=False): "trail_gpx": gpx_count, "retroarch": retro_count, "bgstats": bgstats_count, + "ebird": ebird_count, }, ) - return ko_count, gpx_count, retro_count, bgstats_count + return ko_count, gpx_count, retro_count, bgstats_count, ebird_count def scan_webdav_for_koreader(webdav_client, user_id, restart=False): @@ -343,3 +347,60 @@ def scan_webdav_for_bgstats(webdav_client, user_id): os.unlink(tmp.name) return new_imports + + +def scan_webdav_for_ebird(webdav_client, user_id): + """Download .csv files from WebDAV var/ebird/ and queue imports for new files.""" + from scrobbles.models import EBirdCSVImport + from scrobbles.tasks import process_ebird_csv_import + + ebird_path = DEFAULT_EBIRD_PATH + try: + webdav_client.info(ebird_path) + except: + logger.info("No var/ebird/ directory on webdav", extra={"user_id": user_id}) + return 0 + + try: + files = webdav_client.list(ebird_path) + except Exception as e: + logger.warning( + "Could not list var/ebird/", + extra={"user_id": user_id, "error": str(e)}, + ) + return 0 + + new_imports = 0 + already_imported = set( + EBirdCSVImport.objects.filter(user_id=user_id).values_list( + "original_filename", flat=True + ) + ) + + for fname in files: + fname = os.path.basename(fname) + if not fname.lower().endswith(".csv"): + continue + if fname in already_imported: + logger.debug(f"Skipping already-imported {fname}") + continue + + tmp = tempfile.NamedTemporaryFile(delete=False, suffix=fname) + try: + webdav_client.download_sync( + remote_path=f"{ebird_path}/{fname}", local_path=tmp.name + ) + imp = EBirdCSVImport.objects.create( + user_id=user_id, + original_filename=fname, + ) + with open(tmp.name, "rb") as f: + imp.csv_file.save(fname, f, save=True) + process_ebird_csv_import.delay(imp.id) + new_imports += 1 + except Exception as e: + logger.error(f"Failed to import eBird CSV file {fname}: {e}") + finally: + os.unlink(tmp.name) + + return new_imports diff --git a/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py b/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py index bd176bc..5b254d9 100644 --- a/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py +++ b/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py @@ -14,10 +14,11 @@ class Command(BaseCommand): restart = False if options["restart"]: restart = True - ko_count, gpx_count, retro_count, bgstats_count = ( + ko_count, gpx_count, retro_count, bgstats_count, ebird_count = ( webdav.import_from_webdav_for_all_users(restart=restart) ) print( f"Started {ko_count} KOReader, {gpx_count} Trail GPX, " - f"{retro_count} Retroarch, {bgstats_count} BGStats WebDAV imports" + f"{retro_count} Retroarch, {bgstats_count} BGStats, " + f"{ebird_count} eBird WebDAV imports" ) diff --git a/vrobbler/apps/scrobbles/migrations/0082_ebird_csv_import_model.py b/vrobbler/apps/scrobbles/migrations/0082_ebird_csv_import_model.py new file mode 100644 index 0000000..bdfe37f --- /dev/null +++ b/vrobbler/apps/scrobbles/migrations/0082_ebird_csv_import_model.py @@ -0,0 +1,75 @@ +# Generated by Django 4.2.29 on 2026-05-23 21:24 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import django_extensions.db.fields +import scrobbles.models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("scrobbles", "0081_bgstats_import_model"), + ] + + operations = [ + migrations.CreateModel( + name="EBirdCSVImport", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name="created" + ), + ), + ( + "modified", + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name="modified" + ), + ), + ("uuid", models.UUIDField(default=uuid.uuid4, editable=False)), + ("processing_started", models.DateTimeField(blank=True, null=True)), + ("processed_finished", models.DateTimeField(blank=True, null=True)), + ("process_log", models.TextField(blank=True, null=True)), + ("process_count", models.IntegerField(blank=True, null=True)), + ( + "csv_file", + models.FileField( + blank=True, + null=True, + upload_to=scrobbles.models.EBirdCSVImport.get_path, + ), + ), + ( + "original_filename", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "user", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + related_name="scrobbles_ebirdcsvimport_set", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "verbose_name": "eBird CSV Import", + }, + ), + ] diff --git a/vrobbler/apps/scrobbles/migrations/0083_migrate_birding_data.py b/vrobbler/apps/scrobbles/migrations/0083_migrate_birding_data.py new file mode 100644 index 0000000..5c3e35b --- /dev/null +++ b/vrobbler/apps/scrobbles/migrations/0083_migrate_birding_data.py @@ -0,0 +1,36 @@ +# Generated by Django 4.2.29 on 2026-05-23 21:24 + +from django.db import migrations + + +def copy_birding_data(apps, schema_editor): + OldImport = apps.get_model("birds", "BirdingCSVImport") + NewImport = apps.get_model("scrobbles", "EBirdCSVImport") + db_alias = schema_editor.connection.alias + + for old in OldImport.objects.using(db_alias).iterator(): + NewImport.objects.using(db_alias).create( + id=old.id, + created=old.created, + modified=old.modified, + uuid=old.uuid, + user=old.user, + processing_started=old.processing_started, + processed_finished=old.processed_finished, + process_log=old.process_log, + process_count=old.process_count, + csv_file=old.csv_file, + original_filename=( + old.csv_file.name.rpartition("/")[-1] if old.csv_file else None + ), + ) + + +class Migration(migrations.Migration): + dependencies = [ + ("scrobbles", "0082_ebird_csv_import_model"), + ] + + operations = [ + migrations.RunPython(copy_birding_data, migrations.RunPython.noop), + ] diff --git a/vrobbler/apps/scrobbles/models.py b/vrobbler/apps/scrobbles/models.py index 15accb9..f853d26 100644 --- a/vrobbler/apps/scrobbles/models.py +++ b/vrobbler/apps/scrobbles/models.py @@ -502,6 +502,53 @@ class BGStatsImport(BaseFileImportMixin): self.mark_finished() +class EBirdCSVImport(BaseFileImportMixin): + class Meta: + verbose_name = "eBird CSV Import" + + user = models.ForeignKey( + User, + on_delete=models.DO_NOTHING, + **BNULL, + related_name="scrobbles_ebirdcsvimport_set", + ) + + @property + def import_type(self) -> str: + return "eBird" + + def get_absolute_url(self): + return reverse("scrobbles:ebird-csv-import-detail", kwargs={"slug": self.uuid}) + + def get_path(instance, filename): + extension = filename.split(".")[-1] + uuid = instance.uuid + return f"birding-csv-uploads/{uuid}.{extension}" + + @property + def upload_file_path(self): + if getattr(settings, "USE_S3_STORAGE"): + path = self.csv_file.url + else: + path = self.csv_file.path + return path + + csv_file = models.FileField(upload_to=get_path, **BNULL) + original_filename = models.CharField(max_length=255, **BNULL) + + def process(self, force=False): + from birds.importer import import_birding_csv + + if self.processed_finished and not force: + logger.info(f"{self} already processed on {self.processed_finished}") + return + + self.mark_started() + scrobbles = import_birding_csv(self.upload_file_path, self.user_id) + self.record_log(scrobbles) + self.mark_finished() + + class ScrobbleQuerySet(models.QuerySet): def with_related(self): return self.select_related("user").prefetch_related( diff --git a/vrobbler/apps/scrobbles/tasks.py b/vrobbler/apps/scrobbles/tasks.py index c22e909..bee378e 100644 --- a/vrobbler/apps/scrobbles/tasks.py +++ b/vrobbler/apps/scrobbles/tasks.py @@ -158,6 +158,16 @@ def process_trail_gpx_import(import_id): trail_gpx_import.process() +@shared_task +def process_ebird_csv_import(import_id): + EBirdCSVImport = apps.get_model("scrobbles", "EBirdCSVImport") + birding_import = EBirdCSVImport.objects.filter(id=import_id).first() + if not birding_import: + logger.warn(f"EBirdCSVImport not found with id {import_id}") + return + birding_import.process() + + @shared_task def create_yesterdays_charts(): """Build/update charts for all users starting from last known record.""" diff --git a/vrobbler/apps/scrobbles/urls.py b/vrobbler/apps/scrobbles/urls.py index b3afa06..37813e7 100644 --- a/vrobbler/apps/scrobbles/urls.py +++ b/vrobbler/apps/scrobbles/urls.py @@ -141,6 +141,11 @@ urlpatterns = [ views.ScrobbleTrailGPXImportDetailView.as_view(), name="trail-gpx-import-detail", ), + path( + "imports/ebird-csv//", + views.ScrobbleBirdingCSVImportDetailView.as_view(), + name="ebird-csv-import-detail", + ), path( "long-plays/", views.ScrobbleLongPlaysView.as_view(), diff --git a/vrobbler/apps/scrobbles/views.py b/vrobbler/apps/scrobbles/views.py index 147e1aa..143e1f5 100644 --- a/vrobbler/apps/scrobbles/views.py +++ b/vrobbler/apps/scrobbles/views.py @@ -70,6 +70,7 @@ from scrobbles.forms import ExportScrobbleForm, ScrobbleForm from scrobbles.models import ( AudioScrobblerTSVImport, BGStatsImport, + EBirdCSVImport, KoReaderImport, LastFmImport, RetroarchImport, @@ -464,9 +465,7 @@ class ScrobbleImportListView(TemplateView): context_data["bgstats_imports"] = BGStatsImport.objects.filter( user=self.request.user, ).order_by("-processing_started")[:10] - context_data["birding_csv_imports"] = apps.get_model( - "birds", "BirdingCSVImport" - ).objects.filter( + context_data["birding_csv_imports"] = EBirdCSVImport.objects.filter( user=self.request.user, ).order_by("-processing_started")[:10] context_data["scale_csv_imports"] = ScaleCSVImport.objects.filter( @@ -498,6 +497,8 @@ class BaseScrobbleImportDetailView(DetailView): title = "Retroarch Import" if self.model == BGStatsImport: title = "BG Stats Import" + if self.model == EBirdCSVImport: + title = "eBird CSV Import" if self.model == ScaleCSVImport: title = "Scale CSV Import" if self.model == TrailGPXImport: @@ -534,6 +535,10 @@ class ScrobbleTrailGPXImportDetailView(BaseScrobbleImportDetailView): model = TrailGPXImport +class ScrobbleBirdingCSVImportDetailView(BaseScrobbleImportDetailView): + model = EBirdCSVImport + + class ManualScrobbleView(FormView): form_class = ScrobbleForm template_name = "scrobbles/manual_form.html" diff --git a/vrobbler/apps/videogames/retroarch.py b/vrobbler/apps/videogames/retroarch.py index a8c5aa4..565ce9e 100644 --- a/vrobbler/apps/videogames/retroarch.py +++ b/vrobbler/apps/videogames/retroarch.py @@ -161,6 +161,7 @@ def import_retroarch_lrtl_files(playlog_path: str, user_id: int) -> List[dict]: playback_position_seconds=playback_position_seconds, played_to_completion=True, in_progress=False, + timezone=timestamp.tzinfo.tzname, long_play_seconds=game_data["runtime"], long_play_complete=long_play_complete, user_id=user_id, diff --git a/vrobbler/templates/scrobbles/import_list.html b/vrobbler/templates/scrobbles/import_list.html index bda2086..f7a702c 100644 --- a/vrobbler/templates/scrobbles/import_list.html +++ b/vrobbler/templates/scrobbles/import_list.html @@ -79,6 +79,31 @@ {% endif %} +{% if birding_csv_imports %} +
+

eBird

+
+ + + + + + + + + {% for obj in birding_csv_imports %} + + + + + + {% endfor %} + +
StartedFinishedScrobbles
{{obj.human_start}}{{obj.processed_finished}}{{obj.process_count}}
+
+
+{% endif %} + {% if bgstats_imports %}

BG Stats