diff --git a/data/birding-example.csv b/data/birding-example.csv new file mode 100644 index 0000000..3affce2 --- /dev/null +++ b/data/birding-example.csv @@ -0,0 +1,5 @@ +Species,Count,Location,Observation Type,Observation Date,Start Time,Duration,Distance,Area,Party Size,Complete Checklist,# of species,Details +Canada Goose,6,"120 Perkins Street, Castine, Maine, US (44.384, -68.805)",Stationary,"May 10, 2026",4:15 PM,9 minute(s),,,4,true,4 species, +Common Loon,1,"120 Perkins Street, Castine, Maine, US (44.384, -68.805)",Stationary,"May 10, 2026",4:15 PM,9 minute(s),,,4,true,4 species, +Double-crested Cormorant,1,"120 Perkins Street, Castine, Maine, US (44.384, -68.805)",Stationary,"May 10, 2026",4:15 PM,9 minute(s),,,4,true,4 species, +Boat-tailed Grackle,2,"120 Perkins Street, Castine, Maine, US (44.384, -68.805)",Stationary,"May 10, 2026",4:15 PM,9 minute(s),,,4,true,4 species,Sitting together on roof line of a house on Water Street. 20 meters away. Both birds were mostly black with green accents on the breast with long tails which they were repeatedly fanning out to show the V shape. diff --git a/vrobbler/apps/birds/admin.py b/vrobbler/apps/birds/admin.py index 60f62a6..c0778c0 100644 --- a/vrobbler/apps/birds/admin.py +++ b/vrobbler/apps/birds/admin.py @@ -1,4 +1,4 @@ -from birds.models import Bird, BirdingLocation +from birds.models import Bird, BirdingCSVImport, BirdingLocation from django.contrib import admin from scrobbles.admin import ScrobbleInline @@ -20,3 +20,10 @@ class BirdingLocationAdmin(admin.ModelAdmin): inlines = [ ScrobbleInline, ] + + +@admin.register(BirdingCSVImport) +class BirdingCSVImportAdmin(admin.ModelAdmin): + date_hierarchy = "created" + list_display = ("uuid", "process_count", "processed_finished", "processing_started") + ordering = ("-created",) diff --git a/vrobbler/apps/birds/importer.py b/vrobbler/apps/birds/importer.py new file mode 100644 index 0000000..d9f9b09 --- /dev/null +++ b/vrobbler/apps/birds/importer.py @@ -0,0 +1,152 @@ +import csv +import logging +import re +from collections import defaultdict +from datetime import datetime +from zoneinfo import ZoneInfo + +from django.contrib.auth import get_user_model + +from birds.models import Bird, BirdSightingEntry, BirdSightingLogData, BirdingLocation +from scrobbles.models import Scrobble + +logger = logging.getLogger(__name__) +User = get_user_model() + +LOCATION_COORDS_RE = re.compile(r"\(([\d\.\-]+),\s*([\d\.\-]+)\)") +DURATION_RE = re.compile(r"(\d+)\s*minute") + + +def parse_duration(duration_str): + if not duration_str: + return None + match = DURATION_RE.search(duration_str) + if match: + return int(match.group(1)) + return None + + +def parse_coords(location_str): + match = LOCATION_COORDS_RE.search(location_str) + if match: + return float(match.group(1)), float(match.group(2)) + return None, None + + +def parse_timestamp(date_str, time_str): + try: + dt = datetime.strptime(f"{date_str} {time_str}", "%B %d, %Y %I:%M %p") + return dt.replace(tzinfo=ZoneInfo("UTC")) + except (ValueError, TypeError): + try: + dt = datetime.strptime(date_str, "%B %d, %Y") + return dt.replace(tzinfo=ZoneInfo("UTC")) + except (ValueError, TypeError): + logger.warning(f"Could not parse date/time: {date_str} {time_str}") + return None + + +def import_birding_csv(file_path, user_id): + user = User.objects.get(id=user_id) + new_scrobbles = [] + + with open(file_path, newline="", encoding="utf-8-sig") as f: + reader = csv.DictReader(f) + rows = list(reader) + + groups = defaultdict(list) + for row in rows: + key = ( + row.get("Location", "").strip(), + row.get("Observation Date", "").strip(), + row.get("Start Time", "").strip(), + ) + groups[key].append(row) + + for (location_str, date_str, time_str), sighting_rows in groups.items(): + if not location_str: + logger.warning("Skipping rows with no location") + continue + + timestamp = parse_timestamp(date_str, time_str) + if not timestamp: + continue + + location_title = LOCATION_COORDS_RE.sub("", location_str).strip().rstrip(",").strip() + if not location_title: + location_title = location_str + + location = BirdingLocation.find_or_create(location_title) + lat, lon = parse_coords(location_str) + if lat and lon and not location.geo_location: + from locations.models import GeoLocation + + geo, _ = GeoLocation.objects.get_or_create( + lat=round(lat, 6), + lon=round(lon, 6), + defaults={"altitude": None}, + ) + location.geo_location = geo + location.save(update_fields=["geo_location"]) + + first_row = sighting_rows[0] + duration_minutes = parse_duration(first_row.get("Duration", "")) + party_size = first_row.get("Party Size", "").strip() + + birds_data = [] + for row in sighting_rows: + species = row.get("Species", "").strip() + if not species: + continue + count_str = row.get("Count", "1").strip() + try: + count = int(count_str) if count_str else 1 + except ValueError: + count = 1 + details = row.get("Details", "").strip() + + bird = Bird.find_or_create(species) + entry = BirdSightingEntry( + bird_id=bird.id, quantity=count, sighting_notes=details or None + ) + birds_data.append(entry.asdict) + + logdata = BirdSightingLogData( + birds=birds_data, + duration_minutes=duration_minutes, + ) + + notes = [] + if party_size: + notes.append(f"Party size: {party_size}") + checklist_flag = first_row.get("Complete Checklist", "").strip() + if checklist_flag: + notes.append(f"Complete checklist: {checklist_flag}") + + log_dict = logdata.asdict + if notes: + log_dict["notes"] = notes + + scrobble = Scrobble( + user=user, + timestamp=timestamp, + source="Birding CSV Import", + birding_location=location, + log=log_dict, + played_to_completion=True, + in_progress=False, + media_type=Scrobble.MediaType.BIRDING_LOCATION, + ) + existing = Scrobble.objects.filter( + timestamp=timestamp, + birding_location=location, + user=user, + ).first() + if existing: + logger.debug(f"Skipping existing scrobble for {location}") + continue + new_scrobbles.append(scrobble) + + created = Scrobble.objects.bulk_create(new_scrobbles) + logger.info(f"Created {len(created)} birding scrobbles") + return created diff --git a/vrobbler/apps/birds/migrations/0002_birdingcsvimport.py b/vrobbler/apps/birds/migrations/0002_birdingcsvimport.py new file mode 100644 index 0000000..f83968b --- /dev/null +++ b/vrobbler/apps/birds/migrations/0002_birdingcsvimport.py @@ -0,0 +1,67 @@ +# Generated by Django 4.2.29 on 2026-05-15 15:41 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import django_extensions.db.fields +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("birds", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="BirdingCSVImport", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name="created" + ), + ), + ( + "modified", + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name="modified" + ), + ), + ("uuid", models.UUIDField(default=uuid.uuid4, editable=False)), + ("processing_started", models.DateTimeField(blank=True, null=True)), + ("processed_finished", models.DateTimeField(blank=True, null=True)), + ("process_log", models.TextField(blank=True, null=True)), + ("process_count", models.IntegerField(blank=True, null=True)), + ( + "csv_file", + models.FileField( + blank=True, null=True, upload_to="birding-csv-uploads/" + ), + ), + ( + "user", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "verbose_name": "Birding CSV Import", + }, + ), + ] diff --git a/vrobbler/apps/birds/models.py b/vrobbler/apps/birds/models.py index 57e4d03..07be040 100644 --- a/vrobbler/apps/birds/models.py +++ b/vrobbler/apps/birds/models.py @@ -4,8 +4,12 @@ from functools import cached_property from typing import Optional from uuid import uuid4 +from django.conf import settings +from django.contrib.auth import get_user_model +from django.core.files import File from django.db import models from django.urls import reverse +from django.utils import timezone from django_extensions.db.models import TimeStampedModel from imagekit.models import ImageSpecField from imagekit.processors import ResizeToFit @@ -13,6 +17,8 @@ from locations.models import GeoLocation from scrobbles.dataclasses import BaseLogData, WithPeopleLogData from scrobbles.mixins import ScrobblableConstants, ScrobblableMixin +User = get_user_model() + logger = logging.getLogger(__name__) BNULL = {"blank": True, "null": True} @@ -179,3 +185,88 @@ class BirdingLocation(ScrobblableMixin): if not location: location = cls.objects.create(title=title) return location + + +class BirdingCSVImport(TimeStampedModel): + user = models.ForeignKey(User, on_delete=models.DO_NOTHING, **BNULL) + uuid = models.UUIDField(editable=False, default=uuid4) + processing_started = models.DateTimeField(**BNULL) + processed_finished = models.DateTimeField(**BNULL) + process_log = models.TextField(**BNULL) + process_count = models.IntegerField(**BNULL) + csv_file = models.FileField(upload_to="birding-csv-uploads/", **BNULL) + + class Meta: + verbose_name = "Birding CSV Import" + + def __str__(self): + return f"Birding import on {self.human_start}" + + @property + def human_start(self): + start = "Unknown" + if self.processing_started: + start = self.processing_started.strftime("%B %d, %Y at %H:%M") + return start + + @property + def import_type(self): + return "Birding CSV" + + def get_absolute_url(self): + return reverse("birds:csv_import_detail", kwargs={"slug": self.uuid}) + + @property + def upload_file_path(self): + if getattr(settings, "USE_S3_STORAGE"): + path = self.csv_file.url + else: + path = self.csv_file.path + return path + + def mark_started(self): + self.processing_started = timezone.now() + self.save(update_fields=["processing_started"]) + + def mark_finished(self): + self.processed_finished = timezone.now() + self.save(update_fields=["processed_finished"]) + + def record_log(self, scrobbles): + self.process_log = "" + if not scrobbles: + self.process_count = 0 + self.save(update_fields=["process_log", "process_count"]) + return + for count, scrobble in enumerate(scrobbles): + scrobble_str = ( + f"{scrobble.id}\t{scrobble.timestamp}\t{scrobble.media_obj}" + ) + log_line = f"{scrobble_str}" + if count > 0: + log_line = "\n" + log_line + self.process_log += log_line + self.process_count = len(scrobbles) + self.save(update_fields=["process_log", "process_count"]) + + def scrobbles(self): + from scrobbles.models import Scrobble + + scrobble_ids = [] + if self.process_log: + for line in self.process_log.split("\n"): + sid = line.split("\t")[0] + if sid: + scrobble_ids.append(sid) + return Scrobble.objects.filter(id__in=scrobble_ids) + + def process(self, force=False): + if self.processed_finished and not force: + logger.info(f"{self} already processed on {self.processed_finished}") + return + from birds.importer import import_birding_csv + + self.mark_started() + scrobbles = import_birding_csv(self.upload_file_path, self.user_id) + self.record_log(scrobbles) + self.mark_finished() diff --git a/vrobbler/apps/birds/urls.py b/vrobbler/apps/birds/urls.py index 4fc8fd8..18684b1 100644 --- a/vrobbler/apps/birds/urls.py +++ b/vrobbler/apps/birds/urls.py @@ -24,4 +24,14 @@ urlpatterns = [ views.BirdDetailView.as_view(), name="bird_detail", ), + path( + "upload/birding-csv/", + views.BirdingCSVImportCreateView.as_view(), + name="csv-upload", + ), + path( + "imports/birding-csv//", + views.BirdingCSVImportDetailView.as_view(), + name="csv_import_detail", + ), ] diff --git a/vrobbler/apps/birds/views.py b/vrobbler/apps/birds/views.py index f2dc725..0a82d98 100644 --- a/vrobbler/apps/birds/views.py +++ b/vrobbler/apps/birds/views.py @@ -1,6 +1,13 @@ -from birds.models import Bird, BirdingLocation +from birds.models import Bird, BirdingCSVImport, BirdingLocation +from django.contrib.auth.mixins import LoginRequiredMixin +from django.http import HttpResponseRedirect +from django.urls import reverse_lazy from django.views import generic -from scrobbles.views import ScrobbleableDetailView, ScrobbleableListView +from scrobbles.views import ( + ScrobbleableDetailView, + ScrobbleableListView, + JsonableResponseMixin, +) class BirdingLocationListView(ScrobbleableListView): @@ -20,3 +27,33 @@ class BirdListView(generic.ListView): class BirdDetailView(generic.DetailView): model = Bird slug_field = "uuid" + + +class BirdingCSVImportCreateView( + LoginRequiredMixin, JsonableResponseMixin, generic.CreateView +): + model = BirdingCSVImport + fields = ["csv_file"] + template_name = "scrobbles/upload_form.html" + success_url = reverse_lazy("vrobbler-home") + + def form_valid(self, form): + self.object = form.save(commit=False) + self.object.user = self.request.user + self.object.save() + self.object.process() + return HttpResponseRedirect(self.request.META.get("HTTP_REFERER")) + + +class BirdingCSVImportDetailView(generic.DetailView): + model = BirdingCSVImport + slug_field = "uuid" + template_name = "scrobbles/import_detail.html" + + def get_queryset(self): + return super().get_queryset().filter(user=self.request.user) + + def get_context_data(self, **kwargs): + context_data = super().get_context_data(**kwargs) + context_data["title"] = "Birding CSV Import" + return context_data diff --git a/vrobbler/apps/scrobbles/views.py b/vrobbler/apps/scrobbles/views.py index 5e211ec..293402f 100644 --- a/vrobbler/apps/scrobbles/views.py +++ b/vrobbler/apps/scrobbles/views.py @@ -451,6 +451,11 @@ class ScrobbleImportListView(TemplateView): context_data["retroarch_imports"] = RetroarchImport.objects.filter( user=self.request.user, ).order_by("-processing_started")[:10] + context_data["birding_csv_imports"] = apps.get_model( + "birds", "BirdingCSVImport" + ).objects.filter( + user=self.request.user, + ).order_by("-processing_started")[:10] return context_data diff --git a/vrobbler/templates/scrobbles/import_list.html b/vrobbler/templates/scrobbles/import_list.html index ca360c0..bff0298 100644 --- a/vrobbler/templates/scrobbles/import_list.html +++ b/vrobbler/templates/scrobbles/import_list.html @@ -103,4 +103,29 @@ {% endif %} + +{% if birding_csv_imports %} +
+

Birding CSV

+
+ + + + + + + + + {% for obj in birding_csv_imports %} + + + + + + {% endfor %} + +
StartedFinishedScrobbles
{{obj.human_start}}{{obj.processed_finished}}{{obj.process_count}}
+
+
+{% endif %} {% endblock %}