[trails] Add auto GPX and FIT file importing
This commit is contained in:
BIN
data/sample-trail.fit
Normal file
BIN
data/sample-trail.fit
Normal file
Binary file not shown.
3360
data/sample_trail.gpx
Normal file
3360
data/sample_trail.gpx
Normal file
File diff suppressed because one or more lines are too long
25
poetry.lock
generated
25
poetry.lock
generated
@ -1796,6 +1796,17 @@ files = [
|
||||
packaging = ">=20"
|
||||
platformdirs = ">=4.3.6"
|
||||
|
||||
[[package]]
|
||||
name = "fitparse"
|
||||
version = "1.2.0"
|
||||
description = "Python library to parse ANT/Garmin .FIT files"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "fitparse-1.2.0.tar.gz", hash = "sha256:2d691022452dea6dabad13cc6e017ca467fe8a3a895cd3ac67a50a7bb716b4a9"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "flake8"
|
||||
version = "7.3.0"
|
||||
@ -1953,6 +1964,18 @@ files = [
|
||||
{file = "frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gpxpy"
|
||||
version = "1.6.2"
|
||||
description = "GPX file parser and GPS track manipulation library"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "gpxpy-1.6.2-py3-none-any.whl", hash = "sha256:289bc2d80f116c988d0a1e763fda22838f83005573ece2bbc6521817b26fb40a"},
|
||||
{file = "gpxpy-1.6.2.tar.gz", hash = "sha256:a72c484b97ec42b80834353b029cc8ee1b79f0ffca1179b2210bb3baf26c01ae"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gunicorn"
|
||||
version = "20.1.0"
|
||||
@ -5981,4 +6004,4 @@ cffi = ["cffi (>=1.17,<2.0) ; platform_python_implementation != \"PyPy\" and pyt
|
||||
[metadata]
|
||||
lock-version = "2.1"
|
||||
python-versions = ">=3.11,<3.14"
|
||||
content-hash = "40ffa724f63298ca37279d7f72dc541f372d02244481c225fa748940da8ab468"
|
||||
content-hash = "395bc627d9e2fa0434082fc574693e349cf0c604c47e8554b9a50a28bdd273d0"
|
||||
|
||||
@ -59,6 +59,8 @@ feedparser = "^6.0.12"
|
||||
titlecase = "^2.4.1"
|
||||
bgg-api = "^1.1.13"
|
||||
recipe-scrapers = "^15.11.0"
|
||||
gpxpy = "^1.6.2"
|
||||
fitparse = "^1.2.0"
|
||||
|
||||
[tool.poetry.group.test]
|
||||
optional = true
|
||||
|
||||
0
tests/trails_tests/__init__.py
Normal file
0
tests/trails_tests/__init__.py
Normal file
161
tests/trails_tests/test_gpx_importer.py
Normal file
161
tests/trails_tests/test_gpx_importer.py
Normal file
@ -0,0 +1,161 @@
|
||||
import os
|
||||
import tempfile
|
||||
from datetime import timedelta
|
||||
|
||||
import pytest
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.core.files import File
|
||||
|
||||
from locations.models import GeoLocation
|
||||
from scrobbles.importers.trail_gpx import (
|
||||
import_trail_gpx,
|
||||
parse_trackpoints,
|
||||
)
|
||||
from scrobbles.models import Scrobble, TrailGPXImport
|
||||
from trails.models import Trail
|
||||
|
||||
User = get_user_model()
|
||||
|
||||
SAMPLE_GPX = os.path.join(
|
||||
os.path.dirname(__file__), "..", "..", "data", "sample_trail.gpx"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def user(db):
|
||||
return User.objects.create(email="trailblazer@example.com")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_gpx_path():
|
||||
return SAMPLE_GPX
|
||||
|
||||
|
||||
class TestParseTrackpoints:
|
||||
def test_parses_gpx(self, sample_gpx_path):
|
||||
points, name = parse_trackpoints(sample_gpx_path)
|
||||
assert len(points) == 837
|
||||
assert name == "Morning Run ⛅"
|
||||
lat, lon, ele, t = points[0]
|
||||
assert round(lat, 6) == 34.190598
|
||||
assert round(lon, 6) == -118.844015
|
||||
assert ele == 305.3
|
||||
assert t is not None
|
||||
|
||||
def test_first_and_last_times(self, sample_gpx_path):
|
||||
points, _ = parse_trackpoints(sample_gpx_path)
|
||||
first_time = points[0][3]
|
||||
last_time = points[-1][3]
|
||||
duration = (last_time - first_time).total_seconds()
|
||||
assert duration == pytest.approx(3770, abs=5)
|
||||
|
||||
|
||||
class TestImportTrailGPX:
|
||||
def test_creates_trail(self, user, sample_gpx_path):
|
||||
import_trail_gpx(sample_gpx_path, user.id)
|
||||
assert Trail.objects.filter(title="Morning Run ⛅").exists()
|
||||
|
||||
def test_creates_geolocation(self, user, sample_gpx_path):
|
||||
import_trail_gpx(sample_gpx_path, user.id)
|
||||
assert GeoLocation.objects.filter(lat=34.190598, lon=-118.844015).exists()
|
||||
|
||||
def test_sets_trailhead(self, user, sample_gpx_path):
|
||||
import_trail_gpx(sample_gpx_path, user.id)
|
||||
trail = Trail.objects.filter(title="Morning Run ⛅").first()
|
||||
assert trail.trailhead_location is not None
|
||||
assert round(trail.trailhead_location.lat, 6) == 34.190598
|
||||
|
||||
def test_creates_scrobble(self, user, sample_gpx_path):
|
||||
import_trail_gpx(sample_gpx_path, user.id)
|
||||
assert Scrobble.objects.filter(source="GPX Import").count() == 1
|
||||
|
||||
def test_scrobble_timestamps(self, user, sample_gpx_path):
|
||||
import_trail_gpx(sample_gpx_path, user.id)
|
||||
scrobble = Scrobble.objects.filter(source="GPX Import").first()
|
||||
assert scrobble.timestamp.isoformat().startswith("2022-06-05T13:55:09")
|
||||
assert scrobble.stop_timestamp.isoformat().startswith("2022-06-05T14:57:59")
|
||||
assert scrobble.media_type == Scrobble.MediaType.TRAIL
|
||||
|
||||
def test_scrobble_has_trail_fk(self, user, sample_gpx_path):
|
||||
import_trail_gpx(sample_gpx_path, user.id)
|
||||
scrobble = Scrobble.objects.filter(source="GPX Import").first()
|
||||
assert scrobble.trail is not None
|
||||
assert scrobble.trail.title == "Morning Run ⛅"
|
||||
|
||||
def test_scrobble_has_gpx_file(self, user, sample_gpx_path):
|
||||
import_trail_gpx(sample_gpx_path, user.id)
|
||||
scrobble = Scrobble.objects.filter(source="GPX Import").first()
|
||||
assert scrobble.gpx_file
|
||||
assert scrobble.gpx_file.name.endswith(".gpx")
|
||||
|
||||
def test_lookup_existing_trail_by_trailhead(self, user, sample_gpx_path):
|
||||
geo = GeoLocation.objects.create(lat=34.190598, lon=-118.844015)
|
||||
trail = Trail.objects.create(title="Existing Trail", trailhead_location=geo)
|
||||
import_trail_gpx(sample_gpx_path, user.id)
|
||||
scrobble = Scrobble.objects.filter(source="GPX Import").first()
|
||||
assert scrobble.trail.id == trail.id
|
||||
|
||||
def test_dedup(self, user, sample_gpx_path):
|
||||
import_trail_gpx(sample_gpx_path, user.id)
|
||||
import_trail_gpx(sample_gpx_path, user.id)
|
||||
assert Scrobble.objects.filter(source="GPX Import").count() == 1
|
||||
|
||||
|
||||
class TestTrailGPXImportModel:
|
||||
def test_create_import_model(self, db, user, sample_gpx_path):
|
||||
imp = TrailGPXImport.objects.create(
|
||||
user=user,
|
||||
original_filename="test_trail.gpx",
|
||||
)
|
||||
assert imp.uuid is not None
|
||||
assert imp.import_type == "Trail GPX"
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_process_via_model(self, user, sample_gpx_path):
|
||||
imp = TrailGPXImport.objects.create(
|
||||
user=user,
|
||||
original_filename="Morning Run.gpx",
|
||||
)
|
||||
with open(sample_gpx_path, "rb") as f:
|
||||
imp.gpx_file.save("Morning Run.gpx", File(f), save=True)
|
||||
imp.process()
|
||||
imp.refresh_from_db()
|
||||
assert imp.process_count == 1
|
||||
assert imp.processed_finished is not None
|
||||
|
||||
|
||||
class TestFindByTrailhead:
|
||||
def test_exact_match(self, db):
|
||||
geo = GeoLocation.objects.create(lat=34.190598, lon=-118.844015)
|
||||
trail = Trail.objects.create(title="Test Trail", trailhead_location=geo)
|
||||
found = Trail.find_by_trailhead(34.190598, -118.844015)
|
||||
assert found == trail
|
||||
|
||||
def test_within_tolerance(self, db):
|
||||
geo = GeoLocation.objects.create(lat=34.190598, lon=-118.844015)
|
||||
trail = Trail.objects.create(title="Nearby Trail", trailhead_location=geo)
|
||||
found = Trail.find_by_trailhead(34.191000, -118.844000, tolerance_m=100)
|
||||
assert found == trail
|
||||
|
||||
def test_beyond_tolerance(self, db):
|
||||
geo = GeoLocation.objects.create(lat=34.190598, lon=-118.844015)
|
||||
Trail.objects.create(title="Far Trail", trailhead_location=geo)
|
||||
found = Trail.find_by_trailhead(34.200000, -118.850000, tolerance_m=50)
|
||||
assert found is None
|
||||
|
||||
def test_no_trailhead_returns_none(self, db):
|
||||
Trail.objects.create(title="No Location")
|
||||
found = Trail.find_by_trailhead(34.190598, -118.844015)
|
||||
assert found is None
|
||||
|
||||
|
||||
class TestFindOrCreate:
|
||||
def test_find_existing(self, db):
|
||||
Trail.objects.create(title="Existing Trail")
|
||||
trail = Trail.find_or_create("Existing Trail")
|
||||
assert trail.title == "Existing Trail"
|
||||
|
||||
def test_create_new(self, db):
|
||||
trail = Trail.find_or_create("New Trail")
|
||||
assert trail.title == "New Trail"
|
||||
assert Trail.objects.count() == 1
|
||||
@ -7,6 +7,7 @@ from scrobbles.models import (
|
||||
RetroarchImport,
|
||||
ScaleCSVImport,
|
||||
Scrobble,
|
||||
TrailGPXImport,
|
||||
)
|
||||
from scrobbles.mixins import Genre
|
||||
|
||||
@ -83,6 +84,11 @@ class ScaleCSVImportAdmin(ImportBaseAdmin):
|
||||
...
|
||||
|
||||
|
||||
@admin.register(TrailGPXImport)
|
||||
class TrailGPXImportAdmin(ImportBaseAdmin):
|
||||
...
|
||||
|
||||
|
||||
@admin.register(Genre)
|
||||
class GenreAdmin(admin.ModelAdmin):
|
||||
list_display = (
|
||||
|
||||
178
vrobbler/apps/scrobbles/importers/trail_gpx.py
Normal file
178
vrobbler/apps/scrobbles/importers/trail_gpx.py
Normal file
@ -0,0 +1,178 @@
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
from datetime import timezone as dt_timezone
|
||||
from typing import Optional
|
||||
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.core.files import File
|
||||
|
||||
from locations.models import GeoLocation
|
||||
from scrobbles.models import Scrobble
|
||||
from trails.models import Trail
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
User = get_user_model()
|
||||
|
||||
|
||||
def parse_trackpoints(file_path):
|
||||
_, ext = os.path.splitext(file_path)
|
||||
ext = ext.lower()
|
||||
|
||||
if ext == ".gpx":
|
||||
return _parse_gpx(file_path)
|
||||
elif ext == ".fit":
|
||||
return _parse_fit(file_path)
|
||||
else:
|
||||
raise ValueError(f"Unsupported file type: {ext}")
|
||||
|
||||
|
||||
def _parse_gpx(file_path):
|
||||
import gpxpy
|
||||
|
||||
with open(file_path) as f:
|
||||
gpx = gpxpy.parse(f)
|
||||
|
||||
track_name = None
|
||||
points = []
|
||||
for track in gpx.tracks:
|
||||
if track.name and not track_name:
|
||||
track_name = track.name
|
||||
for seg in track.segments:
|
||||
for pt in seg.points:
|
||||
points.append((pt.latitude, pt.longitude, pt.elevation, pt.time))
|
||||
return points, track_name or os.path.splitext(os.path.basename(file_path))[0]
|
||||
|
||||
|
||||
def _parse_fit(file_path):
|
||||
import fitparse
|
||||
|
||||
fitfile = fitparse.FitFile(file_path)
|
||||
messages = list(fitfile.get_messages("record"))
|
||||
|
||||
points = []
|
||||
track_name = os.path.splitext(os.path.basename(file_path))[0]
|
||||
|
||||
for msg in messages:
|
||||
lat = None
|
||||
lon = None
|
||||
ele = None
|
||||
t = None
|
||||
for field in msg:
|
||||
if field.name == "position_lat":
|
||||
lat = field.value * (180.0 / (2**31)) if field.value else None
|
||||
elif field.name == "position_long":
|
||||
lon = field.value * (180.0 / (2**31)) if field.value else None
|
||||
elif field.name == "altitude":
|
||||
ele = field.value
|
||||
elif field.name == "timestamp":
|
||||
t = field.value.replace(tzinfo=dt_timezone.utc) if field.value else None
|
||||
if lat is not None and lon is not None:
|
||||
points.append((lat, lon, ele, t))
|
||||
|
||||
session_msgs = list(fitfile.get_messages("session"))
|
||||
for msg in session_msgs:
|
||||
for field in msg:
|
||||
if field.name == "sport":
|
||||
track_name = str(field.value)
|
||||
break
|
||||
|
||||
return points, track_name
|
||||
|
||||
|
||||
def convert_fit_to_gpx(file_path):
|
||||
import gpxpy.gpx
|
||||
|
||||
points, track_name = _parse_fit(file_path)
|
||||
gpx = gpxpy.gpx.GPX()
|
||||
gpx_track = gpxpy.gpx.GPXTrack(name=track_name)
|
||||
gpx.tracks.append(gpx_track)
|
||||
gpx_seg = gpxpy.gpx.GPXTrackSegment()
|
||||
gpx_track.segments.append(gpx_seg)
|
||||
|
||||
for lat, lon, ele, t in points:
|
||||
gpx_seg.points.append(gpxpy.gpx.GPXTrackPoint(lat, lon, elevation=ele, time=t))
|
||||
|
||||
tmp = tempfile.NamedTemporaryFile(mode="w", suffix=".gpx", delete=False)
|
||||
tmp.write(gpx.to_xml())
|
||||
tmp.close()
|
||||
return tmp.name
|
||||
|
||||
|
||||
def import_trail_gpx(file_path, user_id, original_filename=None):
|
||||
user = User.objects.get(id=user_id)
|
||||
new_scrobbles = []
|
||||
|
||||
points, track_name = parse_trackpoints(file_path)
|
||||
|
||||
if not points:
|
||||
logger.warning(f"No trackpoints found in {file_path}")
|
||||
return []
|
||||
|
||||
first_lat, first_lon, _, first_time = points[0]
|
||||
_, _, _, last_time = points[-1]
|
||||
|
||||
if first_time is None:
|
||||
logger.warning(f"No timestamps in {file_path}")
|
||||
return []
|
||||
|
||||
geo, _ = GeoLocation.objects.get_or_create(
|
||||
lat=round(first_lat, 6),
|
||||
lon=round(first_lon, 6),
|
||||
defaults={"altitude": None},
|
||||
)
|
||||
|
||||
trail = Trail.find_by_trailhead(first_lat, first_lon, tolerance_m=100)
|
||||
if not trail:
|
||||
trail = Trail.find_or_create(track_name)
|
||||
trail.trailhead_location = geo
|
||||
trail.save(update_fields=["trailhead_location"])
|
||||
|
||||
timestamp = first_time
|
||||
stop_timestamp = last_time
|
||||
|
||||
existing = Scrobble.objects.filter(
|
||||
timestamp=timestamp,
|
||||
trail=trail,
|
||||
user=user,
|
||||
).first()
|
||||
if existing:
|
||||
logger.debug(f"Skipping existing scrobble for trail {trail}")
|
||||
return []
|
||||
|
||||
scrobble = Scrobble(
|
||||
user=user,
|
||||
timestamp=timestamp,
|
||||
stop_timestamp=stop_timestamp,
|
||||
source="GPX Import",
|
||||
trail=trail,
|
||||
log={},
|
||||
played_to_completion=True,
|
||||
in_progress=False,
|
||||
media_type=Scrobble.MediaType.TRAIL,
|
||||
)
|
||||
|
||||
_, ext = os.path.splitext(file_path)
|
||||
if ext.lower() == ".fit":
|
||||
gpx_path = convert_fit_to_gpx(file_path)
|
||||
with open(gpx_path, "rb") as f:
|
||||
scrobble.gpx_file.save(
|
||||
f"{original_filename or 'trail'}.gpx",
|
||||
File(f),
|
||||
save=False,
|
||||
)
|
||||
os.unlink(gpx_path)
|
||||
else:
|
||||
with open(file_path, "rb") as f:
|
||||
scrobble.gpx_file.save(
|
||||
original_filename or os.path.basename(file_path),
|
||||
File(f),
|
||||
save=False,
|
||||
)
|
||||
|
||||
new_scrobbles.append(scrobble)
|
||||
|
||||
created = Scrobble.objects.bulk_create(new_scrobbles)
|
||||
logger.info(f"Created {len(created)} trail scrobbles")
|
||||
return created
|
||||
@ -1,9 +1,11 @@
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
from books.koreader import fetch_file_from_webdav
|
||||
from profiles.models import UserProfile
|
||||
from scrobbles.models import KoReaderImport
|
||||
from scrobbles.tasks import process_koreader_import
|
||||
from scrobbles.models import KoReaderImport, TrailGPXImport
|
||||
from scrobbles.tasks import process_koreader_import, process_trail_gpx_import
|
||||
from scrobbles.utils import get_file_md5_hash
|
||||
from webdav.client import get_webdav_client
|
||||
|
||||
@ -13,7 +15,6 @@ logger = logging.getLogger(__name__)
|
||||
def import_from_webdav_for_all_users(restart=False):
|
||||
"""Grab a list of all users with WebDAV enabled and kickoff imports for them"""
|
||||
|
||||
# WebDavImport = apps.get_model("scrobbles", "WebDavImport")
|
||||
webdav_enabled_user_ids = UserProfile.objects.filter(
|
||||
webdav_url__isnull=False,
|
||||
webdav_user__isnull=False,
|
||||
@ -23,6 +24,7 @@ def import_from_webdav_for_all_users(restart=False):
|
||||
logger.info(f"Start import of {webdav_enabled_user_ids.count()} webdav accounts")
|
||||
|
||||
koreader_import_count = 0
|
||||
trail_gpx_import_count = 0
|
||||
|
||||
for user_id in webdav_enabled_user_ids:
|
||||
webdav_client = get_webdav_client(user_id)
|
||||
@ -78,4 +80,54 @@ def import_from_webdav_for_all_users(restart=False):
|
||||
|
||||
process_koreader_import.delay(koreader_import.id)
|
||||
koreader_import_count += 1
|
||||
return koreader_import_count
|
||||
|
||||
trail_gpx_import_count += scan_webdav_for_gpx(webdav_client, user_id)
|
||||
|
||||
return koreader_import_count, trail_gpx_import_count
|
||||
|
||||
|
||||
def scan_webdav_for_gpx(webdav_client, user_id):
|
||||
try:
|
||||
webdav_client.info("gpx/")
|
||||
except:
|
||||
logger.info("No gpx/ directory on webdav", extra={"user_id": user_id})
|
||||
return 0
|
||||
|
||||
try:
|
||||
files = webdav_client.list("gpx/")
|
||||
except Exception as e:
|
||||
logger.warning("Could not list gpx/", extra={"user_id": user_id, "error": str(e)})
|
||||
return 0
|
||||
|
||||
gpx_extensions = (".gpx", ".fit")
|
||||
new_imports = 0
|
||||
already_imported = set(
|
||||
TrailGPXImport.objects.filter(user_id=user_id)
|
||||
.values_list("original_filename", flat=True)
|
||||
)
|
||||
|
||||
for fname in files:
|
||||
fname = os.path.basename(fname)
|
||||
if not fname.lower().endswith(gpx_extensions):
|
||||
continue
|
||||
if fname in already_imported:
|
||||
logger.debug(f"Skipping already-imported {fname}")
|
||||
continue
|
||||
|
||||
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=fname)
|
||||
try:
|
||||
webdav_client.download_sync(remote_path=f"gpx/{fname}", local_path=tmp.name)
|
||||
imp = TrailGPXImport.objects.create(
|
||||
user_id=user_id,
|
||||
original_filename=fname,
|
||||
)
|
||||
with open(tmp.name, "rb") as f:
|
||||
imp.gpx_file.save(fname, f, save=True)
|
||||
process_trail_gpx_import.delay(imp.id)
|
||||
new_imports += 1
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to import {fname}: {e}")
|
||||
finally:
|
||||
os.unlink(tmp.name)
|
||||
|
||||
return new_imports
|
||||
|
||||
@ -14,5 +14,5 @@ class Command(BaseCommand):
|
||||
restart = False
|
||||
if options["restart"]:
|
||||
restart = True
|
||||
count = webdav.import_from_webdav_for_all_users(restart=restart)
|
||||
print(f"Started {count} WeDAV imports")
|
||||
ko_count, gpx_count = webdav.import_from_webdav_for_all_users(restart=restart)
|
||||
print(f"Started {ko_count} KOReader and {gpx_count} Trail GPX WebDAV imports")
|
||||
|
||||
74
vrobbler/apps/scrobbles/migrations/0078_trailgpximport.py
Normal file
74
vrobbler/apps/scrobbles/migrations/0078_trailgpximport.py
Normal file
@ -0,0 +1,74 @@
|
||||
# Generated by Django 4.2.29 on 2026-05-21 14:36
|
||||
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
import django.db.models.deletion
|
||||
import django_extensions.db.fields
|
||||
import scrobbles.models
|
||||
import uuid
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
("scrobbles", "0077_scalecsvimport"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="TrailGPXImport",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True,
|
||||
primary_key=True,
|
||||
serialize=False,
|
||||
verbose_name="ID",
|
||||
),
|
||||
),
|
||||
(
|
||||
"created",
|
||||
django_extensions.db.fields.CreationDateTimeField(
|
||||
auto_now_add=True, verbose_name="created"
|
||||
),
|
||||
),
|
||||
(
|
||||
"modified",
|
||||
django_extensions.db.fields.ModificationDateTimeField(
|
||||
auto_now=True, verbose_name="modified"
|
||||
),
|
||||
),
|
||||
("uuid", models.UUIDField(default=uuid.uuid4, editable=False)),
|
||||
("processing_started", models.DateTimeField(blank=True, null=True)),
|
||||
("processed_finished", models.DateTimeField(blank=True, null=True)),
|
||||
("process_log", models.TextField(blank=True, null=True)),
|
||||
("process_count", models.IntegerField(blank=True, null=True)),
|
||||
(
|
||||
"gpx_file",
|
||||
models.FileField(
|
||||
blank=True,
|
||||
null=True,
|
||||
upload_to=scrobbles.models.TrailGPXImport.get_path,
|
||||
),
|
||||
),
|
||||
(
|
||||
"original_filename",
|
||||
models.CharField(blank=True, max_length=255, null=True),
|
||||
),
|
||||
(
|
||||
"user",
|
||||
models.ForeignKey(
|
||||
blank=True,
|
||||
null=True,
|
||||
on_delete=django.db.models.deletion.DO_NOTHING,
|
||||
to=settings.AUTH_USER_MODEL,
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"verbose_name": "Trail GPX Import",
|
||||
},
|
||||
),
|
||||
]
|
||||
@ -299,6 +299,48 @@ class ScaleCSVImport(BaseFileImportMixin):
|
||||
self.mark_finished()
|
||||
|
||||
|
||||
class TrailGPXImport(BaseFileImportMixin):
|
||||
class Meta:
|
||||
verbose_name = "Trail GPX Import"
|
||||
|
||||
@property
|
||||
def import_type(self) -> str:
|
||||
return "Trail GPX"
|
||||
|
||||
def get_absolute_url(self):
|
||||
return reverse("scrobbles:trail-gpx-import-detail", kwargs={"slug": self.uuid})
|
||||
|
||||
def get_path(instance, filename):
|
||||
extension = filename.split(".")[-1]
|
||||
uuid = instance.uuid
|
||||
return f"trail-gpx-uploads/{uuid}.{extension}"
|
||||
|
||||
@property
|
||||
def upload_file_path(self):
|
||||
if getattr(settings, "USE_S3_STORAGE"):
|
||||
path = self.gpx_file.url
|
||||
else:
|
||||
path = self.gpx_file.path
|
||||
return path
|
||||
|
||||
gpx_file = models.FileField(upload_to=get_path, **BNULL)
|
||||
original_filename = models.CharField(max_length=255, **BNULL)
|
||||
|
||||
def process(self, force=False):
|
||||
from scrobbles.importers.trail_gpx import import_trail_gpx
|
||||
|
||||
if self.processed_finished and not force:
|
||||
logger.info(f"{self} already processed on {self.processed_finished}")
|
||||
return
|
||||
|
||||
self.mark_started()
|
||||
scrobbles = import_trail_gpx(
|
||||
self.upload_file_path, self.user.id, self.original_filename
|
||||
)
|
||||
self.record_log(scrobbles)
|
||||
self.mark_finished()
|
||||
|
||||
|
||||
class LastFmImport(BaseFileImportMixin):
|
||||
class Meta:
|
||||
verbose_name = "Last.FM Import"
|
||||
|
||||
@ -133,6 +133,16 @@ def process_koreader_import(import_id):
|
||||
koreader_import.process()
|
||||
|
||||
|
||||
@shared_task
|
||||
def process_trail_gpx_import(import_id):
|
||||
TrailGPXImport = apps.get_model("scrobbles", "TrailGPXImport")
|
||||
trail_gpx_import = TrailGPXImport.objects.filter(id=import_id).first()
|
||||
if not trail_gpx_import:
|
||||
logger.warn(f"TrailGPXImport not found with id {import_id}")
|
||||
return
|
||||
trail_gpx_import.process()
|
||||
|
||||
|
||||
@shared_task
|
||||
def create_yesterdays_charts():
|
||||
"""Build/update charts for all users starting from last known record."""
|
||||
|
||||
@ -62,6 +62,11 @@ urlpatterns = [
|
||||
views.ScaleCSVImportCreateView.as_view(),
|
||||
name="scale-csv-upload",
|
||||
),
|
||||
path(
|
||||
"upload/trail-gpx/",
|
||||
views.TrailGPXImportCreateView.as_view(),
|
||||
name="trail-gpx-upload",
|
||||
),
|
||||
path(
|
||||
"lastfm-import/",
|
||||
views.lastfm_import,
|
||||
@ -126,6 +131,11 @@ urlpatterns = [
|
||||
views.ScrobbleScaleCSVImportDetailView.as_view(),
|
||||
name="scale-csv-import-detail",
|
||||
),
|
||||
path(
|
||||
"imports/trail-gpx/<slug:slug>/",
|
||||
views.ScrobbleTrailGPXImportDetailView.as_view(),
|
||||
name="trail-gpx-import-detail",
|
||||
),
|
||||
path(
|
||||
"long-plays/",
|
||||
views.ScrobbleLongPlaysView.as_view(),
|
||||
|
||||
@ -74,11 +74,13 @@ from scrobbles.models import (
|
||||
RetroarchImport,
|
||||
ScaleCSVImport,
|
||||
Scrobble,
|
||||
TrailGPXImport,
|
||||
)
|
||||
from scrobbles.scrobblers import *
|
||||
from scrobbles.tasks import (
|
||||
process_koreader_import,
|
||||
process_lastfm_import,
|
||||
process_trail_gpx_import,
|
||||
process_tsv_import,
|
||||
)
|
||||
from scrobbles.utils import (
|
||||
@ -465,6 +467,9 @@ class ScrobbleImportListView(TemplateView):
|
||||
context_data["scale_csv_imports"] = ScaleCSVImport.objects.filter(
|
||||
user=self.request.user,
|
||||
).order_by("-processing_started")[:10]
|
||||
context_data["trail_gpx_imports"] = TrailGPXImport.objects.filter(
|
||||
user=self.request.user,
|
||||
).order_by("-processing_started")[:10]
|
||||
return context_data
|
||||
|
||||
|
||||
@ -488,6 +493,8 @@ class BaseScrobbleImportDetailView(DetailView):
|
||||
title = "Retroarch Import"
|
||||
if self.model == ScaleCSVImport:
|
||||
title = "Scale CSV Import"
|
||||
if self.model == TrailGPXImport:
|
||||
title = "Trail GPX Import"
|
||||
context_data["title"] = title
|
||||
return context_data
|
||||
|
||||
@ -512,6 +519,10 @@ class ScrobbleScaleCSVImportDetailView(BaseScrobbleImportDetailView):
|
||||
model = ScaleCSVImport
|
||||
|
||||
|
||||
class ScrobbleTrailGPXImportDetailView(BaseScrobbleImportDetailView):
|
||||
model = TrailGPXImport
|
||||
|
||||
|
||||
class ManualScrobbleView(FormView):
|
||||
form_class = ScrobbleForm
|
||||
template_name = "scrobbles/manual_form.html"
|
||||
@ -600,6 +611,23 @@ class ScaleCSVImportCreateView(
|
||||
return HttpResponseRedirect(self.request.META.get("HTTP_REFERER"))
|
||||
|
||||
|
||||
class TrailGPXImportCreateView(
|
||||
LoginRequiredMixin, JsonableResponseMixin, CreateView
|
||||
):
|
||||
model = TrailGPXImport
|
||||
fields = ["gpx_file"]
|
||||
template_name = "scrobbles/upload_form.html"
|
||||
success_url = reverse_lazy("vrobbler-home")
|
||||
|
||||
def form_valid(self, form):
|
||||
self.object = form.save(commit=False)
|
||||
self.object.user = self.request.user
|
||||
self.object.original_filename = form.cleaned_data["gpx_file"].name
|
||||
self.object.save()
|
||||
process_trail_gpx_import.delay(self.object.id)
|
||||
return HttpResponseRedirect(self.request.META.get("HTTP_REFERER"))
|
||||
|
||||
|
||||
@api_view(["GET"])
|
||||
@permission_classes([IsAuthenticated])
|
||||
def lastfm_import(request):
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
from dataclasses import dataclass
|
||||
from math import asin, cos, radians, sin, sqrt
|
||||
from typing import Optional
|
||||
|
||||
from django.apps import apps
|
||||
@ -12,6 +13,14 @@ from scrobbles.mixins import ScrobblableConstants, ScrobblableMixin
|
||||
BNULL = {"blank": True, "null": True}
|
||||
|
||||
|
||||
def haversine(lat1, lon1, lat2, lon2):
|
||||
R = 6371000
|
||||
dlat = radians(lat2 - lat1)
|
||||
dlon = radians(lon2 - lon1)
|
||||
a = sin(dlat / 2) ** 2 + cos(radians(lat1)) * cos(radians(lat2)) * sin(dlon / 2) ** 2
|
||||
return R * 2 * asin(sqrt(a))
|
||||
|
||||
|
||||
@dataclass
|
||||
class TrailLogData(BaseLogData, WithPeopleLogData):
|
||||
effort: Optional[str] = None
|
||||
@ -68,7 +77,25 @@ class Trail(ScrobblableMixin):
|
||||
|
||||
@classmethod
|
||||
def find_or_create(cls, title: str) -> "Trail":
|
||||
return cls.objects.filter(title=title).first()
|
||||
trail = cls.objects.filter(title__iexact=title).first()
|
||||
if not trail:
|
||||
trail = cls.objects.create(title=title)
|
||||
return trail
|
||||
|
||||
@classmethod
|
||||
def find_by_trailhead(cls, lat, lon, tolerance_m=100):
|
||||
candidates = cls.objects.filter(
|
||||
trailhead_location__isnull=False,
|
||||
).select_related("trailhead_location")
|
||||
best = None
|
||||
best_dist = float("inf")
|
||||
for trail in candidates:
|
||||
loc = trail.trailhead_location
|
||||
d = haversine(lat, lon, loc.lat, loc.lon)
|
||||
if d < best_dist and d <= tolerance_m:
|
||||
best = trail
|
||||
best_dist = d
|
||||
return best
|
||||
|
||||
def scrobbles(self, user_id):
|
||||
Scrobble = apps.get_model("scrobbles", "Scrobble")
|
||||
|
||||
@ -160,4 +160,35 @@
|
||||
<input type="submit" value="Import Scale CSV">
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div class="row">
|
||||
<h3>Trail GPX</h3>
|
||||
{% if trail_gpx_imports %}
|
||||
<div class="table-responsive">
|
||||
<table class="table table-striped table-sm">
|
||||
<thead>
|
||||
<tr>
|
||||
<th scope="col">Started</th>
|
||||
<th scope="col">Finished</th>
|
||||
<th scope="col">Scrobbles</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for obj in trail_gpx_imports %}
|
||||
<tr>
|
||||
<td><a href="{{obj.get_absolute_url}}">{{obj.human_start}}</a></td>
|
||||
<td>{{obj.processed_finished}}</td>
|
||||
<td>{{obj.process_count}}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{% endif %}
|
||||
<form action="{% url 'scrobbles:trail-gpx-upload' %}" method="post" enctype="multipart/form-data" style="margin-top: 10px;">
|
||||
{% csrf_token %}
|
||||
<input type="file" name="gpx_file" accept=".gpx,.fit" required>
|
||||
<input type="submit" value="Import Trail GPX">
|
||||
</form>
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
||||
Reference in New Issue
Block a user