[trails] Add auto GPX and FIT file importing
This commit is contained in:
BIN
data/sample-trail.fit
Normal file
BIN
data/sample-trail.fit
Normal file
Binary file not shown.
3360
data/sample_trail.gpx
Normal file
3360
data/sample_trail.gpx
Normal file
File diff suppressed because one or more lines are too long
25
poetry.lock
generated
25
poetry.lock
generated
@ -1796,6 +1796,17 @@ files = [
|
|||||||
packaging = ">=20"
|
packaging = ">=20"
|
||||||
platformdirs = ">=4.3.6"
|
platformdirs = ">=4.3.6"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fitparse"
|
||||||
|
version = "1.2.0"
|
||||||
|
description = "Python library to parse ANT/Garmin .FIT files"
|
||||||
|
optional = false
|
||||||
|
python-versions = "*"
|
||||||
|
groups = ["main"]
|
||||||
|
files = [
|
||||||
|
{file = "fitparse-1.2.0.tar.gz", hash = "sha256:2d691022452dea6dabad13cc6e017ca467fe8a3a895cd3ac67a50a7bb716b4a9"},
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "flake8"
|
name = "flake8"
|
||||||
version = "7.3.0"
|
version = "7.3.0"
|
||||||
@ -1953,6 +1964,18 @@ files = [
|
|||||||
{file = "frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad"},
|
{file = "frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad"},
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "gpxpy"
|
||||||
|
version = "1.6.2"
|
||||||
|
description = "GPX file parser and GPS track manipulation library"
|
||||||
|
optional = false
|
||||||
|
python-versions = ">=3.6"
|
||||||
|
groups = ["main"]
|
||||||
|
files = [
|
||||||
|
{file = "gpxpy-1.6.2-py3-none-any.whl", hash = "sha256:289bc2d80f116c988d0a1e763fda22838f83005573ece2bbc6521817b26fb40a"},
|
||||||
|
{file = "gpxpy-1.6.2.tar.gz", hash = "sha256:a72c484b97ec42b80834353b029cc8ee1b79f0ffca1179b2210bb3baf26c01ae"},
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "gunicorn"
|
name = "gunicorn"
|
||||||
version = "20.1.0"
|
version = "20.1.0"
|
||||||
@ -5981,4 +6004,4 @@ cffi = ["cffi (>=1.17,<2.0) ; platform_python_implementation != \"PyPy\" and pyt
|
|||||||
[metadata]
|
[metadata]
|
||||||
lock-version = "2.1"
|
lock-version = "2.1"
|
||||||
python-versions = ">=3.11,<3.14"
|
python-versions = ">=3.11,<3.14"
|
||||||
content-hash = "40ffa724f63298ca37279d7f72dc541f372d02244481c225fa748940da8ab468"
|
content-hash = "395bc627d9e2fa0434082fc574693e349cf0c604c47e8554b9a50a28bdd273d0"
|
||||||
|
|||||||
@ -59,6 +59,8 @@ feedparser = "^6.0.12"
|
|||||||
titlecase = "^2.4.1"
|
titlecase = "^2.4.1"
|
||||||
bgg-api = "^1.1.13"
|
bgg-api = "^1.1.13"
|
||||||
recipe-scrapers = "^15.11.0"
|
recipe-scrapers = "^15.11.0"
|
||||||
|
gpxpy = "^1.6.2"
|
||||||
|
fitparse = "^1.2.0"
|
||||||
|
|
||||||
[tool.poetry.group.test]
|
[tool.poetry.group.test]
|
||||||
optional = true
|
optional = true
|
||||||
|
|||||||
0
tests/trails_tests/__init__.py
Normal file
0
tests/trails_tests/__init__.py
Normal file
161
tests/trails_tests/test_gpx_importer.py
Normal file
161
tests/trails_tests/test_gpx_importer.py
Normal file
@ -0,0 +1,161 @@
|
|||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
from datetime import timedelta
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from django.contrib.auth import get_user_model
|
||||||
|
from django.core.files import File
|
||||||
|
|
||||||
|
from locations.models import GeoLocation
|
||||||
|
from scrobbles.importers.trail_gpx import (
|
||||||
|
import_trail_gpx,
|
||||||
|
parse_trackpoints,
|
||||||
|
)
|
||||||
|
from scrobbles.models import Scrobble, TrailGPXImport
|
||||||
|
from trails.models import Trail
|
||||||
|
|
||||||
|
User = get_user_model()
|
||||||
|
|
||||||
|
SAMPLE_GPX = os.path.join(
|
||||||
|
os.path.dirname(__file__), "..", "..", "data", "sample_trail.gpx"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def user(db):
|
||||||
|
return User.objects.create(email="trailblazer@example.com")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def sample_gpx_path():
|
||||||
|
return SAMPLE_GPX
|
||||||
|
|
||||||
|
|
||||||
|
class TestParseTrackpoints:
|
||||||
|
def test_parses_gpx(self, sample_gpx_path):
|
||||||
|
points, name = parse_trackpoints(sample_gpx_path)
|
||||||
|
assert len(points) == 837
|
||||||
|
assert name == "Morning Run ⛅"
|
||||||
|
lat, lon, ele, t = points[0]
|
||||||
|
assert round(lat, 6) == 34.190598
|
||||||
|
assert round(lon, 6) == -118.844015
|
||||||
|
assert ele == 305.3
|
||||||
|
assert t is not None
|
||||||
|
|
||||||
|
def test_first_and_last_times(self, sample_gpx_path):
|
||||||
|
points, _ = parse_trackpoints(sample_gpx_path)
|
||||||
|
first_time = points[0][3]
|
||||||
|
last_time = points[-1][3]
|
||||||
|
duration = (last_time - first_time).total_seconds()
|
||||||
|
assert duration == pytest.approx(3770, abs=5)
|
||||||
|
|
||||||
|
|
||||||
|
class TestImportTrailGPX:
|
||||||
|
def test_creates_trail(self, user, sample_gpx_path):
|
||||||
|
import_trail_gpx(sample_gpx_path, user.id)
|
||||||
|
assert Trail.objects.filter(title="Morning Run ⛅").exists()
|
||||||
|
|
||||||
|
def test_creates_geolocation(self, user, sample_gpx_path):
|
||||||
|
import_trail_gpx(sample_gpx_path, user.id)
|
||||||
|
assert GeoLocation.objects.filter(lat=34.190598, lon=-118.844015).exists()
|
||||||
|
|
||||||
|
def test_sets_trailhead(self, user, sample_gpx_path):
|
||||||
|
import_trail_gpx(sample_gpx_path, user.id)
|
||||||
|
trail = Trail.objects.filter(title="Morning Run ⛅").first()
|
||||||
|
assert trail.trailhead_location is not None
|
||||||
|
assert round(trail.trailhead_location.lat, 6) == 34.190598
|
||||||
|
|
||||||
|
def test_creates_scrobble(self, user, sample_gpx_path):
|
||||||
|
import_trail_gpx(sample_gpx_path, user.id)
|
||||||
|
assert Scrobble.objects.filter(source="GPX Import").count() == 1
|
||||||
|
|
||||||
|
def test_scrobble_timestamps(self, user, sample_gpx_path):
|
||||||
|
import_trail_gpx(sample_gpx_path, user.id)
|
||||||
|
scrobble = Scrobble.objects.filter(source="GPX Import").first()
|
||||||
|
assert scrobble.timestamp.isoformat().startswith("2022-06-05T13:55:09")
|
||||||
|
assert scrobble.stop_timestamp.isoformat().startswith("2022-06-05T14:57:59")
|
||||||
|
assert scrobble.media_type == Scrobble.MediaType.TRAIL
|
||||||
|
|
||||||
|
def test_scrobble_has_trail_fk(self, user, sample_gpx_path):
|
||||||
|
import_trail_gpx(sample_gpx_path, user.id)
|
||||||
|
scrobble = Scrobble.objects.filter(source="GPX Import").first()
|
||||||
|
assert scrobble.trail is not None
|
||||||
|
assert scrobble.trail.title == "Morning Run ⛅"
|
||||||
|
|
||||||
|
def test_scrobble_has_gpx_file(self, user, sample_gpx_path):
|
||||||
|
import_trail_gpx(sample_gpx_path, user.id)
|
||||||
|
scrobble = Scrobble.objects.filter(source="GPX Import").first()
|
||||||
|
assert scrobble.gpx_file
|
||||||
|
assert scrobble.gpx_file.name.endswith(".gpx")
|
||||||
|
|
||||||
|
def test_lookup_existing_trail_by_trailhead(self, user, sample_gpx_path):
|
||||||
|
geo = GeoLocation.objects.create(lat=34.190598, lon=-118.844015)
|
||||||
|
trail = Trail.objects.create(title="Existing Trail", trailhead_location=geo)
|
||||||
|
import_trail_gpx(sample_gpx_path, user.id)
|
||||||
|
scrobble = Scrobble.objects.filter(source="GPX Import").first()
|
||||||
|
assert scrobble.trail.id == trail.id
|
||||||
|
|
||||||
|
def test_dedup(self, user, sample_gpx_path):
|
||||||
|
import_trail_gpx(sample_gpx_path, user.id)
|
||||||
|
import_trail_gpx(sample_gpx_path, user.id)
|
||||||
|
assert Scrobble.objects.filter(source="GPX Import").count() == 1
|
||||||
|
|
||||||
|
|
||||||
|
class TestTrailGPXImportModel:
|
||||||
|
def test_create_import_model(self, db, user, sample_gpx_path):
|
||||||
|
imp = TrailGPXImport.objects.create(
|
||||||
|
user=user,
|
||||||
|
original_filename="test_trail.gpx",
|
||||||
|
)
|
||||||
|
assert imp.uuid is not None
|
||||||
|
assert imp.import_type == "Trail GPX"
|
||||||
|
|
||||||
|
@pytest.mark.django_db(transaction=True)
|
||||||
|
def test_process_via_model(self, user, sample_gpx_path):
|
||||||
|
imp = TrailGPXImport.objects.create(
|
||||||
|
user=user,
|
||||||
|
original_filename="Morning Run.gpx",
|
||||||
|
)
|
||||||
|
with open(sample_gpx_path, "rb") as f:
|
||||||
|
imp.gpx_file.save("Morning Run.gpx", File(f), save=True)
|
||||||
|
imp.process()
|
||||||
|
imp.refresh_from_db()
|
||||||
|
assert imp.process_count == 1
|
||||||
|
assert imp.processed_finished is not None
|
||||||
|
|
||||||
|
|
||||||
|
class TestFindByTrailhead:
|
||||||
|
def test_exact_match(self, db):
|
||||||
|
geo = GeoLocation.objects.create(lat=34.190598, lon=-118.844015)
|
||||||
|
trail = Trail.objects.create(title="Test Trail", trailhead_location=geo)
|
||||||
|
found = Trail.find_by_trailhead(34.190598, -118.844015)
|
||||||
|
assert found == trail
|
||||||
|
|
||||||
|
def test_within_tolerance(self, db):
|
||||||
|
geo = GeoLocation.objects.create(lat=34.190598, lon=-118.844015)
|
||||||
|
trail = Trail.objects.create(title="Nearby Trail", trailhead_location=geo)
|
||||||
|
found = Trail.find_by_trailhead(34.191000, -118.844000, tolerance_m=100)
|
||||||
|
assert found == trail
|
||||||
|
|
||||||
|
def test_beyond_tolerance(self, db):
|
||||||
|
geo = GeoLocation.objects.create(lat=34.190598, lon=-118.844015)
|
||||||
|
Trail.objects.create(title="Far Trail", trailhead_location=geo)
|
||||||
|
found = Trail.find_by_trailhead(34.200000, -118.850000, tolerance_m=50)
|
||||||
|
assert found is None
|
||||||
|
|
||||||
|
def test_no_trailhead_returns_none(self, db):
|
||||||
|
Trail.objects.create(title="No Location")
|
||||||
|
found = Trail.find_by_trailhead(34.190598, -118.844015)
|
||||||
|
assert found is None
|
||||||
|
|
||||||
|
|
||||||
|
class TestFindOrCreate:
|
||||||
|
def test_find_existing(self, db):
|
||||||
|
Trail.objects.create(title="Existing Trail")
|
||||||
|
trail = Trail.find_or_create("Existing Trail")
|
||||||
|
assert trail.title == "Existing Trail"
|
||||||
|
|
||||||
|
def test_create_new(self, db):
|
||||||
|
trail = Trail.find_or_create("New Trail")
|
||||||
|
assert trail.title == "New Trail"
|
||||||
|
assert Trail.objects.count() == 1
|
||||||
@ -7,6 +7,7 @@ from scrobbles.models import (
|
|||||||
RetroarchImport,
|
RetroarchImport,
|
||||||
ScaleCSVImport,
|
ScaleCSVImport,
|
||||||
Scrobble,
|
Scrobble,
|
||||||
|
TrailGPXImport,
|
||||||
)
|
)
|
||||||
from scrobbles.mixins import Genre
|
from scrobbles.mixins import Genre
|
||||||
|
|
||||||
@ -83,6 +84,11 @@ class ScaleCSVImportAdmin(ImportBaseAdmin):
|
|||||||
...
|
...
|
||||||
|
|
||||||
|
|
||||||
|
@admin.register(TrailGPXImport)
|
||||||
|
class TrailGPXImportAdmin(ImportBaseAdmin):
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
@admin.register(Genre)
|
@admin.register(Genre)
|
||||||
class GenreAdmin(admin.ModelAdmin):
|
class GenreAdmin(admin.ModelAdmin):
|
||||||
list_display = (
|
list_display = (
|
||||||
|
|||||||
178
vrobbler/apps/scrobbles/importers/trail_gpx.py
Normal file
178
vrobbler/apps/scrobbles/importers/trail_gpx.py
Normal file
@ -0,0 +1,178 @@
|
|||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
from datetime import timezone as dt_timezone
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from django.contrib.auth import get_user_model
|
||||||
|
from django.core.files import File
|
||||||
|
|
||||||
|
from locations.models import GeoLocation
|
||||||
|
from scrobbles.models import Scrobble
|
||||||
|
from trails.models import Trail
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
User = get_user_model()
|
||||||
|
|
||||||
|
|
||||||
|
def parse_trackpoints(file_path):
|
||||||
|
_, ext = os.path.splitext(file_path)
|
||||||
|
ext = ext.lower()
|
||||||
|
|
||||||
|
if ext == ".gpx":
|
||||||
|
return _parse_gpx(file_path)
|
||||||
|
elif ext == ".fit":
|
||||||
|
return _parse_fit(file_path)
|
||||||
|
else:
|
||||||
|
raise ValueError(f"Unsupported file type: {ext}")
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_gpx(file_path):
|
||||||
|
import gpxpy
|
||||||
|
|
||||||
|
with open(file_path) as f:
|
||||||
|
gpx = gpxpy.parse(f)
|
||||||
|
|
||||||
|
track_name = None
|
||||||
|
points = []
|
||||||
|
for track in gpx.tracks:
|
||||||
|
if track.name and not track_name:
|
||||||
|
track_name = track.name
|
||||||
|
for seg in track.segments:
|
||||||
|
for pt in seg.points:
|
||||||
|
points.append((pt.latitude, pt.longitude, pt.elevation, pt.time))
|
||||||
|
return points, track_name or os.path.splitext(os.path.basename(file_path))[0]
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_fit(file_path):
|
||||||
|
import fitparse
|
||||||
|
|
||||||
|
fitfile = fitparse.FitFile(file_path)
|
||||||
|
messages = list(fitfile.get_messages("record"))
|
||||||
|
|
||||||
|
points = []
|
||||||
|
track_name = os.path.splitext(os.path.basename(file_path))[0]
|
||||||
|
|
||||||
|
for msg in messages:
|
||||||
|
lat = None
|
||||||
|
lon = None
|
||||||
|
ele = None
|
||||||
|
t = None
|
||||||
|
for field in msg:
|
||||||
|
if field.name == "position_lat":
|
||||||
|
lat = field.value * (180.0 / (2**31)) if field.value else None
|
||||||
|
elif field.name == "position_long":
|
||||||
|
lon = field.value * (180.0 / (2**31)) if field.value else None
|
||||||
|
elif field.name == "altitude":
|
||||||
|
ele = field.value
|
||||||
|
elif field.name == "timestamp":
|
||||||
|
t = field.value.replace(tzinfo=dt_timezone.utc) if field.value else None
|
||||||
|
if lat is not None and lon is not None:
|
||||||
|
points.append((lat, lon, ele, t))
|
||||||
|
|
||||||
|
session_msgs = list(fitfile.get_messages("session"))
|
||||||
|
for msg in session_msgs:
|
||||||
|
for field in msg:
|
||||||
|
if field.name == "sport":
|
||||||
|
track_name = str(field.value)
|
||||||
|
break
|
||||||
|
|
||||||
|
return points, track_name
|
||||||
|
|
||||||
|
|
||||||
|
def convert_fit_to_gpx(file_path):
|
||||||
|
import gpxpy.gpx
|
||||||
|
|
||||||
|
points, track_name = _parse_fit(file_path)
|
||||||
|
gpx = gpxpy.gpx.GPX()
|
||||||
|
gpx_track = gpxpy.gpx.GPXTrack(name=track_name)
|
||||||
|
gpx.tracks.append(gpx_track)
|
||||||
|
gpx_seg = gpxpy.gpx.GPXTrackSegment()
|
||||||
|
gpx_track.segments.append(gpx_seg)
|
||||||
|
|
||||||
|
for lat, lon, ele, t in points:
|
||||||
|
gpx_seg.points.append(gpxpy.gpx.GPXTrackPoint(lat, lon, elevation=ele, time=t))
|
||||||
|
|
||||||
|
tmp = tempfile.NamedTemporaryFile(mode="w", suffix=".gpx", delete=False)
|
||||||
|
tmp.write(gpx.to_xml())
|
||||||
|
tmp.close()
|
||||||
|
return tmp.name
|
||||||
|
|
||||||
|
|
||||||
|
def import_trail_gpx(file_path, user_id, original_filename=None):
|
||||||
|
user = User.objects.get(id=user_id)
|
||||||
|
new_scrobbles = []
|
||||||
|
|
||||||
|
points, track_name = parse_trackpoints(file_path)
|
||||||
|
|
||||||
|
if not points:
|
||||||
|
logger.warning(f"No trackpoints found in {file_path}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
first_lat, first_lon, _, first_time = points[0]
|
||||||
|
_, _, _, last_time = points[-1]
|
||||||
|
|
||||||
|
if first_time is None:
|
||||||
|
logger.warning(f"No timestamps in {file_path}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
geo, _ = GeoLocation.objects.get_or_create(
|
||||||
|
lat=round(first_lat, 6),
|
||||||
|
lon=round(first_lon, 6),
|
||||||
|
defaults={"altitude": None},
|
||||||
|
)
|
||||||
|
|
||||||
|
trail = Trail.find_by_trailhead(first_lat, first_lon, tolerance_m=100)
|
||||||
|
if not trail:
|
||||||
|
trail = Trail.find_or_create(track_name)
|
||||||
|
trail.trailhead_location = geo
|
||||||
|
trail.save(update_fields=["trailhead_location"])
|
||||||
|
|
||||||
|
timestamp = first_time
|
||||||
|
stop_timestamp = last_time
|
||||||
|
|
||||||
|
existing = Scrobble.objects.filter(
|
||||||
|
timestamp=timestamp,
|
||||||
|
trail=trail,
|
||||||
|
user=user,
|
||||||
|
).first()
|
||||||
|
if existing:
|
||||||
|
logger.debug(f"Skipping existing scrobble for trail {trail}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
scrobble = Scrobble(
|
||||||
|
user=user,
|
||||||
|
timestamp=timestamp,
|
||||||
|
stop_timestamp=stop_timestamp,
|
||||||
|
source="GPX Import",
|
||||||
|
trail=trail,
|
||||||
|
log={},
|
||||||
|
played_to_completion=True,
|
||||||
|
in_progress=False,
|
||||||
|
media_type=Scrobble.MediaType.TRAIL,
|
||||||
|
)
|
||||||
|
|
||||||
|
_, ext = os.path.splitext(file_path)
|
||||||
|
if ext.lower() == ".fit":
|
||||||
|
gpx_path = convert_fit_to_gpx(file_path)
|
||||||
|
with open(gpx_path, "rb") as f:
|
||||||
|
scrobble.gpx_file.save(
|
||||||
|
f"{original_filename or 'trail'}.gpx",
|
||||||
|
File(f),
|
||||||
|
save=False,
|
||||||
|
)
|
||||||
|
os.unlink(gpx_path)
|
||||||
|
else:
|
||||||
|
with open(file_path, "rb") as f:
|
||||||
|
scrobble.gpx_file.save(
|
||||||
|
original_filename or os.path.basename(file_path),
|
||||||
|
File(f),
|
||||||
|
save=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
new_scrobbles.append(scrobble)
|
||||||
|
|
||||||
|
created = Scrobble.objects.bulk_create(new_scrobbles)
|
||||||
|
logger.info(f"Created {len(created)} trail scrobbles")
|
||||||
|
return created
|
||||||
@ -1,9 +1,11 @@
|
|||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
|
||||||
from books.koreader import fetch_file_from_webdav
|
from books.koreader import fetch_file_from_webdav
|
||||||
from profiles.models import UserProfile
|
from profiles.models import UserProfile
|
||||||
from scrobbles.models import KoReaderImport
|
from scrobbles.models import KoReaderImport, TrailGPXImport
|
||||||
from scrobbles.tasks import process_koreader_import
|
from scrobbles.tasks import process_koreader_import, process_trail_gpx_import
|
||||||
from scrobbles.utils import get_file_md5_hash
|
from scrobbles.utils import get_file_md5_hash
|
||||||
from webdav.client import get_webdav_client
|
from webdav.client import get_webdav_client
|
||||||
|
|
||||||
@ -13,7 +15,6 @@ logger = logging.getLogger(__name__)
|
|||||||
def import_from_webdav_for_all_users(restart=False):
|
def import_from_webdav_for_all_users(restart=False):
|
||||||
"""Grab a list of all users with WebDAV enabled and kickoff imports for them"""
|
"""Grab a list of all users with WebDAV enabled and kickoff imports for them"""
|
||||||
|
|
||||||
# WebDavImport = apps.get_model("scrobbles", "WebDavImport")
|
|
||||||
webdav_enabled_user_ids = UserProfile.objects.filter(
|
webdav_enabled_user_ids = UserProfile.objects.filter(
|
||||||
webdav_url__isnull=False,
|
webdav_url__isnull=False,
|
||||||
webdav_user__isnull=False,
|
webdav_user__isnull=False,
|
||||||
@ -23,6 +24,7 @@ def import_from_webdav_for_all_users(restart=False):
|
|||||||
logger.info(f"Start import of {webdav_enabled_user_ids.count()} webdav accounts")
|
logger.info(f"Start import of {webdav_enabled_user_ids.count()} webdav accounts")
|
||||||
|
|
||||||
koreader_import_count = 0
|
koreader_import_count = 0
|
||||||
|
trail_gpx_import_count = 0
|
||||||
|
|
||||||
for user_id in webdav_enabled_user_ids:
|
for user_id in webdav_enabled_user_ids:
|
||||||
webdav_client = get_webdav_client(user_id)
|
webdav_client = get_webdav_client(user_id)
|
||||||
@ -78,4 +80,54 @@ def import_from_webdav_for_all_users(restart=False):
|
|||||||
|
|
||||||
process_koreader_import.delay(koreader_import.id)
|
process_koreader_import.delay(koreader_import.id)
|
||||||
koreader_import_count += 1
|
koreader_import_count += 1
|
||||||
return koreader_import_count
|
|
||||||
|
trail_gpx_import_count += scan_webdav_for_gpx(webdav_client, user_id)
|
||||||
|
|
||||||
|
return koreader_import_count, trail_gpx_import_count
|
||||||
|
|
||||||
|
|
||||||
|
def scan_webdav_for_gpx(webdav_client, user_id):
|
||||||
|
try:
|
||||||
|
webdav_client.info("gpx/")
|
||||||
|
except:
|
||||||
|
logger.info("No gpx/ directory on webdav", extra={"user_id": user_id})
|
||||||
|
return 0
|
||||||
|
|
||||||
|
try:
|
||||||
|
files = webdav_client.list("gpx/")
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Could not list gpx/", extra={"user_id": user_id, "error": str(e)})
|
||||||
|
return 0
|
||||||
|
|
||||||
|
gpx_extensions = (".gpx", ".fit")
|
||||||
|
new_imports = 0
|
||||||
|
already_imported = set(
|
||||||
|
TrailGPXImport.objects.filter(user_id=user_id)
|
||||||
|
.values_list("original_filename", flat=True)
|
||||||
|
)
|
||||||
|
|
||||||
|
for fname in files:
|
||||||
|
fname = os.path.basename(fname)
|
||||||
|
if not fname.lower().endswith(gpx_extensions):
|
||||||
|
continue
|
||||||
|
if fname in already_imported:
|
||||||
|
logger.debug(f"Skipping already-imported {fname}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=fname)
|
||||||
|
try:
|
||||||
|
webdav_client.download_sync(remote_path=f"gpx/{fname}", local_path=tmp.name)
|
||||||
|
imp = TrailGPXImport.objects.create(
|
||||||
|
user_id=user_id,
|
||||||
|
original_filename=fname,
|
||||||
|
)
|
||||||
|
with open(tmp.name, "rb") as f:
|
||||||
|
imp.gpx_file.save(fname, f, save=True)
|
||||||
|
process_trail_gpx_import.delay(imp.id)
|
||||||
|
new_imports += 1
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to import {fname}: {e}")
|
||||||
|
finally:
|
||||||
|
os.unlink(tmp.name)
|
||||||
|
|
||||||
|
return new_imports
|
||||||
|
|||||||
@ -14,5 +14,5 @@ class Command(BaseCommand):
|
|||||||
restart = False
|
restart = False
|
||||||
if options["restart"]:
|
if options["restart"]:
|
||||||
restart = True
|
restart = True
|
||||||
count = webdav.import_from_webdav_for_all_users(restart=restart)
|
ko_count, gpx_count = webdav.import_from_webdav_for_all_users(restart=restart)
|
||||||
print(f"Started {count} WeDAV imports")
|
print(f"Started {ko_count} KOReader and {gpx_count} Trail GPX WebDAV imports")
|
||||||
|
|||||||
74
vrobbler/apps/scrobbles/migrations/0078_trailgpximport.py
Normal file
74
vrobbler/apps/scrobbles/migrations/0078_trailgpximport.py
Normal file
@ -0,0 +1,74 @@
|
|||||||
|
# Generated by Django 4.2.29 on 2026-05-21 14:36
|
||||||
|
|
||||||
|
from django.conf import settings
|
||||||
|
from django.db import migrations, models
|
||||||
|
import django.db.models.deletion
|
||||||
|
import django_extensions.db.fields
|
||||||
|
import scrobbles.models
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||||
|
("scrobbles", "0077_scalecsvimport"),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.CreateModel(
|
||||||
|
name="TrailGPXImport",
|
||||||
|
fields=[
|
||||||
|
(
|
||||||
|
"id",
|
||||||
|
models.BigAutoField(
|
||||||
|
auto_created=True,
|
||||||
|
primary_key=True,
|
||||||
|
serialize=False,
|
||||||
|
verbose_name="ID",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"created",
|
||||||
|
django_extensions.db.fields.CreationDateTimeField(
|
||||||
|
auto_now_add=True, verbose_name="created"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"modified",
|
||||||
|
django_extensions.db.fields.ModificationDateTimeField(
|
||||||
|
auto_now=True, verbose_name="modified"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
("uuid", models.UUIDField(default=uuid.uuid4, editable=False)),
|
||||||
|
("processing_started", models.DateTimeField(blank=True, null=True)),
|
||||||
|
("processed_finished", models.DateTimeField(blank=True, null=True)),
|
||||||
|
("process_log", models.TextField(blank=True, null=True)),
|
||||||
|
("process_count", models.IntegerField(blank=True, null=True)),
|
||||||
|
(
|
||||||
|
"gpx_file",
|
||||||
|
models.FileField(
|
||||||
|
blank=True,
|
||||||
|
null=True,
|
||||||
|
upload_to=scrobbles.models.TrailGPXImport.get_path,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"original_filename",
|
||||||
|
models.CharField(blank=True, max_length=255, null=True),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"user",
|
||||||
|
models.ForeignKey(
|
||||||
|
blank=True,
|
||||||
|
null=True,
|
||||||
|
on_delete=django.db.models.deletion.DO_NOTHING,
|
||||||
|
to=settings.AUTH_USER_MODEL,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
options={
|
||||||
|
"verbose_name": "Trail GPX Import",
|
||||||
|
},
|
||||||
|
),
|
||||||
|
]
|
||||||
@ -299,6 +299,48 @@ class ScaleCSVImport(BaseFileImportMixin):
|
|||||||
self.mark_finished()
|
self.mark_finished()
|
||||||
|
|
||||||
|
|
||||||
|
class TrailGPXImport(BaseFileImportMixin):
|
||||||
|
class Meta:
|
||||||
|
verbose_name = "Trail GPX Import"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def import_type(self) -> str:
|
||||||
|
return "Trail GPX"
|
||||||
|
|
||||||
|
def get_absolute_url(self):
|
||||||
|
return reverse("scrobbles:trail-gpx-import-detail", kwargs={"slug": self.uuid})
|
||||||
|
|
||||||
|
def get_path(instance, filename):
|
||||||
|
extension = filename.split(".")[-1]
|
||||||
|
uuid = instance.uuid
|
||||||
|
return f"trail-gpx-uploads/{uuid}.{extension}"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def upload_file_path(self):
|
||||||
|
if getattr(settings, "USE_S3_STORAGE"):
|
||||||
|
path = self.gpx_file.url
|
||||||
|
else:
|
||||||
|
path = self.gpx_file.path
|
||||||
|
return path
|
||||||
|
|
||||||
|
gpx_file = models.FileField(upload_to=get_path, **BNULL)
|
||||||
|
original_filename = models.CharField(max_length=255, **BNULL)
|
||||||
|
|
||||||
|
def process(self, force=False):
|
||||||
|
from scrobbles.importers.trail_gpx import import_trail_gpx
|
||||||
|
|
||||||
|
if self.processed_finished and not force:
|
||||||
|
logger.info(f"{self} already processed on {self.processed_finished}")
|
||||||
|
return
|
||||||
|
|
||||||
|
self.mark_started()
|
||||||
|
scrobbles = import_trail_gpx(
|
||||||
|
self.upload_file_path, self.user.id, self.original_filename
|
||||||
|
)
|
||||||
|
self.record_log(scrobbles)
|
||||||
|
self.mark_finished()
|
||||||
|
|
||||||
|
|
||||||
class LastFmImport(BaseFileImportMixin):
|
class LastFmImport(BaseFileImportMixin):
|
||||||
class Meta:
|
class Meta:
|
||||||
verbose_name = "Last.FM Import"
|
verbose_name = "Last.FM Import"
|
||||||
|
|||||||
@ -133,6 +133,16 @@ def process_koreader_import(import_id):
|
|||||||
koreader_import.process()
|
koreader_import.process()
|
||||||
|
|
||||||
|
|
||||||
|
@shared_task
|
||||||
|
def process_trail_gpx_import(import_id):
|
||||||
|
TrailGPXImport = apps.get_model("scrobbles", "TrailGPXImport")
|
||||||
|
trail_gpx_import = TrailGPXImport.objects.filter(id=import_id).first()
|
||||||
|
if not trail_gpx_import:
|
||||||
|
logger.warn(f"TrailGPXImport not found with id {import_id}")
|
||||||
|
return
|
||||||
|
trail_gpx_import.process()
|
||||||
|
|
||||||
|
|
||||||
@shared_task
|
@shared_task
|
||||||
def create_yesterdays_charts():
|
def create_yesterdays_charts():
|
||||||
"""Build/update charts for all users starting from last known record."""
|
"""Build/update charts for all users starting from last known record."""
|
||||||
|
|||||||
@ -62,6 +62,11 @@ urlpatterns = [
|
|||||||
views.ScaleCSVImportCreateView.as_view(),
|
views.ScaleCSVImportCreateView.as_view(),
|
||||||
name="scale-csv-upload",
|
name="scale-csv-upload",
|
||||||
),
|
),
|
||||||
|
path(
|
||||||
|
"upload/trail-gpx/",
|
||||||
|
views.TrailGPXImportCreateView.as_view(),
|
||||||
|
name="trail-gpx-upload",
|
||||||
|
),
|
||||||
path(
|
path(
|
||||||
"lastfm-import/",
|
"lastfm-import/",
|
||||||
views.lastfm_import,
|
views.lastfm_import,
|
||||||
@ -126,6 +131,11 @@ urlpatterns = [
|
|||||||
views.ScrobbleScaleCSVImportDetailView.as_view(),
|
views.ScrobbleScaleCSVImportDetailView.as_view(),
|
||||||
name="scale-csv-import-detail",
|
name="scale-csv-import-detail",
|
||||||
),
|
),
|
||||||
|
path(
|
||||||
|
"imports/trail-gpx/<slug:slug>/",
|
||||||
|
views.ScrobbleTrailGPXImportDetailView.as_view(),
|
||||||
|
name="trail-gpx-import-detail",
|
||||||
|
),
|
||||||
path(
|
path(
|
||||||
"long-plays/",
|
"long-plays/",
|
||||||
views.ScrobbleLongPlaysView.as_view(),
|
views.ScrobbleLongPlaysView.as_view(),
|
||||||
|
|||||||
@ -74,11 +74,13 @@ from scrobbles.models import (
|
|||||||
RetroarchImport,
|
RetroarchImport,
|
||||||
ScaleCSVImport,
|
ScaleCSVImport,
|
||||||
Scrobble,
|
Scrobble,
|
||||||
|
TrailGPXImport,
|
||||||
)
|
)
|
||||||
from scrobbles.scrobblers import *
|
from scrobbles.scrobblers import *
|
||||||
from scrobbles.tasks import (
|
from scrobbles.tasks import (
|
||||||
process_koreader_import,
|
process_koreader_import,
|
||||||
process_lastfm_import,
|
process_lastfm_import,
|
||||||
|
process_trail_gpx_import,
|
||||||
process_tsv_import,
|
process_tsv_import,
|
||||||
)
|
)
|
||||||
from scrobbles.utils import (
|
from scrobbles.utils import (
|
||||||
@ -465,6 +467,9 @@ class ScrobbleImportListView(TemplateView):
|
|||||||
context_data["scale_csv_imports"] = ScaleCSVImport.objects.filter(
|
context_data["scale_csv_imports"] = ScaleCSVImport.objects.filter(
|
||||||
user=self.request.user,
|
user=self.request.user,
|
||||||
).order_by("-processing_started")[:10]
|
).order_by("-processing_started")[:10]
|
||||||
|
context_data["trail_gpx_imports"] = TrailGPXImport.objects.filter(
|
||||||
|
user=self.request.user,
|
||||||
|
).order_by("-processing_started")[:10]
|
||||||
return context_data
|
return context_data
|
||||||
|
|
||||||
|
|
||||||
@ -488,6 +493,8 @@ class BaseScrobbleImportDetailView(DetailView):
|
|||||||
title = "Retroarch Import"
|
title = "Retroarch Import"
|
||||||
if self.model == ScaleCSVImport:
|
if self.model == ScaleCSVImport:
|
||||||
title = "Scale CSV Import"
|
title = "Scale CSV Import"
|
||||||
|
if self.model == TrailGPXImport:
|
||||||
|
title = "Trail GPX Import"
|
||||||
context_data["title"] = title
|
context_data["title"] = title
|
||||||
return context_data
|
return context_data
|
||||||
|
|
||||||
@ -512,6 +519,10 @@ class ScrobbleScaleCSVImportDetailView(BaseScrobbleImportDetailView):
|
|||||||
model = ScaleCSVImport
|
model = ScaleCSVImport
|
||||||
|
|
||||||
|
|
||||||
|
class ScrobbleTrailGPXImportDetailView(BaseScrobbleImportDetailView):
|
||||||
|
model = TrailGPXImport
|
||||||
|
|
||||||
|
|
||||||
class ManualScrobbleView(FormView):
|
class ManualScrobbleView(FormView):
|
||||||
form_class = ScrobbleForm
|
form_class = ScrobbleForm
|
||||||
template_name = "scrobbles/manual_form.html"
|
template_name = "scrobbles/manual_form.html"
|
||||||
@ -600,6 +611,23 @@ class ScaleCSVImportCreateView(
|
|||||||
return HttpResponseRedirect(self.request.META.get("HTTP_REFERER"))
|
return HttpResponseRedirect(self.request.META.get("HTTP_REFERER"))
|
||||||
|
|
||||||
|
|
||||||
|
class TrailGPXImportCreateView(
|
||||||
|
LoginRequiredMixin, JsonableResponseMixin, CreateView
|
||||||
|
):
|
||||||
|
model = TrailGPXImport
|
||||||
|
fields = ["gpx_file"]
|
||||||
|
template_name = "scrobbles/upload_form.html"
|
||||||
|
success_url = reverse_lazy("vrobbler-home")
|
||||||
|
|
||||||
|
def form_valid(self, form):
|
||||||
|
self.object = form.save(commit=False)
|
||||||
|
self.object.user = self.request.user
|
||||||
|
self.object.original_filename = form.cleaned_data["gpx_file"].name
|
||||||
|
self.object.save()
|
||||||
|
process_trail_gpx_import.delay(self.object.id)
|
||||||
|
return HttpResponseRedirect(self.request.META.get("HTTP_REFERER"))
|
||||||
|
|
||||||
|
|
||||||
@api_view(["GET"])
|
@api_view(["GET"])
|
||||||
@permission_classes([IsAuthenticated])
|
@permission_classes([IsAuthenticated])
|
||||||
def lastfm_import(request):
|
def lastfm_import(request):
|
||||||
|
|||||||
@ -1,4 +1,5 @@
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
from math import asin, cos, radians, sin, sqrt
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
from django.apps import apps
|
from django.apps import apps
|
||||||
@ -12,6 +13,14 @@ from scrobbles.mixins import ScrobblableConstants, ScrobblableMixin
|
|||||||
BNULL = {"blank": True, "null": True}
|
BNULL = {"blank": True, "null": True}
|
||||||
|
|
||||||
|
|
||||||
|
def haversine(lat1, lon1, lat2, lon2):
|
||||||
|
R = 6371000
|
||||||
|
dlat = radians(lat2 - lat1)
|
||||||
|
dlon = radians(lon2 - lon1)
|
||||||
|
a = sin(dlat / 2) ** 2 + cos(radians(lat1)) * cos(radians(lat2)) * sin(dlon / 2) ** 2
|
||||||
|
return R * 2 * asin(sqrt(a))
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class TrailLogData(BaseLogData, WithPeopleLogData):
|
class TrailLogData(BaseLogData, WithPeopleLogData):
|
||||||
effort: Optional[str] = None
|
effort: Optional[str] = None
|
||||||
@ -68,7 +77,25 @@ class Trail(ScrobblableMixin):
|
|||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def find_or_create(cls, title: str) -> "Trail":
|
def find_or_create(cls, title: str) -> "Trail":
|
||||||
return cls.objects.filter(title=title).first()
|
trail = cls.objects.filter(title__iexact=title).first()
|
||||||
|
if not trail:
|
||||||
|
trail = cls.objects.create(title=title)
|
||||||
|
return trail
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def find_by_trailhead(cls, lat, lon, tolerance_m=100):
|
||||||
|
candidates = cls.objects.filter(
|
||||||
|
trailhead_location__isnull=False,
|
||||||
|
).select_related("trailhead_location")
|
||||||
|
best = None
|
||||||
|
best_dist = float("inf")
|
||||||
|
for trail in candidates:
|
||||||
|
loc = trail.trailhead_location
|
||||||
|
d = haversine(lat, lon, loc.lat, loc.lon)
|
||||||
|
if d < best_dist and d <= tolerance_m:
|
||||||
|
best = trail
|
||||||
|
best_dist = d
|
||||||
|
return best
|
||||||
|
|
||||||
def scrobbles(self, user_id):
|
def scrobbles(self, user_id):
|
||||||
Scrobble = apps.get_model("scrobbles", "Scrobble")
|
Scrobble = apps.get_model("scrobbles", "Scrobble")
|
||||||
|
|||||||
@ -160,4 +160,35 @@
|
|||||||
<input type="submit" value="Import Scale CSV">
|
<input type="submit" value="Import Scale CSV">
|
||||||
</form>
|
</form>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div class="row">
|
||||||
|
<h3>Trail GPX</h3>
|
||||||
|
{% if trail_gpx_imports %}
|
||||||
|
<div class="table-responsive">
|
||||||
|
<table class="table table-striped table-sm">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th scope="col">Started</th>
|
||||||
|
<th scope="col">Finished</th>
|
||||||
|
<th scope="col">Scrobbles</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{% for obj in trail_gpx_imports %}
|
||||||
|
<tr>
|
||||||
|
<td><a href="{{obj.get_absolute_url}}">{{obj.human_start}}</a></td>
|
||||||
|
<td>{{obj.processed_finished}}</td>
|
||||||
|
<td>{{obj.process_count}}</td>
|
||||||
|
</tr>
|
||||||
|
{% endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
<form action="{% url 'scrobbles:trail-gpx-upload' %}" method="post" enctype="multipart/form-data" style="margin-top: 10px;">
|
||||||
|
{% csrf_token %}
|
||||||
|
<input type="file" name="gpx_file" accept=".gpx,.fit" required>
|
||||||
|
<input type="submit" value="Import Trail GPX">
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
{% endblock %}
|
{% endblock %}
|
||||||
|
|||||||
Reference in New Issue
Block a user