From 090bd797950de990e9fc567e5cc235a8677daee7 Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Sun, 24 May 2020 12:03:39 +0200 Subject: [PATCH 1/6] implement basic pbf file uploading --- .gitignore | 1 + Dockerfile | 1 + tileservermapping/osm_data/__init__.py | 0 tileservermapping/osm_data/admin.py | 4 ++ tileservermapping/osm_data/apps.py | 5 +++ ...nitial_squashed_0004_auto_20200524_0937.py | 32 ++++++++++++++++ .../osm_data/migrations/__init__.py | 0 tileservermapping/osm_data/models.py | 37 +++++++++++++++++++ tileservermapping/osm_data/serializers.py | 18 +++++++++ tileservermapping/osm_data/tests.py | 3 ++ tileservermapping/osm_data/urls.py | 8 ++++ tileservermapping/osm_data/views.py | 7 ++++ tileservermapping/settings.py.example | 1 + tileservermapping/settings_base.py | 6 +++ tileservermapping/urls.py | 1 + 15 files changed, 124 insertions(+) create mode 100644 tileservermapping/osm_data/__init__.py create mode 100644 tileservermapping/osm_data/admin.py create mode 100644 tileservermapping/osm_data/apps.py create mode 100644 tileservermapping/osm_data/migrations/0001_initial_squashed_0004_auto_20200524_0937.py create mode 100644 tileservermapping/osm_data/migrations/__init__.py create mode 100644 tileservermapping/osm_data/models.py create mode 100644 tileservermapping/osm_data/serializers.py create mode 100644 tileservermapping/osm_data/tests.py create mode 100644 tileservermapping/osm_data/urls.py create mode 100644 tileservermapping/osm_data/views.py diff --git a/.gitignore b/.gitignore index 34a3256..07957e9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ db.sqlite settings.py .idea +./media # Byte-compiled / optimized / DLL files diff --git a/Dockerfile b/Dockerfile index 56c81f3..d161527 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,3 +25,4 @@ ENV LANG='en_US.UTF-8' EXPOSE 8000/tcp # uwsgi EXPOSE 3003/tcp +VOLUME /app/media diff --git a/tileservermapping/osm_data/__init__.py b/tileservermapping/osm_data/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tileservermapping/osm_data/admin.py b/tileservermapping/osm_data/admin.py new file mode 100644 index 0000000..93185e6 --- /dev/null +++ b/tileservermapping/osm_data/admin.py @@ -0,0 +1,4 @@ +from django.contrib import admin +from . import models + +admin.site.register(models.PlanetDump) \ No newline at end of file diff --git a/tileservermapping/osm_data/apps.py b/tileservermapping/osm_data/apps.py new file mode 100644 index 0000000..d252308 --- /dev/null +++ b/tileservermapping/osm_data/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class OsmDataConfig(AppConfig): + name = 'tileservermapping.osm_data' diff --git a/tileservermapping/osm_data/migrations/0001_initial_squashed_0004_auto_20200524_0937.py b/tileservermapping/osm_data/migrations/0001_initial_squashed_0004_auto_20200524_0937.py new file mode 100644 index 0000000..4ff88e0 --- /dev/null +++ b/tileservermapping/osm_data/migrations/0001_initial_squashed_0004_auto_20200524_0937.py @@ -0,0 +1,32 @@ +# Generated by Django 3.0.5 on 2020-05-24 10:02 + +from django.db import migrations, models +import tileservermapping.osm_data.models + + +class Migration(migrations.Migration): + + replaces = [('osm_data', '0001_initial'), ('osm_data', '0002_auto_20200524_0924'), ('osm_data', '0003_auto_20200524_0930'), ('osm_data', '0004_auto_20200524_0937')] + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='PlanetDump', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('x', models.IntegerField(help_text='Slippy map coordinate X')), + ('y', models.IntegerField(help_text='Slippy map coordinate Y')), + ('z', models.IntegerField(help_text='Slippy map coordinate Z (zoom)')), + ('file', models.FileField(upload_to=tileservermapping.osm_data.models.generate_file_name)), + ], + options={ + 'unique_together': set(), + }, + ), + migrations.AddConstraint( + model_name='planetdump', + constraint=models.UniqueConstraint(fields=('x', 'y', 'z'), name='unique_coordinates'), + ), + ] diff --git a/tileservermapping/osm_data/migrations/__init__.py b/tileservermapping/osm_data/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tileservermapping/osm_data/models.py b/tileservermapping/osm_data/models.py new file mode 100644 index 0000000..e38bf3d --- /dev/null +++ b/tileservermapping/osm_data/models.py @@ -0,0 +1,37 @@ +import posixpath +from django.db import models + + +def generate_file_name(instance, filename): + """ + Generate uploaded filename based on coordinates of the file + + :param instance: Instance of the newly created database object + :type instance: PlanetDump + :param filename: Originally uploaded file name + :type filename: str + """ + return posixpath.join('planet_dumps', f'{instance.z}_{instance.x}_{instance.y}.pbf') + + +class PlanetDump(models.Model): + """ + Planet dump file encoded in PBF format. + Each database object is mapped to one file in the file storage and can be used to manage that file. + + One PlanetDump does not always store the whole planet but only a smaller portion + defined by the `x y` and `z` coordinates. + """ + + x = models.IntegerField(help_text='Slippy map coordinate X') + y = models.IntegerField(help_text='Slippy map coordinate Y') + z = models.IntegerField(help_text='Slippy map coordinate Z (zoom)') + file = models.FileField(upload_to=generate_file_name) + + class Meta: + constraints = [ + models.UniqueConstraint(fields=['x', 'y', 'z'], name='unique_coordinates') + ] + + def __str__(self): + return f'{self.__class__.__name__} of x:{self.x} y:{self.y} z:{self.z}' diff --git a/tileservermapping/osm_data/serializers.py b/tileservermapping/osm_data/serializers.py new file mode 100644 index 0000000..f19a040 --- /dev/null +++ b/tileservermapping/osm_data/serializers.py @@ -0,0 +1,18 @@ +from rest_framework import serializers, validators +from . import models + + +class PlanetDumpSerializer(serializers.ModelSerializer): + class Meta: + model = models.PlanetDump + fields = ['x', 'y', 'z', 'file'] + validators = [ + validators.UniqueTogetherValidator( + queryset=models.PlanetDump.objects.all(), + fields=['x', 'y', 'z'] + ) + ] + + def validate_file(self, value): + # TODO Validate that file is actually a pbf file + return value diff --git a/tileservermapping/osm_data/tests.py b/tileservermapping/osm_data/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/tileservermapping/osm_data/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/tileservermapping/osm_data/urls.py b/tileservermapping/osm_data/urls.py new file mode 100644 index 0000000..c8be2ed --- /dev/null +++ b/tileservermapping/osm_data/urls.py @@ -0,0 +1,8 @@ +from django.urls import path, include, re_path +from rest_framework import routers + +from . import views + +router = routers.DefaultRouter() +router.register(r"planet_dumps", views.PlanetDumpViewset) +urlpatterns = router.urls diff --git a/tileservermapping/osm_data/views.py b/tileservermapping/osm_data/views.py new file mode 100644 index 0000000..240c5e2 --- /dev/null +++ b/tileservermapping/osm_data/views.py @@ -0,0 +1,7 @@ +from rest_framework import viewsets +from . import models, serializers + + +class PlanetDumpViewset(viewsets.ModelViewSet): + queryset = models.PlanetDump.objects.all() + serializer_class = serializers.PlanetDumpSerializer diff --git a/tileservermapping/settings.py.example b/tileservermapping/settings.py.example index a2f8db8..03028ef 100644 --- a/tileservermapping/settings.py.example +++ b/tileservermapping/settings.py.example @@ -36,3 +36,4 @@ DATABASES = { } STATIC_ROOT = os.path.join('/', 'app', 'static') +MEDIA_ROOT = os.path.join('/', 'app', 'media') diff --git a/tileservermapping/settings_base.py b/tileservermapping/settings_base.py index e78fdd2..29fc967 100644 --- a/tileservermapping/settings_base.py +++ b/tileservermapping/settings_base.py @@ -31,6 +31,7 @@ 'rest_framework', 'drf_yasg', 'tileservermapping.mapping', + 'tileservermapping.osm_data', ] MIDDLEWARE = [ @@ -116,4 +117,9 @@ "ALLOWED_VERSIONS": ["v1"], "DEFAULT_VERSIONING_CLASS": "rest_framework.versioning.URLPathVersioning", "DEFAULT_PERMISSION_CLASSES": ["rest_framework.permissions.IsAuthenticated"], + "DEFAULT_PARSER_CLASSES": [ + "rest_framework.parsers.JSONParser", + "rest_framework.parsers.FormParser", + "rest_framework.parsers.MultiPartParser", + ] } diff --git a/tileservermapping/urls.py b/tileservermapping/urls.py index e72163e..5d1d2ed 100644 --- a/tileservermapping/urls.py +++ b/tileservermapping/urls.py @@ -25,6 +25,7 @@ urlpatterns = [ path("api//", include("tileservermapping.mapping.urls")), + path("api//", include("tileservermapping.osm_data.urls")), path("mappings/", include("tileservermapping.mapping.urls")), # included for compatibility to old url schema path("admin/", admin.site.urls), From a994e1a7954f3a5b2450c44286475db7c172ea55 Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Mon, 25 May 2020 17:33:39 +0200 Subject: [PATCH 2/6] use nginx in docker --- .gitignore | 3 ++- Dockerfile | 8 ++++++-- docker/entrypoint.sh | 5 +++++ docker/nginx.conf | 18 ++++++++++++++++++ docker/supervisor.conf | 15 +++++++++++++++ docker/uwsgi.ini | 3 --- 6 files changed, 46 insertions(+), 6 deletions(-) create mode 100755 docker/entrypoint.sh create mode 100644 docker/nginx.conf create mode 100644 docker/supervisor.conf diff --git a/.gitignore b/.gitignore index 07957e9..fb78fd8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,8 @@ db.sqlite settings.py .idea -./media +media +**.pbf # Byte-compiled / optimized / DLL files diff --git a/Dockerfile b/Dockerfile index d161527..c1d65a5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,8 @@ FROM docker.io/debian:buster-slim RUN apt update -RUN apt install -y --no-install-recommends pipenv uwsgi uwsgi-plugin-python3 python3-psycopg2 python3-setuptools +RUN apt install -y --no-install-recommends pipenv uwsgi uwsgi-plugin-python3 python3-psycopg2 python3-setuptools \ + nginx supervisor # install project dependencies and add sources ADD Pipfile Pipfile.lock /app/src/ @@ -14,12 +15,15 @@ RUN mkdir -p /app/config RUN cp /app/src/tileservermapping/settings.py.example /app/config/settings.py RUN ln -sf /app/config/settings.py /app/src/tileservermapping/settings.py RUN ln -s /app/src/docker/uwsgi.ini /etc/uwsgi/tileservermapping.ini +RUN ln -s /app/src/docker/supervisor.conf /etc/supervisor/conf.d/app.conf +RUN ln -sf /app/src/docker/nginx.conf /etc/nginx/sites-enabled/default # collect staticfiles RUN ./manage.py collectstatic --no-input # container metadata -CMD uwsgi /etc/uwsgi/tileservermapping.ini +ENTRYPOINT ["/app/src/docker/entrypoint.sh"] +CMD ["supervisord", "-n", "-c", "/etc/supervisor/supervisord.conf", "-u", "root"] ENV LANG='en_US.UTF-8' # http EXPOSE 8000/tcp diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 0000000..df17312 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -e + +/app/src/manage.py migrate +exec "$@" \ No newline at end of file diff --git a/docker/nginx.conf b/docker/nginx.conf new file mode 100644 index 0000000..1073453 --- /dev/null +++ b/docker/nginx.conf @@ -0,0 +1,18 @@ +server { + listen 8000; + server_name _; + + location / { + include uwsgi_params; + uwsgi_pass localhost:3003; + client_max_body_size 0; + } + + location /media { + alias /app/media; + } + + location /static { + alias /app/static; + } +} \ No newline at end of file diff --git a/docker/supervisor.conf b/docker/supervisor.conf new file mode 100644 index 0000000..ec12694 --- /dev/null +++ b/docker/supervisor.conf @@ -0,0 +1,15 @@ +[program:nginx] +command=nginx -g "daemon off;" +autorestart=true +stdout_logfile = /dev/stdout +stdout_logfile_maxbytes = 0 +stderr_logfile = /dev/stderr +stderr_logfile_maxbytes = 0 + +[program:uwsgi] +command=uwsgi --ini /etc/uwsgi/tileservermapping.ini +autorestart=true +stdout_logfile = /dev/stdout +stdout_logfile_maxbytes = 0 +stderr_logfile = /dev/stderr +stderr_logfile_maxbytes = 0 \ No newline at end of file diff --git a/docker/uwsgi.ini b/docker/uwsgi.ini index e5e91a3..9ead486 100644 --- a/docker/uwsgi.ini +++ b/docker/uwsgi.ini @@ -1,7 +1,6 @@ [uwsgi] master = true socket = :3003 -http-socket = :8000 plugins = python3 chdir = /app/src @@ -13,5 +12,3 @@ cheaper = 2 ; disable uWSGI request logging disable-logging = true - -static-map = /static=/app/static From bd19215f260482a002c10159fbbe1db47e0dbad5 Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Mon, 25 May 2020 17:34:13 +0200 Subject: [PATCH 3/6] make django media location configurable via env --- README.md | 1 + tileservermapping/settings.py.example | 2 +- tileservermapping/settings_base.py | 7 ++----- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index cf06b21..fb54ec0 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ TM\_DEBUG | *empty* | Enables django debug mode when not empty TM\_SECRET\_KEY | v€ry $ecret key | [**Change this in production**](https://docs.djangoproject.com/en/3.0/ref/settings/#std:setting-SECRET_KEY) TM\_HOSTS | localhost | Comma sperated list of hostnames which this server responds to TM\_DB\_HOST | localhost | Hostname of the postgresql database server +TM\_MEDIA\_ROOT | /app/media | Where uploaded files are stored TM\_DB\_PORT | 5432 | Database port TM\_DB\_NAME | osm_tileservermapping | Which database to use on that postgresql server TM\_DB\_USER | osm_tileservermapping | User used to authenticate at the database diff --git a/tileservermapping/settings.py.example b/tileservermapping/settings.py.example index 03028ef..685ca87 100644 --- a/tileservermapping/settings.py.example +++ b/tileservermapping/settings.py.example @@ -36,4 +36,4 @@ DATABASES = { } STATIC_ROOT = os.path.join('/', 'app', 'static') -MEDIA_ROOT = os.path.join('/', 'app', 'media') +MEDIA_ROOT = os.getenv('TM_MEDIA_ROOT', os.path.join('/', 'app', 'media')) diff --git a/tileservermapping/settings_base.py b/tileservermapping/settings_base.py index 29fc967..c9ef1c2 100644 --- a/tileservermapping/settings_base.py +++ b/tileservermapping/settings_base.py @@ -108,6 +108,7 @@ # https://docs.djangoproject.com/en/2.0/howto/static-files/ STATIC_URL = '/static/' +MEDIA_URL = '/media/' # Django Rest Framework and extensions @@ -117,9 +118,5 @@ "ALLOWED_VERSIONS": ["v1"], "DEFAULT_VERSIONING_CLASS": "rest_framework.versioning.URLPathVersioning", "DEFAULT_PERMISSION_CLASSES": ["rest_framework.permissions.IsAuthenticated"], - "DEFAULT_PARSER_CLASSES": [ - "rest_framework.parsers.JSONParser", - "rest_framework.parsers.FormParser", - "rest_framework.parsers.MultiPartParser", - ] + "DEFAULT_PARSER_CLASSES": ["rest_framework.parsers.JSONParser"] } From 1b04e79a8a69f483014612bbaeaed6672af88b13 Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Mon, 25 May 2020 17:38:09 +0200 Subject: [PATCH 4/6] finish uploading of pbf files implementation --- .../0002_alter_planet_dump_file_field.py | 27 +++++++++++++++++++ tileservermapping/osm_data/models.py | 4 ++- tileservermapping/osm_data/serializers.py | 8 ++---- tileservermapping/osm_data/storage.py | 11 ++++++++ tileservermapping/osm_data/urls.py | 2 +- tileservermapping/osm_data/views.py | 14 +++++++++- tileservermapping/urls.py | 2 +- 7 files changed, 58 insertions(+), 10 deletions(-) create mode 100644 tileservermapping/osm_data/migrations/0002_alter_planet_dump_file_field.py create mode 100644 tileservermapping/osm_data/storage.py diff --git a/tileservermapping/osm_data/migrations/0002_alter_planet_dump_file_field.py b/tileservermapping/osm_data/migrations/0002_alter_planet_dump_file_field.py new file mode 100644 index 0000000..86c571d --- /dev/null +++ b/tileservermapping/osm_data/migrations/0002_alter_planet_dump_file_field.py @@ -0,0 +1,27 @@ +# Generated by Django 3.0.5 on 2020-05-25 15:37 + +from django.db import migrations, models +import tileservermapping.osm_data.models +import tileservermapping.osm_data.storage + + +class Migration(migrations.Migration): + + replaces = [('osm_data', '0002_auto_20200525_1451'), ('osm_data', '0003_auto_20200525_1530')] + + dependencies = [ + ('osm_data', '0001_initial_squashed_0004_auto_20200524_0937'), + ] + + operations = [ + migrations.AlterField( + model_name='planetdump', + name='file', + field=models.FileField(default=None, null=True, upload_to=tileservermapping.osm_data.models.generate_file_name), + ), + migrations.AlterField( + model_name='planetdump', + name='file', + field=models.FileField(default=None, null=True, storage=tileservermapping.osm_data.storage.OverwriteStorage(), upload_to=tileservermapping.osm_data.models.generate_file_name), + ), + ] diff --git a/tileservermapping/osm_data/models.py b/tileservermapping/osm_data/models.py index e38bf3d..0029de9 100644 --- a/tileservermapping/osm_data/models.py +++ b/tileservermapping/osm_data/models.py @@ -1,6 +1,8 @@ import posixpath from django.db import models +from tileservermapping.osm_data.storage import OverwriteStorage + def generate_file_name(instance, filename): """ @@ -26,7 +28,7 @@ class PlanetDump(models.Model): x = models.IntegerField(help_text='Slippy map coordinate X') y = models.IntegerField(help_text='Slippy map coordinate Y') z = models.IntegerField(help_text='Slippy map coordinate Z (zoom)') - file = models.FileField(upload_to=generate_file_name) + file = models.FileField(upload_to=generate_file_name, null=True, default=None, storage=OverwriteStorage()) class Meta: constraints = [ diff --git a/tileservermapping/osm_data/serializers.py b/tileservermapping/osm_data/serializers.py index f19a040..47e878a 100644 --- a/tileservermapping/osm_data/serializers.py +++ b/tileservermapping/osm_data/serializers.py @@ -2,17 +2,13 @@ from . import models -class PlanetDumpSerializer(serializers.ModelSerializer): +class PlanetDumpSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = models.PlanetDump - fields = ['x', 'y', 'z', 'file'] + fields = ['url', 'id', 'x', 'y', 'z', 'file'] validators = [ validators.UniqueTogetherValidator( queryset=models.PlanetDump.objects.all(), fields=['x', 'y', 'z'] ) ] - - def validate_file(self, value): - # TODO Validate that file is actually a pbf file - return value diff --git a/tileservermapping/osm_data/storage.py b/tileservermapping/osm_data/storage.py new file mode 100644 index 0000000..4ab6daf --- /dev/null +++ b/tileservermapping/osm_data/storage.py @@ -0,0 +1,11 @@ +from django.core.files.storage import FileSystemStorage + + +class OverwriteStorage(FileSystemStorage): + def _save(self, name, content): + if self.exists(name): + self.delete(name) + return super(OverwriteStorage, self)._save(name, content) + + def get_available_name(self, name, max_length=None): + return name diff --git a/tileservermapping/osm_data/urls.py b/tileservermapping/osm_data/urls.py index c8be2ed..837352f 100644 --- a/tileservermapping/osm_data/urls.py +++ b/tileservermapping/osm_data/urls.py @@ -4,5 +4,5 @@ from . import views router = routers.DefaultRouter() -router.register(r"planet_dumps", views.PlanetDumpViewset) +router.register(r'planet_dumps', views.PlanetDumpViewset) urlpatterns = router.urls diff --git a/tileservermapping/osm_data/views.py b/tileservermapping/osm_data/views.py index 240c5e2..ee35fe4 100644 --- a/tileservermapping/osm_data/views.py +++ b/tileservermapping/osm_data/views.py @@ -1,7 +1,19 @@ -from rest_framework import viewsets +from rest_framework import viewsets, permissions, parsers from . import models, serializers class PlanetDumpViewset(viewsets.ModelViewSet): queryset = models.PlanetDump.objects.all() serializer_class = serializers.PlanetDumpSerializer + + def get_parsers(self): + if self.request.method == 'POST' or self.request.method == 'PATCH' or self.request.method == 'PUT': + return [*super(PlanetDumpViewset, self).get_parsers(), parsers.MultiPartParser()] + else: + return super().get_parsers() + + def get_permissions(self): + if self.action == 'list': + return [permissions.AllowAny()] + else: + return super(PlanetDumpViewset, self).get_permissions() diff --git a/tileservermapping/urls.py b/tileservermapping/urls.py index 5d1d2ed..ceb6c9e 100644 --- a/tileservermapping/urls.py +++ b/tileservermapping/urls.py @@ -32,5 +32,5 @@ path("schema/", schema_view.without_ui(cache_timeout=0), name="schema-json"), path("docs/", schema_view.with_ui("swagger", cache_timeout=0), name="schema-swagger-ui"), - path("", RedirectView.as_view(url="/docs")) + path("", RedirectView.as_view(url="/docs")), ] From f5e7322f2e62aa68751329cdbf162b3224028be3 Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Thu, 28 May 2020 20:56:28 +0200 Subject: [PATCH 5/6] add sql dumps to osm_data management --- tileservermapping/osm_data/admin.py | 3 +- .../osm_data/migrations/0001_initial.py | 42 +++++++++++++++++ ...nitial_squashed_0004_auto_20200524_0937.py | 32 ------------- .../0002_alter_planet_dump_file_field.py | 27 ----------- tileservermapping/osm_data/models.py | 45 ++++++++++++++----- tileservermapping/osm_data/serializers.py | 16 +++++++ tileservermapping/osm_data/urls.py | 1 + tileservermapping/osm_data/views.py | 17 +++++++ 8 files changed, 113 insertions(+), 70 deletions(-) create mode 100644 tileservermapping/osm_data/migrations/0001_initial.py delete mode 100644 tileservermapping/osm_data/migrations/0001_initial_squashed_0004_auto_20200524_0937.py delete mode 100644 tileservermapping/osm_data/migrations/0002_alter_planet_dump_file_field.py diff --git a/tileservermapping/osm_data/admin.py b/tileservermapping/osm_data/admin.py index 93185e6..f4012d4 100644 --- a/tileservermapping/osm_data/admin.py +++ b/tileservermapping/osm_data/admin.py @@ -1,4 +1,5 @@ from django.contrib import admin from . import models -admin.site.register(models.PlanetDump) \ No newline at end of file +admin.site.register(models.PlanetDump) +admin.site.register(models.SqlDump) \ No newline at end of file diff --git a/tileservermapping/osm_data/migrations/0001_initial.py b/tileservermapping/osm_data/migrations/0001_initial.py new file mode 100644 index 0000000..ff76423 --- /dev/null +++ b/tileservermapping/osm_data/migrations/0001_initial.py @@ -0,0 +1,42 @@ +# Generated by Django 3.0.6 on 2020-05-28 18:34 + +from django.db import migrations, models +import tileservermapping.osm_data.models +import tileservermapping.osm_data.storage + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='SqlDump', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('x', models.IntegerField(help_text='Slippy map coordinate X')), + ('y', models.IntegerField(help_text='Slippy map coordinate Z')), + ('z', models.IntegerField(help_text='Slippy map coordinate Z')), + ('file', models.FileField(default=None, null=True, storage=tileservermapping.osm_data.storage.OverwriteStorage(), upload_to=tileservermapping.osm_data.models.gen_sql_dump_location)), + ], + options={ + 'unique_together': {('x', 'y', 'z')}, + }, + ), + migrations.CreateModel( + name='PlanetDump', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('x', models.IntegerField(help_text='Slippy map coordinate X')), + ('y', models.IntegerField(help_text='Slippy map coordinate Y')), + ('z', models.IntegerField(help_text='Slippy map coordinate Z (zoom)')), + ('file', models.FileField(default=None, null=True, storage=tileservermapping.osm_data.storage.OverwriteStorage(), upload_to=tileservermapping.osm_data.models.gen_planet_dump_location)), + ], + options={ + 'unique_together': {('x', 'y', 'z')}, + }, + ), + ] diff --git a/tileservermapping/osm_data/migrations/0001_initial_squashed_0004_auto_20200524_0937.py b/tileservermapping/osm_data/migrations/0001_initial_squashed_0004_auto_20200524_0937.py deleted file mode 100644 index 4ff88e0..0000000 --- a/tileservermapping/osm_data/migrations/0001_initial_squashed_0004_auto_20200524_0937.py +++ /dev/null @@ -1,32 +0,0 @@ -# Generated by Django 3.0.5 on 2020-05-24 10:02 - -from django.db import migrations, models -import tileservermapping.osm_data.models - - -class Migration(migrations.Migration): - - replaces = [('osm_data', '0001_initial'), ('osm_data', '0002_auto_20200524_0924'), ('osm_data', '0003_auto_20200524_0930'), ('osm_data', '0004_auto_20200524_0937')] - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='PlanetDump', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('x', models.IntegerField(help_text='Slippy map coordinate X')), - ('y', models.IntegerField(help_text='Slippy map coordinate Y')), - ('z', models.IntegerField(help_text='Slippy map coordinate Z (zoom)')), - ('file', models.FileField(upload_to=tileservermapping.osm_data.models.generate_file_name)), - ], - options={ - 'unique_together': set(), - }, - ), - migrations.AddConstraint( - model_name='planetdump', - constraint=models.UniqueConstraint(fields=('x', 'y', 'z'), name='unique_coordinates'), - ), - ] diff --git a/tileservermapping/osm_data/migrations/0002_alter_planet_dump_file_field.py b/tileservermapping/osm_data/migrations/0002_alter_planet_dump_file_field.py deleted file mode 100644 index 86c571d..0000000 --- a/tileservermapping/osm_data/migrations/0002_alter_planet_dump_file_field.py +++ /dev/null @@ -1,27 +0,0 @@ -# Generated by Django 3.0.5 on 2020-05-25 15:37 - -from django.db import migrations, models -import tileservermapping.osm_data.models -import tileservermapping.osm_data.storage - - -class Migration(migrations.Migration): - - replaces = [('osm_data', '0002_auto_20200525_1451'), ('osm_data', '0003_auto_20200525_1530')] - - dependencies = [ - ('osm_data', '0001_initial_squashed_0004_auto_20200524_0937'), - ] - - operations = [ - migrations.AlterField( - model_name='planetdump', - name='file', - field=models.FileField(default=None, null=True, upload_to=tileservermapping.osm_data.models.generate_file_name), - ), - migrations.AlterField( - model_name='planetdump', - name='file', - field=models.FileField(default=None, null=True, storage=tileservermapping.osm_data.storage.OverwriteStorage(), upload_to=tileservermapping.osm_data.models.generate_file_name), - ), - ] diff --git a/tileservermapping/osm_data/models.py b/tileservermapping/osm_data/models.py index 0029de9..d146d59 100644 --- a/tileservermapping/osm_data/models.py +++ b/tileservermapping/osm_data/models.py @@ -1,19 +1,29 @@ import posixpath +from typing import * + from django.db import models from tileservermapping.osm_data.storage import OverwriteStorage -def generate_file_name(instance, filename): +def gen_planet_dump_location(instance, filename) -> str: + """ + Generate uploaded filename based on coordinates of the instance + + :param PlanetDump instance: Instance of the newly created database object + :param str filename: Originally uploaded file name + """ + return f'planet_dumps/{instance.z}_{instance.x}_{instance.y}.pbf' + + +def gen_sql_dump_location(instance, filename) -> str: """ Generate uploaded filename based on coordinates of the file - :param instance: Instance of the newly created database object - :type instance: PlanetDump - :param filename: Originally uploaded file name - :type filename: str + :param SqlDump instance: Instance of the newly created database object + :param str filename: Originally uploaded file name """ - return posixpath.join('planet_dumps', f'{instance.z}_{instance.x}_{instance.y}.pbf') + return f'sql_dumps/{instance.z}_{instance.x}_{instance.y}.pg_dump' class PlanetDump(models.Model): @@ -28,12 +38,27 @@ class PlanetDump(models.Model): x = models.IntegerField(help_text='Slippy map coordinate X') y = models.IntegerField(help_text='Slippy map coordinate Y') z = models.IntegerField(help_text='Slippy map coordinate Z (zoom)') - file = models.FileField(upload_to=generate_file_name, null=True, default=None, storage=OverwriteStorage()) + file = models.FileField(upload_to=gen_planet_dump_location, null=True, default=None, storage=OverwriteStorage()) + + class Meta: + unique_together = [['x', 'y', 'z']] + + def __str__(self): + return f'{self.__class__.__name__} of x:{self.x} y:{self.y} z:{self.z}' + + +class SqlDump(models.Model): + """ + PostgreSQL Dump files which hold all the data a Tileserver needs. + """ + + x = models.IntegerField(help_text='Slippy map coordinate X') + y = models.IntegerField(help_text='Slippy map coordinate Z') + z = models.IntegerField(help_text='Slippy map coordinate Z') + file = models.FileField(upload_to=gen_sql_dump_location, null=True, default=None, storage=OverwriteStorage()) class Meta: - constraints = [ - models.UniqueConstraint(fields=['x', 'y', 'z'], name='unique_coordinates') - ] + unique_together = [['x', 'y', 'z']] def __str__(self): return f'{self.__class__.__name__} of x:{self.x} y:{self.y} z:{self.z}' diff --git a/tileservermapping/osm_data/serializers.py b/tileservermapping/osm_data/serializers.py index 47e878a..c62e498 100644 --- a/tileservermapping/osm_data/serializers.py +++ b/tileservermapping/osm_data/serializers.py @@ -12,3 +12,19 @@ class Meta: fields=['x', 'y', 'z'] ) ] + + # TODO: Validate uploaded pbf file + + +class SqlDumpSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = models.SqlDump + fields = ['url', 'id', 'x', 'y', 'z', 'file'] + validators = [ + validators.UniqueTogetherValidator( + queryset=models.SqlDump.objects.all(), + fields=['x', 'y', 'z'] + ) + ] + + # TODO: Validate uploaded postgresql-dump file diff --git a/tileservermapping/osm_data/urls.py b/tileservermapping/osm_data/urls.py index 837352f..72d02a0 100644 --- a/tileservermapping/osm_data/urls.py +++ b/tileservermapping/osm_data/urls.py @@ -5,4 +5,5 @@ router = routers.DefaultRouter() router.register(r'planet_dumps', views.PlanetDumpViewset) +router.register(r'postgresql_dumps', views.SqlDumpViewset) urlpatterns = router.urls diff --git a/tileservermapping/osm_data/views.py b/tileservermapping/osm_data/views.py index ee35fe4..350ab9e 100644 --- a/tileservermapping/osm_data/views.py +++ b/tileservermapping/osm_data/views.py @@ -17,3 +17,20 @@ def get_permissions(self): return [permissions.AllowAny()] else: return super(PlanetDumpViewset, self).get_permissions() + + +class SqlDumpViewset(viewsets.ModelViewSet): + queryset = models.SqlDump.objects.all() + serializer_class = serializers.SqlDumpSerializer + + def get_parsers(self): + if self.request.method == 'POST' or self.request.method == 'PATCH' or self.request.method == 'PUT': + return [*super(SqlDumpViewset, self).get_parsers(), parsers.MultiPartParser()] + else: + return super().get_parsers() + + def get_permissions(self): + if self.action == 'list': + return [permissions.AllowAny()] + else: + return super(SqlDumpViewset, self).get_permissions() From 06a7c17e0a298d5094e6fc90d6b2f571fb9d77c8 Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Fri, 29 May 2020 13:51:26 +0200 Subject: [PATCH 6/6] only add specific sources in dockerfile --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c1d65a5..87178be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,9 @@ RUN apt install -y --no-install-recommends pipenv uwsgi uwsgi-plugin-python3 pyt ADD Pipfile Pipfile.lock /app/src/ WORKDIR /app/src RUN pipenv install --system --deploy --ignore-pipfile -ADD . /app/src/ +ADD manage.py /app/src/ +ADD docker /app/src/docker +ADD tileservermapping /app/src/tileservermapping # put configuration in correct places RUN mkdir -p /app/config