diff --git a/.gitignore b/.gitignore index 34a3256..fb78fd8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ db.sqlite settings.py .idea +media +**.pbf # Byte-compiled / optimized / DLL files diff --git a/Dockerfile b/Dockerfile index 56c81f3..87178be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,27 +1,34 @@ FROM docker.io/debian:buster-slim RUN apt update -RUN apt install -y --no-install-recommends pipenv uwsgi uwsgi-plugin-python3 python3-psycopg2 python3-setuptools +RUN apt install -y --no-install-recommends pipenv uwsgi uwsgi-plugin-python3 python3-psycopg2 python3-setuptools \ + nginx supervisor # install project dependencies and add sources ADD Pipfile Pipfile.lock /app/src/ WORKDIR /app/src RUN pipenv install --system --deploy --ignore-pipfile -ADD . /app/src/ +ADD manage.py /app/src/ +ADD docker /app/src/docker +ADD tileservermapping /app/src/tileservermapping # put configuration in correct places RUN mkdir -p /app/config RUN cp /app/src/tileservermapping/settings.py.example /app/config/settings.py RUN ln -sf /app/config/settings.py /app/src/tileservermapping/settings.py RUN ln -s /app/src/docker/uwsgi.ini /etc/uwsgi/tileservermapping.ini +RUN ln -s /app/src/docker/supervisor.conf /etc/supervisor/conf.d/app.conf +RUN ln -sf /app/src/docker/nginx.conf /etc/nginx/sites-enabled/default # collect staticfiles RUN ./manage.py collectstatic --no-input # container metadata -CMD uwsgi /etc/uwsgi/tileservermapping.ini +ENTRYPOINT ["/app/src/docker/entrypoint.sh"] +CMD ["supervisord", "-n", "-c", "/etc/supervisor/supervisord.conf", "-u", "root"] ENV LANG='en_US.UTF-8' # http EXPOSE 8000/tcp # uwsgi EXPOSE 3003/tcp +VOLUME /app/media diff --git a/README.md b/README.md index 194cdff..79c4a4e 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ TM\_DEBUG | *empty* | Enables django debug mode when not empty TM\_SECRET\_KEY | v€ry $ecret key | [**Change this in production**](https://docs.djangoproject.com/en/3.0/ref/settings/#std:setting-SECRET_KEY) TM\_HOSTS | localhost | Comma sperated list of hostnames which this server responds to TM\_DB\_HOST | localhost | Hostname of the postgresql database server +TM\_MEDIA\_ROOT | /app/media | Where uploaded files are stored TM\_DB\_PORT | 5432 | Database port TM\_DB\_NAME | osm_tileservermapping | Which database to use on that postgresql server TM\_DB\_USER | osm_tileservermapping | User used to authenticate at the database diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 0000000..df17312 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -e + +/app/src/manage.py migrate +exec "$@" \ No newline at end of file diff --git a/docker/nginx.conf b/docker/nginx.conf new file mode 100644 index 0000000..1073453 --- /dev/null +++ b/docker/nginx.conf @@ -0,0 +1,18 @@ +server { + listen 8000; + server_name _; + + location / { + include uwsgi_params; + uwsgi_pass localhost:3003; + client_max_body_size 0; + } + + location /media { + alias /app/media; + } + + location /static { + alias /app/static; + } +} \ No newline at end of file diff --git a/docker/supervisor.conf b/docker/supervisor.conf new file mode 100644 index 0000000..ec12694 --- /dev/null +++ b/docker/supervisor.conf @@ -0,0 +1,15 @@ +[program:nginx] +command=nginx -g "daemon off;" +autorestart=true +stdout_logfile = /dev/stdout +stdout_logfile_maxbytes = 0 +stderr_logfile = /dev/stderr +stderr_logfile_maxbytes = 0 + +[program:uwsgi] +command=uwsgi --ini /etc/uwsgi/tileservermapping.ini +autorestart=true +stdout_logfile = /dev/stdout +stdout_logfile_maxbytes = 0 +stderr_logfile = /dev/stderr +stderr_logfile_maxbytes = 0 \ No newline at end of file diff --git a/docker/uwsgi.ini b/docker/uwsgi.ini index e5e91a3..9ead486 100644 --- a/docker/uwsgi.ini +++ b/docker/uwsgi.ini @@ -1,7 +1,6 @@ [uwsgi] master = true socket = :3003 -http-socket = :8000 plugins = python3 chdir = /app/src @@ -13,5 +12,3 @@ cheaper = 2 ; disable uWSGI request logging disable-logging = true - -static-map = /static=/app/static diff --git a/tileservermapping/osm_data/__init__.py b/tileservermapping/osm_data/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tileservermapping/osm_data/admin.py b/tileservermapping/osm_data/admin.py new file mode 100644 index 0000000..f4012d4 --- /dev/null +++ b/tileservermapping/osm_data/admin.py @@ -0,0 +1,5 @@ +from django.contrib import admin +from . import models + +admin.site.register(models.PlanetDump) +admin.site.register(models.SqlDump) \ No newline at end of file diff --git a/tileservermapping/osm_data/apps.py b/tileservermapping/osm_data/apps.py new file mode 100644 index 0000000..d252308 --- /dev/null +++ b/tileservermapping/osm_data/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class OsmDataConfig(AppConfig): + name = 'tileservermapping.osm_data' diff --git a/tileservermapping/osm_data/migrations/0001_initial.py b/tileservermapping/osm_data/migrations/0001_initial.py new file mode 100644 index 0000000..ff76423 --- /dev/null +++ b/tileservermapping/osm_data/migrations/0001_initial.py @@ -0,0 +1,42 @@ +# Generated by Django 3.0.6 on 2020-05-28 18:34 + +from django.db import migrations, models +import tileservermapping.osm_data.models +import tileservermapping.osm_data.storage + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='SqlDump', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('x', models.IntegerField(help_text='Slippy map coordinate X')), + ('y', models.IntegerField(help_text='Slippy map coordinate Z')), + ('z', models.IntegerField(help_text='Slippy map coordinate Z')), + ('file', models.FileField(default=None, null=True, storage=tileservermapping.osm_data.storage.OverwriteStorage(), upload_to=tileservermapping.osm_data.models.gen_sql_dump_location)), + ], + options={ + 'unique_together': {('x', 'y', 'z')}, + }, + ), + migrations.CreateModel( + name='PlanetDump', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('x', models.IntegerField(help_text='Slippy map coordinate X')), + ('y', models.IntegerField(help_text='Slippy map coordinate Y')), + ('z', models.IntegerField(help_text='Slippy map coordinate Z (zoom)')), + ('file', models.FileField(default=None, null=True, storage=tileservermapping.osm_data.storage.OverwriteStorage(), upload_to=tileservermapping.osm_data.models.gen_planet_dump_location)), + ], + options={ + 'unique_together': {('x', 'y', 'z')}, + }, + ), + ] diff --git a/tileservermapping/osm_data/migrations/__init__.py b/tileservermapping/osm_data/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tileservermapping/osm_data/models.py b/tileservermapping/osm_data/models.py new file mode 100644 index 0000000..d146d59 --- /dev/null +++ b/tileservermapping/osm_data/models.py @@ -0,0 +1,64 @@ +import posixpath +from typing import * + +from django.db import models + +from tileservermapping.osm_data.storage import OverwriteStorage + + +def gen_planet_dump_location(instance, filename) -> str: + """ + Generate uploaded filename based on coordinates of the instance + + :param PlanetDump instance: Instance of the newly created database object + :param str filename: Originally uploaded file name + """ + return f'planet_dumps/{instance.z}_{instance.x}_{instance.y}.pbf' + + +def gen_sql_dump_location(instance, filename) -> str: + """ + Generate uploaded filename based on coordinates of the file + + :param SqlDump instance: Instance of the newly created database object + :param str filename: Originally uploaded file name + """ + return f'sql_dumps/{instance.z}_{instance.x}_{instance.y}.pg_dump' + + +class PlanetDump(models.Model): + """ + Planet dump file encoded in PBF format. + Each database object is mapped to one file in the file storage and can be used to manage that file. + + One PlanetDump does not always store the whole planet but only a smaller portion + defined by the `x y` and `z` coordinates. + """ + + x = models.IntegerField(help_text='Slippy map coordinate X') + y = models.IntegerField(help_text='Slippy map coordinate Y') + z = models.IntegerField(help_text='Slippy map coordinate Z (zoom)') + file = models.FileField(upload_to=gen_planet_dump_location, null=True, default=None, storage=OverwriteStorage()) + + class Meta: + unique_together = [['x', 'y', 'z']] + + def __str__(self): + return f'{self.__class__.__name__} of x:{self.x} y:{self.y} z:{self.z}' + + +class SqlDump(models.Model): + """ + PostgreSQL Dump files which hold all the data a Tileserver needs. + """ + + x = models.IntegerField(help_text='Slippy map coordinate X') + y = models.IntegerField(help_text='Slippy map coordinate Z') + z = models.IntegerField(help_text='Slippy map coordinate Z') + file = models.FileField(upload_to=gen_sql_dump_location, null=True, default=None, storage=OverwriteStorage()) + + class Meta: + unique_together = [['x', 'y', 'z']] + + def __str__(self): + return f'{self.__class__.__name__} of x:{self.x} y:{self.y} z:{self.z}' diff --git a/tileservermapping/osm_data/serializers.py b/tileservermapping/osm_data/serializers.py new file mode 100644 index 0000000..c62e498 --- /dev/null +++ b/tileservermapping/osm_data/serializers.py @@ -0,0 +1,30 @@ +from rest_framework import serializers, validators +from . import models + + +class PlanetDumpSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = models.PlanetDump + fields = ['url', 'id', 'x', 'y', 'z', 'file'] + validators = [ + validators.UniqueTogetherValidator( + queryset=models.PlanetDump.objects.all(), + fields=['x', 'y', 'z'] + ) + ] + + # TODO: Validate uploaded pbf file + + +class SqlDumpSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = models.SqlDump + fields = ['url', 'id', 'x', 'y', 'z', 'file'] + validators = [ + validators.UniqueTogetherValidator( + queryset=models.SqlDump.objects.all(), + fields=['x', 'y', 'z'] + ) + ] + + # TODO: Validate uploaded postgresql-dump file diff --git a/tileservermapping/osm_data/storage.py b/tileservermapping/osm_data/storage.py new file mode 100644 index 0000000..4ab6daf --- /dev/null +++ b/tileservermapping/osm_data/storage.py @@ -0,0 +1,11 @@ +from django.core.files.storage import FileSystemStorage + + +class OverwriteStorage(FileSystemStorage): + def _save(self, name, content): + if self.exists(name): + self.delete(name) + return super(OverwriteStorage, self)._save(name, content) + + def get_available_name(self, name, max_length=None): + return name diff --git a/tileservermapping/osm_data/tests.py b/tileservermapping/osm_data/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/tileservermapping/osm_data/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/tileservermapping/osm_data/urls.py b/tileservermapping/osm_data/urls.py new file mode 100644 index 0000000..72d02a0 --- /dev/null +++ b/tileservermapping/osm_data/urls.py @@ -0,0 +1,9 @@ +from django.urls import path, include, re_path +from rest_framework import routers + +from . import views + +router = routers.DefaultRouter() +router.register(r'planet_dumps', views.PlanetDumpViewset) +router.register(r'postgresql_dumps', views.SqlDumpViewset) +urlpatterns = router.urls diff --git a/tileservermapping/osm_data/views.py b/tileservermapping/osm_data/views.py new file mode 100644 index 0000000..350ab9e --- /dev/null +++ b/tileservermapping/osm_data/views.py @@ -0,0 +1,36 @@ +from rest_framework import viewsets, permissions, parsers +from . import models, serializers + + +class PlanetDumpViewset(viewsets.ModelViewSet): + queryset = models.PlanetDump.objects.all() + serializer_class = serializers.PlanetDumpSerializer + + def get_parsers(self): + if self.request.method == 'POST' or self.request.method == 'PATCH' or self.request.method == 'PUT': + return [*super(PlanetDumpViewset, self).get_parsers(), parsers.MultiPartParser()] + else: + return super().get_parsers() + + def get_permissions(self): + if self.action == 'list': + return [permissions.AllowAny()] + else: + return super(PlanetDumpViewset, self).get_permissions() + + +class SqlDumpViewset(viewsets.ModelViewSet): + queryset = models.SqlDump.objects.all() + serializer_class = serializers.SqlDumpSerializer + + def get_parsers(self): + if self.request.method == 'POST' or self.request.method == 'PATCH' or self.request.method == 'PUT': + return [*super(SqlDumpViewset, self).get_parsers(), parsers.MultiPartParser()] + else: + return super().get_parsers() + + def get_permissions(self): + if self.action == 'list': + return [permissions.AllowAny()] + else: + return super(SqlDumpViewset, self).get_permissions() diff --git a/tileservermapping/settings.py.example b/tileservermapping/settings.py.example index a2f8db8..685ca87 100644 --- a/tileservermapping/settings.py.example +++ b/tileservermapping/settings.py.example @@ -36,3 +36,4 @@ DATABASES = { } STATIC_ROOT = os.path.join('/', 'app', 'static') +MEDIA_ROOT = os.getenv('TM_MEDIA_ROOT', os.path.join('/', 'app', 'media')) diff --git a/tileservermapping/settings_base.py b/tileservermapping/settings_base.py index b353e45..bc788e7 100644 --- a/tileservermapping/settings_base.py +++ b/tileservermapping/settings_base.py @@ -34,6 +34,7 @@ 'drf_yasg', 'tileservermapping.mapping', 'tileservermapping.service_accounts', + 'tileservermapping.osm_data', ] MIDDLEWARE = [ @@ -110,6 +111,7 @@ # https://docs.djangoproject.com/en/2.0/howto/static-files/ STATIC_URL = '/static/' +MEDIA_URL = '/media/' # Django Rest Framework and extensions @@ -123,9 +125,8 @@ 'rest_framework.authentication.SessionAuthentication', 'tileservermapping.service_accounts.authentication_classes.ServiceAccountTokenAuthentication' ], - "DEFAULT_PERMISSION_CLASSES": [ - "rest_framework.permissions.IsAuthenticated" - ], + "DEFAULT_PERMISSION_CLASSES": ["rest_framework.permissions.IsAuthenticated"], + "DEFAULT_PARSER_CLASSES": ["rest_framework.parsers.JSONParser"] } diff --git a/tileservermapping/urls.py b/tileservermapping/urls.py index fbe4ba5..b1e1713 100644 --- a/tileservermapping/urls.py +++ b/tileservermapping/urls.py @@ -25,6 +25,7 @@ urlpatterns = [ path("api//", include("tileservermapping.mapping.urls")), + path("api//", include("tileservermapping.osm_data.urls")), path("api//", include("tileservermapping.service_accounts.urls")), path("mappings/", include("tileservermapping.mapping.urls")), # included for compatibility to old url schema @@ -32,5 +33,5 @@ path("schema/", schema_view.without_ui(cache_timeout=0), name="schema-json"), path("docs/", schema_view.with_ui("swagger", cache_timeout=0), name="schema-swagger-ui"), - path("", RedirectView.as_view(url="/docs")) + path("", RedirectView.as_view(url="/docs")), ]