Backend: Re-organize code, improve Dockerfile, add script for demo data

- Code is now in subdirectory to separate from dockerfile etc.
- script to fill dev container with data now works remotely
- settings accepts more input from environment for better config

Backend/.dockerignore

+# Docker
+Dockerfile
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# Editor backups
+*~
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/

Backend/Dockerfile

-from python:3.6-alpine
+FROM python:3.6-alpine
 
-ENV FLASK_APP app.py
-ENV FLASK_DEBUG true
-EXPOSE 80
+# Create user with home directory and no password and change workdir
+RUN adduser -Dh /pvk pvk
+WORKDIR /pvk
+# API will run on port 80
+EXPOSE 8080
+# Environment variable for config, use path for docker secrets as default
+ENV pvk_CONFIG=/run/secrets/pvk_config
 
-COPY ./requirements.txt /requirements.txt
-RUN pip install -r requirements.txt
+# Install bjoern and dependencies for install (we need to keep libev)
+RUN apk add --no-cache --virtual .deps \
+        musl-dev python-dev gcc git && \
+    apk add --no-cache libev-dev && \
+    pip install bjoern
 
-COPY ./app.py /app.py
-COPY ./settings.py /settings.py
-COPY ./security.py /security.py
-COPY ./signups.py /signups.py
-COPY ./validation.py /validation.py
+# Copy files to /api directory, install requirements
+COPY ./ /pvk
+RUN pip install -r /pvk/requirements.txt
 
-COPY ./demo_server.py /demo_server.py
+# Cleanup dependencies
+RUN apk del .deps
 
-WORKDIR /
+# Switch user
+USER pvk
 
-# Run database init before starting dev server
-ENTRYPOINT ["python"]
-CMD ["demo_server.py"]
+# Start bjoern
+CMD ["python3", "server.py"]

Backend/app.py → Backend/backend/app.py

@@ -21,9 +21,9 @@ from os import getcwd
 from eve import Eve
 from flask import Config
 
-from security import APIAuth, only_own_nethz
-from validation import APIValidator
-from signups import (
+from backend.security import APIAuth, only_own_nethz
+from backend.validation import APIValidator
+from backend.signups import (
     new_signups,
     deleted_signup,
     patched_signup,
@@ -40,7 +40,7 @@ def create_app(settings=None):
     makes this easy) and updated settings from the function call, if provided.
     """
     config = Config(getcwd())
-    config.from_object('settings')
+    config.from_object('backend.settings')
     if settings is not None:
         config.update(settings)
 

Backend/settings.py → Backend/backend/settings.py

@@ -11,16 +11,13 @@ schema directly.
 
 from os import environ
 
-# prefix everything with /api
-URL_PREFIX = 'api'
-
 # CORS
 X_DOMAINS = '*'
 X_HEADERS = ['Authorization', 'If-Match', 'If-Modified-Since', 'Content-Type']
 
 # AMIVAPI URL and Admin Group
-AMIVAPI_URL = "https://amiv-api.ethz.ch"
-ADMIN_GROUP_NAME = 'PVK Admins'
+AMIVAPI_URL = environ.get('AMIVAPI_URL', 'https://amiv-api.ethz.ch')
+ADMIN_GROUP_NAME = environ.get('AMIVAPI_GROUP', 'PVK Admins')
 
 # DB
 MONGO_HOST = environ.get('MONGO_HOST', 'localhost')

Backend/validation.py → Backend/backend/validation.py

@@ -12,7 +12,7 @@ TODO: Several validation rules still need to be implemented. See `settings.py`.
 from flask import request, current_app
 from eve.io.mongo import Validator
 
-from security import is_admin, get_nethz
+from backend.security import is_admin, get_nethz
 
 
 class APIValidator(Validator):

Backend/demo_server.py → Backend/create_demo_data.py

-"""Demo Server.
+"""Demo Data.
 
-Add some demo data to the database and run the app.
+This script can be used to create demo data on the development server.
+
+TODO: Accept user and password from command line, provide output
 """
 
-from os import getenv
+import sys
 from random import randint
 from datetime import datetime as dt, timedelta
-import json
-
-from flask import g
-
-from app import create_app
-
+import requests
 
-HOST = getenv('PVK_HOST', '0.0.0.0')
-PORT = getenv('PVK_PORT', '80')
+AMIVAPI_DEV_URL = "https://amiv-api.ethz.ch"
+PVK_DEV_URL = 'http://localhost:8080'  # 'http://pvk-api-dev.amiv.ethz.ch'
 
-APP = create_app()
-CLIENT = APP.test_client()
-DATE_FORMAT = APP.config['DATE_FORMAT']
+DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
 ASSISTANTS = ['pablo', 'assi', 'anon', 'mongo']
 MIN_SPOTS = 20
 MAX_SPOTS = 40
 
-"""
-# Clear everything
-from pymongo import MongoClient
 
-connection = MongoClient(APP.config['MONGO_HOST'],
-                         APP.config['MONGO_PORT'])
-connection.drop_database(APP.config['MONGO_DBNAME'])
-connection.close()
-"""
+def login(username, password):
+    """Login user, return token."""
+    data = {
+        'username': username,
+        'password': password,
+    }
+    return requests.post("%s/%s" % (AMIVAPI_DEV_URL, 'sessions'),
+                         json=data).json()['token']
 
 
-def post(resource, data):
+def post(resource, data, token):
     """Create something, ignoring auth."""
-    with APP.test_request_context():
-        g.apiuser = 'Not None :)'
-        g.nethz = 'Something'
-        g.admin = True
+    response = requests.post('%s/%s' % (PVK_DEV_URL, resource),
+                             json=data,
+                             headers={'Authorization': token})
+    data = response.json()
 
-        response = CLIENT.post('%s/%s' % (APP.config['URL_PREFIX'], resource),
-                               data=json.dumps(data),
-                               content_type="application/json",
-                               headers={'Authorization': 'Token Lala'})
+    if response.status_code != 201:
+        status = data['_error']['code']
+        message = str(data['_error'].get('message', ''))
+        issues = str(data.get('_issues', ''))
+        print('%s:' % status, issues or message)
+        return {}
 
-        if response.status_code != 201:
-            error = json.loads(response.get_data(as_text=True))
-            status = error['_error']['code']
-            message = str(error['_error'].get('message', ''))
-            issues = str(error.get('_issues', ''))
-            print('%s:' % status, issues or message)
-            return {}
+    return data
 
-        return json.loads(response.get_data(as_text=True))
 
-
-def create_lectures(department):
+def create_lectures(department, token):
     """Create a few lectures, return names."""
     lectures = []
     for lecture in ['Some %s Lecture', 'Cool %s Lecture', 'Boring %s Lecture']:
@@ -70,7 +59,7 @@ def create_lectures(department):
                 'department': department,
                 'assistants': ASSISTANTS,
             }
-            response = post('lectures', data)
+            response = post('lectures', data, token)
             if response:
                 lectures.append(response['_id'])
 
@@ -105,7 +94,7 @@ ROOM = room_gen()
 TIME = time_gen(dt.now() + timedelta(days=90))  # Sometime in the future
 
 
-def create_course(lecture, assistant, open_signup=True):
+def create_course(lecture, assistant, token, open_signup=True):
     """Create several course for each lecture."""
     signup_diff = timedelta(days=-5) if open_signup else timedelta(15)
     start = dt.utcnow() + signup_diff
@@ -124,40 +113,52 @@ def create_course(lecture, assistant, open_signup=True):
         'room': next(ROOM),
         'spots': randint(MIN_SPOTS, MAX_SPOTS),
     }
-    return post('courses', data)['_id']
+    return post('courses', data, token)['_id']
 
 
-def create_signups(course):
+def create_signups(course, token):
     """Create random number of signups to a course."""
-    for n in range(randint(0, 2*MAX_SPOTS)):
+    for ind in range(randint(0, 2*MAX_SPOTS)):
         data = {
-            'nethz': 'student%d' % n,
+            'nethz': 'student%d' % ind,
             'course': course
         }
-        post('signups', data=data)
+        post('signups', data, token)
 
 
-if __name__ == '__main__':
+def main():
+    """Login and create demo data."""
+    if len(sys.argv) != 3:
+        print('Usage: python %s USERNAME PASSWORD' % sys.argv[0])
+        sys.exit(-1)
+
+    print('Logging in...')
+    token = login(sys.argv[1], sys.argv[2])
+
     # Create courses
     # (Repeat for itet and mavt)
     for department in ['itet', 'mavt']:
-        # Create a few lectures
-        lectures = create_lectures(department)
+        print('Department: %s' % department)
 
-        # Create a few courses with closed signup
+        print('Creating lectures...')
+        lectures = create_lectures(department, token)
+
+        print('Creating courses with closed signup...')
         for lecture in lectures:
-            for n in range(randint(0, 2)):
-                create_course(lecture, ASSISTANTS[n], open_signup=False)
+            for ind in range(randint(0, 2)):
+                create_course(lecture, ASSISTANTS[ind],
+                              token, open_signup=False)
 
-        # Create a few courses with open signup
+        print('Creating courses with closed signup...')
         courses = []
         for lecture in lectures:
-            for n in range(randint(2, len(ASSISTANTS))):
-                courses.append(create_course(lecture, ASSISTANTS[n]))
+            for ind in range(randint(2, len(ASSISTANTS))):
+                courses.append(create_course(lecture, token, ASSISTANTS[ind]))
 
-        # Create signups for courses
+        print('Creating signups...')
         for course in courses:
-            create_signups(course)
+            create_signups(course, token)
+
 
-    # Start the app
-    APP.run(host=HOST, port=int(PORT))
+if __name__ == '__main__':
+    main()

Backend/server.py

+# wsgi server (used in docker container)
+# [bjoern](https://github.com/jonashaag/bjoern) required.
+
+from backend.app import create_app
+import bjoern
+
+if __name__ == '__main__':
+    print('Starting bjoern on port 8080...', flush=True)
+    bjoern.run(create_app(), '0.0.0.0', 8080)

Backend/tests/conftest.py

@@ -16,7 +16,7 @@ import pytest
 
 from flask import g
 from flask.testing import FlaskClient
-from app import create_app
+from backend.app import create_app
 
 
 TEST_SETTINGS = {

Backend/tests/test_api.py

@@ -19,7 +19,7 @@ So we just put the provided token into g and see if the functions work.
 
 from flask import g
 
-from security import get_user, get_nethz, is_admin
+from backend.security import get_user, get_nethz, is_admin
 
 
 def test_user_found(app, usertoken):

Backend/tests/test_signups.py

@@ -10,7 +10,7 @@ from datetime import datetime as dt
 from unittest.mock import patch, call
 import pytest
 
-from signups import update_signups
+from backend.signups import update_signups
 
 
 def test_success(app):
@@ -198,7 +198,7 @@ def course(app):
 @pytest.fixture
 def mock_update():
     """Mock the actual updating of spots for a test."""
-    with patch('signups.update_signups', return_value=[]) as update:
+    with patch('backend.signups.update_signups', return_value=[]) as update:
         yield update
 
 

Backend/tox.ini

@@ -13,10 +13,8 @@ deps =
     -rrequirements.txt
 passenv = MONGO_HOST
 
-# Pylint config is a bit hacky, for some reason *.py doesnt work in tox
 [testenv:pylint]
-commands = pylint app.py security.py settings.py signups.py validation.py \
-    setup.py tests
+commands = pylint backend tests
 deps =
     pylint
     pytest