diff --git a/.gitignore b/.gitignore index 6fcd1562..71e567fd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ *.swp *.swo -settings.py # ---> Python # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/.run/makemigrations.run.xml b/.run/makemigrations.run.xml new file mode 100644 index 00000000..a15b200b --- /dev/null +++ b/.run/makemigrations.run.xml @@ -0,0 +1,25 @@ + + + + + \ No newline at end of file diff --git a/.run/migrate.run.xml b/.run/migrate.run.xml new file mode 100644 index 00000000..b547f375 --- /dev/null +++ b/.run/migrate.run.xml @@ -0,0 +1,25 @@ + + + + + \ No newline at end of file diff --git a/.run/runzipdaemon.run.xml b/.run/runzipdaemon.run.xml new file mode 100644 index 00000000..0a525470 --- /dev/null +++ b/.run/runzipdaemon.run.xml @@ -0,0 +1,25 @@ + + + + + \ No newline at end of file diff --git a/.run/serve.run.xml b/.run/serve.run.xml new file mode 100644 index 00000000..305ddf8b --- /dev/null +++ b/.run/serve.run.xml @@ -0,0 +1,26 @@ + + + + + \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index fdf3ae3d..2260ae9c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update && \ # add requirements file WORKDIR /app/src -COPY requirements.txt /app/src/requirements.txt +COPY imagetagger/requirements.txt /app/src/requirements.txt # install python dependencies RUN pip3 install -r /app/src/requirements.txt @@ -22,10 +22,12 @@ RUN apt-get clean # add remaining sources COPY imagetagger /app/src/imagetagger -# confiure runtime environment -RUN mkdir /app/data /app/static /app/config -RUN cp /app/src/imagetagger/imagetagger/settings.py.example /app/config/settings.py -RUN ln -sf /app/config/settings.py /app/src/imagetagger/imagetagger/settings.py +# configure imagetagger.settings_local to be importable but 3rd party providable +RUN mkdir -p /app/data /app/config/ +RUN touch /app/config/settings.py +RUN ln -sf /app/config/settings.py /app/src/imagetagger/imagetagger/settings_local.py + +# configure runtime environment RUN sed -i 's/env python/env python3/g' /app/src/imagetagger/manage.py ARG UID_WWW_DATA=5008 @@ -33,14 +35,15 @@ ARG GID_WWW_DATA=33 RUN usermod -u $UID_WWW_DATA -g $GID_WWW_DATA -d /app/data/ www-data RUN chown -R www-data /app -RUN /app/src/imagetagger/manage.py collectstatic --no-input - COPY docker/uwsgi_imagetagger.ini /etc/uwsgi/imagetagger.ini COPY docker/nginx.conf /etc/nginx/sites-enabled/default COPY docker/update_points docker/zip_daemon docker/run /app/bin/ RUN ln -sf /app/bin/* /usr/local/bin ENTRYPOINT ["/usr/local/bin/run"] ENV IN_DOCKER=true +ENV DJANGO_CONFIGURATION=Prod +ENV IT_FS_URL=/app/data +ENV IT_STATIC_ROOT=/var/www/imagetagger # add image metadata EXPOSE 3008 diff --git a/README.md b/README.md index dda60c00..23b131e7 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,26 @@ If you are participating in RoboCup, you should not install your own instance bu For a short overview of the functions please have a look at the following poster: https://robocup.informatik.uni-hamburg.de/wp-content/uploads/2017/11/imagetagger-poster.pdf +Table of Contents: +- [ImageTagger](#imagetagger) + + - [Features](#features) + + - [Planned Features](#planned-features) + + - [Reference](#reference) + + - [Installing and running ImageTagger](#installing-and-running-imagetagger) + + - [Locally](#locally) + - [In-Docker](#in-docker) + - [On Kubernetes](#on-kubernetes) + + - [Configuration](#configuration) + + - [Used dependencies](#used-dependencies) + + ## Features * team creation @@ -27,7 +47,7 @@ For a short overview of the functions please have a look at the following poster ## Reference -This paper describes the Bit-Bots Imagetagger more in depth. Please cite if you use this tool in your research: +This paper describes the Bit-Bots ImageTagger more in depth. Please cite if you use this tool in your research: FIEDLER, Niklas, et al. [ImageTagger: An Open Source Online Platform for Collaborative Image Labeling.](https://robocup.informatik.uni-hamburg.de/wp-content/uploads/2018/11/imagetagger_paper.pdf) In: RoboCup 2018: Robot World Cup XXII. Springer, 2018. @@ -40,131 +60,204 @@ FIEDLER, Niklas, et al. [ImageTagger: An Open Source Online Platform for Collabo organization={Springer} } ``` -## Upgrade -``` -pip install -U -r requirements.txt +## Installing and running ImageTagger +ImageTagger can be installed and run locally (best for development), in a docker container or in Kubernetes +(used in our deployment). + +### Locally + +In some of the following code snippets, the `DJANGO_CONFIGURATION` environment variable is exported. +This defines the type of deployment by selecting one of our predefined configuration presets. +If ImageTagger is running in a development environment, no export is necessary. + +1. #### Install the latest release + + You should probably do this in a [virtual environment](https://virtualenv.pypa.io/en/stable/) + + Replace `v…` with the latest release tag. + ```shell + git checkout v… + cd imagetagger + pip3 install -r requirements.txt + ``` + +2. #### Setup a database server + + As a database server [postgresql](https://www.postgresql.org/) is required. + Please seek a guide specific to your operating system on how to install a server and get it running. + + Once postgresql is installed, a user and database need to be set up for ImageTagger. + Of course, the user and password can be changed to something else. + ```postgresql + CREATE USER imagetagger PASSWORD 'imagetagger'; + CREATE DATABASE imagetagger WITH OWNER imagetagger; + ``` + +3. ### Select a Configuration preset + + When running ImageTagger as a developer, no step is necessary because a development configuration is used per + default when not running as a docker based deployment. + However if this is supposed to be a production deployment, export the following environment variable. + + Currently available presets are `Dev` and `Prod` + + ```shell + export DJANGO_CONFIGURATION=Prod + ``` + +3. #### Configuring ImageTagger to connect to the database + + Please see the lower [Configuration](#configuration) section on how to configure ImageTagger for your specific + database credentials. + +4. #### Initialize the database + + Run the following to create and setup all necessary database tables: + ```shell + ./manage.py migrate + ``` + +5. #### Create a user + + ```shell + ./manage.py createsuperuser + ``` + +6. #### Run the server + + ```shell + ./manage.py runserver + ``` + +**In a production deployment** it is necessary to run the following commands after each upgrade: +```shell ./manage.py migrate +./manage.py compilemessages +./manage.py collectstatic ``` -for additional steps on some releases see instructions -in [UPGRADE.md](https://github.com/bit-bots/imagetagger/blob/master/UPGRADE.md) - -## Install - -Checkout the latest release: - -``` -git checkout v0.x -``` - -In our production Senty is used for error reporting (pip install sentry-sdk). -django-auth-ldap is used for login via ldap -uwsgi is used to serve the app to nginx - -Install Python Dependencies: - -``` -pip3 install -r requirements.txt -``` - -Copy settings.py.example to settings.py in the imagetagger folder: - -``` -cp imagetagger/settings.py.example imagetagger/settings.py +for additional steps on some releases see instructions in [UPGRADE.md](https://github.com/bit-bots/imagetagger/blob/master/UPGRADE.md) + +### In-Docker + +1. #### Checkout the latest release + + ```shell + git checkout v… + ``` + +2. #### Build the container image + + *Note:* the Dockerfile has been tested with [podman](https://podman.io/) as well. + ```shell + docker build -t imagetagger . + ``` + +3. #### Run the container + + ```shell + docker run -it -p 8000:80 --name imagetagger imagetagger + ``` + + This step will not work out of the box because configuration still needs to be done. + See the lower [section about configuring](#configuration) ImageTagger on how to fix this. + +4. #### Create a user + + *Note: This step requires a container running in the background.* + ```shell + docker exec -it imagetagger /app/src/imagetagger/manage.py createsuperuser + ``` + +#### About the Container + +| Kind | Description | +|---|---| +| Volume | `/app/data` is where persistent data (like images) are stored +| Volume | `/app/config` is where additional custom configuration files can be placed. See the [Configuration section](#configuration) below +| Environment | ImageTagger can mostly be configured via environment variables +| Ports | The container internal webserver listens on port 80 for incoming connections. + +### On Kubernetes + +1. Follow the steps for [In-Docker](#in-docker) on how to build a container image + +2. **Apply kubernetes configuration** + + *Note: This assumes you have access to a kubernetes cluster configured and can use kubectl* + + We use [kustomize](https://kustomize.io/) for management of our kubernetes configuration. + It can be applied by running the following commands: + ```shell + kustomize build . > k8s.yml + kubectl apply -f k8s.yml + ``` + +#### Generated Kubernetes resources + +| Kind | Name | Description +|---|---|--- +| [Namespace](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/) | imagetagger | The namespace in which each resource resides +| [Deployment](https://kubernetes.io/es/docs/concepts/workloads/controllers/deployment/) + [Service](https://kubernetes.io/docs/concepts/services-networking/service/) | imagetagger-postgres | postgresql server which is required by ImageTagger. The replica count can be dialed down to 0 if an the usage of an external server is desired. +| [PersistentVolumeClaim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) | imagetagger-database | Where the postgresql server stores its data +| [ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) | imagetagger-postgres | Configuration of the postgresql server. Also available inside the application server deployment so that settings can be [referenced](https://kubernetes.io/docs/tasks/inject-data-application/define-interdependent-environment-variables/). +| [Deployment](https://kubernetes.io/es/docs/concepts/workloads/controllers/deployment/) + [Service](https://kubernetes.io/docs/concepts/services-networking/service/) | imagetagger-web | application server. Per default this deployment references the image `imagetagger:local` which is probably not resolvable and should be replaced by a reference to where your previously built container image is available. +| [PersistentVolumeClaim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) | imagetagger-image-data | Where the application server stores its images (and tools). +| [ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) | imagetagger-web | Configuration of the application server. Mounted as environment variables. See [Configuration](#configuration) for details. + +## Configuration + +ImageTagger is a Django application and uses [django-configurations](https://django-configurations.readthedocs.io/en/stable/) +for better configuration management. + +ImageTagger is configured to use a *Dev* configuration when running locally and *Prod* when running in a container. +This can be overridden via the environment variable `DJANGO_CONFIGURATION`. + +For a list of available configuration values see [settings.py](https://github.com/bit-bots/imagetagger/blob/master/imagetagger/imagetagger/settings.py). +Towards the bottom is a list of *values*. These are taken from environment variables where the key is the variable name +but with an `IT_` prefix. + +If completely custom configuration is desired, `imagetagger/imagetagger/settings_local.py` can be created in which +a custom configuration class may be created. In Docker this file may be located at `/app/config/settings.py` so that +mounting it should be simple. +To use this custom configuration class, the environment variables `DJANGO_SETTINGS_MODULE=imagetagger.settings_local` +and `DJANGO_CONFIGURATION=MyCustomClass` must be set. + +If downloading zip files of Imagesets is desired, the feature can be enabled by settings `IT_ENABLE_ZIP_DOWNLOAD` to `true`. +A separate zip generation daemon must then be started via the following command. +This feature is enabled and running automatically in Docker based deployments. +```shell +export DJANGO_CONFIGURATION=Prod +./manage.py runzipdaemon ``` -and customize the settings.py. - -The following settings should probably be changed: - -+ The secret key -+ The DEBUG setting -+ The ALLOWED\_HOSTS -+ The database settings -+ The UPLOAD\_FS\_GROUP to the id of the group that should access and create the uploaded images - -For the database, postgresql is used. Install it by running `sudo apt install postgresql` on Debian based operating systems. A default database cluster is automatically initialized. - -Other systems may require different commands to install the package and the database cluster may -have to be initialized manually (e.g. using `sudo -iu postgres initdb --locale en_US.UTF-8 -D '/var/lib/postgres/data'`). - -To start the postgresql server, run `sudo systemctl start postgresql.service`. If the server should always be started on boot, run `sudo systemctl enable postgresql.service`. - -Then, create the user and the database by running - -`sudo -iu postgres psql` - -and then, in the postgres environment - -``` -CREATE USER imagetagger PASSWORD 'imagetagger'; -CREATE DATABASE imagetagger WITH OWNER imagetagger ENCODING UTF8; -``` - -where of course the password and the user should be adapted to the ones specified in the database settings in the settings.py. - -To initialize the database, run `./manage.py migrate` - -To create an administrator user, run `./manage.py createsuperuser`. - -`./manage.py runserver` starts the server with the configuration given in the settings.py file. - To create annotation types, log into the application and click on Administration at the very bottom of the home page. -For **production** systems it is necessary to run the following commands after each upgrade -```bash -./manage.py migrate -./manage.py compilemessages -./manage.py collectstatic -``` +### Minimal production Configuration -Our production uwisgi config can be found at https://github.com/fsinfuhh/mafiasi-rkt/blob/master/imagetagger/uwsgi-imagetagger.ini +In production, the following configuration values **must** be defined (as environment variables) -Example Nginx Config: +| Key | Description +|---|--- +| `IT_SECRET_KEY` | The [django secret key](https://docs.djangoproject.com/en/3.0/ref/settings/#std:setting-SECRET_KEY) used by ImageTagger. It is used for password hashing and other cryptographic operations. +| `IT_ALLOWED_HOSTS` | [django ALLOWED_HOSTS](https://docs.djangoproject.com/en/3.0/ref/settings/#allowed-hosts) as comma separated list of values. It defines the hostnames which this application will allow to be used under. +| `IT_DB_HOST` | Hostname (or IP-address) of the postgresql server. When deploying on kubernetes, the provided Kustomization sets this to reference the database deployment. +| `IT_DB_PASSWORD ` | Password used for authenticating against the postgresql server. +| `IT_DOWNLOAD_BASE_URL` | Base-URL under which this application is reachable. It defines the prefix for generated download links. -``` -server { - listen 443; - server_name imagetagger.bit-bots.de; - - ssl_certificate /etc/letsencrypt/certs/imagetagger.bit-bots.de/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/certs/imagetagger.bit-bots.de/privkey.pem; - include /etc/nginx/ssl.conf; - include /etc/nginx/acme.conf; - ssl on; - - client_max_body_size 4G; - - access_log /var/log/nginx/imagetagger.bit-bots.de.access.log; - error_log /var/log/nginx/imagetagger.bit-bots.de.error.log; - - location /static { - expires 1h; - alias /var/www/imagetagger; - } - - location /ngx_static_dn/ { - internal; - alias /srv/data/imagetagger/storage/pictures/; - } - - location / { - include uwsgi_params; - uwsgi_pass 127.0.0.1:4819; - uwsgi_read_timeout 120; - } -} -``` +### Database Configuration -If you want to provide zip files of image sets, set `ENABLE_ZIP_DOWNLOAD = True` in your `settings.py`. -A daemon that creates and updates the zip files is necessary, you can start it with `./manage.py runzipdaemon`. -Please take into account that the presence of zip files will double your storage requirement. +The following environment variables configure how the postgresql server is accessed -Zip archive download via a script is also possible. The URL is `/images/imageset//download/`. A successful request -returns HTTP 200 OK and the zip file. When the file generation is still in progress, HTTP 202 ACCEPTED is returned. -For an empty image set, HTTP 204 NO CONTENT is returned instead of an empty zip archive. +| Key | Required | Default | Description +|---|---|---|--- +| `IT_DB_HOST` | yes | | Hostname (or IP-address) of the postgresql server. When deploying on kubernetes, the provided Kustomization sets this to reference the database deployment. +| `IT_DB_PORT` | no | 5432 | Port on which the postgresql server listens. +| `IT_DB_NAME` | no | imagetagger | Database name on the postgresql server. ImageTagger requires full access to it. +| `IT_DB_USER` | no | imagetagger | User as which to authenticate on the postgresql server. +| `IT_DB_PASSWORD` | yes | | Password used for authentication. ## Used dependencies diff --git a/UPGRADE.md b/UPGRADE.md index a38a3ccf..39210e85 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -1,6 +1,17 @@ Upgrade instructions ==================== +Upgrading 0.5 to 0.6 +-------------------- + +Release 0.6 changed the application configuration to be largely based on environment variables instead of the operator +providing a custom `settings.py` file. + +Most settings which previously had to be defined can now be given as environment variables but with an `IT_` prefix. +See [the README](https://github.com/bit-bots/imagetagger#configuration) on more details. + +If desired, a custom configuration file can still be supplied. This is explained in the README as well. + Upgrading 0.4 to 0.5 -------------------- diff --git a/docker/run b/docker/run index bf12f668..2f8e8b9f 100755 --- a/docker/run +++ b/docker/run @@ -1,6 +1,7 @@ #!/bin/bash set -e +/app/src/imagetagger/manage.py collectstatic --no-input /app/src/imagetagger/manage.py migrate nginx diff --git a/imagetagger/imagetagger/base/__init__.py b/imagetagger/imagetagger/base/__init__.py index e69de29b..f7930fdf 100644 --- a/imagetagger/imagetagger/base/__init__.py +++ b/imagetagger/imagetagger/base/__init__.py @@ -0,0 +1 @@ +from imagetagger.base.checks import * diff --git a/imagetagger/imagetagger/base/checks.py b/imagetagger/imagetagger/base/checks.py new file mode 100644 index 00000000..f74c5ea8 --- /dev/null +++ b/imagetagger/imagetagger/base/checks.py @@ -0,0 +1,40 @@ +from typing import List +from os.path import join, dirname +from django.core.checks import register, CheckMessage, Error +from django.conf import settings +from fs.base import FS +from . import filesystem + + +def _create_check_file(fs: FS, path: str): + fs.makedirs(dirname(path), recreate=True) + fs.create(path, wipe=True) + fs.writebytes(path, bytes('This is a demo file content', encoding='ASCII')) + fs.remove(path) + + +@register +def check_fs_root_config(app_configs, **kwargs) -> List[CheckMessage]: + try: + _create_check_file(filesystem.root(), join(settings.IMAGE_PATH, 'check.tmp.jpeg')) + _create_check_file(filesystem.root(), join(settings.TOOLS_PATH, 'check.tmp.py')) + return [] + except Exception as e: + return [Error( + 'Persistent filesystem is incorrectly configured. Could not create and delete a temporary check file', + hint=f'Check your FS_URL settings (currently {settings.FS_URL})', + obj=e, + )] + + +@register +def check_tmp_fs_config(app_configs, **kwargs) -> List[CheckMessage]: + try: + _create_check_file(filesystem.tmp(), join(settings.TMP_IMAGE_PATH, 'check.tmp.jpeg')) + return [] + except Exception as e: + return [Error( + 'Persistent filesystem is incorrectly configured. Could not create and delete a temporary check file', + hint=f'Check your TMP_FS_URL settings (currently {settings.TMP_FS_URL})', + obj=e, + )] diff --git a/imagetagger/imagetagger/base/filesystem.py b/imagetagger/imagetagger/base/filesystem.py index a99eabe5..eabc1c91 100644 --- a/imagetagger/imagetagger/base/filesystem.py +++ b/imagetagger/imagetagger/base/filesystem.py @@ -6,7 +6,7 @@ def root() -> fs.base.FS: """Get the root filesystem object or create one by opening `settings.FS_URL`.""" if root._instance is None: - root._instance = fs.open_fs(settings.FS_URL) + root._instance = fs.open_fs(settings.FS_URL, writeable=True) return root._instance root._instance = None @@ -15,7 +15,7 @@ def root() -> fs.base.FS: def tmp() -> fs.base.FS: """Get the tmp filesystem object or create one by opening `settings.TMP_FS_URL`.""" if tmp._instance is None: - tmp._instance = fs.open_fs(settings.TMP_FS_URL) + tmp._instance = fs.open_fs(settings.TMP_FS_URL, writeable=True) return tmp._instance tmp._instance = None \ No newline at end of file diff --git a/imagetagger/imagetagger/settings.py b/imagetagger/imagetagger/settings.py new file mode 100644 index 00000000..0edd694e --- /dev/null +++ b/imagetagger/imagetagger/settings.py @@ -0,0 +1,231 @@ +import os +from os.path import join as path_join +from configurations import Configuration, values +from django.contrib import messages +from django.core.exceptions import ImproperlyConfigured + +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def is_in_docker() -> bool: + return os.getenv('IN_DOCKER', '') != '' + + +class Base(Configuration): + ######################################################################## + # + # General django settings + # https://docs.djangoproject.com/en/3.1/topics/settings/ + # + ######################################################################## + INSTALLED_APPS = [ + 'imagetagger.annotations', + 'imagetagger.base', + 'imagetagger.images', + 'imagetagger.users', + 'imagetagger.tools', + 'imagetagger.administration', + 'django.contrib.admin', + 'imagetagger.tagger_messages', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'rest_framework', + 'widget_tweaks', + 'friendlytagloader', + ] + + MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'django.middleware.locale.LocaleMiddleware', + ] + + TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + 'imagetagger.base.context_processors.base_data', + ], + }, + }, + ] + + ROOT_URLCONF = 'imagetagger.urls' + WSGI_APPLICATION = 'imagetagger.wsgi.application' + + FILE_UPLOAD_HANDLERS = [ + "django.core.files.uploadhandler.MemoryFileUploadHandler", + "django.core.files.uploadhandler.TemporaryFileUploadHandler", + ] + + # Password validation + # https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators + + AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, + ] + + # Internationalization + # https://docs.djangoproject.com/en/1.10/topics/i18n/ + + USE_I18N = True + + USE_L10N = True + + USE_TZ = True + + AUTH_USER_MODEL = 'users.User' + + PROBLEMS_URL = 'https://github.com/bit-bots/imagetagger/issues' + PROBLEMS_TEXT = '' + + LOGIN_URL = '/user/login/' + LOGIN_REDIRECT_URL = '/images/' + + # Flash Messages + # https://docs.djangoproject.com/en/3.1/ref/contrib/messages/ + + MESSAGE_STORAGE = 'django.contrib.messages.storage.session.SessionStorage' + MESSAGE_TAGS = { + messages.INFO: 'info', + messages.ERROR: 'danger', + messages.WARNING: 'warning', + messages.SUCCESS: 'success', + } + + # Sets the default expire time for new messages in days + DEFAULT_EXPIRE_TIME = 7 + + # Sets the default number of messages per page + MESSAGES_PER_PAGE = 10 + + # Static files (CSS, JavaScript, Images) + # https://docs.djangoproject.com/en/1.10/howto/static-files/ + + STATIC_URL = '/static/' + + EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' + + ######################################################################## + # + # Computed properties and hooks + # + ######################################################################## + @property + def DATABASES(self): + """https://docs.djangoproject.com/en/1.10/ref/settings/#databases""" + return { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'HOST': self.DB_HOST, + 'PORT': self.DB_PORT, + 'NAME': self.DB_NAME, + 'USER': self.DB_USER, + 'PASSWORD': self.DB_PASSWORD + } + } + + @classmethod + def post_setup(cls): + super().post_setup() + + if cls.SENTRY_REPORTING_ENABLED: + try: + import sentry_sdk + from sentry_sdk.integrations.django import DjangoIntegration + sentry_sdk.init( + dsn=cls.SENTRY_DSN, + integrations=[DjangoIntegration()], + # If you wish to associate users to errors you may enable sending PII data. + send_default_pii=cls.SENTRY_SEND_DEFAULT_PII + ) + except ImportError: + raise ImproperlyConfigured("Could not import sentry although the server is configured to use it") + + ######################################################################## + # + # User-adaptable settings + # + ######################################################################## + DEBUG = values.BooleanValue(environ_prefix='IT', default=False) + SECRET_KEY = values.SecretValue(environ_prefix='IT') + DB_HOST = values.Value(environ_prefix='IT', environ_required=True) + DB_PORT = values.PositiveIntegerValue(environ_prefix='IT', default=5432) + DB_NAME = values.Value(environ_prefix='IT', default='imagetagger') + DB_USER = values.Value(environ_prefix='IT', default=DB_NAME) + DB_PASSWORD = values.Value(environ_prefix='IT', environ_required=True) + ALLOWED_HOSTS = values.ListValue(environ_prefix='IT', environ_required=True) + LANGUAGE_CODE = values.Value(environ_prefix='IT', default='en-us') + TIME_ZONE = values.Value(environ_prefix='IT', default='Europe/Berlin') + STATIC_ROOT = values.Value(environ_prefix='IT', default=path_join(BASE_DIR, 'static')) + USE_IMPRINT = values.BooleanValue(environ_prefix='IT', default=False) + IMPRINT_NAME = values.Value(environ_prefix='IT') + IMPRINT_URL = values.Value(environ_prefix='IT') + # the URL where the ImageTagger is hosted e.g. https://imagetagger.bit-bots.de + DOWNLOAD_BASE_URL = values.Value(environ_prefix='IT', environ_required=True) + TOOLS_ENABLED = values.BooleanValue(environ_prefix='IT', default=True) + TOOL_UPLOAD_NOTICE = values.Value(environ_prefix='IT', default='') + ENABLE_ZIP_DOWNLOAD = values.BooleanValue(environ_prefix='IT', default=is_in_docker()) + USE_NGINX_IMAGE_PROVISION = values.BooleanValue(environ_prefix='IT', default=is_in_docker()) + FS_URL = values.Value(environ_prefix='IT', default=path_join(BASE_DIR, 'data')) + TMP_FS_URL = values.Value(environ_prefix='IT', default='temp://imagetagger') + UPLOAD_NOTICE = values.Value(environ_prefix='IT', default='By uploading images to this tool you accept that ' + 'the images get published under creative commons license ' + 'and confirm that you have the permission to do so.') + EXPORT_SEPARATOR = values.Value(environ_prefix='IT', default='|') + # the path to the folder with the imagesets relative to the filesystem root (see FS_URL) + IMAGE_PATH = values.Value(environ_prefix='IT', default='images') + # the path to use for temporary image files relative to the temp filesystem (see TMP_FS_URL) + TMP_IMAGE_PATH = values.Value(environ_prefix='IT', default='images') + # the path to the folder with the tools relative to the filesystem root (see FS_URL) + TOOLS_PATH = values.Value(environ_prefix='IT', default='tools') + # filename extension of accepted imagefiles + IMAGE_EXTENSION = values.ListValue(environ_prefix='IT', default=['png', 'jpeg']) + ACCOUNT_ACTIVATION_DAYS = values.PositiveIntegerValue(environ_prefix='IT', default=7) + + SENTRY_REPORTING_ENABLED = values.BooleanValue(environ_prefix='IT', default=False) + SENTRY_DSN = values.Value(environ_prefix='IT', environ_required=SENTRY_REPORTING_ENABLED) + SENTRY_SEND_DEFAULT_PII = values.BooleanValue(environ_prefix='IT', default=False) + + +class Dev(Base): + DEBUG = values.BooleanValue(environ_prefix='IT', default=True) + SECRET_KEY = values.Value(environ_prefix='IT', default='DEV-KEY ONLY! DONT USE IN PRODUCTION!') + DB_HOST = values.Value(environ_prefix='IT', default='localhost') + DB_PASSWORD = values.Value(environ_prefix='IT', default='imagetagger') + ALLOWED_HOSTS = values.ListValue(environ_prefix='IT', default=['localhost', '127.0.0.1']) + DOWNLOAD_BASE_URL = values.Value(environ_prefix='IT', default='localhost') + + +class Prod(Base): + EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' + EMAIL_HOST = values.Value(environ_prefix='IT', environ_required=True) + EMAIL_PORT = values.Value(environ_prefix='IT') + EMAIL_HOST_USER = values.Value(environ_prefix='IT') + EMAIL_HOST_PASSWORD = values.Value(environ_prefix='IT') diff --git a/imagetagger/imagetagger/settings.py.example b/imagetagger/imagetagger/settings.py.example deleted file mode 100644 index 96bc15c7..00000000 --- a/imagetagger/imagetagger/settings.py.example +++ /dev/null @@ -1,129 +0,0 @@ -from .settings_base import * -from fs import path - -# Settings for Imagetagger -# Copy this file as settings.py and customise as necessary - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'DEV KEY PLEASE CHANGE IN PRODUCTION INTO SOMETHING RANDOM' - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True - -# Allowed Host headers this site can server -ALLOWED_HOSTS = ['localhost', '127.0.0.1'] - -# Additional installed apps -INSTALLED_APPS += [ - # 'raven.contrib.django.raven_compat', # uncomment if sentry is used -] - -# Database -# https://docs.djangoproject.com/en/1.10/ref/settings/#databases - -DATABASES = { - 'default': { - # Imagetagger relies on some Postgres features so other Databses will _not_ work - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'HOST': 'imagetagger-postgres', - 'NAME': 'imagetagger', - 'PASSWORD': 'imagetagger', - 'USER': 'imagetagger', - } -} - -# Internationalization -# https://docs.djangoproject.com/en/1.10/topics/i18n/ - -LANGUAGE_CODE = 'en-us' - -# TIME_ZONE = 'Europe/Berlin' #Timezone of your server - -STATIC_ROOT = '/var/www/imagetagger' - -# EXPORT_SEPARATOR = '|' - -# IMAGE_PATH = 'images' # the path to the folder with the imagesets relative to the filesystem root (see FS_URL) -# TMP_IMAGE_PATH = 'images' # the path to use for temporary image files relative to the temp filesystem (see TMP_FS_URL) - -# filename extension of accepted imagefiles -# IMAGE_EXTENSION = { -# 'png', -# 'jpg', -# } - -# defines if images get provided directly via nginx which generally improves imageset download performance -USE_NGINX_IMAGE_PROVISION = True if os.getenv("IN_DOCKER", "") != "" else False - -# The 'report a problem' page on an internal server error can either be a custom url or a text that can be defined here. -# PROBLEMS_URL = 'https://problems.example.com' -# PROBLEMS_TEXT = 'To report a problem, contact admin@example.com' - -USE_IMPRINT = False -IMPRINT_NAME = '' -IMPRINT_URL = '' -UPLOAD_NOTICE = 'By uploading images to this tool you accept that the images get published under creative commons license and confirm that you have the permission to do so.' - -DOWNLOAD_BASE_URL = '' # the URL where the ImageTagger is hosted e.g. https://imagetagger.bit-bots.de - -ACCOUNT_ACTIVATION_DAYS = 7 - -UPLOAD_FS_GROUP = 33 # www-data on debian - -# If enabled, run manage.py runzipdaemon to create the zip files and keep them up to date -ENABLE_ZIP_DOWNLOAD = True if os.getenv("IN_DOCKER", "") != "" else False - -# Test mail functionality by printing mails to console: -EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' -TOOLS_ENABLED = True -TOOL_UPLOAD_NOTICE = '' - -IMAGE_PATH = 'images' -TOOLS_PATH = 'tools' - -# Use ldap login -# -# import ldap -# from django_auth_ldap.config import LDAPSearch -# -# AUTHENTICATION_BACKENDS = ( -# 'django_auth_ldap.backend.LDAPBackend', -# 'django.contrib.auth.backends.ModelBackend', -# ) -# -# AUTH_LDAP_SERVER_URI = "ldap_host" -# AUTH_LDAP_BIND_DN = "ldap_bind_dn" -# AUTH_LDAP_BIND_PASSWORD = "ldap_bind_pw" -# AUTH_LDAP_USER_SEARCH = LDAPSearch("ou=People,dc=mafiasi,dc=de", -# ldap.SCOPE_SUBTREE, "(uid=%(user)s)") -# AUTH_LDAP_ALWAYS_UPDATE_USER = True -# -# from django_auth_ldap.config import LDAPSearch, PosixGroupType -# -# AUTH_LDAP_GROUP_SEARCH = LDAPSearch("ou=groups,dc=mafiasi,dc=de", -# ldap.SCOPE_SUBTREE, "(objectClass=posixGroup)" -# ) -# AUTH_LDAP_GROUP_TYPE = PosixGroupType() -# AUTH_LDAP_FIND_GROUP_PERMS = True -# AUTH_LDAP_MIRROR_GROUPS = True -# -# AUTH_LDAP_USER_ATTR_MAP = {"first_name": "givenName", "last_name": "sn", "email": "mail"} -# -# AUTH_LDAP_USER_FLAGS_BY_GROUP = { -# "is_staff": ["cn=Editoren,ou=groups,dc=mafiasi,dc=de", -# "cn=Server-AG,ou=groups,dc=mafiasi,dc=de"], -# "is_superuser": "cn=Server-AG,ou=groups,dc=mafiasi,dc=de" -# } - - -# Usage of sentry for error reporting -# import sentry_sdk -# from sentry_sdk.integrations.django import DjangoIntegration -# -# sentry_sdk.init( -# dsn="https://...", -# integrations=[DjangoIntegration()], -# -# # If you wish to associate users to errors you may enable sending PII data. -# send_default_pii=True -# ) diff --git a/imagetagger/imagetagger/settings_base.py b/imagetagger/imagetagger/settings_base.py deleted file mode 100644 index 6b0667b8..00000000 --- a/imagetagger/imagetagger/settings_base.py +++ /dev/null @@ -1,154 +0,0 @@ -""" -Django settings for imagetagger project. - -Generated by 'django-admin startproject' using Django 1.10.3. - -For more information on this file, see -https://docs.djangoproject.com/en/1.10/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/1.10/ref/settings/ -""" - -import os -from django.contrib.messages import constants as messages - -FS_URL = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -TMP_FS_URL = 'temp://imagetagger' - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = False - -ALLOWED_HOSTS = [] - -# Application definition - -INSTALLED_APPS = [ - 'imagetagger.annotations', - 'imagetagger.base', - 'imagetagger.images', - 'imagetagger.users', - 'imagetagger.tools', - 'imagetagger.administration', - 'django.contrib.admin', - 'imagetagger.tagger_messages', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'rest_framework', - 'widget_tweaks', - 'friendlytagloader', -] - -MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django.middleware.locale.LocaleMiddleware', -] - -ROOT_URLCONF = 'imagetagger.urls' - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - 'imagetagger.base.context_processors.base_data', - ], - }, - }, -] - -WSGI_APPLICATION = 'imagetagger.wsgi.application' - -FILE_UPLOAD_HANDLERS = ( - "django.core.files.uploadhandler.MemoryFileUploadHandler", - "django.core.files.uploadhandler.TemporaryFileUploadHandler", -) - -# Password validation -# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators - -AUTH_PASSWORD_VALIDATORS = [ - { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', - }, -] - -AUTH_USER_MODEL = 'users.User' - - -# Internationalization -# https://docs.djangoproject.com/en/1.10/topics/i18n/ - -LANGUAGE_CODE = 'en-us' - -TIME_ZONE = 'Europe/Berlin' # Timezone of your server - -USE_I18N = True - -USE_L10N = True - -USE_TZ = True - -PROBLEMS_URL = 'https://github.com/bit-bots/imagetagger/issues' -PROBLEMS_TEXT = '' - -LOGIN_URL = '/user/login/' -LOGIN_REDIRECT_URL = '/images/' - -MESSAGE_STORAGE = 'django.contrib.messages.storage.session.SessionStorage' -MESSAGE_TAGS = { - messages.INFO: 'info', - messages.ERROR: 'danger', - messages.WARNING: 'warning', - messages.SUCCESS: 'success', -} - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/1.10/howto/static-files/ - -STATIC_URL = '/static/' -STATIC_ROOT = 'static/' - -EXPORT_SEPARATOR = '|' - -IMAGE_PATH = 'images' # the path to the folder with the imagesets relative to the filesystem root (see FS_URL) -TMP_IMAGE_PATH = 'images' # the path to use for temporary image files relative to the temp filesystem (see TMP_FS_URL) -TOOLS_PATH = 'tools' # the path to the folder with the tools relative to the filesystem root (see FS_URL) -TOOLS_ENABLED = True -TOOL_UPLOAD_NOTICE = '' - -# filename extension of accepted imagefiles -IMAGE_EXTENSION = { - 'png', - 'jpeg', -} - -# Sets the default expire time for new messages in days -DEFAULT_EXPIRE_TIME = 7 - -# Sets the default number of messages per page -MESSAGES_PER_PAGE = 10 diff --git a/imagetagger/imagetagger/wsgi.py b/imagetagger/imagetagger/wsgi.py index 080b3097..595fd039 100644 --- a/imagetagger/imagetagger/wsgi.py +++ b/imagetagger/imagetagger/wsgi.py @@ -9,8 +9,9 @@ import os -from django.core.wsgi import get_wsgi_application +from configurations.wsgi import get_wsgi_application os.environ.setdefault("DJANGO_SETTINGS_MODULE", "imagetagger.settings") +os.environ.setdefault("DJANGO_CONFIGURATION", "Prod") application = get_wsgi_application() diff --git a/imagetagger/manage.py b/imagetagger/manage.py index b7689a88..42e41ffe 100755 --- a/imagetagger/manage.py +++ b/imagetagger/manage.py @@ -4,8 +4,9 @@ if __name__ == "__main__": os.environ.setdefault("DJANGO_SETTINGS_MODULE", "imagetagger.settings") + os.environ.setdefault("DJANGO_CONFIGURATION", "Dev") try: - from django.core.management import execute_from_command_line + from configurations.management import execute_from_command_line except ImportError: # The above import may fail for some other reason. Ensure that the # issue is really that Django is missing to avoid masking other diff --git a/requirements.in b/imagetagger/requirements.in similarity index 85% rename from requirements.in rename to imagetagger/requirements.in index 61ab412d..75037b28 100644 --- a/requirements.in +++ b/imagetagger/requirements.in @@ -7,3 +7,4 @@ psycopg2-binary django-registration django-friendly-tag-loader fs +django-configurations diff --git a/imagetagger/requirements.txt b/imagetagger/requirements.txt new file mode 100644 index 00000000..42f2c742 --- /dev/null +++ b/imagetagger/requirements.txt @@ -0,0 +1,25 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile requirements.in +# +appdirs==1.4.4 # via fs +confusable-homoglyphs==3.2.0 # via django-registration +django-configurations==2.2 # via -r requirements.in +django-friendly-tag-loader==1.3.1 # via -r requirements.in +django-registration==3.1.1 # via -r requirements.in +django-widget-tweaks==1.4.8 # via -r requirements.in +django==2.2.17 # via -r requirements.in, django-registration, djangorestframework +djangorestframework==3.12.2 # via -r requirements.in +fasteners==0.15 # via -r requirements.in +fs==2.4.11 # via -r requirements.in +monotonic==1.5 # via fasteners +pillow==8.0.1 # via -r requirements.in +psycopg2-binary==2.8.6 # via -r requirements.in +pytz==2020.4 # via django, fs +six==1.15.0 # via django-configurations, fasteners, fs +sqlparse==0.4.1 # via django + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/k8s/web_app.yml b/k8s/web_app.yml index 89a8347a..a5fa6cf5 100644 --- a/k8s/web_app.yml +++ b/k8s/web_app.yml @@ -30,9 +30,6 @@ spec: tier: web spec: volumes: - - name: config - configMap: - name: imagetagger-web - name: data persistentVolumeClaim: claimName: imagetagger-image-data @@ -44,10 +41,12 @@ spec: containerPort: 3008 - name: http containerPort: 80 + envFrom: + - configMapRef: + name: imagetagger-web + - configMapRef: + name: imagetagger-postgres volumeMounts: - - name: config - mountPath: /app/config - readOnly: true - name: data mountPath: /app/data livenessProbe: diff --git a/kustomization.yml b/kustomization.yml index 290734e0..ef203336 100644 --- a/kustomization.yml +++ b/kustomization.yml @@ -12,8 +12,13 @@ resources: configMapGenerator: - name: imagetagger-web - files: - - "settings.py=./imagetagger/imagetagger/settings.py.example" + literals: + # service hostname + - "IT_DB_HOST=imagetagger-postgres" + # references to below postgres configMap which is also mounted in the pod + - "IT_DB_USER=$(POSTGRES_USER)" + - "IT_DB_PASSWORD=$(POSTGRES_PASSWORD)" + - "IT_DB_NAME=$(POSTGRES_DB)" - name: imagetagger-postgres literals: - "POSTGRES_PASSWORD=imagetagger" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 93045299..00000000 --- a/requirements.txt +++ /dev/null @@ -1,24 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements.txt requirements.in -# -appdirs==1.4.3 # via fs -confusable-homoglyphs==3.2.0 # via django-registration -django-friendly-tag-loader==1.3.1 # via -r requirements.in -django-registration==3.1 # via -r requirements.in -django-widget-tweaks==1.4.8 # via -r requirements.in -django==2.2.13 # via -r requirements.in, django-registration, djangorestframework -djangorestframework==3.11.0 # via -r requirements.in -fasteners==0.15 # via -r requirements.in -fs==2.4.11 # via -r requirements.in -monotonic==1.5 # via fasteners -pillow==7.1.2 # via -r requirements.in -psycopg2-binary==2.8.5 # via -r requirements.in -pytz==2020.1 # via django, fs -six==1.14.0 # via fasteners, fs -sqlparse==0.3.1 # via django - -# The following packages are considered to be unsafe in a requirements file: -# setuptools