Merge pull request #283 from ngoduykhanh/flask_migrate

Switching current db migration scripts to Flask-Migrate

.gitignore

@@ -28,6 +28,7 @@ advanced_settings.json
 idp.crt
 log.txt
 
+migrations/*
 db_repository/*
 upload/avatar/*
 tmp/*

app/__init__.py

@@ -2,6 +2,7 @@ from werkzeug.contrib.fixers import ProxyFix
 from flask import Flask, request, session, redirect, url_for
 from flask_login import LoginManager
 from flask_sqlalchemy import SQLAlchemy as SA
+from flask_migrate import Migrate
 
 
 # subclass SQLAlchemy to enable pool_pre_ping
@@ -26,6 +27,7 @@ logging = logger('powerdns-admin', app.config['LOG_LEVEL'], app.config['LOG_FILE
 login_manager = LoginManager()
 login_manager.init_app(app)
 db = SQLAlchemy(app)
+migrate = Migrate(app, db) # used for flask-migrate
 
 def enable_github_oauth(GITHUB_ENABLE):
     if not GITHUB_ENABLE:

app/models.py

@@ -1665,7 +1665,8 @@ class Server(object):
 class History(db.Model):
     id = db.Column(db.Integer, primary_key = True)
     msg = db.Column(db.String(256))
-    detail = db.Column(db.Text().with_variant(db.Text(length=2**24-2), 'mysql'))
+    # detail = db.Column(db.Text().with_variant(db.Text(length=2**24-2), 'mysql'))
+    detail = db.Column(db.Text())
     created_by = db.Column(db.String(128))
     created_on = db.Column(db.DateTime, default=datetime.utcnow)
 

app/views.py

@@ -465,7 +465,6 @@ def dashboard():
     else:
         logging.debug('Update domains in background')
 
-
     # stats for dashboard
     domain_count = Domain.query.count()
     users = User.query.all()

config_template.py

@@ -31,7 +31,7 @@ SQLA_DB_NAME = 'powerdnsadmin'
 #SQLALCHEMY_DATABASE_URI = 'mysql://'+SQLA_DB_USER+':'\
 #    +SQLA_DB_PASSWORD+'@'+SQLA_DB_HOST+'/'+SQLA_DB_NAME
 #SQLite
-SQLALCHEMY_DATABASE_URI = 'sqlite:///pdns.db'
+SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(basedir, 'pdns.db')
 SQLALCHEMY_MIGRATE_REPO = os.path.join(basedir, 'db_repository')
 SQLALCHEMY_TRACK_MODIFICATIONS = True
 

create_db.py

@@ -1,121 +0,0 @@
-#!/usr/bin/env python3
-
-import sys
-import time
-import os.path
-import traceback
-
-from migrate.versioning import api
-from config import SQLALCHEMY_DATABASE_URI
-from config import SQLALCHEMY_MIGRATE_REPO
-from app import db
-from app.models import Role, Setting, DomainTemplate
-
-
-def start():
-    wait_time = get_waittime_from_env()
-
-    if not connect_db(wait_time):
-        print("ERROR: Couldn't connect to database server")
-        exit(1)
-
-    init_records()
-
-def get_waittime_from_env():
-    return int(os.environ.get('WAITFOR_DB', 1))
-
-def connect_db(wait_time):
-    for i in range(0, wait_time):
-        print("INFO: Wait for database server")
-        sys.stdout.flush()
-        try:
-            db.create_all()
-            return True
-        except:
-            traceback.print_exc()
-            time.sleep(1)
-
-    return False
-
-def init_roles(db, role_names):
-
-    # Get key name of data
-    name_of_roles = [r.name for r in role_names]
-
-    # Query to get current data
-    rows = db.session.query(Role).filter(Role.name.in_(name_of_roles)).all()
-    name_of_rows = [r.name for r in rows]
-
-    # Check which data that need to insert
-    roles = [r for r in role_names if r.name not in name_of_rows]
-
-    # Insert data
-    for role in roles:
-        db.session.add(role)
-
-def init_settings(db, setting_names):
-
-    # Get key name of data
-    name_of_settings = [r.name for r in setting_names]
-
-    # Query to get current data
-    rows = db.session.query(Setting).filter(Setting.name.in_(name_of_settings)).all()
-
-    # Check which data that need to insert
-    name_of_rows = [r.name for r in rows]
-    settings = [r for r in setting_names if r.name not in name_of_rows]
-
-    # Insert data
-    for setting in settings:
-        db.session.add(setting)
-
-
-def init_domain_templates(db, domain_template_names):
-
-    # Get key name of data
-    name_of_domain_templates = map(lambda r: r.name, domain_template_names)
-
-    # Query to get current data
-    rows = db.session.query(DomainTemplate).filter(DomainTemplate.name.in_(name_of_domain_templates)).all()
-
-    # Check which data that need to insert
-    name_of_rows = map(lambda r: r.name, rows)
-    domain_templates = filter(lambda r: r.name not in name_of_rows, domain_template_names)
-
-    # Insert data
-    for domain_template in domain_templates:
-        db.session.add(domain_template)
-
-def init_records():
-    # Create initial user roles and turn off maintenance mode
-    init_roles(db, [
-        Role('Administrator', 'Administrator'),
-        Role('User', 'User')
-    ])
-    init_settings(db, [
-        Setting('maintenance', 'False'),
-        Setting('fullscreen_layout', 'True'),
-        Setting('record_helper', 'True'),
-        Setting('login_ldap_first', 'True'),
-        Setting('default_record_table_size', '15'),
-        Setting('default_domain_table_size', '10'),
-        Setting('auto_ptr','False')
-    ])
-    # TODO: add sample records to sample templates
-    init_domain_templates(db, [
-        DomainTemplate('basic_template_1', 'Basic Template #1'),
-        DomainTemplate('basic_template_2', 'Basic Template #2'),
-        DomainTemplate('basic_template_3', 'Basic Template #3')
-    ])
-    db_commit = db.session.commit()
-    commit_version_control(db_commit)
-
-def commit_version_control(db_commit):
-    if not os.path.exists(SQLALCHEMY_MIGRATE_REPO):
-        api.create(SQLALCHEMY_MIGRATE_REPO, 'database repository')
-        api.version_control(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO)
-    elif db_commit is not None:
-        api.version_control(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO, api.version(SQLALCHEMY_MIGRATE_REPO))
-
-if __name__ == '__main__':
-    start()

db_downgrade.py

@@ -1,8 +0,0 @@
-#!/usr/bin/env python3
-from migrate.versioning import api
-from config import SQLALCHEMY_DATABASE_URI
-from config import SQLALCHEMY_MIGRATE_REPO
-v = api.db_version(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO)
-api.downgrade(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO, v - 1)
-v = api.db_version(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO)
-print('Current database version: ' + str(v))

db_migrate.py

@@ -1,17 +0,0 @@
-#!/usr/bin/env python3
-import imp
-from migrate.versioning import api
-from app import db
-from config import SQLALCHEMY_DATABASE_URI
-from config import SQLALCHEMY_MIGRATE_REPO
-v = api.db_version(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO)
-migration = SQLALCHEMY_MIGRATE_REPO + ('/versions/%03d_migration.py' % (v+1))
-tmp_module = imp.new_module('old_model')
-old_model = api.create_model(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO)
-exec(old_model, tmp_module.__dict__)
-script = api.make_update_script_for_model(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO, tmp_module.meta, db.metadata)
-open(migration, "wt").write(script)
-api.upgrade(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO)
-v = api.db_version(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO)
-print('New migration saved as ' + migration)
-print('Current database version: ' + str(v))

db_upgrade.py

@@ -1,7 +0,0 @@
-#!/usr/bin/env python3
-from migrate.versioning import api
-from config import SQLALCHEMY_DATABASE_URI
-from config import SQLALCHEMY_MIGRATE_REPO
-api.upgrade(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO)
-v = api.db_version(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO)
-print('Current database version: ' + str(v))

docker-compose.yml

@@ -30,6 +30,7 @@ services:
       - PDA_DB_PASSWORD=${PDA_DB_PASSWORD}
       - PDNS_HOST=${PDNS_HOST}
       - PDNS_API_KEY=${PDNS_API_KEY}
+      - FLASK_APP=/powerdns-admin/app/__init__.py
     depends_on:
       powerdns-admin-mysql:
         condition: service_healthy

docker/PowerDNS-Admin/Dockerfile

@@ -1,5 +1,5 @@
 FROM ubuntu:latest
-MAINTAINER Khanh Ngo "ngokhanhit@gmail.com"
+MAINTAINER Khanh Ngo "k@ndk.name"
 ARG ENVIRONMENT=development
 ENV ENVIRONMENT=${ENVIRONMENT}
 
@@ -7,6 +7,12 @@ WORKDIR /powerdns-admin
 
 RUN apt-get update -y
 RUN apt-get install -y apt-transport-https
+
+RUN apt-get install -y locales locales-all
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US.UTF-8
+
 RUN apt-get install -y python3-pip python3-dev supervisor curl
 
 RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -

docker/PowerDNS-Admin/entrypoint.sh

@@ -1,11 +1,19 @@
 #!/bin/sh
 
-export LC_ALL=C.UTF-8
-export LANG=C.UTF-8
-
 cd /powerdns-admin
-./create_db.py
+
+if [ ! -d "/powerdns-admin/migrations" ]; then
+    flask db init --directory /powerdns-admin/migrations
+    flask db migrate -m "Init DB" --directory /powerdns-admin/migrations
+    flask db upgrade --directory /powerdns-admin/migrations
+    ./init_data.py
+
+else
+    /usr/local/bin/flask db migrate -m "Upgrade BD Schema" --directory /powerdns-admin/migrations
+    /usr/local/bin/flask db upgrade --directory /powerdns-admin/migrations
+fi
+
 yarn install --pure-lockfile
-FLASK_APP=app/__init__.py flask assets build
+flask assets build
 
 /usr/bin/supervisord -c /etc/supervisord.conf

init_data.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+
+from app import app, db
+from app.models import Role, Setting, DomainTemplate
+
+admin_role = Role(name='Administrator', description='Administrator')
+user_role = Role(name='User', description='User')
+
+setting_1 = Setting(name='maintenance', value='False')
+setting_2 = Setting(name='fullscreen_layout', value='True')
+setting_3 = Setting(name='record_helper', value='True')
+setting_4 = Setting(name='login_ldap_first', value='True')
+setting_5 = Setting(name='default_record_table_size', value='15')
+setting_6 = Setting(name='default_domain_table_size', value='10')
+setting_7 = Setting(name='auto_ptr', value='False')
+
+template_1 = DomainTemplate(name='basic_template_1', description='Basic Template #1')
+template_2 = DomainTemplate(name='basic_template_2', description='Basic Template #2')
+template_3 = DomainTemplate(name='basic_template_3', description='Basic Template #3')
+
+db.session.add(admin_role)
+db.session.add(user_role)
+
+db.session.add(setting_1)
+db.session.add(setting_2)
+db.session.add(setting_3)
+db.session.add(setting_4)
+db.session.add(setting_5)
+db.session.add(setting_6)
+db.session.add(setting_7)
+
+db.session.add(template_1)
+db.session.add(template_2)
+db.session.add(template_3)
+
+db.session.commit()

requirements.txt

@@ -4,8 +4,8 @@ Flask-WTF==0.14.2
 Flask-Login==0.4.1
 Flask-OAuthlib==0.9.4
 Flask-SQLAlchemy==2.3.2
+Flask-Migrate==2.1.1
 SQLAlchemy==1.2.5
-sqlalchemy-migrate==0.10.0
 mysqlclient==1.3.12
 configobj==5.0.6
 bcrypt==3.1.4