From 0ef57b2f9fd8b45141cdebb81caa6fcf324d6b25 Mon Sep 17 00:00:00 2001 From: Nick Douma Date: Thu, 6 Aug 2020 15:16:18 +0200 Subject: [PATCH 1/5] Implement account update method Allow syncing of all known accounts from PowerDNS, in the same way that Domain().update() does for domains. --- powerdnsadmin/models/account.py | 64 +++++++++++++++++++++++++++++++++ powerdnsadmin/models/domain.py | 8 ++--- update_accounts.py | 33 +++++++++++++++++ update_zones.py | 4 +-- 4 files changed, 103 insertions(+), 6 deletions(-) create mode 100644 update_accounts.py diff --git a/powerdnsadmin/models/account.py b/powerdnsadmin/models/account.py index 523ac1d..cb394e2 100644 --- a/powerdnsadmin/models/account.py +++ b/powerdnsadmin/models/account.py @@ -1,6 +1,10 @@ +import traceback from flask import current_app +from urllib.parse import urljoin +from ..lib import utils from .base import db +from .setting import Setting from .user import User from .account_user import AccountUser @@ -20,6 +24,12 @@ class Account(db.Model): self.contact = contact self.mail = mail + # PDNS configs + self.PDNS_STATS_URL = Setting().get('pdns_api_url') + self.PDNS_API_KEY = Setting().get('pdns_api_key') + self.PDNS_VERSION = Setting().get('pdns_version') + self.API_EXTENDED_URL = utils.pdns_api_extended_uri(self.PDNS_VERSION) + if self.name is not None: self.name = ''.join(c for c in self.name.lower() if c in "abcdefghijklmnopqrstuvwxyz0123456789") @@ -200,3 +210,57 @@ class Account(db.Model): 'Cannot revoke user privileges on account {0}. DETAIL: {1}'. format(self.name, e)) return False + + def update(self): + """ + Fetch accounts from PowerDNS and syncs them into DB + """ + db_accounts = Account.query.all() + list_db_accounts = [d.name for d in db_accounts] + current_app.logger.info("Found {} accounts in PowerDNS-Admin".format( + len(list_db_accounts))) + headers = {'X-API-Key': self.PDNS_API_KEY} + try: + jdata = utils.fetch_json( + urljoin(self.PDNS_STATS_URL, + self.API_EXTENDED_URL + '/servers/localhost/zones'), + headers=headers, + timeout=int(Setting().get('pdns_api_timeout')), + verify=Setting().get('verify_ssl_connections')) + list_jaccount = set(d['account'] for d in jdata if d['account']) + + try: + # Remove accounts that don't exist any more + should_removed_db_account = list( + set(list_db_accounts).difference(list_jaccount)) + for account_name in should_removed_db_account: + account_id = self.get_id_by_name(account_name) + if not account_id: + continue + current_app.logger.info("Deleting account for {0}".format(account_name)) + account = Account.query.get(account_id) + db.session.delete(account) + except Exception as e: + current_app.logger.error( + 'Can not delete account from DB. DETAIL: {0}'.format(e)) + current_app.logger.debug(traceback.format_exc()) + + for account_name in list_jaccount: + account_id = self.get_id_by_name(account_name) + if account_id: + continue + current_app.logger.info("Creating account for {0}".format(account_name)) + account = Account(name=account_name) + db.session.add(account) + + db.session.commit() + current_app.logger.info('Update accounts finished') + return { + 'status': 'ok', + 'msg': 'Account table has been updated successfully' + } + except Exception as e: + db.session.rollback() + current_app.logger.error( + 'Cannot update account table. Error: {0}'.format(e)) + return {'status': 'error', 'msg': 'Cannot update account table'} diff --git a/powerdnsadmin/models/domain.py b/powerdnsadmin/models/domain.py index 40453b5..6cd259a 100644 --- a/powerdnsadmin/models/domain.py +++ b/powerdnsadmin/models/domain.py @@ -116,7 +116,7 @@ class Domain(db.Model): db_domain = Domain.query.all() list_db_domain = [d.name for d in db_domain] dict_db_domain = dict((x.name, x) for x in db_domain) - current_app.logger.info("Found {} entries in PowerDNS-Admin".format( + current_app.logger.info("Found {} domains in PowerDNS-Admin".format( len(list_db_domain))) headers = {'X-API-Key': self.PDNS_API_KEY} try: @@ -128,7 +128,7 @@ class Domain(db.Model): verify=Setting().get('verify_ssl_connections')) list_jdomain = [d['name'].rstrip('.') for d in jdata] current_app.logger.info( - "Found {} entries in PowerDNS server".format(len(list_jdomain))) + "Found {} zones in PowerDNS server".format(len(list_jdomain))) try: # domains should remove from db since it doesn't exist in powerdns anymore @@ -166,8 +166,8 @@ class Domain(db.Model): except Exception as e: db.session.rollback() current_app.logger.error( - 'Can not update domain table. Error: {0}'.format(e)) - return {'status': 'error', 'msg': 'Can not update domain table'} + 'Cannot update domain table. Error: {0}'.format(e)) + return {'status': 'error', 'msg': 'Cannot update domain table'} def update_pdns_admin_domain(self, domain, account_id, data, do_commit=True): # existing domain, only update if something actually has changed diff --git a/update_accounts.py b/update_accounts.py new file mode 100644 index 0000000..1229c16 --- /dev/null +++ b/update_accounts.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 + +#################################################################################################################################### +# A CLI Script to update list of accounts. Can be useful for people who want to execute updates from a cronjob +# +# Tip: +# When running from a cron, use flock (you might need to install it) to be sure only one process is running a time. eg: +# */5 * * * * flock -xn "/tmp/pdns-update-zones.lock" python /var/www/html/apps/poweradmin/update_accounts.py >/dev/null 2>&1 +# +############################################################## + +### Imports +import sys +import logging + +from powerdnsadmin import create_app +from powerdnsadmin.models.account import Account +from powerdnsadmin.models.domain import Domain +from powerdnsadmin.models.setting import Setting + +app = create_app() +app.logger.setLevel(logging.INFO) + +with app.app_context(): + status = Setting().get('bg_domain_updates') + + ### Check if bg_domain_updates is set to true + if not status: + app.logger.error('Please turn on "bg_domain_updates" setting to run this job.') + sys.exit(1) + + Account().update() + Domain().update() diff --git a/update_zones.py b/update_zones.py index f13b873..5da542f 100644 --- a/update_zones.py +++ b/update_zones.py @@ -29,6 +29,6 @@ with app.app_context(): sys.exit(1) ### Start the update process - app.logger.info('Update zones from nameserver API') + app.logger.info('Update domains from nameserver API') - d = Domain().update() + Domain().update() From 9632898b403ba3802ef2663d8c6316b43c557c48 Mon Sep 17 00:00:00 2001 From: Nick Douma Date: Thu, 6 Aug 2020 15:40:11 +0200 Subject: [PATCH 2/5] Domains should not be updated in update_accounts.py --- update_accounts.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/update_accounts.py b/update_accounts.py index 1229c16..0578ce0 100644 --- a/update_accounts.py +++ b/update_accounts.py @@ -15,7 +15,6 @@ import logging from powerdnsadmin import create_app from powerdnsadmin.models.account import Account -from powerdnsadmin.models.domain import Domain from powerdnsadmin.models.setting import Setting app = create_app() @@ -30,4 +29,3 @@ with app.app_context(): sys.exit(1) Account().update() - Domain().update() From b4d7f66e29780306736912f3def2ba52a1876395 Mon Sep 17 00:00:00 2001 From: Nick Douma Date: Thu, 6 Aug 2020 16:33:00 +0200 Subject: [PATCH 3/5] Use Account.delete_account to also handle unlinking of Users --- powerdnsadmin/models/account.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/powerdnsadmin/models/account.py b/powerdnsadmin/models/account.py index cb394e2..54bc48e 100644 --- a/powerdnsadmin/models/account.py +++ b/powerdnsadmin/models/account.py @@ -98,7 +98,7 @@ class Account(db.Model): db.session.commit() return {'status': True, 'msg': 'Account updated successfully'} - def delete_account(self): + def delete_account(self, commit=True): """ Delete an account """ @@ -107,7 +107,8 @@ class Account(db.Model): try: Account.query.filter(Account.name == self.name).delete() - db.session.commit() + if commit: + db.session.commit() return True except Exception as e: db.session.rollback() @@ -239,7 +240,7 @@ class Account(db.Model): continue current_app.logger.info("Deleting account for {0}".format(account_name)) account = Account.query.get(account_id) - db.session.delete(account) + account.delete_account(commit=False) except Exception as e: current_app.logger.error( 'Can not delete account from DB. DETAIL: {0}'.format(e)) From 3c59ba6f84a6ff923d6cdcad7002c6b52b4f8718 Mon Sep 17 00:00:00 2001 From: Nick Douma Date: Thu, 6 Aug 2020 17:44:54 +0200 Subject: [PATCH 4/5] Account does not have username --- powerdnsadmin/models/account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/powerdnsadmin/models/account.py b/powerdnsadmin/models/account.py index 54bc48e..566b07c 100644 --- a/powerdnsadmin/models/account.py +++ b/powerdnsadmin/models/account.py @@ -114,7 +114,7 @@ class Account(db.Model): db.session.rollback() current_app.logger.error( 'Cannot delete account {0} from DB. DETAIL: {1}'.format( - self.username, e)) + self.name, e)) return False def get_user(self): From 70c2744f291368a8fb6608a98f317580f2b4a0f6 Mon Sep 17 00:00:00 2001 From: Nick Douma Date: Thu, 6 Aug 2020 17:45:05 +0200 Subject: [PATCH 5/5] Log amount of accounts found in PowerDNS --- powerdnsadmin/models/account.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/powerdnsadmin/models/account.py b/powerdnsadmin/models/account.py index 566b07c..ad46ef9 100644 --- a/powerdnsadmin/models/account.py +++ b/powerdnsadmin/models/account.py @@ -229,6 +229,8 @@ class Account(db.Model): timeout=int(Setting().get('pdns_api_timeout')), verify=Setting().get('verify_ssl_connections')) list_jaccount = set(d['account'] for d in jdata if d['account']) + current_app.logger.info("Found {} accounts in PowerDNS".format( + len(list_jaccount))) try: # Remove accounts that don't exist any more