From 0efab0abb4ee6d2109275d1fb59b890bac29e7ea Mon Sep 17 00:00:00 2001 From: Omar Sanchez Date: Tue, 21 Apr 2020 14:23:11 -0400 Subject: [PATCH 1/6] added serial number mapping for devices --- racktables2device42.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/racktables2device42.py b/racktables2device42.py index 71d5338..0be4666 100644 --- a/racktables2device42.py +++ b/racktables2device42.py @@ -727,6 +727,7 @@ def process_data(self, data, dev_id): devicedata = {} device2rack = {} name = None + serial_no = None opsys = None hardware = None note = None @@ -740,6 +741,7 @@ def process_data(self, data, dev_id): rlocation_id, rlocation_name, rparent_name = x name = x[1] + serial_no = x[3] note = x[-7] if 'Operating System' in x: @@ -782,6 +784,8 @@ def process_data(self, data, dev_id): if name: # set device data devicedata.update({'name': name}) + if serial_no: + devicedata.update({'serial_no': serial_no}) if hardware: devicedata.update({'hardware': hardware[:48]}) if opsys: From 0dfc505705db466de573200e727e9dd83f590cff Mon Sep 17 00:00:00 2001 From: Omar Sanchez Date: Tue, 21 Apr 2020 16:32:07 -0400 Subject: [PATCH 2/6] upgraded script for python 3 support --- conf.sample | 21 -------- conf.sample.cfg | 28 +++++++++++ racktables2device42.py | 111 ++++++++++++++++++++++------------------- 3 files changed, 89 insertions(+), 71 deletions(-) delete mode 100644 conf.sample create mode 100644 conf.sample.cfg diff --git a/conf.sample b/conf.sample deleted file mode 100644 index 5ceb3ae..0000000 --- a/conf.sample +++ /dev/null @@ -1,21 +0,0 @@ -# ====== MySQL Source (Racktables) ====== # -DB_IP = 'racktables server IP' -DB_PORT = '3306' -DB_NAME = 'racktables database name' -DB_USER = 'racktables user' -DB_PWD = 'racktables password' -# ====== Log settings ==================== # -LOGFILE = 'migration.log' -STDOUT = True # print to STDOUT -DEBUG = True # write debug log -DEBUG_LOG = 'debug.log' -# ====== Device42 upload settings ========= # -D42_USER = 'device42 user' -D42_PWD = 'device42 password' -D42_URL = 'https:// device42 server IP address' -# ====== Other settings ========= # -CHILD_AS_BUILDING = True # use RT's sub-location as Device42 building. If False, use it as a Device42 room. -ROW_AS_ROOM = True # use RT's row as Device42 room. -# Note: Rooms are required because racks are mounted to rooms, not to buildings! -PDU_MOUNT = 'left' # Can be one of: left, right, above, below. Used for Zero-U PDU migration. Default is 'left' -PDU_ORIENTATION = 'front' # can be one of: front, back. Used for Zero-U PDU migration. Default is 'front' \ No newline at end of file diff --git a/conf.sample.cfg b/conf.sample.cfg new file mode 100644 index 0000000..8c84160 --- /dev/null +++ b/conf.sample.cfg @@ -0,0 +1,28 @@ +[Racktables_MySQL_Source] +DB_IP = racktables_ip +DB_PORT = 3306 +DB_NAME = racktables_db_name +DB_USER = racktables_user +DB_PWD = racktables_password + +[Log_Settings] +LOGFILE = migration.log +STDOUT = True +DEBUG = True +DEBUG_LOG = debug.log + +[Device42_Settings] +D42_USER = device42_user +D42_PWD = device42_password +D42_URL = http://10.0.0.0 + +[Other] +# use RT's sub-location as Device42 building. If False, use it as a Device42 room. +CHILD_AS_BUILDING = True +# use RT's row as Device42 room. +ROW_AS_ROOM = True +# Note: Rooms are required because racks are mounted to rooms, not to buildings! +# Can be one of: left, right, above, below. Used for Zero-U PDU migration. Default is 'left' +PDU_MOUNT = left +# can be one of: front, back. Used for Zero-U PDU migration. Default is 'front' +PDU_ORIENTATION = front \ No newline at end of file diff --git a/racktables2device42.py b/racktables2device42.py index 0be4666..ead70a0 100644 --- a/racktables2device42.py +++ b/racktables2device42.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -__version__ = 5.33 +__version__ = 6.00 """ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, @@ -13,33 +13,42 @@ """ ############################################################################################################# -# v5.0 of python script that connects to RackTables DB and migrates data to Device42 appliance using APIs +# v6.0 of python script that connects to RackTables DB and migrates data to Device42 appliance using APIs # Refer to README for further instructions ############################################################################################################# - +import configparser import sys -import imp import os import pymysql as sql import codecs import requests -import base64 import struct import socket import json +from requests.auth import HTTPBasicAuth + +dir_path = os.path.dirname(os.path.realpath(__file__)) try: requests.packages.urllib3.disable_warnings() except: pass -conf = imp.load_source('conf', 'conf') +conf = configparser.ConfigParser() + +try: + config_reader = conf.read(dir_path + '\conf.cfg') + print(conf.sections()) +except Exception as e: + print(e) + print('failed to read conf.cfg file, did you rename conf.cfg.sample.cfg to conf.cfg?') + exit(1) class Logger: def __init__(self, logfile, stdout): - print '[!] Version %s' % __version__ + print('[!] Version %s' % __version__) self.logfile = logfile self.stdout = stdout self.check_log_file() @@ -47,7 +56,8 @@ def __init__(self, logfile, stdout): def check_log_file(self): while 1: if os.path.exists(self.logfile): - reply = raw_input("[!] Log file already exists. Overwrite or append [O|A]? ") + reply = input("[!] Log file already exists. Overwrite or append [O|A]? ") + print(reply) if reply.lower().strip() == 'o': with open(self.logfile, 'w'): pass @@ -56,25 +66,25 @@ def check_log_file(self): break else: break - if conf.DEBUG and os.path.exists(conf.DEBUG_LOG): - with open(conf.DEBUG_LOG, 'w'): + if conf['Log_Settings']['DEBUG'] and os.path.exists(conf['Log_Settings']['DEBUG_LOG']): + with open(conf['Log_Settings']['DEBUG_LOG'], 'w'): pass def writer(self, msg): - if conf.LOGFILE and conf.LOGFILE != '': + if conf['Log_Settings']['LOGFILE'] and conf['Log_Settings']['LOGFILE'] != '': with codecs.open(self.logfile, 'a', encoding='utf-8') as f: - msg = msg.decode('UTF-8', 'ignore') + msg = str(msg) f.write(msg + '\r\n') # \r\n for notepad if self.stdout: try: - print msg + print(msg) except: - print msg.encode('ascii', 'ignore') + ' # < non-ASCII chars detected! >' + print(msg.encode('ascii', 'ignore') + ' # < non-ASCII chars detected! >') @staticmethod def debugger(msg): - if conf.DEBUG_LOG and conf.DEBUG_LOG != '': - with codecs.open(conf.DEBUG_LOG, 'a', encoding='utf-8') as f: + if conf['Log_Settings']['DEBUG_LOG'] and conf['Log_Settings']['DEBUG_LOG'] != '': + with codecs.open(conf['Log_Settings']['DEBUG_LOG'], 'a', encoding='utf-8') as f: title, message = msg row = '\n-----------------------------------------------------\n%s\n%s' % (title, message) f.write(row + '\r\n\r\n') # \r\n for notepad @@ -82,22 +92,22 @@ def debugger(msg): class REST: def __init__(self): - self.password = conf.D42_PWD - self.username = conf.D42_USER - self.base_url = conf.D42_URL + + self.password = conf['Device42_Settings']['D42_PWD'] + self.username = conf['Device42_Settings']['D42_USER'] + self.base_url = conf['Device42_Settings']['D42_URL'] def uploader(self, data, url): payload = data headers = { - 'Authorization': 'Basic ' + base64.b64encode(self.username + ':' + self.password), 'Content-Type': 'application/x-www-form-urlencoded' } if 'custom_fields' in url: - r = requests.put(url, data=payload, headers=headers, verify=False) + r = requests.put(url, data=payload, headers=headers, auth=HTTPBasicAuth(self.username, self.password), verify=False) else: - r = requests.post(url, data=payload, headers=headers, verify=False) - msg = unicode(payload) + r = requests.post(url, data=payload, headers=headers, auth=HTTPBasicAuth(self.username, self.password), verify=False) + msg = str(payload).encode() logger.writer(msg) msg = 'Status code: %s' % str(r.status_code) logger.writer(msg) @@ -106,18 +116,16 @@ def uploader(self, data, url): try: return r.json() - except Exception as e: - - print '\n[*] Exception: %s' % str(e) + except Exception as err: + print('\n[*] Exception: %s' % str(err)) pass def fetcher(self, url): headers = { - 'Authorization': 'Basic ' + base64.b64encode(self.username + ':' + self.password), 'Content-Type': 'application/x-www-form-urlencoded' } - r = requests.get(url, headers=headers, verify=False) + r = requests.get(url, headers=headers, verify=False, auth=HTTPBasicAuth(self.username, self.password)) msg = 'Status code: %s' % str(r.status_code) logger.writer(msg) msg = str(r.text) @@ -280,8 +288,11 @@ def connect(self): Connection to RT database :return: """ - self.con = sql.connect(host=conf.DB_IP, port=int(conf.DB_PORT), - db=conf.DB_NAME, user=conf.DB_USER, passwd=conf.DB_PWD) + self.con = sql.connect(host=conf['Racktables_MySQL_Source']['DB_IP'], + port=int(conf['Racktables_MySQL_Source']['DB_PORT']), + db=conf['Racktables_MySQL_Source']['DB_NAME'], + user=conf['Racktables_MySQL_Source']['DB_USER'], + passwd=conf['Racktables_MySQL_Source']['DB_PWD']) @staticmethod def convert_ip(ip_raw): @@ -306,7 +317,7 @@ def get_ips(self): q = 'SELECT * FROM IPv4Address WHERE IPv4Address.name != ""' cur.execute(q) ips = cur.fetchall() - if conf.DEBUG: + if conf['Log_Settings']['DEBUG']: msg = ('IPs', str(ips)) logger.debugger(msg) @@ -338,7 +349,7 @@ def get_subnets(self): q = "SELECT * FROM IPv4Network" cur.execute(q) subnets = cur.fetchall() - if conf.DEBUG: + if conf['Log_Settings']['DEBUG']: msg = ('Subnets', str(subnets)) logger.debugger(msg) for line in subnets: @@ -369,7 +380,7 @@ def get_infrastructure(self): q = """select id,name, parent_id, parent_name from Location""" cur.execute(q) raw = cur.fetchall() - if conf.CHILD_AS_BUILDING: + if conf['Other']['CHILD_AS_BUILDING']: for rec in raw: building_id, building_name, parent_id, parent_name = rec buildings_map.update({building_id: building_name}) @@ -385,7 +396,7 @@ def get_infrastructure(self): self.d42_racks.update({d42_rack['name']: d42_rack['rack_id']}) # upload buildings - if conf.DEBUG: + if conf['Log_Settings']['DEBUG']: msg = ('Buildings', str(buildings_map)) logger.debugger(msg) bdata = {} @@ -395,7 +406,7 @@ def get_infrastructure(self): # upload rooms buildings = json.loads((rest.get_buildings()))['buildings'] - if not conf.CHILD_AS_BUILDING: + if not conf['Other']['CHILD_AS_BUILDING']: for room, parent in rooms_map.items(): roomdata = {} roomdata.update({'name': room}) @@ -421,7 +432,7 @@ def get_infrastructure(self): rack.update({'rack_id': self.d42_racks[rack_name]}) rack.update({'size': height}) rack.update({'rt_id': rack_id}) # we will remove this later - if conf.ROW_AS_ROOM: + if conf['Other']['ROW_AS_ROOM']: rack.update({'room': row_name}) rack.update({'building': location_name}) else: @@ -436,8 +447,8 @@ def get_infrastructure(self): racks.append(rack) # upload rows as rooms - if conf.ROW_AS_ROOM: - if conf.DEBUG: + if conf['Other']['ROW_AS_ROOM']: + if conf['Log_Settings']['DEBUG']: msg = ('Rooms', str(rows_map)) logger.debugger(msg) for room, parent in rows_map.items(): @@ -447,7 +458,7 @@ def get_infrastructure(self): rest.post_room(roomdata) # upload racks - if conf.DEBUG: + if conf['Log_Settings']['DEBUG']: msg = ('Racks', str(racks)) logger.debugger(msg) for rack in racks: @@ -482,7 +493,7 @@ def get_hardware(self): cur.execute(q) data = cur.fetchall() - if conf.DEBUG: + if conf['Log_Settings']['DEBUG']: msg = ('Hardware', str(data)) logger.debugger(msg) @@ -941,11 +952,11 @@ def get_device_to_ip(self): IPv4Allocation.ip,IPv4Allocation.name, Object.name as hostname FROM %s.`IPv4Allocation` - LEFT JOIN Object ON Object.id = object_id""" % conf.DB_NAME + LEFT JOIN Object ON Object.id = object_id""" % conf['Racktables_MySQL_Source']['DB_NAME'] cur.execute(q) data = cur.fetchall() - if conf.DEBUG: + if conf['Log_Settings']['DEBUG']: msg = ('Device to IP', str(data)) logger.debugger(msg) @@ -978,7 +989,7 @@ def get_pdus(self): cur.execute(q) data = cur.fetchall() - if conf.DEBUG: + if conf['Log_Settings']['DEBUG']: msg = ('PDUs', str(data)) logger.debugger(msg) @@ -1063,12 +1074,12 @@ def get_pdus(self): \n[!] INFO: Cannot mount pdu "%s" (RT id = %d) to the rack.\ \n\tWrong rack id map value: %s' % (name, pdu_id, str(rack_id)) logger.writer(msg) - if conf.PDU_MOUNT.lower() in ('left', 'right', 'above', 'below'): - where = conf.PDU_MOUNT.lower() + if conf['Other']['PDU_MOUNT'].lower() in ('left', 'right', 'above', 'below'): + where = conf['Other']['PDU_MOUNT'].lower() else: where = 'left' - if conf.PDU_ORIENTATION.lower() in ('front', 'back'): - mount = conf.PDU_ORIENTATION.lower() + if conf['Other']['PDU_ORIENTATION'].lower() in ('front', 'back'): + mount = conf['Other']['PDU_ORIENTATION'].lower() else: mount = 'front' rdata = {} @@ -1102,7 +1113,7 @@ def get_patch_panels(self): cur.execute(q) data = cur.fetchall() - if conf.DEBUG: + if conf['Log_Settings']['DEBUG']: msg = ('PDUs', str(data)) logger.debugger(msg) @@ -1263,8 +1274,8 @@ def main(): if __name__ == '__main__': - logger = Logger(conf.LOGFILE, conf.STDOUT) + logger = Logger(conf['Log_Settings']['LOGFILE'], conf['Log_Settings']['STDOUT']) rest = REST() main() - print '\n[!] Done!' + print('\n[!] Done!') sys.exit() From 9ce203286fd70eba545427f8612112bae412ed68 Mon Sep 17 00:00:00 2001 From: Omar Sanchez Date: Tue, 21 Apr 2020 16:34:08 -0400 Subject: [PATCH 3/6] debug statement fix --- racktables2device42.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/racktables2device42.py b/racktables2device42.py index ead70a0..b9e465f 100644 --- a/racktables2device42.py +++ b/racktables2device42.py @@ -42,7 +42,7 @@ print(conf.sections()) except Exception as e: print(e) - print('failed to read conf.cfg file, did you rename conf.cfg.sample.cfg to conf.cfg?') + print('failed to read conf.cfg file, did you rename conf.sample.cfg to conf.cfg?') exit(1) From 79e0c81d64de88cb2feb1695e527ac9acde5dbf0 Mon Sep 17 00:00:00 2001 From: Omar Sanchez Date: Tue, 21 Apr 2020 16:40:38 -0400 Subject: [PATCH 4/6] updated readme --- README.md | 64 +++++++++++++++++++++++++++---------------------------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 03aa2fe..fb3b0a5 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ that integrates centralized password management, impact charts and applications This repository contains sample script to take Inventory information from a RackTables installation and send it to Device42 appliance using the REST APIs. ## Version ----------------------------- - * 5.33 + * 6.00 ## Assumptions ----------------------------- @@ -13,47 +13,45 @@ This repository contains sample script to take Inventory information from a Rack ### Requirements ----------------------------- - * python 2.7.x - * pymysql (you can install it on ubuntu with `sudo pip install pymysql`, other similar commands for other OS) - * requests (you can install it with sudo pip install requests or sudo apt-get install python-requests) + * python 3.7.x * allow remote connections to RackTables MySQL port 3306 ### Usage ----------------------------- - - * copy conf.sample to conf - * in conf add D42 URL/credentials -``` -# ====== Device42 upload settings ========= # -D42_USER = 'device42 user' -D42_PWD = 'device42 password' -D42_URL = 'https:// device42 server IP address' +* Install dependencies by running the following + +```python +pip3 install -r requirements.txt ``` - * add RackTables DB info/credentials -``` -# ====== MySQL Source (Racktables) ====== # -DB_IP = 'racktables server IP' -DB_PORT = '3306' -DB_NAME = 'racktables database name' -DB_USER = 'racktables user' -DB_PWD = 'racktables password' -``` - * adjust log settings -``` -# ====== Log settings ==================== # -LOGFILE = 'migration.log' -STDOUT = True # print to STDOUT -DEBUG = True # write debug log -DEBUG_LOG = 'debug.log' -``` - * other setings +* copy conf.sample.cfg to conf.cfg +* in conf.cfg add D42 URL/credentials + +***conf.cfg*** ``` -# ====== Other settings ========= # +[Racktables_MySQL_Source] +DB_IP = racktables_ip +DB_PORT = 3306 +DB_NAME = racktables_db_name +DB_USER = racktables_user +DB_PWD = racktables_password + +[Log_Settings] +LOGFILE = migration.log +STDOUT = True +DEBUG = True +DEBUG_LOG = debug.log + +[Device42_Settings] +D42_USER = device42_user +D42_PWD = device42_password +D42_URL = http://10.0.0.0 + +[Other] CHILD_AS_BUILDING = True ROW_AS_ROOM = True -PDU_MOUNT = 'left' -PDU_ORIENTATION = 'front' +PDU_MOUNT = left +PDU_ORIENTATION = front ``` - CHILD_AS_BUILDING: Racktables uses a concept of location and sub-location (container within container). Device42 uses a concept of Buildings and Rooms. In case CHILD_AS_BUILDING is set to True, sub-locations will be uploaded as Rooms to Device42. Otherwise, sub-locations will be uploaded as Buildings. From aa55a87e378c0e7c0602b00212306d467ff9550e Mon Sep 17 00:00:00 2001 From: Omar Sanchez Date: Wed, 22 Apr 2020 17:33:40 -0400 Subject: [PATCH 5/6] added option to create all IPs for subnets --- conf.sample.cfg | 3 +++ racktables2device42.py | 28 ++++++++++++++++++++++------ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/conf.sample.cfg b/conf.sample.cfg index 8c84160..a508bd5 100644 --- a/conf.sample.cfg +++ b/conf.sample.cfg @@ -17,6 +17,9 @@ D42_PWD = device42_password D42_URL = http://10.0.0.0 [Other] +# By default, a subnet's show ip list will only display used IPs, this is because IP objects are not created for unused IPs +# If this option is set to true, these IP objects will be created, this will however increase script run time greatly +CREATE_AVAILABLE_IPS = False # use RT's sub-location as Device42 building. If False, use it as a Device42 room. CHILD_AS_BUILDING = True # use RT's row as Device42 room. diff --git a/racktables2device42.py b/racktables2device42.py index b9e465f..787271e 100644 --- a/racktables2device42.py +++ b/racktables2device42.py @@ -26,6 +26,7 @@ import struct import socket import json +import ipaddress from requests.auth import HTTPBasicAuth dir_path = os.path.dirname(os.path.realpath(__file__)) @@ -39,7 +40,6 @@ try: config_reader = conf.read(dir_path + '\conf.cfg') - print(conf.sections()) except Exception as e: print(e) print('failed to read conf.cfg file, did you rename conf.sample.cfg to conf.cfg?') @@ -360,6 +360,19 @@ def get_subnets(self): subs.update({'name': name}) rest.post_subnet(subs) + if conf['Other']['CREATE_AVAILABLE_IPS']: + for ip in [str(ip) for ip in ipaddress.IPv4Network(str(subnet) + '/' + str(mask))]: + if conf['Log_Settings']['DEBUG']: + msg = ('IPs', ipaddress.IPv4Network(str(subnet) + '/' + str(mask))) + logger.debugger(msg) + + ip_data = { + 'ipaddress': ip, + 'subnet': name + } + + rest.post_ip(ip_data) + def get_infrastructure(self): """ Get locations, rows and racks from RT, convert them to buildings and rooms and send to uploader. @@ -1274,8 +1287,11 @@ def main(): if __name__ == '__main__': - logger = Logger(conf['Log_Settings']['LOGFILE'], conf['Log_Settings']['STDOUT']) - rest = REST() - main() - print('\n[!] Done!') - sys.exit() + try: + logger = Logger(conf['Log_Settings']['LOGFILE'], conf['Log_Settings']['STDOUT']) + rest = REST() + main() + print('\n[!] Done!') + except KeyError: + print('\n[!] Ensure that the script has been properly configured. Did you create the conf.cfg file?') + print('\n[!] Done!') From 69d42f530a50f0b1b91e5b061b6cb8d4dd742862 Mon Sep 17 00:00:00 2001 From: Omar Sanchez Date: Wed, 22 Apr 2020 17:37:34 -0400 Subject: [PATCH 6/6] updated version --- README.md | 3 ++- racktables2device42.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fb3b0a5..01468c3 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ that integrates centralized password management, impact charts and applications This repository contains sample script to take Inventory information from a RackTables installation and send it to Device42 appliance using the REST APIs. ## Version ----------------------------- - * 6.00 + * 6.10 ## Assumptions ----------------------------- @@ -48,6 +48,7 @@ D42_PWD = device42_password D42_URL = http://10.0.0.0 [Other] +CREATE_AVAILABLE_IPS = False CHILD_AS_BUILDING = True ROW_AS_ROOM = True PDU_MOUNT = left diff --git a/racktables2device42.py b/racktables2device42.py index 787271e..30c23f1 100644 --- a/racktables2device42.py +++ b/racktables2device42.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -__version__ = 6.00 +__version__ = 6.10 """ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,