#!/usr/bin/env python """ Omada SDN Query Script This script queries the OMADA SDN to populate NetAlertX with Omada switches, access points, and clients. It attempts to identify and populate their connections by switch/access points and ports/SSID, and tries to differentiate root bridges from accessories. Author: ffsb Version: 0.2 - Added logic to retry Omada API call once as it sometimes fails, and improved error handling. """ __author__ = "ffsb" __version__ = "0.1" #initial __version__ = "0.2" # added logic to retry omada api call once as it seems to sometimes fail for some reasons, and error handling logic... __version__ = "0.3" # adding parallelism # # sample code to update unbound on opnsense - for reference... # curl -X POST -d '{"host":{"enabled":"1","hostname":"test","domain":"testdomain.com","rr":"A","mxprio":"","mx":"","server":"10.0.1.1","description":""}}' -H "Content-Type: application/json" -k -u $OPNS_KEY:$OPNS_SECRET https://$IPFW/api/unbound/settings/AddHostOverride # import os import pathlib import sys import json import sqlite3 import tplink_omada_client import importlib.util import time import io import re import concurrent.futures from queue import Queue import multiprocessing from multiprocessing import Pool, Manager import os # Define the installation path and extend the system path for plugin imports INSTALL_PATH = "/app" sys.path.extend([f"{INSTALL_PATH}/front/plugins", f"{INSTALL_PATH}/server"]) from plugin_helper import Plugin_Object, Plugin_Objects, decodeBase64 from plugin_utils import get_plugins_configs from logger import mylog from const import pluginsPath, fullDbPath from helper import timeNowTZ, get_setting_value from notification import write_notification # Define the current path and log file paths CUR_PATH = str(pathlib.Path(__file__).parent.resolve()) LOG_FILE = os.path.join(CUR_PATH, 'script.log') RESULT_FILE = os.path.join(CUR_PATH, 'last_result.log') # Initialize the Plugin obj output file plugin_objects = Plugin_Objects(RESULT_FILE) pluginName = 'OMDSDN' # # sample target output: # 0 MAC, 1 IP, 2 Name, 3 switch/AP, 4 port/SSID, 5 TYPE #17:27:10 [] token: "['9C-04-A0-82-67-45', '192.168.0.217', '9C-04-A0-82-67-45', '17', '40-AE-30-A5-A7-50, 'Switch']" # Constants for array indices MAC, IP, NAME, SWITCH_AP, PORT_SSID, TYPE = range(6) # sample omada devices input format: # # 0.MAC 1.IP 2.type 3.status 4.name 5.model #40-AE-30-A5-A7-50 192.168.0.11 ap CONNECTED ompapaoffice EAP773(US) v1.0 #B0-95-75-46-0C-39 192.168.0.4 switch CONNECTED pantry12 T1600G-52PS v4.0 dMAC, dIP, dTYPE, dSTATUS, dNAME, dMODEL = range(6) # sample omada clients input format: # 0 MAC, 1 IP, 2 Name, 3 switch/AP, 4 port/SSID, #17:27:10 [] token: "['9C-04-A0-82-67-45', '192.168.0.217', '9C-04-A0-82-67-45', 'froggies2', '(ompapaoffice)']" #17:27:10 [] token: "['50-02-91-29-E7-53', '192.168.0.153', 'frontyard_ESP_29E753', 'pantry12', '(48)']" #17:27:10 [] token: "['00-E2-59-00-A0-8E', '192.168.0.1', 'bastion', 'office24', '(23)']" #17:27:10 [] token: "['60-DD-8E-CA-A4-B3', '192.168.0.226', 'brick', 'froggies3', '(ompapaoffice)']" cMAC, cIP, cNAME, cSWITCH_AP, cPORT_SSID = range(5) OMDLOGLEVEL = 'verbose' def ieee2ietf_mac_formater(inputmac): """Translate MAC address from standard IEEE model to IETF draft.""" return inputmac.lower().replace('-', ':') def ietf2ieee_mac_formater(inputmac): """Translate MAC address from IETF draft to standard IEEE model.""" return inputmac.upper().replace(':', '-') def get_mac_from_IP(target_IP): """Get MAC address from IP using ARP.""" from scapy.all import ARP, Ether, srp try: arp_request = ARP(pdst=target_IP) ether = Ether(dst="ff:ff:ff:ff:ff:ff") packet = ether/arp_request result = srp(packet, timeout=3, verbose=0)[0] if result: return result[0][1].hwsrc else: return None except Exception as e: mylog('minimal', [f'[{pluginName}] get_mac_from_IP ERROR:{e}']) return None def callomada(myargs): """Wrapper to call the Omada python library's own wrapper.""" arguments = " ".join(myargs) mylog('verbose', [f'[{pluginName}] callomada START:{arguments}']) from tplink_omada_client.cli import main as omada from contextlib import redirect_stdout omada_output = '' retries = 2 while omada_output == '' and retries > 0: retries -= 1 try: mf = io.StringIO() with redirect_stdout(mf): omada(myargs) omada_output = mf.getvalue() except Exception as e: mylog('minimal', [f'[{pluginName}] ERROR WHILE CALLING callomada:{arguments}\n {e}']) omada_output = '' mylog('verbose', [f'[{pluginName}] callomada END:{arguments}']) return omada_output def extract_mac_addresses(text): """Extract all the MAC addresses from multiline text.""" mac_pattern = r"([0-9A-Fa-f]{2}[:-][0-9A-Fa-f]{2}[:-][0-9A-Fa-f]{2}[:-][0-9A-Fa-f]{2}[:-][0-9A-Fa-f]{2}[:-][0-9A-Fa-f]{2})" return re.findall(mac_pattern, text) def find_default_gateway_ip(): """Find the default gateway IP address.""" from scapy.all import conf, Route default_route = conf.route.route("0.0.0.0") return default_route[2] if default_route[2] else None def add_uplink(uplink_mac, switch_mac, device_data_bymac, sadevices_linksbymac, port_byswitchmac_byclientmac): """Add uplink information to switches recursively.""" mylog(OMDLOGLEVEL, [f'[{pluginName}] trying to add uplink="{uplink_mac}" to switch="{switch_mac}"']) mylog(OMDLOGLEVEL, [f'[{pluginName}] before adding:"{device_data_bymac[switch_mac]}"']) mylog(OMDLOGLEVEL, [f'[{pluginName}] this are the port_byswitchmac:"{port_byswitchmac_byclientmac[switch_mac]}"']) if device_data_bymac[switch_mac][SWITCH_AP] == 'null': device_data_bymac[switch_mac][SWITCH_AP] = uplink_mac if device_data_bymac[switch_mac][TYPE] == 'Switch' and device_data_bymac[uplink_mac][TYPE] == 'Switch': port_to_uplink = port_byswitchmac_byclientmac[switch_mac][uplink_mac] else: port_to_uplink = device_data_bymac[uplink_mac][PORT_SSID] device_data_bymac[switch_mac][PORT_SSID] = port_to_uplink mylog(OMDLOGLEVEL, [f'[{pluginName}] after adding:"{device_data_bymac[switch_mac]}"']) for link in sadevices_linksbymac[switch_mac]: if device_data_bymac[link][SWITCH_AP] == 'null' and device_data_bymac[switch_mac][TYPE] == 'Switch': add_uplink(switch_mac, link, device_data_bymac, sadevices_linksbymac, port_byswitchmac_byclientmac) def main(): """Main function to execute the script.""" start_time = time.time() mylog('verbose', [f'[{pluginName}] starting execution']) from database import DB from device import Device_obj db = DB() db.open() device_handler = Device_obj(db) # Retrieve configuration settings omada_username = get_setting_value('OMDSDN_username') omada_password = get_setting_value('OMDSDN_password') omada_sites = get_setting_value('OMDSDN_sites') omada_site = omada_sites[0] omada_url = get_setting_value('OMDSDN_url') # Login to Omada omada_login = callomada(['-t', 'myomada', 'target', '--url', omada_url, '--user', omada_username, '--password', omada_password, '--site', omada_site, '--set-default']) mylog('verbose', [f'[{pluginName}] login to omada result is: {omada_login}']) # Get clients and devices clients_list = callomada(['-t', 'myomada', 'clients']) mylog('verbose', [f'[{pluginName}] clients found:"{clients_list.count("\n")}"\n{clients_list}']) switches_and_aps = callomada(['-t', 'myomada', 'devices']) mylog('verbose', [f'[{pluginName}] omada devices (switches, access points) found:"{switches_and_aps.count("\n")}" \n {switches_and_aps}']) # Process data device_data = get_device_data(clients_list, switches_and_aps, device_handler) mylog('verbose', [f'[{pluginName}] New entries to create: "{len(device_data)}"']) if len(device_data) > 0: for device in device_data: mylog(OMDLOGLEVEL, [f'[{pluginName}] main parsing device: "{device}"']) myport = device[PORT_SSID] if device[PORT_SSID].isdigit() else '' myssid = device[PORT_SSID] if not device[PORT_SSID].isdigit() else '' ParentNetworkNode = ieee2ietf_mac_formater(device[SWITCH_AP]) if device[SWITCH_AP] != 'Internet' else 'Internet' plugin_objects.add_object( primaryId = ieee2ietf_mac_formater(device[MAC]), secondaryId = device[IP], watched1 = device[NAME] if device[NAME] != 'null' else '', watched2 = ParentNetworkNode, watched3 = myport, watched4 = myssid, extra = device[TYPE] if device[TYPE] != 'null' else '', foreignKey = ieee2ietf_mac_formater(device[MAC]) ) mylog(OMDLOGLEVEL, [f'[{pluginName}] New entries: "{len(device_data)}"']) # Write results plugin_objects.write_result_file() end_time = time.time() mylog('verbose', [f'[{pluginName}] execution completed in {end_time - start_time:.2f} seconds']) return 0 ''' # version 0.3b def get_omada_devices_details(sadevice_data,switch_details,switch_dumps): """Get device details from Omada. saved into a dictionary of strings""" mylog(OMDLOGLEVEL, [f'[{pluginName}]getting the omada devices details: "{sadevice_data}"']) thisswitch = sadevice_data[dMAC] if sadevice_data[dTYPE] == 'ap': switch_details[thisswitch] = callomada(['access-point', thisswitch]) elif sadevice_data[dTYPE] == 'switch': switch_details[thisswitch] = callomada(['switch', thisswitch]) switch_dumps[thisswitch] = callomada(['-t','myomada','switch','-d',thisswitch]) else: switch_details[thisswitch] = 'null' switch_dumps[thisswitch] = 'null' return ''' ''' # version 0.3c def get_omada_devices_details(sadevice_data): mthisswitch = sadevice_data[dMAC] mswitch_detail = '' mswitch_dump = '' if sadevice_data[dTYPE] == 'ap': mswitch_detail = callomada(['access-point', mthisswitch]) elif sadevice_data[dTYPE] == 'switch': mswitch_detail = callomada(['switch', mthisswitch]) mswitch_dump = callomada(['-t','myomada','switch','-d',mthisswitch]) else: mswitch_detail = 'null' nswitch_dump = 'null' return mthisswitch, mswitch_detail, mswitch_dump ''' def get_omada_devices_details(sadevice_data): thisswitch = sadevice_data[dMAC] try: if sadevice_data[dTYPE] == 'ap': switch_detail = callomada(['access-point', thisswitch]) return thisswitch, switch_detail, None elif sadevice_data[dTYPE] == 'switch': switch_detail = callomada(['switch', thisswitch]) switch_dump = callomada(['-t','myomada','switch','-d',thisswitch]) return thisswitch, switch_detail, switch_dump else: return thisswitch, 'null', 'null' except Exception as e: mylog('error', [f'[{pluginName}] Error processing {thisswitch}: {str(e)}']) return thisswitch, 'error', 'error' def get_device_data(omada_clients_output, switches_and_aps, device_handler): """Process and return device data from Omada output.""" """ switch_dumps = {} switch_details = {} sadevices_macbyname = {} sadevices_macbymac = {} sadevices_linksbymac = {} port_byswitchmac_byclientmac = {} device_data_bymac = {} device_data_mac_byip = {} omada_force_overwrite = get_setting_value('OMDSDN_force_overwrite') """ manager = Manager() switch_dumps = manager.dict() switch_details = manager.dict() sadevices_macbyname = manager.dict() sadevices_macbymac = manager.dict() sadevices_linksbymac = manager.dict() port_byswitchmac_byclientmac = manager.dict() device_data_bymac = manager.dict() device_data_mac_byip = manager.dict() omada_force_overwrite = get_setting_value('OMDSDN_force_overwrite') sadevices = switches_and_aps.splitlines() mylog(OMDLOGLEVEL, [f'[{pluginName}] switches_and_aps rows: "{len(sadevices)}"']) ''' for sadevice in sadevices: sadevice_data = sadevice.split() get_omada_devices_details(sadevice_data,switch_details,switch_dumps) ''' ''' # Create a ThreadPoolExecutor # version 0.3b with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: # Submit tasks for each device futures = [] for sadevice in sadevices: sadevice_data = sadevice.split() future = executor.submit(get_omada_devices_details, sadevice_data, switch_details, switch_dumps) futures.append(future) # Wait for all tasks to complete concurrent.futures.wait(futures) ''' ''' # version 0.3c # Create a ThreadPoolExecutor with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor: # Submit tasks for each device future_to_device = {executor.submit(get_omada_devices_details, sadevice.split()): sadevice for sadevice in sadevices} # Process results as they complete for future in concurrent.futures.as_completed(future_to_device): csadevice = future_to_device[future] try: mylog('verbose', [f'[{pluginName}] processing results of: {csadevice}']) cthisswitch, cswitch_detail, cswitch_dump = future.result() switch_details[cthisswitch] = cswitch_detail switch_dumps[cthisswitch] = cswitch_dump except Exception as exc: mylog('error', [f'[{pluginName}] {csadevice} generated an exception: {exc}']) ''' # Use multiprocessing Pool with Pool(processes=3) as pool: results = pool.map(get_omada_devices_details, [sadevice.split() for sadevice in sadevices]) mylog(OMDLOGLEVEL, [f'[{pluginName}] All API calls completed. Processing results...']) # Process results for thisswitch, switch_detail, switch_dump in results: switch_details[thisswitch] = switch_detail if switch_dump is not None: switch_dumps[thisswitch] = switch_dump mylog(OMDLOGLEVEL, [f'[{pluginName}] Finished collecting device details. Processing data...']) # Now process the collected data for sadevice in sadevices: sadevice_data = sadevice.split() thisswitch = sadevice_data[dMAC] sadevices_macbyname[sadevice_data[4]] = thisswitch if sadevice_data[dTYPE] == 'ap': sadevice_type = 'AP' #sadevice_details = callomada(['access-point', thisswitch]) sadevice_details = switch_details[thisswitch] if sadevice_details == '': sadevice_links = [thisswitch] else: sadevice_links = extract_mac_addresses(sadevice_details) sadevices_linksbymac[thisswitch] = sadevice_links[1:] mylog(OMDLOGLEVEL, [f'[{pluginName}]adding switch details: "{sadevice_details}"']) mylog(OMDLOGLEVEL, [f'[{pluginName}]links are: "{sadevice_links}"']) mylog(OMDLOGLEVEL, [f'[{pluginName}]linksbymac are: "{sadevices_linksbymac[thisswitch]}"']) elif sadevice_data[dTYPE] == 'switch': sadevice_type = 'Switch' #sadevice_details=callomada(['switch', thisswitch]) sadevice_details = switch_details[thisswitch] if sadevice_details == '': sadevice_links = [thisswitch] else: sadevice_links=extract_mac_addresses(sadevice_details) sadevices_linksbymac[thisswitch] = sadevice_links[1:] # recovering the list of switches connected to sadevice switch and on which port... #switchdump = callomada(['-t','myomada','switch','-d',thisswitch]) switchdump = switch_dumps[thisswitch] port_byswitchmac_byclientmac[thisswitch] = {} for link in sadevices_linksbymac[thisswitch]: port_pattern = r"(?:{[^}]*\"port\"\: )([0-9]+)(?=[^}]*"+re.escape(link)+r")" myport = re.findall(port_pattern, switchdump,re.DOTALL) port_byswitchmac_byclientmac[thisswitch][link] = myport[0] mylog(OMDLOGLEVEL, [f'[{pluginName}]links are: "{sadevice_links}"']) mylog(OMDLOGLEVEL, [f'[{pluginName}]linksbymac are: "{sadevices_linksbymac[thisswitch]}"']) mylog(OMDLOGLEVEL, [f'[{pluginName}]ports of each links are: "{port_byswitchmac_byclientmac[thisswitch]}"']) mylog(OMDLOGLEVEL, [f'[{pluginName}]adding switch details: "{sadevice_details}"']) else: sadevice_type = 'null' sadevice_details='null' device_data_bymac[thisswitch] = [thisswitch, sadevice_data[dIP], sadevice_data[dNAME], 'null', 'null',sadevice_type] device_data_mac_byip[sadevice_data[dIP]] = thisswitch foo=[thisswitch, sadevice_data[1], sadevice_data[4], 'null', 'null'] mylog(OMDLOGLEVEL, [f'[{pluginName}]adding switch: "{foo}"']) # sadevices_macbymac[thisswitch] = thisswitch mylog(OMDLOGLEVEL, [f'[{pluginName}] switch_macbyname: "{sadevices_macbyname}"']) mylog(OMDLOGLEVEL, [f'[{pluginName}] switches: "{device_data_bymac}"']) # do some processing, call exteranl APIs, and return a device list # ... """ MAC = 0 IP = 1 NAME = 2 SWITCH_AP = 3 PORT_SSID = 4 TYPE = 5 """ # sample target output: # 0 MAC, 1 IP, 2 Name, 3 MAC of switch/AP, 4 port/SSID, 5 TYPE #17:27:10 [] token: "['9C-04-A0-82-67-45', '192.168.0.217', 'brick', 'ompapaoffice','froggies2', , 'Switch']" odevices = omada_clients_output.splitlines() mylog(OMDLOGLEVEL, [f'[{pluginName}] omada_clients_outputs rows: "{len(odevices)}"']) for odevice in odevices: odevice_data = odevice.split() odevice_data_reordered = [ MAC, IP, NAME, SWITCH_AP, PORT_SSID, TYPE] odevice_data_reordered[MAC]=odevice_data[cMAC] odevice_data_reordered[IP]=odevice_data[cIP] real_naxname = device_handler.getValueWithMac('dev_Name',ieee2ietf_mac_formater(odevice_data[cMAC])) # # if the name stored in Nax for a device is empty or the MAC addres or has some parenthhesis or is the same as in omada # don't bother updating omada's name at all. # naxname = real_naxname if real_naxname != None: if '(' in real_naxname: # removing parenthesis and domains from the name naxname = real_naxname.split('(')[0] if naxname != None and '.' in naxname: naxname = naxname.split('.')[0] if naxname in ( None, 'null', '' ): naxname = odevice_data[cNAME] if odevice_data[cNAME] != '' else odevice_data[cMAC] naxname = naxname.strip() mylog('debug', [f'[{pluginName}] TEST name from MAC: {naxname}']) if odevice_data[cNAME] in (odevice_data[cMAC], 'null', ''): mylog('verbose', [f'[{pluginName}] updating omada server because odevice_data is: {odevice_data[cNAME]} and naxname is: "{naxname}"']) callomada(['set-client-name', odevice_data[cMAC], naxname]) odevice_data_reordered[NAME] = naxname else: if omada_force_overwrite and naxname != odevice_data[cNAME] : mylog('verbose', [f'[{pluginName}] updating omada server because odevice_data is: "{odevice_data[cNAME]} and naxname is: "{naxname}"']) callomada(['set-client-name', odevice_data[cMAC], naxname]) odevice_data_reordered[NAME] = naxname mightbeport = odevice_data[cPORT_SSID].lstrip('(') mightbeport = mightbeport.rstrip(')') if mightbeport.isdigit(): odevice_data_reordered[SWITCH_AP] = odevice_data[cSWITCH_AP] odevice_data_reordered[PORT_SSID] = mightbeport else: odevice_data_reordered[SWITCH_AP] = mightbeport odevice_data_reordered[PORT_SSID] = odevice_data[cSWITCH_AP] # replacing the switch name with its MAC... try: mightbemac = sadevices_macbyname[odevice_data_reordered[SWITCH_AP]] odevice_data_reordered[SWITCH_AP] = mightbemac except KeyError: mylog(OMDLOGLEVEL, [f'[{pluginName}] could not find the mac adddress for: "{odevice_data_reordered[SWITCH_AP]}"']) # adding the type odevice_data_reordered[TYPE] = 'null' device_data_bymac[odevice_data_reordered[MAC]] = odevice_data_reordered device_data_mac_byip[odevice_data_reordered[IP]] = odevice_data_reordered[MAC] mylog(OMDLOGLEVEL, [f'[{pluginName}] tokens: "{odevice_data}"']) mylog(OMDLOGLEVEL, [f'[{pluginName}] tokens_reordered: "{odevice_data_reordered}"']) # populating the uplinks nodes of the omada switches and access points manually # since OMADA SDN makes is unreliable if the gateway is not their own tplink hardware... # step1 let's find the the default router # default_router_ip = find_default_gateway_ip() default_router_mac = ietf2ieee_mac_formater(get_mac_from_IP(default_router_ip)) device_data_bymac[default_router_mac][TYPE] = 'Firewall' # step2 let's find the first switch and set the default router parent to internet first_switch=device_data_bymac[default_router_mac][SWITCH_AP] device_data_bymac[default_router_mac][SWITCH_AP] = 'Internet' # step3 let's set the switch connected to the default gateway uplink to the default gateway and hardcode port to 1 for now: #device_data_bymac[first_switch][SWITCH_AP]=default_router_mac #device_data_bymac[first_switch][SWITCH_AP][PORT_SSID] = '1' # step4, let's go recursively through switches other links to mark update their uplinks # and pray it ends one day... # add_uplink(default_router_mac,first_switch, device_data_bymac,sadevices_linksbymac,port_byswitchmac_byclientmac) return device_data_bymac.values() if __name__ == '__main__': main()