NetAlertX/front/plugins/omada_sdn_imp/omada_sdn.py.0.3

#!/usr/bin/env python
"""
Omada SDN Query Script

This script queries the OMADA SDN to populate NetAlertX with Omada switches, access points, and clients.
It attempts to identify and populate their connections by switch/access points and ports/SSID,
and tries to differentiate root bridges from accessories.

Author: ffsb
Version: 0.2 - Added logic to retry Omada API call once as it sometimes fails, and improved error handling.
"""
__author__ = "ffsb"
__version__ = "0.1"  #initial
__version__ = "0.2"  # added logic to retry omada api call once as it seems to sometimes fail for some reasons, and error handling logic...
__version__ = "0.3"  # adding parallelism

#
# sample code to update unbound on opnsense - for reference...
# curl -X POST -d '{"host":{"enabled":"1","hostname":"test","domain":"testdomain.com","rr":"A","mxprio":"","mx":"","server":"10.0.1.1","description":""}}' -H "Content-Type: application/json" -k -u $OPNS_KEY:$OPNS_SECRET https://$IPFW/api/unbound/settings/AddHostOverride
#
import os
import pathlib
import sys
import json
import sqlite3
import tplink_omada_client
import importlib.util
import time
import io
import re
import concurrent.futures
from queue import Queue
import multiprocessing
from multiprocessing import Pool, Manager
import os

# Define the installation path and extend the system path for plugin imports
INSTALL_PATH = "/app"
sys.path.extend([f"{INSTALL_PATH}/front/plugins", f"{INSTALL_PATH}/server"])

from plugin_helper import Plugin_Object, Plugin_Objects, decodeBase64
from plugin_utils import get_plugins_configs
from logger import mylog
from const import pluginsPath, fullDbPath
from helper import timeNowTZ, get_setting_value 
from notification import write_notification

# Define the current path and log file paths
CUR_PATH = str(pathlib.Path(__file__).parent.resolve())
LOG_FILE = os.path.join(CUR_PATH, 'script.log')
RESULT_FILE = os.path.join(CUR_PATH, 'last_result.log')

# Initialize the Plugin obj output file
plugin_objects = Plugin_Objects(RESULT_FILE)

pluginName = 'OMDSDN'
#
# sample target output:
#  0 MAC, 1 IP, 2 Name, 3 switch/AP, 4 port/SSID, 5 TYPE    
#17:27:10 [<unique_prefix>] token: "['9C-04-A0-82-67-45', '192.168.0.217', '9C-04-A0-82-67-45', '17', '40-AE-30-A5-A7-50, 'Switch']"
    
# Constants for array indices
MAC, IP, NAME, SWITCH_AP, PORT_SSID, TYPE = range(6)

# sample omada devices input format:
# 
#  0.MAC                    1.IP    2.type  3.status           4.name              5.model
#40-AE-30-A5-A7-50    192.168.0.11      ap CONNECTED        ompapaoffice         EAP773(US) v1.0
#B0-95-75-46-0C-39     192.168.0.4  switch CONNECTED        pantry12             T1600G-52PS v4.0
dMAC, dIP, dTYPE, dSTATUS, dNAME, dMODEL = range(6) 

# sample omada clients input format:
#  0 MAC, 1 IP, 2 Name, 3 switch/AP, 4 port/SSID, 
#17:27:10 [<unique_prefix>] token: "['9C-04-A0-82-67-45', '192.168.0.217', '9C-04-A0-82-67-45', 'froggies2', '(ompapaoffice)']"
#17:27:10 [<unique_prefix>] token: "['50-02-91-29-E7-53', '192.168.0.153', 'frontyard_ESP_29E753', 'pantry12', '(48)']"
#17:27:10 [<unique_prefix>] token: "['00-E2-59-00-A0-8E', '192.168.0.1', 'bastion', 'office24', '(23)']"
#17:27:10 [<unique_prefix>] token: "['60-DD-8E-CA-A4-B3', '192.168.0.226', 'brick', 'froggies3', '(ompapaoffice)']"
cMAC, cIP, cNAME, cSWITCH_AP, cPORT_SSID = range(5)

OMDLOGLEVEL = 'verbose'

def ieee2ietf_mac_formater(inputmac):
    """Translate MAC address from standard IEEE model to IETF draft."""
    return inputmac.lower().replace('-', ':')

def ietf2ieee_mac_formater(inputmac):
    """Translate MAC address from IETF draft to standard IEEE model."""
    return inputmac.upper().replace(':', '-')

def get_mac_from_IP(target_IP):
    """Get MAC address from IP using ARP."""
    from scapy.all import ARP, Ether, srp
    try:
        arp_request = ARP(pdst=target_IP)
        ether = Ether(dst="ff:ff:ff:ff:ff:ff")
        packet = ether/arp_request
        result = srp(packet, timeout=3, verbose=0)[0]
        if result:
            return result[0][1].hwsrc
        else:
            return None
    except Exception as e:
        mylog('minimal', [f'[{pluginName}] get_mac_from_IP ERROR:{e}'])
        return None

def callomada(myargs):
    """Wrapper to call the Omada python library's own wrapper."""
    arguments = " ".join(myargs)
    mylog('verbose', [f'[{pluginName}] callomada START:{arguments}'])
    from tplink_omada_client.cli import main as omada 
    from contextlib import redirect_stdout
    
    omada_output = ''
    retries = 2
    while omada_output == '' and retries > 0:
        retries -= 1
        try:
            mf = io.StringIO()
            with redirect_stdout(mf):
                omada(myargs)
            omada_output = mf.getvalue()
        except Exception as e:
            mylog('minimal', [f'[{pluginName}] ERROR WHILE CALLING callomada:{arguments}\n {e}'])
            omada_output = ''
    mylog('verbose', [f'[{pluginName}] callomada END:{arguments}'])
    return omada_output

def extract_mac_addresses(text):
    """Extract all the MAC addresses from multiline text."""
    mac_pattern = r"([0-9A-Fa-f]{2}[:-][0-9A-Fa-f]{2}[:-][0-9A-Fa-f]{2}[:-][0-9A-Fa-f]{2}[:-][0-9A-Fa-f]{2}[:-][0-9A-Fa-f]{2})"
    return re.findall(mac_pattern, text)

def find_default_gateway_ip():
    """Find the default gateway IP address."""
    from scapy.all import conf, Route
    default_route = conf.route.route("0.0.0.0")
    return default_route[2] if default_route[2] else None

def add_uplink(uplink_mac, switch_mac, device_data_bymac, sadevices_linksbymac, port_byswitchmac_byclientmac):
    """Add uplink information to switches recursively."""
    mylog(OMDLOGLEVEL, [f'[{pluginName}] trying to add uplink="{uplink_mac}" to switch="{switch_mac}"'])
    mylog(OMDLOGLEVEL, [f'[{pluginName}] before adding:"{device_data_bymac[switch_mac]}"'])
    mylog(OMDLOGLEVEL, [f'[{pluginName}] this are the port_byswitchmac:"{port_byswitchmac_byclientmac[switch_mac]}"'])
    
    if device_data_bymac[switch_mac][SWITCH_AP] == 'null':
        device_data_bymac[switch_mac][SWITCH_AP] = uplink_mac
        if device_data_bymac[switch_mac][TYPE] == 'Switch' and device_data_bymac[uplink_mac][TYPE] == 'Switch':
            port_to_uplink = port_byswitchmac_byclientmac[switch_mac][uplink_mac]
        else:
            port_to_uplink = device_data_bymac[uplink_mac][PORT_SSID]
        device_data_bymac[switch_mac][PORT_SSID] = port_to_uplink
        mylog(OMDLOGLEVEL, [f'[{pluginName}] after adding:"{device_data_bymac[switch_mac]}"'])
    
    for link in sadevices_linksbymac[switch_mac]:
        if device_data_bymac[link][SWITCH_AP] == 'null' and device_data_bymac[switch_mac][TYPE] == 'Switch':
            add_uplink(switch_mac, link, device_data_bymac, sadevices_linksbymac, port_byswitchmac_byclientmac)

def main():
    """Main function to execute the script."""
    start_time = time.time()
    mylog('verbose', [f'[{pluginName}] starting execution'])
    
    from database import DB
    from device import Device_obj
    
    db = DB()
    db.open()
    device_handler = Device_obj(db)
    
    # Retrieve configuration settings
    omada_username = get_setting_value('OMDSDN_username')
    omada_password = get_setting_value('OMDSDN_password')
    omada_sites = get_setting_value('OMDSDN_sites')
    omada_site = omada_sites[0]
    omada_url = get_setting_value('OMDSDN_url')
    
    # Login to Omada
    omada_login = callomada(['-t', 'myomada', 'target', '--url', omada_url, '--user', omada_username,
                             '--password', omada_password, '--site', omada_site, '--set-default'])
    mylog('verbose', [f'[{pluginName}] login to omada result is: {omada_login}'])
    
    # Get clients and devices
    clients_list = callomada(['-t', 'myomada', 'clients'])
    mylog('verbose', [f'[{pluginName}] clients found:"{clients_list.count("\n")}"\n{clients_list}'])
    
    switches_and_aps = callomada(['-t', 'myomada', 'devices'])
    mylog('verbose', [f'[{pluginName}] omada devices (switches, access points) found:"{switches_and_aps.count("\n")}" \n  {switches_and_aps}'])
    
    # Process data
    device_data = get_device_data(clients_list, switches_and_aps, device_handler)
    
    mylog('verbose', [f'[{pluginName}] New entries to create: "{len(device_data)}"'])
    if len(device_data) > 0:
        for device in device_data:
            mylog(OMDLOGLEVEL, [f'[{pluginName}] main parsing device: "{device}"'])
            myport = device[PORT_SSID] if device[PORT_SSID].isdigit() else ''
            myssid = device[PORT_SSID] if not device[PORT_SSID].isdigit() else ''
            ParentNetworkNode = ieee2ietf_mac_formater(device[SWITCH_AP]) if device[SWITCH_AP] != 'Internet' else 'Internet'
            plugin_objects.add_object(
                primaryId = ieee2ietf_mac_formater(device[MAC]),
                secondaryId = device[IP],
                watched1 = device[NAME] if device[NAME] != 'null' else '',
                watched2 = ParentNetworkNode,
                watched3 = myport,
                watched4 = myssid,
                extra = device[TYPE] if device[TYPE] != 'null' else '',
                foreignKey = ieee2ietf_mac_formater(device[MAC])
            )
        mylog(OMDLOGLEVEL, [f'[{pluginName}] New entries: "{len(device_data)}"'])
    
    # Write results
    plugin_objects.write_result_file()
    
    end_time = time.time()
    mylog('verbose', [f'[{pluginName}] execution completed in {end_time - start_time:.2f} seconds'])
    
    return 0
'''
# version 0.3b
def get_omada_devices_details(sadevice_data,switch_details,switch_dumps):
    """Get device details from Omada. saved into a dictionary of strings"""
    mylog(OMDLOGLEVEL, [f'[{pluginName}]getting the omada devices details: "{sadevice_data}"'])
    thisswitch = sadevice_data[dMAC]
    if sadevice_data[dTYPE] == 'ap':
        switch_details[thisswitch] = callomada(['access-point', thisswitch])
    elif sadevice_data[dTYPE] == 'switch':        
        switch_details[thisswitch] = callomada(['switch', thisswitch])
        switch_dumps[thisswitch] = callomada(['-t','myomada','switch','-d',thisswitch])
    else:
        switch_details[thisswitch] = 'null'
        switch_dumps[thisswitch] = 'null'
    return 
'''
'''
# version 0.3c
def get_omada_devices_details(sadevice_data):
    mthisswitch = sadevice_data[dMAC]
    mswitch_detail = ''
    mswitch_dump = ''
    if sadevice_data[dTYPE] == 'ap':
        mswitch_detail = callomada(['access-point', mthisswitch])
    elif sadevice_data[dTYPE] == 'switch':        
        mswitch_detail = callomada(['switch', mthisswitch])
        mswitch_dump = callomada(['-t','myomada','switch','-d',mthisswitch])
    else:
        mswitch_detail = 'null'
        nswitch_dump = 'null'
    return mthisswitch, mswitch_detail, mswitch_dump
'''

def get_omada_devices_details(sadevice_data):
    thisswitch = sadevice_data[dMAC]
    try:
        if sadevice_data[dTYPE] == 'ap':
            switch_detail = callomada(['access-point', thisswitch])
            return thisswitch, switch_detail, None
        elif sadevice_data[dTYPE] == 'switch':        
            switch_detail = callomada(['switch', thisswitch])
            switch_dump = callomada(['-t','myomada','switch','-d',thisswitch])
            return thisswitch, switch_detail, switch_dump
        else:
            return thisswitch, 'null', 'null'
    except Exception as e:
        mylog('error', [f'[{pluginName}] Error processing {thisswitch}: {str(e)}'])
        return thisswitch, 'error', 'error'



def get_device_data(omada_clients_output, switches_and_aps, device_handler):
    """Process and return device data from Omada output."""
    """ 
    switch_dumps = {}
    switch_details = {}
    sadevices_macbyname = {}
    sadevices_macbymac = {}
    sadevices_linksbymac = {}
    port_byswitchmac_byclientmac = {}
    device_data_bymac = {}
    device_data_mac_byip = {}
    omada_force_overwrite = get_setting_value('OMDSDN_force_overwrite')
    """
    manager = Manager()
    switch_dumps = manager.dict()
    switch_details = manager.dict()
    sadevices_macbyname = manager.dict()
    sadevices_macbymac = manager.dict()
    sadevices_linksbymac = manager.dict()
    port_byswitchmac_byclientmac = manager.dict()
    device_data_bymac = manager.dict()
    device_data_mac_byip = manager.dict()
    omada_force_overwrite = get_setting_value('OMDSDN_force_overwrite')

    
    sadevices = switches_and_aps.splitlines()
    mylog(OMDLOGLEVEL, [f'[{pluginName}] switches_and_aps rows: "{len(sadevices)}"'])
    ''' 
    for sadevice in sadevices:
        sadevice_data = sadevice.split()
        get_omada_devices_details(sadevice_data,switch_details,switch_dumps)
    '''

    '''
    # Create a ThreadPoolExecutor
    # version 0.3b
    with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
        # Submit tasks for each device
        futures = []
        for sadevice in sadevices:
            sadevice_data = sadevice.split()
            future = executor.submit(get_omada_devices_details, sadevice_data, switch_details, switch_dumps)
            futures.append(future)
        
        # Wait for all tasks to complete
        concurrent.futures.wait(futures)
    '''
    '''
    # version 0.3c
     # Create a ThreadPoolExecutor
    with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor:
        # Submit tasks for each device
        future_to_device = {executor.submit(get_omada_devices_details, sadevice.split()): sadevice for sadevice in sadevices}
        
        # Process results as they complete
        for future in concurrent.futures.as_completed(future_to_device):
            csadevice = future_to_device[future]
            try:
                mylog('verbose', [f'[{pluginName}] processing results  of: {csadevice}'])
                cthisswitch, cswitch_detail, cswitch_dump = future.result()
                switch_details[cthisswitch] = cswitch_detail
                switch_dumps[cthisswitch] = cswitch_dump
            except Exception as exc:
                mylog('error', [f'[{pluginName}] {csadevice} generated an exception: {exc}'])
    '''
    # Use multiprocessing Pool
    with Pool(processes=3) as pool:
        results = pool.map(get_omada_devices_details, [sadevice.split() for sadevice in sadevices])



    mylog(OMDLOGLEVEL, [f'[{pluginName}] All API calls completed. Processing results...'])

        # Process results
    for thisswitch, switch_detail, switch_dump in results:
        switch_details[thisswitch] = switch_detail
        if switch_dump is not None:
            switch_dumps[thisswitch] = switch_dump

    mylog(OMDLOGLEVEL, [f'[{pluginName}] Finished collecting device details. Processing data...'])



    # Now process the collected data
   

    for sadevice in sadevices:
        sadevice_data = sadevice.split()
        thisswitch = sadevice_data[dMAC]
        sadevices_macbyname[sadevice_data[4]] = thisswitch
        if sadevice_data[dTYPE] == 'ap':
            sadevice_type = 'AP'
            #sadevice_details = callomada(['access-point', thisswitch])
            sadevice_details = switch_details[thisswitch]
            if sadevice_details == '':
                sadevice_links = [thisswitch]
            else:
                sadevice_links = extract_mac_addresses(sadevice_details)
            sadevices_linksbymac[thisswitch] = sadevice_links[1:]
            mylog(OMDLOGLEVEL, [f'[{pluginName}]adding switch details: "{sadevice_details}"'])
            mylog(OMDLOGLEVEL, [f'[{pluginName}]links are: "{sadevice_links}"'])
            mylog(OMDLOGLEVEL, [f'[{pluginName}]linksbymac are: "{sadevices_linksbymac[thisswitch]}"'])
        elif sadevice_data[dTYPE] == 'switch':
            sadevice_type = 'Switch'
            #sadevice_details=callomada(['switch', thisswitch])
            sadevice_details = switch_details[thisswitch]
            if sadevice_details == '':
                sadevice_links = [thisswitch]
            else:
                sadevice_links=extract_mac_addresses(sadevice_details)
            sadevices_linksbymac[thisswitch] = sadevice_links[1:]
            # recovering the list of switches connected to sadevice switch and on which port...
            #switchdump = callomada(['-t','myomada','switch','-d',thisswitch])
            switchdump = switch_dumps[thisswitch]
            port_byswitchmac_byclientmac[thisswitch]  = {}
            for link in sadevices_linksbymac[thisswitch]:
                port_pattern = r"(?:{[^}]*\"port\"\: )([0-9]+)(?=[^}]*"+re.escape(link)+r")"
                myport = re.findall(port_pattern, switchdump,re.DOTALL)
                port_byswitchmac_byclientmac[thisswitch][link] = myport[0]
            mylog(OMDLOGLEVEL, [f'[{pluginName}]links are: "{sadevice_links}"'])
            mylog(OMDLOGLEVEL, [f'[{pluginName}]linksbymac are: "{sadevices_linksbymac[thisswitch]}"'])
            mylog(OMDLOGLEVEL, [f'[{pluginName}]ports of each links  are: "{port_byswitchmac_byclientmac[thisswitch]}"'])
            mylog(OMDLOGLEVEL, [f'[{pluginName}]adding switch details: "{sadevice_details}"'])
        else:
            sadevice_type = 'null'
            sadevice_details='null'
        device_data_bymac[thisswitch] = [thisswitch, sadevice_data[dIP], sadevice_data[dNAME], 'null', 'null',sadevice_type]
        device_data_mac_byip[sadevice_data[dIP]] = thisswitch
        foo=[thisswitch, sadevice_data[1], sadevice_data[4], 'null', 'null']
        mylog(OMDLOGLEVEL, [f'[{pluginName}]adding switch: "{foo}"'])
        
    
        

    #    sadevices_macbymac[thisswitch] = thisswitch
    
    mylog(OMDLOGLEVEL, [f'[{pluginName}] switch_macbyname: "{sadevices_macbyname}"'])
    mylog(OMDLOGLEVEL, [f'[{pluginName}] switches: "{device_data_bymac}"'])


    # do some processing, call exteranl APIs, and return a device list
    #  ...
    """  MAC = 0
    IP = 1
    NAME = 2
    SWITCH_AP = 3
    PORT_SSID = 4
    TYPE = 5 """

    # sample target output:
    #  0 MAC, 1 IP, 2 Name, 3 MAC of switch/AP, 4 port/SSID, 5 TYPE    
    #17:27:10 [<unique_prefix>] token: "['9C-04-A0-82-67-45', '192.168.0.217', 'brick', 'ompapaoffice','froggies2', , 'Switch']"

    odevices = omada_clients_output.splitlines()
    mylog(OMDLOGLEVEL, [f'[{pluginName}] omada_clients_outputs rows: "{len(odevices)}"'])
    for odevice in odevices:
        odevice_data = odevice.split()
        odevice_data_reordered = [ MAC, IP, NAME, SWITCH_AP, PORT_SSID, TYPE]
        odevice_data_reordered[MAC]=odevice_data[cMAC]
        odevice_data_reordered[IP]=odevice_data[cIP]
        real_naxname = device_handler.getValueWithMac('dev_Name',ieee2ietf_mac_formater(odevice_data[cMAC]))
            
        #
        # if the name stored in Nax for a device is empty or the MAC addres or has some parenthhesis or is the same as in omada
        # don't bother updating omada's name at all.
        #
        naxname = real_naxname
        if real_naxname != None:
            if '(' in real_naxname:
            # removing parenthesis and domains from the name
                naxname = real_naxname.split('(')[0]
            if naxname != None and '.' in naxname:
                naxname = naxname.split('.')[0]
        if naxname in ( None, 'null', '' ):
            naxname = odevice_data[cNAME] if odevice_data[cNAME] != '' else odevice_data[cMAC]
        naxname = naxname.strip()
        mylog('debug', [f'[{pluginName}] TEST name from MAC: {naxname}'])    
        if odevice_data[cNAME] in (odevice_data[cMAC], 'null', ''):
            mylog('verbose', [f'[{pluginName}] updating omada server because odevice_data is: {odevice_data[cNAME]} and naxname is: "{naxname}"'])
            callomada(['set-client-name', odevice_data[cMAC], naxname])
            odevice_data_reordered[NAME] = naxname
        else:
            if omada_force_overwrite and naxname != odevice_data[cNAME] :
                mylog('verbose', [f'[{pluginName}] updating omada server because odevice_data is: "{odevice_data[cNAME]} and naxname is: "{naxname}"'])
                callomada(['set-client-name', odevice_data[cMAC], naxname])
            odevice_data_reordered[NAME] = naxname
        mightbeport = odevice_data[cPORT_SSID].lstrip('(')
        mightbeport = mightbeport.rstrip(')')
        if mightbeport.isdigit():
            odevice_data_reordered[SWITCH_AP] = odevice_data[cSWITCH_AP]
            odevice_data_reordered[PORT_SSID] = mightbeport
        else:
            odevice_data_reordered[SWITCH_AP] = mightbeport
            odevice_data_reordered[PORT_SSID] = odevice_data[cSWITCH_AP]
        
        # replacing the switch name with its MAC...
        try:
            mightbemac = sadevices_macbyname[odevice_data_reordered[SWITCH_AP]]
            odevice_data_reordered[SWITCH_AP] = mightbemac
        except KeyError:
            mylog(OMDLOGLEVEL, [f'[{pluginName}] could not find the mac adddress for: "{odevice_data_reordered[SWITCH_AP]}"'])
        # adding the type
        odevice_data_reordered[TYPE] = 'null'
        device_data_bymac[odevice_data_reordered[MAC]]  = odevice_data_reordered
        device_data_mac_byip[odevice_data_reordered[IP]] = odevice_data_reordered[MAC]
        mylog(OMDLOGLEVEL, [f'[{pluginName}] tokens: "{odevice_data}"'])
        mylog(OMDLOGLEVEL, [f'[{pluginName}] tokens_reordered: "{odevice_data_reordered}"'])
    # populating the uplinks nodes of the omada switches and access points manually 
    # since OMADA SDN makes is unreliable if the gateway is not their own tplink hardware...

    
    # step1 let's find the the default router
    # 
    default_router_ip =  find_default_gateway_ip()
    default_router_mac = ietf2ieee_mac_formater(get_mac_from_IP(default_router_ip))
    device_data_bymac[default_router_mac][TYPE] = 'Firewall'           
    # step2 let's find the first switch and set the default router parent to internet
    first_switch=device_data_bymac[default_router_mac][SWITCH_AP]
    device_data_bymac[default_router_mac][SWITCH_AP] = 'Internet'             
    # step3 let's set the switch connected to the default gateway uplink to the default gateway and hardcode port to 1 for now:
    #device_data_bymac[first_switch][SWITCH_AP]=default_router_mac                
    #device_data_bymac[first_switch][SWITCH_AP][PORT_SSID] = '1'                
    # step4, let's go recursively through switches other links to mark update their uplinks
    #  and pray it ends one day... 
    # 
    add_uplink(default_router_mac,first_switch, device_data_bymac,sadevices_linksbymac,port_byswitchmac_byclientmac)                       
    return device_data_bymac.values()

if __name__ == '__main__':
    main()