vh/NetAlertX
1
0
Fork 0
mirror of https://github.com/jokob-sk/NetAlertX.git synced 2025-12-07 09:36:05 -08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

0.4 saving api to files

Browse Source
This commit is contained in:
ffsb
2024-07-16 17:47:34 -04:00
parent bf2ce3262d
commit 147166e46e
4 changed files with 665 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
front/plugins/omada_sdn_imp/README.md
View File

@@ -6,11 +6,11 @@ The OMADA SDN plugin aims at synchronizing data between NetAlertX and a TPLINK O
2. extract list of OAMDA Devices (switches and access points) and sync them up with NetAlertX
> [!TIP]
> Some tip.
> some omada devices are apparently not fully compatible with the API which might lead to partial results.
### Quick setup guide
1. You SHOULD (ie: strongly recommend) setting up a dedicated account in your OMADA SDN console dedicated to NetAlertX OMADA_SDN plugin.
1. You SHOULD (ie: strongly recommend) set up an account in your OMADA SDN console dedicated to NetAlertX OMADA_SDN plugin.
- you should set USER TYPE = Local USer
- you should set USER ROLE = Administrator (if you use a read-only role you won't be able to sync names from NetAlerX to OMADA SDN)
- you can set Site Privileges = All Sites (or limit it to specific sites )
@@ -19,12 +19,15 @@ The OMADA SDN plugin aims at synchronizing data between NetAlertX and a TPLINK O
To set up the plugin correctly, make sure...
#### Required Settings
- When to run `PREF_RUN`
-
- OMDSDN_url
- OMDSDN_sites
- OMDSDN_username
- OMDSDN_password
- OMDSDN_force_overwrite
### Usage

163
front/plugins/omada_sdn_imp/omada_sdn.py
View File

@@ -2,6 +2,7 @@
__author__ = "ffsb"
__version__ = "0.1" #initial
__version__ = "0.2" # added logic to retry omada api call once as it seems to sometimes fail for some reasons, and error handling logic...
__version__ = "0.3" # split devices API calls to allow multithreading but had to stop due to concurency issues.
# query OMADA SDN to populate NetAlertX witch omada switches, access points, clients.
# try to identify and populate their connections by switch/accesspoints and ports/SSID
# try to differentiate root bridges from accessory
@@ -21,6 +22,8 @@ import importlib.util
import time
import io
import re
import concurrent.futures
#import netifaces
# Define the installation path and extend the system path for plugin imports
@@ -38,22 +41,35 @@ from notification import write_notification
CUR_PATH = str(pathlib.Path(__file__).parent.resolve())
LOG_FILE = os.path.join(CUR_PATH, 'script.log')
RESULT_FILE = os.path.join(CUR_PATH, 'last_result.log')
OMADA_API_RETURN_FILE = os.path.join(CUR_PATH, 'omada_api_return')
# Initialize the Plugin obj output file
plugin_objects = Plugin_Objects(RESULT_FILE)
pluginName = 'OMDSDN'
#
# sample target output:
# 0)MAC, 1)IP, 2)Name, 3)switch/AP, 4)port/SSID, 5)TYPE
# "['9C-04-A0-82-67-45', '192.168.0.217', 'foo', '40-AE-30-A5-A7-50, '17', 'Switch']"
# constants:
MAC = 0
IP = 1
NAME = 2
SWITCH_AP = 3
PORT_SSID = 4
TYPE = 5
OMDLOGLEVEL='debug'
# 0 MAC, 1 IP, 2 Name, 3 switch/AP, 4 port/SSID, 5 TYPE
#17:27:10 [<unique_prefix>] token: "['9C-04-A0-82-67-45', '192.168.0.217', '9C-04-A0-82-67-45', '17', '40-AE-30-A5-A7-50, 'Switch']"
# Constants for array indices
MAC, IP, NAME, SWITCH_AP, PORT_SSID, TYPE = range(6)
# sample omada devices input format:
#
# 0.MAC 1.IP 2.type 3.status 4.name 5.model
#40-AE-30-A5-A7-50 192.168.0.11 ap CONNECTED ompapaoffice EAP773(US) v1.0
#B0-95-75-46-0C-39 192.168.0.4 switch CONNECTED pantry12 T1600G-52PS v4.0
dMAC, dIP, dTYPE, dSTATUS, dNAME, dMODEL = range(6)
# sample omada clients input format:
# 0 MAC, 1 IP, 2 Name, 3 switch/AP, 4 port/SSID,
#17:27:10 [<unique_prefix>] token: "['9C-04-A0-82-67-45', '192.168.0.217', '9C-04-A0-82-67-45', 'froggies2', '(ompapaoffice)']"
#17:27:10 [<unique_prefix>] token: "['50-02-91-29-E7-53', '192.168.0.153', 'frontyard_ESP_29E753', 'pantry12', '(48)']"
#17:27:10 [<unique_prefix>] token: "['00-E2-59-00-A0-8E', '192.168.0.1', 'bastion', 'office24', '(23)']"
#17:27:10 [<unique_prefix>] token: "['60-DD-8E-CA-A4-B3', '192.168.0.226', 'brick', 'froggies3', '(ompapaoffice)']"
cMAC, cIP, cNAME, cSWITCH_AP, cPORT_SSID = range(5)
OMDLOGLEVEL = 'debug'
pluginName = 'OMDSDN'
#
# translate MAC address from standard ieee model to ietf draft
# AA-BB-CC-DD-EE-FF to aa:bb:cc:dd:ee:ff
@@ -217,18 +233,12 @@ def main():
for device in device_data:
mylog(OMDLOGLEVEL, [f'[{pluginName}] main parsing device: "{device}"'])
if device[PORT_SSID].isdigit():
myport = device[PORT_SSID]
myssid = 'null'
else:
myssid = device[PORT_SSID]
myport = 'null'
if device[SWITCH_AP] != 'Internet':
ParentNetworkNode = ieee2ietf_mac_formater(device[SWITCH_AP])
else:
ParentNetworkNode = device[SWITCH_AP]
myport = device[PORT_SSID] if device[PORT_SSID].isdigit() else ''
myssid = device[PORT_SSID] if not device[PORT_SSID].isdigit() else ''
ParentNetworkNode = ieee2ietf_mac_formater(device[SWITCH_AP]) if device[SWITCH_AP] != 'Internet' else 'Internet'
mymac = ieee2ietf_mac_formater(device[MAC])
plugin_objects.add_object(
primaryId = ieee2ietf_mac_formater(device[MAC]), # MAC
primaryId = mymac, # MAC
secondaryId = device[IP], # IP
watched1 = device[NAME], # NAME/HOSTNAME
watched2 = ParentNetworkNode, # PARENT NETWORK NODE MAC
@@ -237,17 +247,46 @@ def main():
extra = device[TYPE],
#omada_site, # SITENAME (cur_NetworkSite) or VENDOR (cur_Vendor) (PICK one and adjust config.json -> "column": "Extra")
foreignKey = device[MAC].lower().replace('-',':')) # usually MAC
mylog(OMDLOGLEVEL, [f'[{pluginName}] New entries: "{len(device_data)}"'])
mylog('verbose', [f'[{pluginName}] New entries: "{mymac:<18}, {device[IP]:<16}, {device[NAME]:<63}, {ParentNetworkNode:<18}, {myport:<4}, {myssid:<32}, {device[TYPE]}"'])
mylog('verbose', [f'[{pluginName}] New entries: "{len(device_data)}"'])
# log result
plugin_objects.write_result_file()
#mylog(OMDLOGLEVEL, [f'[{pluginName}] TEST name from MAC: {device_handler.getValueWithMac('dev_Name','00:e2:59:00:a0:8e')}'])
#mylog(OMDLOGLEVEL, [f'[{pluginName}] TEST MAC from IP: {get_mac_from_IP('192.168.0.1')} also {ietf2ieee_mac_formater(get_mac_from_IP('192.168.0.1'))}'])
end_time = time.time()
mylog('verbose', [f'[{pluginName}] execution completed in {end_time - start_time:.2f} seconds'])
return 0
def get_omada_devices_details(msadevice_data):
mthisswitch = msadevice_data[dMAC]
mtype = msadevice_data[dTYPE]
mswitch_detail = ''
mswitch_dump = ''
if mtype == 'ap':
mswitch_detail = callomada(['access-point', mthisswitch])
elif mtype == 'switch':
mswitch_detail = callomada(['switch', mthisswitch])
mswitch_dump = callomada(['-t','myomada','switch','-d',mthisswitch])
else:
mswitch_detail = ''
nswitch_dump = ''
details_outfile = OMADA_API_RETURN_FILE+"_"+mthisswitch+"_det"
dump_outfile = OMADA_API_RETURN_FILE+"_"+mthisswitch+"_dmp"
for tmpdfle in [details_outfile+".tmp", dump_outfile+".tmp", details_outfile+".txt", dump_outfile+".txt"]:
if os.path.exists(tmpdfle):
os.remove(tmpdfle)
with open(details_outfile+".tmp", 'w') as f:
f.write(mswitch_detail)
with open(dump_outfile+".tmp", 'w') as f:
f.write(mswitch_dump)
os.rename(details_outfile+".tmp", details_outfile+".txt")
os.rename(dump_outfile+".tmp", dump_outfile+".txt")
return mswitch_detail, mswitch_dump
@@ -265,12 +304,6 @@ def get_device_data(omada_clients_output,switches_and_aps,device_handler):
# 0 MAC, 1 IP, 2 Name, 3 switch/AP, 4 port/SSID, 5 TYPE
#17:27:10 [<unique_prefix>] token: "['9C-04-A0-82-67-45', '192.168.0.217', '9C-04-A0-82-67-45', '17', '40-AE-30-A5-A7-50, 'Switch']"
#constants
dMAC = 0
dIP = 1
dTYPE = 2
dSTATUS = 3
dNAME = 4
dMODEL = 5
sadevices_macbyname = {}
sadevices_macbymac = {}
sadevices_linksbymac = {}
@@ -278,16 +311,30 @@ def get_device_data(omada_clients_output,switches_and_aps,device_handler):
device_data_bymac = {}
device_data_mac_byip = {}
omada_force_overwrite = get_setting_value('OMDSDN_force_overwrite')
switch_details = {}
switch_dumps = {}
sadevices = switches_and_aps.splitlines()
mylog(OMDLOGLEVEL, [f'[{pluginName}] switches_and_aps rows: "{len(sadevices)}"'])
for sadevice in sadevices:
sadevice_data = sadevice.split()
thisswitch = sadevice_data[dMAC]
thistype = sadevice_data[dTYPE]
switch_details[thisswitch], switch_dumps[thisswitch] = get_omada_devices_details(sadevice_data)
mylog('verbose', [f'[{pluginName}] switches details collected "{len(switch_details)}"'])
mylog('verbose', [f'[{pluginName}] dump details collected "{len(switch_details)}"'])
# Using ThreadPoolExecutor for parallel execution
for sadevice in sadevices:
sadevice_data = sadevice.split()
thisswitch = sadevice_data[dMAC]
sadevices_macbyname[sadevice_data[4]] = thisswitch
if sadevice_data[dTYPE] == 'ap':
sadevice_type = 'AP'
sadevice_details = callomada(['access-point', thisswitch])
#sadevice_details = callomada(['access-point', thisswitch])
sadevice_details = switch_details[thisswitch]
if sadevice_details == '':
sadevice_links = [thisswitch]
else:
@@ -298,23 +345,27 @@ def get_device_data(omada_clients_output,switches_and_aps,device_handler):
mylog(OMDLOGLEVEL, [f'[{pluginName}]linksbymac are: "{sadevices_linksbymac[thisswitch]}"'])
elif sadevice_data[dTYPE] == 'switch':
sadevice_type = 'Switch'
sadevice_details=callomada(['switch', thisswitch])
#sadevice_details=callomada(['switch', thisswitch])
sadevice_details = switch_details[thisswitch]
if sadevice_details == '':
sadevice_links = [thisswitch]
else:
sadevice_links=extract_mac_addresses(sadevice_details)
sadevices_linksbymac[thisswitch] = sadevice_links[1:]
# recovering the list of switches connected to sadevice switch and on which port...
switchdump = callomada(['-t','myomada','switch','-d',thisswitch])
#switchdump = callomada(['-t','myomada','switch','-d',thisswitch])
switchdump = switch_dumps[thisswitch]
mylog(OMDLOGLEVEL, [f'[{pluginName}] switchdump: {switchdump}'])
port_byswitchmac_byclientmac[thisswitch] = {}
for link in sadevices_linksbymac[thisswitch]:
port_pattern = r"(?:{[^}]*\"port\"\: )([0-9]+)(?=[^}]*"+re.escape(link)+r")"
myport = re.findall(port_pattern, switchdump,re.DOTALL)
port_byswitchmac_byclientmac[thisswitch][link] = myport[0]
mylog(OMDLOGLEVEL, [f'[{pluginName}]links are: "{sadevice_links}"'])
mylog(OMDLOGLEVEL, [f'[{pluginName}]linksbymac are: "{sadevices_linksbymac[thisswitch]}"'])
mylog(OMDLOGLEVEL, [f'[{pluginName}]ports of each links are: "{port_byswitchmac_byclientmac[thisswitch]}"'])
mylog(OMDLOGLEVEL, [f'[{pluginName}]adding switch details: "{sadevice_details}"'])
mylog(OMDLOGLEVEL, [f'[{pluginName}] switchdump: link={link} myport:{myport}'])
port_byswitchmac_byclientmac[thisswitch][link] = myport[0] if myport else ''
#mylog(OMDLOGLEVEL, [f'[{pluginName}]links are: "{sadevice_links}"'])
#mylog(OMDLOGLEVEL, [f'[{pluginName}]linksbymac are: "{sadevices_linksbymac[thisswitch]}"'])
#mylog(OMDLOGLEVEL, [f'[{pluginName}]ports of each links are: "{port_byswitchmac_byclientmac[thisswitch]}"'])
#mylog(OMDLOGLEVEL, [f'[{pluginName}]adding switch details: "{sadevice_details}"'])
else:
sadevice_type = 'null'
sadevice_details='null'
@@ -334,24 +385,13 @@ def get_device_data(omada_clients_output,switches_and_aps,device_handler):
# do some processing, call exteranl APIs, and return a device list
# ...
""" MAC = 0
IP = 1
NAME = 2
SWITCH_AP = 3
PORT_SSID = 4
TYPE = 5 """
# sample omada clients input format:
# 0 MAC, 1 IP, 2 Name, 3 switch/AP, 4 port/SSID,
#17:27:10 [<unique_prefix>] token: "['9C-04-A0-82-67-45', '192.168.0.217', '9C-04-A0-82-67-45', 'froggies2', '(ompapaoffice)']"
#17:27:10 [<unique_prefix>] token: "['50-02-91-29-E7-53', '192.168.0.153', 'frontyard_ESP_29E753', 'pantry12', '(48)']"
#17:27:10 [<unique_prefix>] token: "['00-E2-59-00-A0-8E', '192.168.0.1', 'bastion', 'office24', '(23)']"
#17:27:10 [<unique_prefix>] token: "['60-DD-8E-CA-A4-B3', '192.168.0.226', 'brick', 'froggies3', '(ompapaoffice)']"
cMAC = 0
cIP = 1
cNAME = 2
cSWITCH_AP = 3
cPORT_SSID = 4
# sample target output:
# 0 MAC, 1 IP, 2 Name, 3 MAC of switch/AP, 4 port/SSID, 5 TYPE
#17:27:10 [<unique_prefix>] token: "['9C-04-A0-82-67-45', '192.168.0.217', 'brick', 'ompapaoffice','froggies2', , 'Switch']"
@@ -369,6 +409,7 @@ def get_device_data(omada_clients_output,switches_and_aps,device_handler):
# if the name stored in Nax for a device is empty or the MAC addres or has some parenthhesis or is the same as in omada
# don't bother updating omada's name at all.
#
'''
if real_naxname == None or ietf2ieee_mac_formater(real_naxname) == odevice_data[cMAC] or '('in real_naxname or real_naxname == odevice_data[cNAME] or real_naxname == 'null':
naxname = None
else:
@@ -384,6 +425,28 @@ def get_device_data(omada_clients_output,switches_and_aps,device_handler):
if omada_force_overwrite and naxname != None:
callomada(['set-client-name', odevice_data[cMAC], naxname])
odevice_data_reordered[NAME] = odevice_data[cNAME]
'''
naxname = real_naxname
if real_naxname != None:
if '(' in real_naxname:
# removing parenthesis and domains from the name
naxname = real_naxname.split('(')[0]
if naxname != None and '.' in naxname:
naxname = naxname.split('.')[0]
if naxname in ( None, 'null', '' ):
naxname = odevice_data[cNAME] if odevice_data[cNAME] != '' else odevice_data[cMAC]
naxname = naxname.strip()
mylog('debug', [f'[{pluginName}] TEST name from MAC: {naxname}'])
if odevice_data[cNAME] in (odevice_data[cMAC], 'null', ''):
mylog('verbose', [f'[{pluginName}] updating omada server because odevice_data is: {odevice_data[cNAME]} and naxname is: "{naxname}"'])
callomada(['set-client-name', odevice_data[cMAC], naxname])
odevice_data_reordered[NAME] = naxname
else:
if omada_force_overwrite and naxname != odevice_data[cNAME] :
mylog('verbose', [f'[{pluginName}] updating omada server because odevice_data is: "{odevice_data[cNAME]} and naxname is: "{naxname}"'])
callomada(['set-client-name', odevice_data[cMAC], naxname])
odevice_data_reordered[NAME] = naxname
mightbeport = odevice_data[cPORT_SSID].lstrip('(')
mightbeport = mightbeport.rstrip(')')
if mightbeport.isdigit():

498
front/plugins/omada_sdn_imp/omada_sdn.py.0.3 Executable file
View File

@@ -0,0 +1,498 @@
#!/usr/bin/env python
"""
Omada SDN Query Script
This script queries the OMADA SDN to populate NetAlertX with Omada switches, access points, and clients.
It attempts to identify and populate their connections by switch/access points and ports/SSID,
and tries to differentiate root bridges from accessories.
Author: ffsb
Version: 0.2 - Added logic to retry Omada API call once as it sometimes fails, and improved error handling.
"""
__author__ = "ffsb"
__version__ = "0.1" #initial
__version__ = "0.2" # added logic to retry omada api call once as it seems to sometimes fail for some reasons, and error handling logic...
__version__ = "0.3" # adding parallelism
#
# sample code to update unbound on opnsense - for reference...
# curl -X POST -d '{"host":{"enabled":"1","hostname":"test","domain":"testdomain.com","rr":"A","mxprio":"","mx":"","server":"10.0.1.1","description":""}}' -H "Content-Type: application/json" -k -u $OPNS_KEY:$OPNS_SECRET https://$IPFW/api/unbound/settings/AddHostOverride
#
import os
import pathlib
import sys
import json
import sqlite3
import tplink_omada_client
import importlib.util
import time
import io
import re
import concurrent.futures
from queue import Queue
import multiprocessing
from multiprocessing import Pool, Manager
import os
# Define the installation path and extend the system path for plugin imports
INSTALL_PATH = "/app"
sys.path.extend([f"{INSTALL_PATH}/front/plugins", f"{INSTALL_PATH}/server"])
from plugin_helper import Plugin_Object, Plugin_Objects, decodeBase64
from plugin_utils import get_plugins_configs
from logger import mylog
from const import pluginsPath, fullDbPath
from helper import timeNowTZ, get_setting_value
from notification import write_notification
# Define the current path and log file paths
CUR_PATH = str(pathlib.Path(__file__).parent.resolve())
LOG_FILE = os.path.join(CUR_PATH, 'script.log')
RESULT_FILE = os.path.join(CUR_PATH, 'last_result.log')
# Initialize the Plugin obj output file
plugin_objects = Plugin_Objects(RESULT_FILE)
pluginName = 'OMDSDN'
#
# sample target output:
# 0 MAC, 1 IP, 2 Name, 3 switch/AP, 4 port/SSID, 5 TYPE
#17:27:10 [<unique_prefix>] token: "['9C-04-A0-82-67-45', '192.168.0.217', '9C-04-A0-82-67-45', '17', '40-AE-30-A5-A7-50, 'Switch']"
# Constants for array indices
MAC, IP, NAME, SWITCH_AP, PORT_SSID, TYPE = range(6)
# sample omada devices input format:
#
# 0.MAC 1.IP 2.type 3.status 4.name 5.model
#40-AE-30-A5-A7-50 192.168.0.11 ap CONNECTED ompapaoffice EAP773(US) v1.0
#B0-95-75-46-0C-39 192.168.0.4 switch CONNECTED pantry12 T1600G-52PS v4.0
dMAC, dIP, dTYPE, dSTATUS, dNAME, dMODEL = range(6)
# sample omada clients input format:
# 0 MAC, 1 IP, 2 Name, 3 switch/AP, 4 port/SSID,
#17:27:10 [<unique_prefix>] token: "['9C-04-A0-82-67-45', '192.168.0.217', '9C-04-A0-82-67-45', 'froggies2', '(ompapaoffice)']"
#17:27:10 [<unique_prefix>] token: "['50-02-91-29-E7-53', '192.168.0.153', 'frontyard_ESP_29E753', 'pantry12', '(48)']"
#17:27:10 [<unique_prefix>] token: "['00-E2-59-00-A0-8E', '192.168.0.1', 'bastion', 'office24', '(23)']"
#17:27:10 [<unique_prefix>] token: "['60-DD-8E-CA-A4-B3', '192.168.0.226', 'brick', 'froggies3', '(ompapaoffice)']"
cMAC, cIP, cNAME, cSWITCH_AP, cPORT_SSID = range(5)
OMDLOGLEVEL = 'verbose'
def ieee2ietf_mac_formater(inputmac):
"""Translate MAC address from standard IEEE model to IETF draft."""
return inputmac.lower().replace('-', ':')
def ietf2ieee_mac_formater(inputmac):
"""Translate MAC address from IETF draft to standard IEEE model."""
return inputmac.upper().replace(':', '-')
def get_mac_from_IP(target_IP):
"""Get MAC address from IP using ARP."""
from scapy.all import ARP, Ether, srp
try:
arp_request = ARP(pdst=target_IP)
ether = Ether(dst="ff:ff:ff:ff:ff:ff")
packet = ether/arp_request
result = srp(packet, timeout=3, verbose=0)[0]
if result:
return result[0][1].hwsrc
else:
return None
except Exception as e:
mylog('minimal', [f'[{pluginName}] get_mac_from_IP ERROR:{e}'])
return None
def callomada(myargs):
"""Wrapper to call the Omada python library's own wrapper."""
arguments = " ".join(myargs)
mylog('verbose', [f'[{pluginName}] callomada START:{arguments}'])
from tplink_omada_client.cli import main as omada
from contextlib import redirect_stdout
omada_output = ''
retries = 2
while omada_output == '' and retries > 0:
retries -= 1
try:
mf = io.StringIO()
with redirect_stdout(mf):
omada(myargs)
omada_output = mf.getvalue()
except Exception as e:
mylog('minimal', [f'[{pluginName}] ERROR WHILE CALLING callomada:{arguments}\n {e}'])
omada_output = ''
mylog('verbose', [f'[{pluginName}] callomada END:{arguments}'])
return omada_output
def extract_mac_addresses(text):
"""Extract all the MAC addresses from multiline text."""
mac_pattern = r"([0-9A-Fa-f]{2}[:-][0-9A-Fa-f]{2}[:-][0-9A-Fa-f]{2}[:-][0-9A-Fa-f]{2}[:-][0-9A-Fa-f]{2}[:-][0-9A-Fa-f]{2})"
return re.findall(mac_pattern, text)
def find_default_gateway_ip():
"""Find the default gateway IP address."""
from scapy.all import conf, Route
default_route = conf.route.route("0.0.0.0")
return default_route[2] if default_route[2] else None
def add_uplink(uplink_mac, switch_mac, device_data_bymac, sadevices_linksbymac, port_byswitchmac_byclientmac):
"""Add uplink information to switches recursively."""
mylog(OMDLOGLEVEL, [f'[{pluginName}] trying to add uplink="{uplink_mac}" to switch="{switch_mac}"'])
mylog(OMDLOGLEVEL, [f'[{pluginName}] before adding:"{device_data_bymac[switch_mac]}"'])
mylog(OMDLOGLEVEL, [f'[{pluginName}] this are the port_byswitchmac:"{port_byswitchmac_byclientmac[switch_mac]}"'])
if device_data_bymac[switch_mac][SWITCH_AP] == 'null':
device_data_bymac[switch_mac][SWITCH_AP] = uplink_mac
if device_data_bymac[switch_mac][TYPE] == 'Switch' and device_data_bymac[uplink_mac][TYPE] == 'Switch':
port_to_uplink = port_byswitchmac_byclientmac[switch_mac][uplink_mac]
else:
port_to_uplink = device_data_bymac[uplink_mac][PORT_SSID]
device_data_bymac[switch_mac][PORT_SSID] = port_to_uplink
mylog(OMDLOGLEVEL, [f'[{pluginName}] after adding:"{device_data_bymac[switch_mac]}"'])
for link in sadevices_linksbymac[switch_mac]:
if device_data_bymac[link][SWITCH_AP] == 'null' and device_data_bymac[switch_mac][TYPE] == 'Switch':
add_uplink(switch_mac, link, device_data_bymac, sadevices_linksbymac, port_byswitchmac_byclientmac)
def main():
"""Main function to execute the script."""
start_time = time.time()
mylog('verbose', [f'[{pluginName}] starting execution'])
from database import DB
from device import Device_obj
db = DB()
db.open()
device_handler = Device_obj(db)
# Retrieve configuration settings
omada_username = get_setting_value('OMDSDN_username')
omada_password = get_setting_value('OMDSDN_password')
omada_sites = get_setting_value('OMDSDN_sites')
omada_site = omada_sites[0]
omada_url = get_setting_value('OMDSDN_url')
# Login to Omada
omada_login = callomada(['-t', 'myomada', 'target', '--url', omada_url, '--user', omada_username,
'--password', omada_password, '--site', omada_site, '--set-default'])
mylog('verbose', [f'[{pluginName}] login to omada result is: {omada_login}'])
# Get clients and devices
clients_list = callomada(['-t', 'myomada', 'clients'])
mylog('verbose', [f'[{pluginName}] clients found:"{clients_list.count("\n")}"\n{clients_list}'])
switches_and_aps = callomada(['-t', 'myomada', 'devices'])
mylog('verbose', [f'[{pluginName}] omada devices (switches, access points) found:"{switches_and_aps.count("\n")}" \n {switches_and_aps}'])
# Process data
device_data = get_device_data(clients_list, switches_and_aps, device_handler)
mylog('verbose', [f'[{pluginName}] New entries to create: "{len(device_data)}"'])
if len(device_data) > 0:
for device in device_data:
mylog(OMDLOGLEVEL, [f'[{pluginName}] main parsing device: "{device}"'])
myport = device[PORT_SSID] if device[PORT_SSID].isdigit() else ''
myssid = device[PORT_SSID] if not device[PORT_SSID].isdigit() else ''
ParentNetworkNode = ieee2ietf_mac_formater(device[SWITCH_AP]) if device[SWITCH_AP] != 'Internet' else 'Internet'
plugin_objects.add_object(
primaryId = ieee2ietf_mac_formater(device[MAC]),
secondaryId = device[IP],
watched1 = device[NAME] if device[NAME] != 'null' else '',
watched2 = ParentNetworkNode,
watched3 = myport,
watched4 = myssid,
extra = device[TYPE] if device[TYPE] != 'null' else '',
foreignKey = ieee2ietf_mac_formater(device[MAC])
)
mylog(OMDLOGLEVEL, [f'[{pluginName}] New entries: "{len(device_data)}"'])
# Write results
plugin_objects.write_result_file()
end_time = time.time()
mylog('verbose', [f'[{pluginName}] execution completed in {end_time - start_time:.2f} seconds'])
return 0
'''
# version 0.3b
def get_omada_devices_details(sadevice_data,switch_details,switch_dumps):
"""Get device details from Omada. saved into a dictionary of strings"""
mylog(OMDLOGLEVEL, [f'[{pluginName}]getting the omada devices details: "{sadevice_data}"'])
thisswitch = sadevice_data[dMAC]
if sadevice_data[dTYPE] == 'ap':
switch_details[thisswitch] = callomada(['access-point', thisswitch])
elif sadevice_data[dTYPE] == 'switch':
switch_details[thisswitch] = callomada(['switch', thisswitch])
switch_dumps[thisswitch] = callomada(['-t','myomada','switch','-d',thisswitch])
else:
switch_details[thisswitch] = 'null'
switch_dumps[thisswitch] = 'null'
return
'''
'''
# version 0.3c
def get_omada_devices_details(sadevice_data):
mthisswitch = sadevice_data[dMAC]
mswitch_detail = ''
mswitch_dump = ''
if sadevice_data[dTYPE] == 'ap':
mswitch_detail = callomada(['access-point', mthisswitch])
elif sadevice_data[dTYPE] == 'switch':
mswitch_detail = callomada(['switch', mthisswitch])
mswitch_dump = callomada(['-t','myomada','switch','-d',mthisswitch])
else:
mswitch_detail = 'null'
nswitch_dump = 'null'
return mthisswitch, mswitch_detail, mswitch_dump
'''
def get_omada_devices_details(sadevice_data):
thisswitch = sadevice_data[dMAC]
try:
if sadevice_data[dTYPE] == 'ap':
switch_detail = callomada(['access-point', thisswitch])
return thisswitch, switch_detail, None
elif sadevice_data[dTYPE] == 'switch':
switch_detail = callomada(['switch', thisswitch])
switch_dump = callomada(['-t','myomada','switch','-d',thisswitch])
return thisswitch, switch_detail, switch_dump
else:
return thisswitch, 'null', 'null'
except Exception as e:
mylog('error', [f'[{pluginName}] Error processing {thisswitch}: {str(e)}'])
return thisswitch, 'error', 'error'
def get_device_data(omada_clients_output, switches_and_aps, device_handler):
"""Process and return device data from Omada output."""
"""
switch_dumps = {}
switch_details = {}
sadevices_macbyname = {}
sadevices_macbymac = {}
sadevices_linksbymac = {}
port_byswitchmac_byclientmac = {}
device_data_bymac = {}
device_data_mac_byip = {}
omada_force_overwrite = get_setting_value('OMDSDN_force_overwrite')
"""
manager = Manager()
switch_dumps = manager.dict()
switch_details = manager.dict()
sadevices_macbyname = manager.dict()
sadevices_macbymac = manager.dict()
sadevices_linksbymac = manager.dict()
port_byswitchmac_byclientmac = manager.dict()
device_data_bymac = manager.dict()
device_data_mac_byip = manager.dict()
omada_force_overwrite = get_setting_value('OMDSDN_force_overwrite')
sadevices = switches_and_aps.splitlines()
mylog(OMDLOGLEVEL, [f'[{pluginName}] switches_and_aps rows: "{len(sadevices)}"'])
'''
for sadevice in sadevices:
sadevice_data = sadevice.split()
get_omada_devices_details(sadevice_data,switch_details,switch_dumps)
'''
'''
# Create a ThreadPoolExecutor
# version 0.3b
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
# Submit tasks for each device
futures = []
for sadevice in sadevices:
sadevice_data = sadevice.split()
future = executor.submit(get_omada_devices_details, sadevice_data, switch_details, switch_dumps)
futures.append(future)
# Wait for all tasks to complete
concurrent.futures.wait(futures)
'''
'''
# version 0.3c
# Create a ThreadPoolExecutor
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor:
# Submit tasks for each device
future_to_device = {executor.submit(get_omada_devices_details, sadevice.split()): sadevice for sadevice in sadevices}
# Process results as they complete
for future in concurrent.futures.as_completed(future_to_device):
csadevice = future_to_device[future]
try:
mylog('verbose', [f'[{pluginName}] processing results of: {csadevice}'])
cthisswitch, cswitch_detail, cswitch_dump = future.result()
switch_details[cthisswitch] = cswitch_detail
switch_dumps[cthisswitch] = cswitch_dump
except Exception as exc:
mylog('error', [f'[{pluginName}] {csadevice} generated an exception: {exc}'])
'''
# Use multiprocessing Pool
with Pool(processes=3) as pool:
results = pool.map(get_omada_devices_details, [sadevice.split() for sadevice in sadevices])
mylog(OMDLOGLEVEL, [f'[{pluginName}] All API calls completed. Processing results...'])
# Process results
for thisswitch, switch_detail, switch_dump in results:
switch_details[thisswitch] = switch_detail
if switch_dump is not None:
switch_dumps[thisswitch] = switch_dump
mylog(OMDLOGLEVEL, [f'[{pluginName}] Finished collecting device details. Processing data...'])
# Now process the collected data
for sadevice in sadevices:
sadevice_data = sadevice.split()
thisswitch = sadevice_data[dMAC]
sadevices_macbyname[sadevice_data[4]] = thisswitch
if sadevice_data[dTYPE] == 'ap':
sadevice_type = 'AP'
#sadevice_details = callomada(['access-point', thisswitch])
sadevice_details = switch_details[thisswitch]
if sadevice_details == '':
sadevice_links = [thisswitch]
else:
sadevice_links = extract_mac_addresses(sadevice_details)
sadevices_linksbymac[thisswitch] = sadevice_links[1:]
mylog(OMDLOGLEVEL, [f'[{pluginName}]adding switch details: "{sadevice_details}"'])
mylog(OMDLOGLEVEL, [f'[{pluginName}]links are: "{sadevice_links}"'])
mylog(OMDLOGLEVEL, [f'[{pluginName}]linksbymac are: "{sadevices_linksbymac[thisswitch]}"'])
elif sadevice_data[dTYPE] == 'switch':
sadevice_type = 'Switch'
#sadevice_details=callomada(['switch', thisswitch])
sadevice_details = switch_details[thisswitch]
if sadevice_details == '':
sadevice_links = [thisswitch]
else:
sadevice_links=extract_mac_addresses(sadevice_details)
sadevices_linksbymac[thisswitch] = sadevice_links[1:]
# recovering the list of switches connected to sadevice switch and on which port...
#switchdump = callomada(['-t','myomada','switch','-d',thisswitch])
switchdump = switch_dumps[thisswitch]
port_byswitchmac_byclientmac[thisswitch] = {}
for link in sadevices_linksbymac[thisswitch]:
port_pattern = r"(?:{[^}]*\"port\"\: )([0-9]+)(?=[^}]*"+re.escape(link)+r")"
myport = re.findall(port_pattern, switchdump,re.DOTALL)
port_byswitchmac_byclientmac[thisswitch][link] = myport[0]
mylog(OMDLOGLEVEL, [f'[{pluginName}]links are: "{sadevice_links}"'])
mylog(OMDLOGLEVEL, [f'[{pluginName}]linksbymac are: "{sadevices_linksbymac[thisswitch]}"'])
mylog(OMDLOGLEVEL, [f'[{pluginName}]ports of each links are: "{port_byswitchmac_byclientmac[thisswitch]}"'])
mylog(OMDLOGLEVEL, [f'[{pluginName}]adding switch details: "{sadevice_details}"'])
else:
sadevice_type = 'null'
sadevice_details='null'
device_data_bymac[thisswitch] = [thisswitch, sadevice_data[dIP], sadevice_data[dNAME], 'null', 'null',sadevice_type]
device_data_mac_byip[sadevice_data[dIP]] = thisswitch
foo=[thisswitch, sadevice_data[1], sadevice_data[4], 'null', 'null']
mylog(OMDLOGLEVEL, [f'[{pluginName}]adding switch: "{foo}"'])
# sadevices_macbymac[thisswitch] = thisswitch
mylog(OMDLOGLEVEL, [f'[{pluginName}] switch_macbyname: "{sadevices_macbyname}"'])
mylog(OMDLOGLEVEL, [f'[{pluginName}] switches: "{device_data_bymac}"'])
# do some processing, call exteranl APIs, and return a device list
# ...
""" MAC = 0
IP = 1
NAME = 2
SWITCH_AP = 3
PORT_SSID = 4
TYPE = 5 """
# sample target output:
# 0 MAC, 1 IP, 2 Name, 3 MAC of switch/AP, 4 port/SSID, 5 TYPE
#17:27:10 [<unique_prefix>] token: "['9C-04-A0-82-67-45', '192.168.0.217', 'brick', 'ompapaoffice','froggies2', , 'Switch']"
odevices = omada_clients_output.splitlines()
mylog(OMDLOGLEVEL, [f'[{pluginName}] omada_clients_outputs rows: "{len(odevices)}"'])
for odevice in odevices:
odevice_data = odevice.split()
odevice_data_reordered = [ MAC, IP, NAME, SWITCH_AP, PORT_SSID, TYPE]
odevice_data_reordered[MAC]=odevice_data[cMAC]
odevice_data_reordered[IP]=odevice_data[cIP]
real_naxname = device_handler.getValueWithMac('dev_Name',ieee2ietf_mac_formater(odevice_data[cMAC]))
#
# if the name stored in Nax for a device is empty or the MAC addres or has some parenthhesis or is the same as in omada
# don't bother updating omada's name at all.
#
naxname = real_naxname
if real_naxname != None:
if '(' in real_naxname:
# removing parenthesis and domains from the name
naxname = real_naxname.split('(')[0]
if naxname != None and '.' in naxname:
naxname = naxname.split('.')[0]
if naxname in ( None, 'null', '' ):
naxname = odevice_data[cNAME] if odevice_data[cNAME] != '' else odevice_data[cMAC]
naxname = naxname.strip()
mylog('debug', [f'[{pluginName}] TEST name from MAC: {naxname}'])
if odevice_data[cNAME] in (odevice_data[cMAC], 'null', ''):
mylog('verbose', [f'[{pluginName}] updating omada server because odevice_data is: {odevice_data[cNAME]} and naxname is: "{naxname}"'])
callomada(['set-client-name', odevice_data[cMAC], naxname])
odevice_data_reordered[NAME] = naxname
else:
if omada_force_overwrite and naxname != odevice_data[cNAME] :
mylog('verbose', [f'[{pluginName}] updating omada server because odevice_data is: "{odevice_data[cNAME]} and naxname is: "{naxname}"'])
callomada(['set-client-name', odevice_data[cMAC], naxname])
odevice_data_reordered[NAME] = naxname
mightbeport = odevice_data[cPORT_SSID].lstrip('(')
mightbeport = mightbeport.rstrip(')')
if mightbeport.isdigit():
odevice_data_reordered[SWITCH_AP] = odevice_data[cSWITCH_AP]
odevice_data_reordered[PORT_SSID] = mightbeport
else:
odevice_data_reordered[SWITCH_AP] = mightbeport
odevice_data_reordered[PORT_SSID] = odevice_data[cSWITCH_AP]
# replacing the switch name with its MAC...
try:
mightbemac = sadevices_macbyname[odevice_data_reordered[SWITCH_AP]]
odevice_data_reordered[SWITCH_AP] = mightbemac
except KeyError:
mylog(OMDLOGLEVEL, [f'[{pluginName}] could not find the mac adddress for: "{odevice_data_reordered[SWITCH_AP]}"'])
# adding the type
odevice_data_reordered[TYPE] = 'null'
device_data_bymac[odevice_data_reordered[MAC]] = odevice_data_reordered
device_data_mac_byip[odevice_data_reordered[IP]] = odevice_data_reordered[MAC]
mylog(OMDLOGLEVEL, [f'[{pluginName}] tokens: "{odevice_data}"'])
mylog(OMDLOGLEVEL, [f'[{pluginName}] tokens_reordered: "{odevice_data_reordered}"'])
# populating the uplinks nodes of the omada switches and access points manually
# since OMADA SDN makes is unreliable if the gateway is not their own tplink hardware...
# step1 let's find the the default router
#
default_router_ip = find_default_gateway_ip()
default_router_mac = ietf2ieee_mac_formater(get_mac_from_IP(default_router_ip))
device_data_bymac[default_router_mac][TYPE] = 'Firewall'
# step2 let's find the first switch and set the default router parent to internet
first_switch=device_data_bymac[default_router_mac][SWITCH_AP]
device_data_bymac[default_router_mac][SWITCH_AP] = 'Internet'
# step3 let's set the switch connected to the default gateway uplink to the default gateway and hardcode port to 1 for now:
#device_data_bymac[first_switch][SWITCH_AP]=default_router_mac
#device_data_bymac[first_switch][SWITCH_AP][PORT_SSID] = '1'
# step4, let's go recursively through switches other links to mark update their uplinks
# and pray it ends one day...
#
add_uplink(default_router_mac,first_switch, device_data_bymac,sadevices_linksbymac,port_byswitchmac_byclientmac)
return device_data_bymac.values()
if __name__ == '__main__':
main()

47
front/plugins/omada_sdn_imp/testre.py
View File

@@ -136,7 +136,52 @@ print(d.values())
foo = 2
#while foo > 0:
# foo = 'toto'
print("foo is ",foo)
if foo in ( 'bar', '', 'null'):
print("foo is bar")
else:
print("foo is not bar")
foo='192-168-0-150.local'
bar = foo.split('.')[0]
print("bar=",bar,"-")
bar2 = 'toto'
print("bar2=",bar2,"-")
import concurrent.futures
import time
import random
def phello(arg):
print('running phell',arg)
time.sleep(random.uniform(0, 6))
return f"parallel hello : {arg}"
def testparalel():
arguments = ["Alice", "Bob", "Charlie", "David"]
results = {}
para = 10
# Using ThreadPoolExecutor for parallel execution
with concurrent.futures.ThreadPoolExecutor(max_workers=para) as executor:
# Submit tasks to the executor
future_to_arg = {executor.submit(phello, arg): arg for arg in arguments}
concurrent.futures.wait(future_to_arg)
# Retrieve results as they complete
for future in concurrent.futures.as_completed(future_to_arg):
arg = future_to_arg[future]
try:
result = future.result()
results[arg] = result
except Exception as exc:
print(f"{arg} generated an exception: {exc}")
# Print results
for arg, result in results.items():
print(f"{arg}: {result}")
testparalel()