Docs

Update DEVICE_MANAGEMENT.md

.github/ISSUE_TEMPLATE/i-have-an-issue.md

@@ -12,35 +12,31 @@ assignees: ''
 
 **Paste last few lines from `pialert.log`**
 
+> You can use `tail -20 /home/pi/pialert/front/log/pialert.log`
+
 ```
-
-paste here
-
+paste_here
 ```
 
 **Paste your `pialert.conf` (remove personal info)**
 
 ```
-
-paste here
-
+paste_here
 ```
 
 **Paste your `docker-compose.yml` and `.env` (remove personal info)**
 
 `docker-compose.yml` 
-```
-
-paste here
 
 ```
-
+paste_here
+```
 
 `.env` 
-```
-
-paste here
 
 ```
+paste_here
+```
+
 **Screenshots**
 If applicable, add screenshots to help explain your problem.

Dockerfile

@@ -7,7 +7,7 @@ ENV USER=pi USER_ID=1000 USER_GID=1000 TZ=Europe/London PORT=20211
 # Todo, do we still need all these packages? I can already see sudo which isn't needed
 
 RUN apt-get update \
-    && apt-get install --no-install-recommends tini ca-certificates curl libwww-perl arp-scan perl apt-utils cron sudo nginx-light php php-cgi php-fpm php-sqlite3 php-curl sqlite3 dnsutils net-tools python3 iproute2 nmap python3-pip zip -y \
+    && apt-get install --no-install-recommends tini snmp ca-certificates curl libwww-perl arp-scan perl apt-utils cron sudo nginx-light php php-cgi php-fpm php-sqlite3 php-curl sqlite3 dnsutils net-tools python3 iproute2 nmap python3-pip zip -y \
     && pip3 install requests paho-mqtt scapy cron-converter pytz json2table dhcp-leases pyunifi \
     && update-alternatives --install /usr/bin/python python /usr/bin/python3 10 \
     && apt-get clean autoclean \

README.md

@@ -54,10 +54,10 @@ The system continuously scans the network for, **New devices**, **New connection
   - 🌟(Experimental) [Plugin system](https://github.com/jokob-sk/Pi.Alert/tree/main/front/plugins)
     - Create custom plugins with automatically generated settings and UI. 
     - Monitor anything for changes
-    - Check the instructions carefully if you are up for a challenge! Current plugins include:
-      - Detecting Rogue DHCP servers
+    - Check the [instructions](https://github.com/jokob-sk/Pi.Alert/tree/main/front/plugins) carefully if you are up for a challenge! Current plugins include:
+      - Detecting Rogue DHCP servers via NMAP
       - Monitoring HTTP status changes of domains/URLs 
-      - Import devices from DHCP.leases files or a UniFi controller
+      - Import devices from DHCP.leases files, a UniFi controller, or an SNMP enabled router
 
   | ![Screen 1][screen1] | ![Screen 2][screen2] | ![Screen 5][screen5]  |
   |----------------------|----------------------| ----------------------| 
@@ -70,9 +70,10 @@ The system continuously scans the network for, **New devices**, **New connection
   - [WatchYourLAN](https://github.com/aceberg/WatchYourLAN) - Lightweight network IP scanner with web GUI (Open source)
   - [Fing](https://www.fing.com/) - Network scanner app for your Internet security (Commercial, Phone App, Proprietary hardware)
 
-### Old docs
+### 📚 Documentation
 
-Device Management [instructions](docs/DEVICE_MANAGEMENT.md) | Old Versions [History](docs/VERSIONS_HISTORY.md)
+- Initial Docker Setup: [Docker instructions](https://github.com/jokob-sk/Pi.Alert/blob/main/dockerfiles/README.md)
+- App Usage and Configuration: [All Documentation](https://github.com/jokob-sk/Pi.Alert/blob/main/docs/README.md)
 
 ### License
   GPL 3.0 | [Read more here](LICENSE.txt) | Source of the [animated GIF (Loading Animation)](https://commons.wikimedia.org/wiki/File:Loading_Animation.gif) | Source of the [selfhosted Fonts](https://github.com/adobe-fonts/source-sans)

back/pialert.py

@@ -41,6 +41,7 @@ from cron_converter import Cron
 from pytz import timezone
 from json2table import convert
 import hashlib
+import multiprocessing
 
 #===============================================================================
 # SQL queries
@@ -124,12 +125,13 @@ def mylog(requestedDebugLevel, n):
 #-------------------------------------------------------------------------------
 def file_print (*args):
 
-    result = ''
-    
-    file = open(logPath + "/pialert.log", "a")    
+    result = ''    
+       
     for arg in args:                
         result += str(arg)
     print(result)
+
+    file = open(logPath + "/pialert.log", "a") 
     file.write(result + '\n')
     file.close()
 
@@ -294,7 +296,7 @@ def commitDB ():
 def ccd(key, default, config, name, inputtype, options, group, events=[], desc = "", regex = ""):
     result = default
 
-    # use existing value if already supplied, otehrwise default value is used
+    # use existing value if already supplied, otherwise default value is used
     if key in config:
         result =  config[key]
 
@@ -655,7 +657,31 @@ def main ():
             if last_network_scan + datetime.timedelta(minutes=SCAN_CYCLE_MINUTES) < time_started:
                 last_network_scan = time_started
                 cycle = 1 # network scan
-                scan_network() 
+                # scan_network() 
+
+                #  DEBUG start ++++++++++++++++++++++++++++++++++++++++++++++++++++++
+                # Start scan_network as a process                
+
+                p = multiprocessing.Process(target=scan_network)
+                p.start()
+
+                # Wait for 3600 seconds (max 1h) or until process finishes
+                p.join(3600)
+
+                # If thread is still active
+                if p.is_alive():
+                    print("DEBUG scan_network running too long - let\'s kill it")
+                    mylog('info', ['    DEBUG scan_network running too long - let\'s kill it'])
+
+                    # Terminate - may not work if process is stuck for good
+                    p.terminate()
+                    # OR Kill - will work for sure, no chance for process to finish nicely however
+                    # p.kill()
+
+                    p.join()
+
+                #  DEBUG end ++++++++++++++++++++++++++++++++++++++++++++++++++++++
+                
             
             # Reporting   
             if cycle in check_report:
@@ -951,7 +977,10 @@ def update_devices_MAC_vendors (pArg = ''):
 
     # All devices loop
     mylog('verbose', ['    Searching devices vendor'])    
-    for device in sql.execute ("SELECT * FROM Devices") :
+    for device in sql.execute ("""SELECT * FROM Devices
+                                  WHERE dev_Vendor = '(unknown)' 
+                                     OR dev_Vendor =''
+                                     OR dev_Vendor IS NULL""") :
         # Search vendor in HW Vendors DB
         vendor = query_MAC_vendor (device['dev_MAC'])
         if vendor == -1 :
@@ -1081,9 +1110,9 @@ def scan_network ():
     update_devices_names()
 
     # Void false connection - disconnections
-    mylog('verbose', ['    Voiding false (ghost) disconnections'])
+    mylog('verbose', ['    Voiding false (ghost) disconnections'])    
     void_ghost_disconnections ()
-  
+
     # Pair session events (Connection / Disconnection)
     mylog('verbose', ['    Pairing session events (connection / disconnection) '])
     pair_sessions_events()  
@@ -3299,7 +3328,8 @@ def update_api(isNotification = False, updateOnlyDataSources = []):
         write_file(folder + 'notification_json_final.json'  , json.dumps(json_final))  
 
     # Save plugins
-    write_file(folder + 'plugins.json'  , json.dumps({"data" : plugins}))  
+    if ENABLE_PLUGINS: 
+        write_file(folder + 'plugins.json'  , json.dumps({"data" : plugins}))  
 
     #  prepare database tables we want to expose 
     dataSourcesSQLs = [
@@ -3824,7 +3854,7 @@ def execute_plugin(plugin):
     else:     
         set_RUN_TIMEOUT = set["value"] 
 
-    mylog('debug', ['     [Plugins] Timeout: ', set_RUN_TIMEOUT])    
+    mylog('debug', ['     [Plugins] Timeout: ', set_RUN_TIMEOUT])     
 
     #  Prepare custom params
     params = []
@@ -3851,18 +3881,18 @@ def execute_plugin(plugin):
                 params.append( [param["name"], resolved] )
     
 
-    # ------- prepare params --------
-    # prepare command from plugin settings, custom parameters  
-    command = resolve_wildcards(set_CMD, params).split()
-
     # build SQL query parameters to insert into the DB
     sqlParams = []
 
     # python-script 
     if plugin['data_source'] == 'python-script':
+        # ------- prepare params --------
+        # prepare command from plugin settings, custom parameters  
+        command = resolve_wildcards_arr(set_CMD.split(), params)        
 
         # Execute command
         mylog('verbose', ['     [Plugins] Executing: ', set_CMD])
+        mylog('debug',   ['     [Plugins] Resolved : ', command])        
 
         try:
             # try runnning a subprocess with a forced timeout in case the subprocess hangs
@@ -3909,7 +3939,7 @@ def execute_plugin(plugin):
         for row in arr:
             # There has to be always 9 columns
             if len(row) == 9 and (row[0] in ['','null']) == False :
-                sqlParams.append((plugin["unique_prefix"], row[0], row[1], 'null', row[2], row[3], row[4], row[5], row[6], 0, row[7], 'null', row[8]))
+                sqlParams.append((plugin["unique_prefix"], row[0], handle_empty(row[1]), 'null', row[2], row[3], row[4], handle_empty(row[5]), handle_empty(row[6]), 0, row[7], 'null', row[8]))
             else:
                 mylog('none', ['        [Plugins]: Skipped invalid sql result'])
 
@@ -3934,6 +3964,14 @@ def execute_plugin(plugin):
         update_api(False, ["plugins_events","plugins_objects"])
 
 
+#-------------------------------------------------------------------------------
+# Handle empty value 
+def handle_empty(value):    
+    if value == '' or value is None:
+        value = 'null'
+
+    return value
+
 #-------------------------------------------------------------------------------
 # Check if watched values changed for the given plugin
 def process_plugin_events(plugin):    
@@ -4161,19 +4199,23 @@ def combine_plugin_objects(old, new):
 
 #-------------------------------------------------------------------------------
 # Replace {wildcars} with parameters
-def resolve_wildcards(command, params):
+def resolve_wildcards_arr(commandArr, params):
 
-    mylog('debug', ['        [Plugins]: Pre-Resolved CMD: ', command])
-    
+    mylog('debug', ['        [Plugins]: Pre-Resolved CMD: '] + commandArr)   
 
     for param in params:
-        mylog('debug', ['        [Plugins]: key     : {', param[0], '}'])
-        mylog('debug', ['        [Plugins]: resolved: ', param[1]])
-        command = command.replace('{' + param[0] + '}', param[1])
+        # mylog('debug', ['        [Plugins]: key     : {', param[0], '}'])
+        # mylog('debug', ['        [Plugins]: resolved: ', param[1]])
 
-    mylog('debug', ['        [Plugins]: Resolved     CMD: ', command])
+        i = 0
+        
+        for comPart in commandArr:
 
-    return command
+            commandArr[i] = comPart.replace('{' + param[0] + '}', param[1]).replace('{s-quote}',"'")
+
+            i += 1
+
+    return commandArr
 
 #-------------------------------------------------------------------------------
 # Flattens a setting to make it passable to a script
@@ -4253,13 +4295,16 @@ def flatten_array(arr):
     
     tmp = ''
 
+    mylog('debug', arr)
+
     for arrayItem in arr:        
         # only one column flattening is supported
         if isinstance(arrayItem, list):            
             arrayItem = str(arrayItem[0])
 
         tmp += arrayItem + ','
-        tmp = tmp.replace("'","").replace(' ','') # No single quotes or empty spaces allowed
+        # tmp = tmp.replace("'","").replace(' ','') # No single quotes or empty spaces allowed
+        tmp = tmp.replace("'","") # No single quotes allowed
 
     return tmp[:-1] # Remove last comma ','
 

dockerfiles/README.md

@@ -53,28 +53,39 @@ docker run -d --rm --network=host \
 
 ### Config (`pialert.conf`)
 
-- The preferred wy is to manage the configuration via Settings 
-- YOu can modify [pialert.conf](https://github.com/jokob-sk/Pi.Alert/tree/main/config) directly if needed
-- ❗ To use the arp-scan method, you need to set the `SCAN_SUBNETS` variable. ()
+- If unavailable, the app generates a default `pialert.conf` and `pialert.db` file on the first run.
+- The preferred way is to manage the configuration via the Settings section in the UI.
+- You can modify [pialert.conf](https://github.com/jokob-sk/Pi.Alert/tree/main/config) directly, if needed.
+
+#### Important settings
+
+These are the most important settings to get at least some output in your Devices screen. Usually, only one approach is used, but you should be able to combine these approaches.
+
+##### For arp-scan: ENABLE_ARPSCAN, SCAN_SUBNETS
+
+- ❗ To use the arp-scan method, you need to set the `SCAN_SUBNETS` variable.
    * The adapter will probably be `eth0` or `eth1`. (Run `iwconfig` to find your interface name(s)) 
    * Specify the network filter (which **significantly** speeds up the scan process). For example, the filter `192.168.1.0/24` covers IP ranges 192.168.1.0 to 192.168.1.255.
    * Examples for one and two subnets  (❗ Note the `['...', '...']` format):
      * One subnet: `SCAN_SUBNETS    = ['192.168.1.0/24 --interface=eth0']`
-     * Two subnets:  `SCAN_SUBNETS    = ['192.168.1.0/24 --interface=eth0', '192.168.1.0/24 --interface=eth1']` 
-   * More documentation on how to [setup vlans](https://github.com/jokob-sk/Pi.Alert/blob/main/docs/SUBNETS.md) 
+     * Two subnets:  `SCAN_SUBNETS    = ['192.168.1.0/24 --interface=eth0', '192.168.1.0/24 --interface=eth1 -vlan=107']` 
+   * More documentation on how to e.g. [setup vlans & limitations](https://github.com/jokob-sk/Pi.Alert/blob/main/docs/SUBNETS.md) 
 
+##### For pihole: PIHOLE_ACTIVE, DHCP_ACTIVE
 
-### 🛑 **Common issues** 
+* `PIHOLE_ACTIVE`: You need to map `:/etc/pihole/pihole-FTL.db in the docker-compose.yml` file if you enable this setting.
+* `DHCP_ACTIVE` : You need to map `:/etc/pihole/dhcp.leases in the docker-compose.yml` file if you enable this setting.
+
+### **Common issues** 
 
 💡 Before creating a new issue, please check if a similar issue was [already resolved](https://github.com/jokob-sk/Pi.Alert/issues?q=is%3Aissue+is%3Aclosed). 
 
 **Permissions**
 
 * If facing issues (AJAX errors, can't write to DB, empty screen, etc,) make sure permissions are set correctly, and check the logs under `/home/pi/pialert/front/log`. 
-* To solve permission issues you can also try to create a DB backup and then run a DB Restore via the **Maintenance > Backup/Restore** section.
-* You can try also setting the owner and group of the `pialert.db` by executing the following on the host system: `docker exec pialert chown -R www-data:www-data /home/pi/pialert/db/pialert.db`. 
+* To solve permission issues you can try setting the owner and group of the `pialert.db` by executing the following on the host system: `docker exec pialert chown -R www-data:www-data /home/pi/pialert/db/pialert.db`. 
 * Map to local User and Group IDs. Specify the enviroment variables `HOST_USER_ID` and `HOST_USER_GID` if needed.
-* Map the pialert.db file (⚠ not folder) to `:/home/pi/pialert/db/pialert.db` (see Examples below for details)
+* If still facing issues, try to map the pialert.db file (⚠ not folder) to `:/home/pi/pialert/db/pialert.db` (see Examples below for details)
 
 **Container restarts / crashes**
 

docs/API.md

@@ -5,12 +5,7 @@ PiAlert comes with a simple API. These API endpoints are static files, that are
 
 ### When are the endpoints updated
 
-Once you enable the API (`ENABLE_API` setting), the endpoints are updated during these events:
-
-1) Always during a notification event.
-2) (optional) If `API_RUN` is set to `schedule` on a specified cron-like schedule specified by the `API_RUN_SCHD` setting.
-3) (optional) If `API_RUN` is set to `interval` every N seconds specified by the `API_RUN_INTERVAL` setting (minimum 5).
-
+Once you enable the API (`ENABLE_API` setting), the endpoints are updated when objects in the API endpoints are changed:
 
 ### Location of the endpoints
 

docs/DEVICE_MANAGEMENT.md

@@ -18,7 +18,7 @@ To edit device information:
   - **Owner**: Device owner (The list is self-populated with existing owners)
   - **Type**: Select a device type from the dropdown list (Smartphone, Table,
       Laptop, TV, router, ....) or type a new device type
-  - **Vendor**: Automatically updated by Pi.Alert
+  - **Vendor**: Automatically updated by Pi.Alert when empty or unknown
   - **Favorite**: Mark the device as favorite and then it will appears at the
       begining of the device list
   - **Group**: Select a grouper ('Always on', 'Personal', Friends') or type

docs/ICONS.md

@@ -0,0 +1,20 @@
+## Icons overview
+
+Icons are used to visually distinguish devices in the app in most of the device listing tables and the [network tree](/docs/NETWORK_TREE.md). Currently only free [Font Awesome](https://fontawesome.com/search?o=r&m=free) icons (up-to v 6.4.0) are supported (I have an unblockable [sponsorship goal](https://github.com/sponsors/jokob-sk) to add the material design icon pack).  
+
+![Raspberry Pi with a brand icon](/docs/img/ICONS/devices-icons.png)
+
+## ⚙ How to use custom device Icons
+
+You can assign icons individually on each device in the Details tab.
+
+![Raspberry Pi device details](/docs/img/ICONS/device-icon.png)
+
+- You can click into the `Icon` field or click the Pencil (2) icon in the above screenshot to enter any text. Only [free Font Awesome](https://fontawesome.com/search?o=r&m=free) icons in the following format will work:
+
+  1. For any value that is only prefixed with `fa-`, you can enter the value directly, such as `server`, `tv`, `ethernet`. 
+  2. If you want to add another classname, e.g. `fa-brands`, you can enter `brands fa-[fontawesome-icon-name]`, so for `apple` that is using the syntax`fa-brands fa-apple`, you would enter `brands fa-apple`.
+
+- If you want to mass-apply an icon to all devices of the same device type (Field marked (4) in the above screenshot), you can click the copy button (Marked (1) in the above screenshot). A confirmation prompt is displayed. If you proceed, icons of all devices set to the same device type as the current device, will be overwritten with the current device's icon.
+
+- The dropdown (3) contains all icons already used in the app for device icons. You need to navigate away or refresh the page once you add a new icon. 

docs/NETWORK_TREE.md

@@ -1,8 +1,19 @@
 ## How to setup your Network page
 
-Make sure you have a root device with the MAC `Internet` (No other MAC addresses are currently support as root)
+Make sure you have a root device with the MAC `Internet` (No other MAC addresses are currently supported as the root node).
 
-To setup a device named `rapberrypi` as a `Switch` in our network. 
+## ⚡Quick setup:
+
+* Go to Devices > Device Details. 
+* Find the device(s) you want to use as network devices (network nodes). 
+* Set the Type of such a device to one of the following: AP, Firewall, Gateway, PLC, Powerline, Router, Switch, USB LAN Adapter, USB WIFI Adapter and WLAN.
+* Save and go to Network where the devices you've marked as network devices (by selecting the Type as mentioned above) will show up as tabs.
+* You can now assign the Unassigend devices to the correct network node.
+
+
+## 🔍Detailed example:
+
+In this example you will setup a device named `rapberrypi` as a `Switch` in our network. 
 
 ### 1) Device details page
 

docs/README.md

@@ -0,0 +1,91 @@
+## Documentation overview
+
+In the app hover-over settings or fields/labels or click blue in-app ❔ (question-mark) icons to get to relevant documentation pages.
+
+![In-app help](/docs/img/GENERAL/in-app-help.png)
+
+There is also an in-app Help / FAQ section that should be answering frequently asked questions.
+
+### 📥 Installation
+
+ ⚠ Only tested as a [docker container - follow these instructions here](https://github.com/jokob-sk/Pi.Alert/blob/main/dockerfiles/README.md). 
+ > Check out [leiweibau's fork](https://github.com/leiweibau/Pi.Alert/) if you want to install Pi.Alert on the server directly or original instructions for [pucherot's original code](https://github.com/pucherot/Pi.Alert/)
+
+
+### 📚 Table of contents
+
+#### Popular/Suggested
+
+- [API endpoints details](/docs/API.md)
+- [Plugin system details and how to develop your own](/front/plugins/README.md)
+- [Network tree map configuration](/docs/NETWORK_TREE.md)
+- [Gmail as SMTP server for sending emails](/docs/SMTP_GMAIL.md)
+- [Subnets and vlans configuration for arp-scan](/docs/SUBNETS.md)
+
+#### System Management
+
+- [Manage devices (legacy docs)](/docs/DEVICE_MANAGEMENT.md)
+- [Random MAC/MAC icon meaning (legacy docs)](/docs/RANDOM_MAC.md)
+- [Custom Icons configuration and support](/docs/ICONS.md)
+
+#### Examples
+
+- [N8N webhook example](/docs/WEBHOOK_N8N.md)
+
+#### Misc
+
+- [New Version notifications](/docs/VERSIONS.md)
+- [Version history (legacy)](/docs/VERSIONS_HISTORY.md)
+- [Invalid JSON errors debug help](/docs/DEBUG_INVALID_JSON.md)
+
+Feel free to suggest or submit new docs via a PR. 
+
+## 👨‍💻 Development priorities
+
+Highest to lowest:
+
+* Fixing core functionality bugs not solvable with workarounds
+* New core functionality unlocking other opportunities (e.g.: plugins) 
+* Refactoring enabling faster implementation of future functionality 
+* UI improvements
+
+Design philosophy: Focus on core functionality and leverage existing apps and tools to make PiAlert integratable into other workflows. 
+
+Examples: 
+
+    1. Supporting apprise makes more sense than implementing multiple individual notification gateways
+    2. Implementing regular expressions support across settings for validation makes more sense than validating one setting with a specific expression. 
+
+UI specific requests are low priority as the framework picked by the original developer is not very extensible (and afaik doesn't support components) and has limited mobile support. Also I argue the value proposition is smaller than working on something else.
+
+Feel free to submit PRs if interested. try to **keep the PRs small/on topic** so they are easier to review and approve. 
+
+That being said, I'd reconsider if more people and or recurring sponsors file a request 😉.
+
+## 🙏 Feature requests
+
+Please be as detailed as possible with **workarounds** you considered and why a native feature is the better way. This gives me better context and will make it more likely to be implemented. Ideally a feature request should be in the format "I want to be able to do XYZ so that ZYX. I considered these approaches XYZ".
+
+## ➕ Pull-requests (PRs)
+
+If you submit a PR please:
+
+1. Check that your changes are backward compatible with existing installations and with a blank setup. 
+2. Existing features should always be preserved. 
+3. Keep the PR small, on-topic and don't change code that is not necessary for the PR to work
+4. New features code should ideally be re-usable for different purposes, not be for a very narrow use-case.
+5. New functionality should ideally be implemented via the Plugins system, if possible.
+
+Soem additional context:
+
+* Permanent settings/config is stored in the `pialert.conf` file
+* Currently temporary (session?) settings are stored in the `Parameters` DB table as key - value pairs. This table is wiped during a container rebuild/restart and it's values re-initialized from cookies / session data from the browser. 
+
+## 🐛 Submitting an issue or bug
+
+Before submitting a new issue please spend a couple of minutes on research:
+
+* Check [🛑 Common issues](https://github.com/jokob-sk/Pi.Alert/tree/main/dockerfiles#-common-issues) 
+* Check [💡 Closed issues](https://github.com/jokob-sk/Pi.Alert/issues?q=is%3Aissue+is%3Aclosed) if a similar issue was solved in the past.
+
+⚠ Please follow the pre-defined issue template to resolve your issue faster.

docs/SUBNETS.md

@@ -9,7 +9,7 @@ For example, a `/24` mask results in 256 IPs to check, where as a `/16` mask che
 - Run `iwconfig` in your container to find your interface name(s) (e.g.: `eth0`, `eth1`). 
 - Append e.g.: ` -vlan=107` to the interface field (e.g.: `eth0 -vlan=107`) for multiple vlans. More details in this [comment in this issue](https://github.com/jokob-sk/Pi.Alert/issues/170#issuecomment-1419902988)
 
-### Example:
+### 🔍Example:
 
 ![Vlan configuration example](/docs/img/SUBNETS/subnets_vlan.png)
 

docs/VERSIONS.md

@@ -0,0 +1,25 @@
+## Am I running the latest released version?
+
+Since version 23.01.14 PiAlert uses a simple timestamp-based version check to verify if a new version is available. You can check the [current and past releases here](https://github.com/jokob-sk/Pi.Alert/releases), or have a look at what I'm [currently working on](https://github.com/jokob-sk/Pi.Alert/issues/138). 
+
+If you are not on the latest version, the app will notify you, that a new released version is avialable the following way:
+
+### 📧 Via email on a notification event
+
+If any notification occurs and an email is sent, the email will contain a note that a new version is available. See the sample email below:
+
+![Sample email if a new version is available](/docs/img/VERSIONS/new-version-available-email.png)
+
+### 🆕 In the UI
+
+In the UI via a notification Icon and via a custom message in the Maintenance section.
+
+![UI screenshot if a new version is available](/docs/img/VERSIONS/new-version-available-maintenance.png)
+
+For a comparison, this is how the UI looks like if you are on the latest stable image:
+
+![UI screenshot if on latest version](/docs/img/VERSIONS/latest-version-maintenance.png)
+
+## Implementation details
+
+During build a [/home/pi/pialert/front/buildtimestamp.txt](https://github.com/jokob-sk/Pi.Alert/blob/092797e75ccfa8359444ad149e727358ac4da05f/Dockerfile#L44) file is created. The app then periodically checks if a new release is available with a newer timestamp in GitHub's rest-based JSON endpoint (check the `def isNewVersion():` method in `pialert.py` for details).   

front/deviceDetails.php

@@ -194,7 +194,7 @@
                       <div class="form-group">
                         <label class="col-sm-3 control-label">
                           <?= lang('DevDetail_Icon');?> 
-                          <a href="https://fontawesome.com/search?q=laptop&o=r&m=free" target="_blank"> <span><i class="fa fa-fw fa-arrow-up-right-from-square"></i></a><span>
+                          <a href="https://github.com/jokob-sk/Pi.Alert/blob/main/docs/ICONS.md" target="_blank"> <span><i class="fa fa-circle-question"></i></a><span>
                         </label>
                         <div class="col-sm-9">
                           <div class="input-group">

front/devices.php

@@ -225,6 +225,8 @@ function main () {
       // save the columns order in the Devices page 
       tableColumnOrder = numberArrayFromString(data);
 
+
+
       //initialize the table headers in the correct order
       var headersDefaultOrder = [ '<?= lang('Device_TableHead_Name');?>',
                                   '<?= lang('Device_TableHead_Owner');?>',
@@ -308,7 +310,7 @@ function mapIndx(oldIndex)
 function initializeDatatable () {
   for(i = 0; i < tableColumnOrder.length; i++)
   {    
-    // hide this column if not in the tableColumnVisible variable
+    // hide this column if not in the tableColumnVisible variable (we need to keep the MAC address (index 11) for functionality reasons)    
     if(tableColumnVisible.includes(tableColumnOrder[i]) == false)
     {
       tableColumnHide.push(mapIndx(tableColumnOrder[i]));
@@ -348,7 +350,7 @@ function initializeDatatable () {
 
       // Device Name
       {targets: [mapIndx(0)],
-        'createdCell': function (td, cellData, rowData, row, col) {
+        'createdCell': function (td, cellData, rowData, row, col) {            
             $(td).html ('<b class="anonymizeDev"><a href="deviceDetails.php?mac='+ rowData[mapIndx(11)] +'" class="">'+ cellData +'</a></b>');
       } },
 

front/maintenance.php

@@ -131,8 +131,11 @@ if (isset($_POST['submit']) && submit && isset($_POST['skinselector_set'])) {
               </div>
               <div class="box-body" style="padding-bottom: 5px;">
                 <div class="db_info_table">
-                    <div class="db_info_table_row">
-                        <div class="db_info_table_cell" style="min-width: 140px"><?= lang('Maintenance_version');?></div>
+                    <div class="db_info_table_row">                      
+                        <div class="db_info_table_cell" style="min-width: 140px"><?= lang('Maintenance_version');?>
+                          <a href="https://github.com/jokob-sk/Pi.Alert/blob/main/docs/VERSIONS.md" target="_blank"> <span><i class="fa fa-circle-question"></i></a><span>
+
+                        </div>
                         <div class="db_info_table_cell">
                         <div class="version" id="version" data-build-time="<?php echo file_get_contents( "buildtimestamp.txt");?>"><?php echo '<span id="new-version-text" class="myhidden">' .lang('Maintenance_new_version').'</span>'.'<span id="current-version-text" class="myhidden">' .lang('Maintenance_current_version').'</span>';?></div>
                         </div>
@@ -707,7 +710,11 @@ function scrollDown()
   for (let i = 0; i < areaIDs.length; i++) {
 
     var tempArea = $('#' + areaIDs[i]);
-    $(tempArea[0]).scrollTop(tempArea[0].scrollHeight);
+    
+    if (tempArea.length > 0)
+    {
+      $(tempArea[0]).scrollTop(tempArea[0].scrollHeight);
+    }
 
   }
 }
@@ -715,8 +722,8 @@ function scrollDown()
 // --------------------------------------------------------
 // Manage displayed columns
 // --------------------------------------------------------
-colDefaultOrder = ['0','1','2','3','4','5','6','7','8','9','10','12','13','14','15','16','17'];
-colDefaultOrderTxt = '[0,1,2,3,4,5,6,7,8,9,10,12,13,14,15,16,17]';
+colDefaultOrder = ['0','1','2','3','4','5','6','7','8','9','10','11','12','13','14','15','16','17'];
+colDefaultOrderTxt = '[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17]';
 
 
 function saveSelectedColumns () { 
@@ -724,9 +731,7 @@ function saveSelectedColumns () {
     // save full order of all columns to simplify mapping later on
     
     colDisplayed = $('#columnsSelect').val();
-
     
-
     colNewOrder = colDisplayed;
 
     // append the remaining columns in the previous order

front/php/server/devices.php

@@ -570,72 +570,6 @@ function getDevicesTotals() {
 }
 
 
-//------------------------------------------------------------------------------
-//  Query the List of devices in a determined Status
-//------------------------------------------------------------------------------
-// function getDevicesListForNetworkTree() {
-//   global $db;
-
-//   $sql = 'SELECT *, CASE
-//               WHEN t1.dev_AlertDeviceDown=1 AND t1.dev_PresentLastScan=0 THEN "Down"
-//               WHEN t1.dev_NewDevice=1 THEN "New"
-//               WHEN t1.dev_PresentLastScan=1 THEN "On-line"
-//               ELSE "Off-line"  END AS dev_Status
-//               FROM (Devices ) t1
-//           LEFT JOIN
-//                       (
-//                             SELECT *,
-//                                   count() as connected_devices 
-//                             FROM Devices b 
-//                             WHERE b.dev_Network_Node_MAC_ADDR NOT NULL group by b.dev_Network_Node_MAC_ADDR
-//                       ) t2
-//                       ON (t1.dev_MAC = t2.dev_MAC); ';
-
-//   $result = $db->query($sql);
-
-//   // arrays of rows
-//   $tableData = array();
-//   while ($row = $result -> fetchArray (SQLITE3_ASSOC)) {
-
-//   $defaultOrder = array ($row['dev_Name'],
-//                     $row['dev_Owner'],
-//                     handleNull($row['dev_DeviceType']),
-//                     handleNull($row['dev_Icon'], "laptop"),
-//                     $row['dev_Favorite'],
-//                     $row['dev_Group'],
-//                     formatDate ($row['dev_FirstConnection']),
-//                     formatDate ($row['dev_LastConnection']),
-//                     $row['dev_LastIP'],
-//                     ( in_array($row['dev_MAC'][1], array("2","6","A","E","a","e")) ? 1 : 0),
-//                     $row['dev_Status'],
-//                     $row['dev_MAC'], // MAC (hidden)
-//                     formatIPlong ($row['dev_LastIP']), // IP orderable
-//                     $row['rowid'], // Rowid (hidden)      
-//                     handleNull($row['dev_Network_Node_MAC_ADDR']), // 
-//                     handleNull($row['connected_devices']) // 
-//                     );
-
-//   $newOrder = array();
-
-//   // reorder columns based on user settings
-//   for($index = 0; $index  < count($columnOrderMapping); $index++)
-//   {
-//   array_push($newOrder, $defaultOrder[$columnOrderMapping[$index][2]]);      
-//   }
-
-//   $tableData['data'][] = $newOrder;
-//   }
-
-//   // Control no rows
-//   if (empty($tableData['data'])) {
-//   $tableData['data'] = '';
-//   }
-
-//   // Return json
-//   echo (json_encode ($tableData));
-
-// }
-
 //------------------------------------------------------------------------------
 //  Query the List of devices in a determined Status
 //------------------------------------------------------------------------------
@@ -649,7 +583,7 @@ function getDevicesList() {
     $forceDefaultOrder = TRUE;
   }
 
-  // This object is used to map from the old order ( second parameter, first number) to the 3rd parameter (Second number (here initialized to -1))
+  // This object is used to map from the old order ( second parameter, first number) to the new mapping, that is represented by the 3rd parameter (Second number)
   $columnOrderMapping = array(
     array("dev_Name", 0, 0),               
     array("dev_Owner", 1, 1),              
@@ -684,9 +618,11 @@ function getDevicesList() {
       $orderedColumns = createArray($row[0]);
 
       // init ordered columns    
-      for($i = 0; $i < count($orderedColumns); $i++) {           
-        $columnOrderMapping[$i][2] = $orderedColumns[$i];      
+      for($i = 0; $i < count($orderedColumns); $i++) { 
+
+        $columnOrderMapping[$i][2] = $orderedColumns[$i];        
       }
+
     }
   }
 

front/php/templates/footer.php

@@ -12,9 +12,14 @@
     <!-- Default to the left -->
 
     <!-- &copy; 2020 Puche -->
-    <?php
-      echo '<span style="display:inline-block; transform: rotate(180deg)">&copy;</span> 2020 Puche (2022+ <a href="mailto:jokob@duck.com?subject=PiAlert">jokob-sk</a>)';
-    ?>
+    <span style="display:inline-block; transform: rotate(180deg)">&copy;</span>
+    
+       2020 Puche (2022+ <a href="mailto:jokob@duck.com?subject=PiAlert">jokob-sk</a>)  
+
+      <a href="https://github.com/jokob-sk/Pi.Alert/tree/main/docs" target="_blank"> 
+        <span>Docs <i class="fa fa-circle-question"></i>
+      </a><span>
+    
     <!-- To the right -->
     <div class="pull-right no-hidden-xs">
 

front/php/templates/language/en_us.php

@@ -214,7 +214,7 @@ $lang['en_us'] = array(
 'DevDetail_button_Reset' => 'Reset Changes',
 'DevDetail_button_Save' => 'Save',
 'DevDetail_button_OverwriteIcons' => 'Overwrite Icons',
-'DevDetail_button_OverwriteIcons_Tooltip' => 'Overwrite icons of all devices with the same type',
+'DevDetail_button_OverwriteIcons_Tooltip' => 'Overwrite icons of all devices with the same device type',
 'DevDetail_button_OverwriteIcons_Warning' => 'Are you sure you want to overwrite all icons of all devices with the same device type as the current device type?',
 'DevDetail_SessionTable_Order' => 'Order',
 'DevDetail_SessionTable_Connection' => 'Connection',
@@ -515,7 +515,7 @@ $lang['en_us'] = array(
 'ENABLE_ARPSCAN_description' => 'Arp-scan is a command-line tool that uses the ARP protocol to discover and fingerprint IP hosts on the local network. An alternative to ARP scan is to enable the <a onclick="toggleAllSettings()" href="#PIHOLE_ACTIVE"><code>PIHOLE_ACTIVE</code>PiHole integration settings</a>.',
 'SCAN_SUBNETS_name' => 'Subnets to scan',
 'SCAN_SUBNETS_description' => '
-The arp-scan time itself depends on the number of IP addresses to check so set this up carefully with the appropriate network mask and interface. Check the <a href="https://github.com/jokob-sk/Pi.Alert/blob/main/docs/SUBNETS.md" target="_blank">subnets documentation</a> for details.
+The arp-scan time itself depends on the number of IP addresses to check so set this up carefully with the appropriate network mask and interface. Check the <a href="https://github.com/jokob-sk/Pi.Alert/blob/main/docs/SUBNETS.md" target="_blank">subnets documentation</a> for help on setting up VLANs, what VLANs are supported, or how to figure out the network mask and your interface.
 ',
 'LOG_LEVEL_name' => 'Print additional logging',
 'LOG_LEVEL_description' => 'This setting will enable more verbose logging. Useful for debugging events writing into the database.',

front/plugins.php

@@ -129,9 +129,7 @@ function localize (obj, key) {
     {
         for(i=0;i<obj[key].length;i++)
         {
-            code = obj[key][i]["language_code"]
-
-            // console.log(code)
+            code = obj[key][i]["language_code"]            
 
             if( code == 'en_us')
             {
@@ -146,7 +144,7 @@ function localize (obj, key) {
         }
     }
 
-    result == "" ? en_us : result ;
+    result == "" ? result = en_us : result ;
 
     return result;
 }
@@ -190,6 +188,9 @@ function generateTabs()
     activetab = 'active'
 
     $.each(pluginDefinitions, function(index, obj) {
+
+        // console.log(obj)
+
         $('#tabs-location').append(
             `<li class=" ${activetab}">
                 <a href="#${obj.unique_prefix}" data-plugin-prefix="${obj.unique_prefix}" id="${obj.unique_prefix}_id" data-toggle="tab" >

front/plugins/README.md

@@ -1,17 +1,9 @@
-# ⚠ Disclaimer
-
-Highly experimental feature. Follow the below very carefully and check example plugin(s). Plugin UI is not my priority right now, happy to approve PRs if you are interested in extending/improvintg the UI experience (e.g. making the tables sortable/filterable). 
-
-## ❗ Known issues:
-
-These issues will be hopefully fixed with time, so please don't report them. Instead, if you know how, feel free to investigate and submit a PR to fix the below. Keep the PRs small as it's easier to approve them:
-
-* Existing plugin objects sometimes not interpreted correctly and a new object is created instead, resulting in duplicate entries.
-* Occasional (experienced twice) hanging of processing plugin script file.
-* UI displaying outdated values until the API endpoints get refreshed. 
-
 ## Overview
 
+| ![Screen 1][screen1] | ![Screen 2][screen2] | 
+|----------------------|----------------------| 
+| ![Screen 3][screen3] | ![Screen 4][screen4] | 
+
 PiAlert comes with a plugin system to feed events from third-party scripts into the UI and then send notifications, if desired. The highlighted functionality this plugin system supports, is dynamic creation of a simple UI to interact with the discovered objects, a mechanism to surface settings of plugins in the UI, or to import objects into existing PiAlert database tables. (Currently update/overwriting of existing objects is not supported.)
 
 Example use cases for plugins could be:
@@ -27,6 +19,23 @@ If you wish to develop a plugin, please check the existing plugin structure. Onc
 
 Again, please read the below carefully if you'd like to contribute with a plugin yourself. This documentation file might be outdated, so double check the sample plugins as well. 
 
+## ⚠ Disclaimer
+
+Experimental feature used also to speed up development  and to make the app more maintainable. Follow the below very carefully and check example plugin(s) if you'd like to write one yourself. Plugin UI is not my priority right now, happy to approve PRs if you are interested in extending/improvintg the UI experience. Example improvements for the taking:
+
+* Making the tables sortable/filterable
+* Using the same approach to display table  data as in the Devices section (solves above)
+* Adding form controls supported to display the data (Currently supported ones are listed in the section "UI settings in database_column_definitions" below)
+* ...
+
+## ❗ Known issues:
+
+These issues will be hopefully fixed with time, so please don't report them. Instead, if you know how, feel free to investigate and submit a PR to fix the below. Keep the PRs small as it's easier to approve them:
+
+* Existing plugin objects sometimes not interpreted correctly and a new object is created instead, resulting in duplicate entries.
+* Occasional (experienced twice) hanging of processing plugin script file.
+* UI displaying outdated values until the API endpoints get refreshed. 
+
 ## Plugin file structure overview 
 
 > Folder name must be the same as the code name value in: `"code_name": "<value>"`
@@ -351,15 +360,16 @@ Example:
 The UI will adjust how columns are displayed in the UI based on the definition of the `database_column_definitions` object. Thease are the supported form controls and related functionality:
 
 - Only columns with `"show": true` and also with at least an English translation will be shown in the UI.
-- Supported types: `label`, `text`, `threshold`, `replace`
+- Supported types: `label`, `text`, `threshold`, `replace`, `deviceip`, `devicemac`, `url`. Check for details below, how columns behave based on the type.
   - `label` makes a column display only
-  - `text` makes a column editable
+  - `text` makes a column editable and a save icon is displayed next to it.
   - See below for information on `threshold`, `replace`
 - The `options` property is used in conjunction with these types:
   - `threshold` - The `options` array contains objects from lowest `maximum` to highest with corresponding `hexColor` used for the value background color if it's less than the specified `maximum`, but more than the previous one in the `options` array
   - `replace` - The `options` array contains objects with an `equals` property, that is compared to the "value" and if the values are the same, the string in `replacement` is displayed in the UI instead of the actual "value"
-  - `devicemac` - The value is considered to be a mac adress and a link pointing to the device with the given mac address is generated.
-  - `url` - The value is considered to be a url so a link is generated.
+- `devicemac` - The value is considered to be a mac address and a link pointing to the device with the given mac address is generated.
+- `deviceip` - The value is considered to be an IP address and a link pointing to the device with the given IP is generated. The IP is cheked against the last detected IP addresses and translated into a mac address that is then used for the link itself.
+- `url` - The value is considered to be a url so a link is generated.
 
 
 ```json
@@ -431,16 +441,13 @@ The UI will adjust how columns are displayed in the UI based on the definition o
 - [website_monitor (WEBMON) config.json](https://github.com/jokob-sk/Pi.Alert/blob/main/front/plugins/website_monitor/config.json) 
 - [dhcp_servers (DHCPSRVS) config.json](https://github.com/jokob-sk/Pi.Alert/blob/main/front/plugins/dhcp_servers/config.json) 
 - [dhcp_leases (DHCPLSS) config.json](https://github.com/jokob-sk/Pi.Alert/blob/main/front/plugins/dhcp_leases/config.json) 
-- [unifi_import (UNFIMP) config.json](https://github.com/jokob-sk/Pi.Alert/blob/main/front/plugins/unifi_import/config.json) 
+- [unifi_import (UNFIMP) config.json](https://github.com/jokob-sk/Pi.Alert/blob/main/front/plugins/unifi_import/config.json)
+- [snmp_discovery (SNMPDSC) config.json](https://github.com/jokob-sk/Pi.Alert/blob/main/front/plugins/snmp_discovery/config.json)
 
 ### SQL query based plugins
 - [nmap_services (NMAPSERV) config.json](https://github.com/jokob-sk/Pi.Alert/blob/main/front/plugins/nmap_services/config.json) 
 
-### Screenshots 
 
-| ![Screen 1][screen1] | ![Screen 2][screen2] | 
-|----------------------|----------------------| 
-| ![Screen 3][screen3] | ![Screen 4][screen4] | 
 
 [screen1]: https://raw.githubusercontent.com/jokob-sk/Pi.Alert/main/docs/img/plugins.png                "Screen 1"
 [screen2]: https://raw.githubusercontent.com/jokob-sk/Pi.Alert/main/docs/img/plugins_settings.png       "Screen 2"

front/plugins/nmap_services/config.json

@@ -49,7 +49,7 @@
             "column": "Object_PrimaryID",
             "css_classes": "col-sm-2",
             "show": true,
-            "type": "label",            
+            "type": "devicemac",            
             "default_value":"",
             "options": [],
             "localized": ["name"],
@@ -220,7 +220,7 @@
         {
             "function": "CMD",
             "type": "text",
-            "default_value":"SELECT  dv.dev_Name as Object_PrimaryID, cast('http://' || dv.dev_LastIP as VARCHAR(100)) || ':' || cast( SUBSTR(ns.Port ,0, INSTR(ns.Port , '/')) as VARCHAR(100)) as Object_SecondaryID,  datetime() as DateTime,  ns.Service as Watched_Value1,        ns.State as Watched_Value2,        'null' as Watched_Value3,        'null' as Watched_Value4,        ns.Extra as Extra, dv.dev_MAC as ForeignKey        FROM (SELECT * FROM Nmap_Scan) ns LEFT JOIN (SELECT dev_Name, dev_MAC, dev_LastIP FROM Devices) dv   ON ns.MAC = dv.dev_MAC",
+            "default_value":"SELECT  ns.MAC as Object_PrimaryID, cast('http://' || dv.dev_LastIP as VARCHAR(100)) || ':' || cast( SUBSTR(ns.Port ,0, INSTR(ns.Port , '/')) as VARCHAR(100)) as Object_SecondaryID,  datetime() as DateTime,  ns.Service as Watched_Value1, ns.State as Watched_Value2, dv.dev_Name as Watched_Value3,        'null' as Watched_Value4,        ns.Extra as Extra, ns.MAC as ForeignKey FROM (SELECT * FROM Nmap_Scan) ns left JOIN (SELECT dev_Name, dev_MAC, dev_LastIP FROM Devices) dv  ON ns.MAC = dv.dev_MAC",
             "options": [],
             "localized": ["name", "description"],
             "name" : [{

front/plugins/snmp_discovery/README.md

@@ -0,0 +1,36 @@
+## Overview
+
+A plugin for importing devices from an SNMP enabled router or switch.  Using SNMP offers an efficient way to discover IPv4 devices across one or more networks/subnets/vlans.
+
+### Usage
+
+Specify the following settings in the Settings section of PiAlert:
+
+- `SNMPDSC_routers` - A list of `snmpwalk` commands to execute against IP addresses of roputers/switches with SNMP turned on. For example: `snmpwalk -v 2c -c public -OXsq 192.168.1.1 .1.3.6.1.2.1.3.1.1.2`
+
+
+### Setup Cisco IOS
+
+Enable IOS SNMP service and restrict to selected (internal) IP/Subnet.
+
+````
+! Add standard ip access-list 10
+ip access-list standard 10
+ permit 192.168.1.0 0.0.0.255
+ permit host 192.168.2.10
+!
+! Enable IOS snmp server with Read Only community 'mysnmpcommunitysecret' name.
+! Restrict connections to access-list 10
+snmp-server community mysnmpcommunitysecret RO 10
+````
+
+Confirm SNMP enabled
+````
+show snmp
+````
+
+### Notes
+
+- Only IPv4 supported.
+- The SNMP OID `.1.1.1.3.6.1.2.1.3.1.1.2` is specifically for devices IPv4 ARP table. This OID has been tested on Cisco ISRs and other L3 devices. Support may vary between other vendors / devices.
+- Expected output (ingestion) in format `iso.3.6.1.2.1.3.1.1.2.3.1.192.168.1.2 "6C 6C 6C 6C 6C 6C "`.

front/plugins/snmp_discovery/config.json

@@ -0,0 +1,328 @@
+{
+    "code_name": "snmp_discovery",
+    "unique_prefix": "SNMPDSC",
+    "enabled": true,    
+    "data_source":  "python-script",
+    "localized": ["display_name", "description", "icon"],
+    "mapped_to_table": "DHCP_Leases",    
+    "display_name" : [{
+        "language_code":"en_us",
+        "string" : "SNMP discovery"
+    }],
+    "icon":[{
+        "language_code":"en_us",
+        "string" : "<i class=\"fa-solid fa-s\"></i>"
+    }],     
+    "description": [{
+        "language_code":"en_us",
+        "string" : "This plugin is used to discover devices via the arp table(s) of a RFC1213 compliant router or switch."
+    }],
+    "params" : [
+        {
+            "name"  : "routers",
+            "type"  : "setting",
+            "value" : "SNMPDSC_routers"
+        }
+    ], 
+    "database_column_definitions":
+    [          
+        {
+            "column": "Index",            
+            "css_classes": "col-sm-2",
+            "show": false,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code":"en_us",
+                "string" : "N/A"
+                }]
+        } ,
+        {
+            "column": "Plugin",
+            "css_classes": "col-sm-2",
+            "show": false,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code":"en_us",
+                "string" : "N/A"
+                }]
+        },
+        {
+            "column": "Object_PrimaryID",
+            "mapped_to_column": "DHCP_MAC", 
+            "css_classes": "col-sm-2",
+            "show": true,
+            "type": "devicemac",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code":"en_us",
+                "string" : "MAC address"
+                }]
+        },
+        {
+            "column": "Object_SecondaryID",
+            "mapped_to_column": "DHCP_IP", 
+            "css_classes": "col-sm-2",
+            "show": true,
+            "type": "deviceip",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code":"en_us",
+                "string" : "IP"
+                }]
+        } ,
+        {
+            "column": "DateTimeCreated",
+            "css_classes": "col-sm-2",
+            "show": true,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code":"en_us",
+                "string" : "Created"
+                }]
+        },
+        {
+            "column": "DateTimeChanged",
+            "mapped_to_column": "DHCP_DateTime", 
+            "css_classes": "col-sm-2",
+            "show": true,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code":"en_us",
+                "string" : "Changed"
+                }]
+        },
+        {
+            "column": "Watched_Value1",
+            "mapped_to_column": "DHCP_Name", 
+            "css_classes": "col-sm-2",
+            "show": true,
+            "type": "label",            
+            "default_value":"(unknown)",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code":"en_us",
+                "string" : "Hostname"
+                }]
+        },
+        {
+            "column": "Watched_Value2",            
+            "css_classes": "col-sm-2",
+            "show": true,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code":"en_us",
+                "string" : "Router IP"
+                }]
+        },
+        {
+            "column": "Watched_Value3",
+            "css_classes": "col-sm-2",
+            "show": false,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code":"en_us",
+                "string" : "Type"
+                }]
+        } ,
+        {
+            "column": "Watched_Value4",
+            "css_classes": "col-sm-2",
+            "show": false,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code":"en_us",
+                "string" : "Network"
+                }]
+        } ,       
+        {
+            "column": "UserData",
+            "css_classes": "col-sm-2",
+            "show": false,
+            "type": "textboxsave",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code":"en_us",
+                "string" : "Comments"
+                }]
+        },        
+        {
+            "column": "Extra",
+            "css_classes": "col-sm-3",
+            "show": true,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code":"en_us",
+                "string" : "RAW output"
+                }]
+        },        
+        {
+            "column": "Status",
+            "css_classes": "col-sm-1",
+            "show": true,
+            "type": "replace",            
+            "default_value":"",
+            "options": [
+                {
+                    "equals": "watched-not-changed",
+                    "replacement": "<div style='text-align:center'><i class='fa-solid fa-square-check'></i><div></div>"
+                },
+                {
+                    "equals": "watched-changed",
+                    "replacement": "<div style='text-align:center'><i class='fa-solid fa-triangle-exclamation'></i></div>"
+                },
+                {
+                    "equals": "new",
+                    "replacement": "<div style='text-align:center'><i class='fa-solid fa-circle-plus'></i></div>"
+                }
+            ],
+            "localized": ["name"],
+            "name":[{
+                "language_code":"en_us",
+                "string" : "Status"
+                }]
+        }            
+    ],
+    "settings":[
+        {
+            "function": "RUN",            
+            "type": "selecttext",            
+            "default_value":"disabled",
+            "options": ["disabled", "once", "schedule", "always_after_scan", "on_new_device"],
+            "localized": ["name", "description"],
+            "name" :[{
+                "language_code":"en_us",
+                "string" : "When to run"
+            }],
+            "description": [{
+                "language_code":"en_us",
+                "string" : "Enable import of devices from a SNMP enabled device. If you select <code>schedule</code> the scheduling settings from below are applied. If you select <code>once</code> the scan is run only once on start of the application (container) or after you update your settings."
+            }]
+        },
+        {
+            "function": "CMD",
+            "type": "text",
+            "default_value":"python3 /home/pi/pialert/front/plugins/snmp_discovery/script.py routers={s-quote}{routers}{s-quote}",
+            "options": [],
+            "localized": ["name", "description"],
+            "name" : [{
+                "language_code":"en_us",
+                "string" : "Command"
+            }],
+            "description": [{
+                "language_code":"en_us",
+                "string" : "Command to run. Not recommended to change."
+            }]
+        },
+        {
+            "function": "routers",
+            "type": "list",
+            "default_value":["snmpwalk -v 2c -c public -OXsq 192.168.1.1 .1.3.6.1.2.1.3.1.1.2"],
+            "options": [],
+            "localized": ["name", "description"],
+            "name" : [{
+                "language_code":"en_us",
+                "string" : "Routers"
+            }],
+            "description": [{
+                "language_code":"en_us",
+                "string" : "A list of <code>snmpwalk</code> commands to execute against IP addresses of roputers/switches with SNMP turned on. <br/> <br/> Example with the router on the IP <code>192.168.1.1</code>: <br/> <code>snmpwalk -v 2c -c public -OXsq 192.168.1.1 .1.3.6.1.2.1.3.1.1.2</code> <br/><br/> Only IPv4 supported. Authentication is not supported. More info on the plugin <a href='https://github.com/jokob-sk/Pi.Alert/tree/main/front/plugins/snmp_discovery' target='_blank'>here</a>."
+            }]
+        },
+        {
+            "function": "RUN_SCHD",
+            "type": "text",
+            "default_value":"0 2 * * *",
+            "options": [],
+            "localized": ["name", "description"],
+            "name" : [{
+                "language_code":"en_us",
+                "string" : "Schedule"
+            }],
+            "description": [{
+                "language_code":"en_us",
+                "string" : "Only enabled if you select <code>schedule</code> in the <a href=\"#DHCPLSS_RUN\"><code>DHCPLSS_RUN</code> setting</a>. Make sure you enter the schedule in the correct cron-like format (e.g. validate at <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). For example entering <code>0 4 * * *</code> will run the scan after 4 am in the <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</code> you set above</a>. Will be run NEXT time the time passes."
+            }]
+        },
+        {
+            "function": "RUN_TIMEOUT",
+            "type": "integer",
+            "default_value":5,
+            "options": [],
+            "localized": ["name", "description"],
+            "name" : [{
+                "language_code":"en_us",
+                "string" : "Run timeout"
+            },
+            {
+                "language_code":"de_de",
+                "string" : "Wartezeit"
+            }],
+            "description": [{
+                "language_code":"en_us",
+                "string" : "Maximum time in seconds to wait for the script to finish. If this time is exceeded the script is aborted."
+            }]
+        },
+        {
+            "function": "WATCH",
+            "type": "multiselect",
+            "default_value":["Watched_Value1"],
+            "options": ["Watched_Value1","Watched_Value2","Watched_Value3","Watched_Value4"],
+            "localized": ["name", "description"],
+            "name" :[{
+                "language_code":"en_us",
+                "string" : "Watched"
+            }] ,
+            "description":[{
+                "language_code":"en_us",
+                "string" : "Send a notification if selected values change. Use <code>CTRL + Click</code> to select/deselect. <ul> <li><code>Watched_Value1</code> is Hostname (not discoverable) </li><li><code>Watched_Value2</code> is Router IP </li><li><code>Watched_Value3</code> is not used </li><li><code>Watched_Value4</code> is not used </li></ul>"
+            }] 
+        },
+        {
+            "function": "REPORT_ON",
+            "type": "multiselect",
+            "default_value":["new","watched-changed"],
+            "options": ["new","watched-changed","watched-not-changed"],
+            "localized": ["name", "description"],
+            "name" :[{
+                "language_code":"en_us",
+                "string" : "Report on"
+            }] ,
+            "description":[{
+                "language_code":"en_us",
+                "string" : "Send a notification only on these statuses. <code>new</code> means a new unique (unique combination of PrimaryId and SecondaryId) object was discovered. <code>watched-changed</code> means that selected <code>Watched_ValueN</code> columns changed."
+            }] 
+        }
+    ]
+}
+

front/plugins/snmp_discovery/script.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python
+
+# Example call
+# python3 /home/pi/pialert/front/plugins/snmp_discovery/script.py routers='snmpwalk -v 2c -c public -OXsq 192.168.1.1 .1.3.6.1.2.1.3.1.1.2'
+
+from __future__ import unicode_literals
+from time import sleep, time, strftime
+import requests
+from requests                               import Request, Session, packages
+import pathlib
+import threading
+import subprocess
+import socket
+import json
+import argparse
+import io
+import sys
+from requests.packages.urllib3.exceptions import InsecureRequestWarning
+import pwd
+import os
+
+
+curPath = str(pathlib.Path(__file__).parent.resolve())
+log_file = curPath + '/script.log'
+last_run = curPath + '/last_result.log'
+
+# Workflow
+
+def main():    
+
+    # init global variables
+    global ROUTERS
+
+    # empty file
+    open(last_run , 'w').close()   
+
+    last_run_logfile = open(last_run, 'a') 
+    
+
+    parser = argparse.ArgumentParser(description='This plugin is used to discover devices via the arp table(s) of a RFC1213 compliant router or switch.')
+    
+    parser.add_argument('routers',  action="store",  help="IP(s) of routers, separated by comma (,) if passing multiple")                        
+
+    values = parser.parse_args()
+
+    # parse output
+    newEntries = []
+
+    if values.routers:        
+
+        ROUTERS = values.routers.split('=')[1].replace('\'','')  
+        
+        newEntries = get_entries(newEntries)  
+   
+    for e in newEntries:        
+        # Insert list into the log            
+        service_monitoring_log(e.primaryId, e.secondaryId, e.created, e.watched1, e.watched2, e.watched3, e.watched4, e.extra, e.foreignKey )
+
+# -----------------------------------------------------------------------------
+def get_entries(newEntries):
+
+    routers = []
+
+    if ',' in ROUTERS:
+        # multiple
+        routers = ROUTERS.split(',')
+    
+    else:
+        # only one
+        routers.append(ROUTERS)
+
+    for router in routers:
+        # snmpwalk -v 2c -c public -OXsq 192.168.1.1 .1.3.6.1.2.1.3.1.1.2
+
+        print(router)
+
+        timeoutSec = 10
+
+        snmpwalkArgs = router.split(' ')
+
+        # Execute N probes and insert in list
+        probes = 1 #  N probes
+        newLines = []                
+        for _ in range(probes):
+            output = subprocess.check_output (snmpwalkArgs, universal_newlines=True,  stderr=subprocess.STDOUT, timeout=(timeoutSec ))
+            newLines = newLines + output.split("\n")
+
+        # Process outputs
+        # Sample: iso.3.6.1.2.1.3.1.1.2.3.1.192.168.1.2 "6C 6C 6C 6C 6C 6C "
+
+        with open(log_file, 'a') as run_logfile:
+            for line in newLines:              
+                # debug
+                run_logfile.write(line)
+
+                tmpSplt = line.split('"')   
+
+                if len(tmpSplt) == 3:
+
+                    ipStr   = tmpSplt[0].split('.') # contains IP
+
+                    macStr  = tmpSplt[1].split(' ') # contains MAC
+
+                    if 'iso.' in line and len(ipStr) == 16:
+                        tmpEntry = plugin_object_class(
+                            f'{macStr[0]}:{macStr[1]}:{macStr[2]}:{macStr[3]}:{macStr[4]}:{macStr[5]}',
+                            f'{ipStr[12]}.{ipStr[13]}.{ipStr[14]}.{ipStr[15]}'.strip(),
+                            watched1='(unknown)',
+                            watched2=snmpwalkArgs[6], # router IP
+                            extra=line
+                        )
+                        newEntries.append(tmpEntry) 
+        
+    return newEntries
+
+
+# -------------------------------------------------------------------
+class plugin_object_class:
+    def __init__(self, primaryId = '',secondaryId = '', watched1 = '',watched2 = '',watched3 = '',watched4 = '',extra = '',foreignKey = ''):        
+        self.pluginPref   = ''
+        self.primaryId    = primaryId
+        self.secondaryId  = secondaryId
+        self.created      = strftime("%Y-%m-%d %H:%M:%S")
+        self.changed      = ''
+        self.watched1     = watched1
+        self.watched2     = watched2
+        self.watched3     = watched3
+        self.watched4     = watched4
+        self.status       = ''
+        self.extra        = extra
+        self.userData     = ''
+        self.foreignKey   = foreignKey
+
+# -----------------------------------------------------------------------------
+def service_monitoring_log(primaryId, secondaryId, created, watched1, watched2 = 'null', watched3 = 'null', watched4 = 'null', extra ='null', foreignKey ='null'  ):
+    
+    if watched1 == '':
+        watched1 = 'null'
+    if watched2 == '':
+        watched2 = 'null'
+    if watched3 == '':
+        watched3 = 'null'
+    if watched4 == '':
+        watched4 = 'null'
+    if extra == '':
+        extra = 'null'
+    if foreignKey == '':
+        foreignKey = 'null'
+
+    with open(last_run, 'a') as last_run_logfile:        
+        last_run_logfile.write("{}|{}|{}|{}|{}|{}|{}|{}|{}\n".format(
+                                                primaryId,
+                                                secondaryId,
+                                                created,                                                
+                                                watched1,
+                                                watched2,
+                                                watched3,
+                                                watched4,
+                                                extra,
+                                                foreignKey
+                                                )
+                             )
+
+
+#===============================================================================
+# BEGIN
+#===============================================================================
+if __name__ == '__main__':    
+    main()  
+

front/plugins/unifi_import/README.md

@@ -4,7 +4,7 @@ A plugin allowing for importing devices from a UniFi controller.
 
 ### Usage
 
-Spedify the following settings in the Settings section of PiAlert:
+Specify the following settings in the Settings section of PiAlert:
 
 - `UNFIMP_username` - Username used to login into the UNIFI controller.
 - `UNFIMP_password` - Password used to login into the UNIFI controller.
@@ -15,4 +15,5 @@ Spedify the following settings in the Settings section of PiAlert:
 
 ### Notes
 
-- Currently only used to import devices, not their status, type or network map.
+- Currently only used to import devices, not their status, type or network map.
+- It is recommend to create a read-only user in your UniFi controller 

front/plugins/unifi_import/config.json

@@ -47,6 +47,11 @@
             "name"  : "port",
             "type"  : "setting",
             "value" : "UNFIMP_port"
+        },
+        {
+            "name"  : "version",
+            "type"  : "setting",
+            "value" : "UNFIMP_version"
         }
     ], 
     "database_column_definitions":
@@ -182,7 +187,7 @@
             "localized": ["name"],
             "name":[{
                 "language_code":"en_us",
-                "string" : "Network"
+                "string" : "Is online?"
                 }]
         } ,       
         {
@@ -208,7 +213,7 @@
             "localized": ["name"],
             "name":[{
                 "language_code":"en_us",
-                "string" : "Hostname"
+                "string" : "Network"
                 }]
         },        
         {
@@ -257,7 +262,7 @@
         {
             "function": "CMD",
             "type": "text",
-            "default_value":"python3 /home/pi/pialert/front/plugins/unifi_import/script.py username={username} password={password}  host={host} sites={sites}  protocol={protocol} port={port}",
+            "default_value":"python3 /home/pi/pialert/front/plugins/unifi_import/script.py username={username} password={password}  host={host} sites={sites}  protocol={protocol} port={port} version={version}",
             "options": [],
             "localized": ["name", "description"],
             "name" : [{
@@ -344,6 +349,21 @@
                 "string" : "The port number where the UNIFI controller is runnig. Usually it is <code>8443</code>."
             }]
         },
+        {
+            "function": "version",
+            "type": "text",
+            "default_value":"",
+            "options": [],
+            "localized": ["name", "description"],
+            "name" : [{
+                "language_code":"en_us",
+                "string" : "API version"
+            }],
+            "description": [{
+                "language_code":"en_us",
+                "string" : "The base version of the Unify controller API. Supported values as of time of writing are <code>v4|v5|unifiOS|UDMP-unifiOS</code>."
+            }]
+        },
         {
             "function": "sites",
             "type": "list",

front/plugins/unifi_import/script.py

@@ -32,7 +32,7 @@ def main():
 
     # init global variables
     global UNIFI_USERNAME, UNIFI_PASSWORD, UNIFI_HOST
-    global UNIFI_SITES, PORT, PROTOCOL
+    global UNIFI_SITES, PORT, PROTOCOL, VERSION
 
     last_run_logfile = open(last_run, 'a') 
 
@@ -47,6 +47,7 @@ def main():
     parser.add_argument('sites',  action="store",  help="Name of the sites (usually 'default', check the URL in your UniFi controller UI). Separated by comma (,) if passing multiple sites")      
     parser.add_argument('protocol',  action="store",  help="https:// or http://")  
     parser.add_argument('port',  action="store",  help="Usually 8443")  
+    parser.add_argument('version',  action="store",  help="The base version of the controller API [v4|v5|unifiOS|UDMP-unifiOS]")  
 
     values = parser.parse_args()
 
@@ -61,6 +62,7 @@ def main():
         UNIFI_SITES = values.sites.split('=')[1]  
         PROTOCOL = values.protocol.split('=')[1]
         PORT = values.port.split('=')[1]
+        VERSION = values.version.split('=')[1]
         
         newEntries = get_entries(newEntries)  
 
@@ -85,7 +87,7 @@ def get_entries(newEntries):
 
     for site in sites:
 
-        c = Controller(UNIFI_HOST, UNIFI_USERNAME, UNIFI_PASSWORD, ssl_verify=False, site_id=site )
+        c = Controller(UNIFI_HOST, UNIFI_USERNAME, UNIFI_PASSWORD, version=VERSION, ssl_verify=False, site_id=site )
 
         for ap in c.get_aps():
 

front/plugins/website_monitor/script.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # Based on the work of https://github.com/leiweibau/Pi.Alert
 
-# /home/pi/pialert/front/plugins/website_monitor/script.py urls=http://google.com,http://bing.com
+# python3 /home/pi/pialert/front/plugins/website_monitor/script.py urls=http://google.com,http://bing.com
 from __future__ import unicode_literals
 from time import sleep, time, strftime
 import requests