🔃 Sync Hub v0.8

2025-12-07 09:36:05 -08:00 · 2024-06-08 12:29:03 +10:00
parent b613775031
commit c591ab9ce3
11 changed files with 157 additions and 96 deletions
--- a/front/plugins/sync/README.md
+++ b/front/plugins/sync/README.md
@@ -1,6 +1,52 @@
 ## Overview

-Synchronization plugin to synchronize multiple app instances. The plugin sends encrypted `last_result.log` files for individual plugins. 
+Synchronization plugin to synchronize multiple app instances. The Plugin can sychronize 2 types of data:
+
+1. 💻 Devices: The plugin sends an encrypted `table_devices.json` file to synchronize the whole Devices DB table.
+1. 🔌 Plugin data:  The plugin sends encrypted `last_result.log` files for individual plugins. 
+
+### Synchronizing 💻 Devices data
+
+This is probably what most of the setups will use. Required settings follow.
+
+#### Node (Source) Settings
+
+- When to run `SYNC_RUN`
+- Schedule `SYNC_RUN_SCHD`
+- API token `SYNC_api_token`
+- Encryption Key `SYNC_encryption_key`
+- Node name `SYNC_node_name`
+- Hub URL `SYNC_hub_url`
+- Send Devices `SYNC_devices` 👈
+
+#### Hub (Target) Settings
+
+- When to run `SYNC_RUN`
+- Schedule `SYNC_RUN_SCHD`
+- API token `SYNC_api_token`
+- Encryption Key `SYNC_encryption_key`
+
+### Synchronizing 🔌 Plugins data
+
+This mechanism will be probably used in special use cases. Required settings follow.
+
+#### Node (Source) Settings
+
+- When to run `SYNC_RUN`
+- Schedule `SYNC_RUN_SCHD`
+- API token `SYNC_api_token`
+- Encryption Key `SYNC_encryption_key`
+- Node name `SYNC_node_name`
+- Hub URL `SYNC_hub_url`
+- Send Plugins `SYNC_plugins` 👈
+
+#### Hub (Target) Settings
+
+- When to run `SYNC_RUN`
+- Schedule `SYNC_RUN_SCHD`
+- API token `SYNC_api_token`
+- Encryption Key `SYNC_encryption_key`
+

 ### Usage

--- a/front/plugins/sync/config.json
+++ b/front/plugins/sync/config.json
@@ -81,6 +81,82 @@
        "string" : "When the node sync should run. Data might be lost if you run the sync less frequently. Good options are <code>always_after_scan</code>, <code>on_new_device</code>, <code>on_notification</code>"
    }]
  },
+  {
+    "function": "RUN_SCHD",
+    "type": "text",
+    "display_condition": {
+      "type" : "setting",
+      "name" : "SYNC_instance_type",
+      "value": "hub"
+    },   
+    "default_value":"*/5 * * * *",
+    "options": [],
+    "localized": ["name", "description"],
+    "name" : [{
+        "language_code":"en_us",
+        "string" : "Schedule"
+    },
+    {
+        "language_code":"es_es",
+        "string" : "Schedule"
+    },
+    {
+        "language_code":"de_de",
+        "string" : "Schedule"
+    }],
+    "description": [{
+        "language_code":"en_us",
+        "string" : "Only enabled if you select <code>schedule</code> in the <a href=\"#SYNC_RUN\"><code>SYNC_RUN</code> setting</a>. Make sure you enter the schedule in the correct cron-like format (e.g. validate at <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). For example entering <code>0 4 * * *</code> will run the scan after 4 am in the <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</code> you set above</a>. Will be run NEXT time the time passes."
+    },
+    {
+        "language_code":"es_es",
+        "string" : "Solo está habilitado si selecciona <code>schedule</code> en la configuración <a href=\"#SYNC_RUN\"><code>SYNC_RUN</code></a>. Asegúrese de ingresar la programación en el formato similar a cron correcto (por ejemplo, valide en <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). Por ejemplo, ingresar <code>0 4 * * *</code> ejecutará el escaneo después de las 4 a.m. en el <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</ código> que configuró arriba</a>. Se ejecutará la PRÓXIMA vez que pase el tiempo."
+    },
+    {
+        "language_code":"de_de",
+        "string" : "Nur aktiviert, wenn Sie <code>schedule</code> in der <a href=\"#SYNC_RUN\"><code>SYNC_RUN</code>-Einstellung</a> auswählen. Stellen Sie sicher, dass Sie den Zeitplan im richtigen Cron-ähnlichen Format eingeben (z. B. validieren unter <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). Wenn Sie beispielsweise <code>0 4 * * *</code> eingeben, wird der Scan nach 4 Uhr morgens in der <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</ ausgeführt. Code> den Sie oben festgelegt haben</a>. Wird das NÄCHSTE Mal ausgeführt, wenn die Zeit vergeht."
+    }]
+  },    
+  {
+    "function": "api_token",
+    "type": "text",
+    "maxLength": 50,
+    "default_value": "",
+    "options": [],
+    "localized": ["name", "description"],
+    "name": [
+      {
+        "language_code": "en_us",
+        "string": "API token"
+      }
+    ],
+    "description": [
+      {
+        "language_code": "en_us",
+        "string": "API token to secure communication. The API token needs to be the same on the hub and on the nodes."
+      } 
+    ]
+  },
+  {
+    "function": "encryption_key",
+    "type": "text",
+    "maxLength": 50,
+    "default_value": "",
+    "options": [],
+    "localized": ["name", "description"],
+    "name": [
+      {
+        "language_code": "en_us",
+        "string": "Encryption Key"
+      }
+    ],
+    "description": [
+      {
+        "language_code": "en_us",
+        "string": "Encryption key used to encrypt the sent data. The key needs to be the same on the hub and on the nodes."
+      } 
+    ]
+  },
  {
    "function": "hub_url",
    "type": "text",
@@ -183,46 +259,6 @@
      } 
    ]
  },
-  {
-    "function": "api_token",
-    "type": "text",
-    "maxLength": 50,
-    "default_value": "",
-    "options": [],
-    "localized": ["name", "description"],
-    "name": [
-      {
-        "language_code": "en_us",
-        "string": "API token"
-      }
-    ],
-    "description": [
-      {
-        "language_code": "en_us",
-        "string": "API token to secure communication. The API token needs to be the same on the hub and on the nodes."
-      } 
-    ]
-  },
-  {
-    "function": "encryption_key",
-    "type": "text",
-    "maxLength": 50,
-    "default_value": "",
-    "options": [],
-    "localized": ["name", "description"],
-    "name": [
-      {
-        "language_code": "en_us",
-        "string": "Encryption Key"
-      }
-    ],
-    "description": [
-      {
-        "language_code": "en_us",
-        "string": "Encryption key used to encrypt the sent data. The key needs to be the same on the hub and on the nodes."
-      } 
-    ]
-  },
  {
    "function": "CMD",
    "type": "readonly",
@@ -258,42 +294,6 @@
      }	        
    ]
  },
-  {
-    "function": "RUN_SCHD",
-    "type": "text",
-    "display_condition": {
-      "type" : "setting",
-      "name" : "SYNC_instance_type",
-      "value": "hub"
-    },   
-    "default_value":"*/5 * * * *",
-    "options": [],
-    "localized": ["name", "description"],
-    "name" : [{
-        "language_code":"en_us",
-        "string" : "Schedule"
-    },
-    {
-        "language_code":"es_es",
-        "string" : "Schedule"
-    },
-    {
-        "language_code":"de_de",
-        "string" : "Schedule"
-    }],
-    "description": [{
-        "language_code":"en_us",
-        "string" : "Only enabled if you select <code>schedule</code> in the <a href=\"#SYNC_RUN\"><code>SYNC_RUN</code> setting</a>. Make sure you enter the schedule in the correct cron-like format (e.g. validate at <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). For example entering <code>0 4 * * *</code> will run the scan after 4 am in the <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</code> you set above</a>. Will be run NEXT time the time passes."
-    },
-    {
-        "language_code":"es_es",
-        "string" : "Solo está habilitado si selecciona <code>schedule</code> en la configuración <a href=\"#SYNC_RUN\"><code>SYNC_RUN</code></a>. Asegúrese de ingresar la programación en el formato similar a cron correcto (por ejemplo, valide en <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). Por ejemplo, ingresar <code>0 4 * * *</code> ejecutará el escaneo después de las 4 a.m. en el <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</ código> que configuró arriba</a>. Se ejecutará la PRÓXIMA vez que pase el tiempo."
-    },
-    {
-        "language_code":"de_de",
-        "string" : "Nur aktiviert, wenn Sie <code>schedule</code> in der <a href=\"#SYNC_RUN\"><code>SYNC_RUN</code>-Einstellung</a> auswählen. Stellen Sie sicher, dass Sie den Zeitplan im richtigen Cron-ähnlichen Format eingeben (z. B. validieren unter <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). Wenn Sie beispielsweise <code>0 4 * * *</code> eingeben, wird der Scan nach 4 Uhr morgens in der <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</ ausgeführt. Code> den Sie oben festgelegt haben</a>. Wird das NÄCHSTE Mal ausgeführt, wenn die Zeit vergeht."
-    }]
-  },    
  {
    "function": "RUN_TIMEOUT",
    "type": "integer",
--- a/front/plugins/sync/hub.php
+++ b/front/plugins/sync/hub.php
@@ -35,8 +35,12 @@ if ($_SERVER['REQUEST_METHOD'] === 'POST') {
    }

    // Generate a unique file path to avoid overwriting existing files
-    $files = glob("{$storage_path}/last_result.{encoded,decoded}.{$node_name}.*.log", GLOB_BRACE);
+    $encoded_files = glob("{$storage_path}/last_result.encoded.{$node_name}.*.log");
+    $decoded_files = glob("{$storage_path}/last_result.decoded.{$node_name}.*.log");
+
+    $files = array_merge($encoded_files, $decoded_files);
    $file_count = count($files) + 1;
+
    $file_path = "{$storage_path}/last_result.encoded.{$node_name}.{$file_count}.log";


--- a/front/plugins/sync/sync.py
+++ b/front/plugins/sync/sync.py
@@ -128,24 +128,18 @@ def main():
                        unique_mac_addresses.add(device['dev_MAC'])
                        device_data.append(device)

+    
+
    if len(device_data) > 0:
        # Retrieve existing dev_MAC values from the Devices table
        placeholders = ', '.join('?' for _ in unique_mac_addresses)
        cursor.execute(f'SELECT dev_MAC FROM Devices WHERE dev_MAC IN ({placeholders})', tuple(unique_mac_addresses))
        existing_mac_addresses = set(row[0] for row in cursor.fetchall())
+        

-        # Filter out existing devices
-        new_devices = [device for device in device_data if device['dev_MAC'] not in existing_mac_addresses]
-
-        #  Remove 'rowid' key if it exists 
-        for device in new_devices:
-            device.pop('rowid', None)
-
-
-        # Prepare the insert statement
-        if new_devices:
-            # insert devices into the lats_result.log to manage state
-            for device in new_devices:
+        # insert devices into the lats_result.log to manage state
+        for device in device_data:
+            if device['dev_PresentLastScan'] == 1:
                plugin_objects.add_object(
                    primaryId   = device['dev_MAC'],
                    secondaryId = device['dev_LastIP'],
@@ -156,6 +150,19 @@ def main():
                    extra       = '',
                    foreignKey  = device['dev_GUID'])

+        # Filter out existing devices
+        new_devices = [device for device in device_data if device['dev_MAC'] not in existing_mac_addresses]
+
+        #  Remove 'rowid' key if it exists 
+        for device in new_devices:
+            device.pop('rowid', None)
+
+        mylog('verbose', [f'[{pluginName}] All devices: "{len(device_data)}"'])
+        mylog('verbose', [f'[{pluginName}] New devices: "{len(new_devices)}"'])
+
+        # Prepare the insert statement
+        if new_devices:
+
            columns = ', '.join(k for k in new_devices[0].keys() if k != 'rowid')
            placeholders = ', '.join('?' for k in new_devices[0] if k != 'rowid')
            sql = f'INSERT INTO Devices ({columns}) VALUES ({placeholders})'