MERGE: resolve conflicts

Signed-off-by: jokob-sk <jokob.sk@gmail.com>

server/utils/crypto_utils.py

@@ -0,0 +1,72 @@
+from Crypto.Cipher import AES
+from Crypto.Util.Padding import pad, unpad
+import base64
+import os
+import hashlib
+import uuid
+
+
+# SIMPLE CRYPT - requeres C compiler -------------------------------------------------------------------------
+
+# def prepare_key(encryption_key):
+#     if len(encryption_key) < 32:
+#         encryption_key = (encryption_key * ((32 // len(encryption_key)) + 1))[:32]
+#     return encryption_key
+
+# def encrypt_data(data, encryption_key):
+#     key = prepare_key(encryption_key)
+#     encrypted_data = encrypt(key, data)
+#     return encrypted_data
+
+# def decrypt_data(data, encryption_key):
+#     key = prepare_key(encryption_key)
+#     decrypted_data = decrypt(key, data).decode('utf-8')
+#     return decrypted_data
+
+# pycryptodome -------------------------------------------------------------------------
+
+
+def prepare_key(encryption_key):
+    key = hashlib.sha256(encryption_key.encode()).digest()
+    return key
+
+
+def encrypt_data(data, encryption_key):
+    key = prepare_key(encryption_key)
+    cipher = AES.new(key, AES.MODE_CBC)
+    ct_bytes = cipher.encrypt(pad(data.encode("utf-8"), AES.block_size))
+    iv = base64.b64encode(cipher.iv).decode("utf-8")
+    ct = base64.b64encode(ct_bytes).decode("utf-8")
+    return iv + ct
+
+
+def decrypt_data(data, encryption_key):
+    key = prepare_key(encryption_key)
+    iv = base64.b64decode(data[:24])
+    ct = base64.b64decode(data[24:])
+    cipher = AES.new(key, AES.MODE_CBC, iv)
+    pt = unpad(cipher.decrypt(ct), AES.block_size)
+    return pt.decode("utf-8")
+
+
+# -------------------------------------------------------------------------------
+def get_random_bytes(length):
+    # Generate random bytes
+    random_bytes = os.urandom(length)
+
+    # Convert bytes to hexadecimal string
+    hex_string = random_bytes.hex()
+
+    # Format hexadecimal string with hyphens
+    formatted_hex = "-".join(
+        hex_string[i : i + 2] for i in range(0, len(hex_string), 2)
+    )
+
+    return formatted_hex
+
+
+# -------------------------------------------------------------------------------
+def generate_deterministic_guid(plugin, primary_id, secondary_id):
+    """Generates a deterministic GUID based on plugin, primary ID, and secondary ID."""
+    data = f"{plugin}-{primary_id}-{secondary_id}".encode("utf-8")
+    return str(uuid.UUID(hashlib.md5(data).hexdigest()))

server/utils/datetime_utils.py

@@ -0,0 +1,211 @@
+#!/usr/bin/env python
+
+import os
+import pathlib
+import sys
+from datetime import datetime
+from dateutil import parser
+import datetime
+import re
+import pytz
+from pytz import timezone
+from typing import Union
+from zoneinfo import ZoneInfo
+import email.utils
+
+# Register NetAlertX directories
+INSTALL_PATH="/app"
+sys.path.extend([f"{INSTALL_PATH}/front/plugins", f"{INSTALL_PATH}/server"])
+
+import conf
+from const import *
+
+
+
+#-------------------------------------------------------------------------------
+# DateTime
+#-------------------------------------------------------------------------------
+
+DATETIME_PATTERN = "%Y-%m-%d %H:%M:%S"
+DATETIME_REGEX = re.compile(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$')
+
+def timeNowTZ():
+    if conf.tz:
+        return datetime.datetime.now(conf.tz).replace(microsecond=0)
+    else:
+        return datetime.datetime.now().replace(microsecond=0)
+
+def timeNow():
+    return datetime.datetime.now().replace(microsecond=0)
+
+def get_timezone_offset():    
+    now = datetime.datetime.now(conf.tz)
+    offset_hours = now.utcoffset().total_seconds() / 3600        
+    offset_formatted =  "{:+03d}:{:02d}".format(int(offset_hours), int((offset_hours % 1) * 60))
+    return offset_formatted
+
+def timeNowDB(local=True):
+    """
+    Return the current time (local or UTC) as ISO 8601 for DB storage.
+    Safe for SQLite, PostgreSQL, etc.
+
+    Example local: '2025-11-04 18:09:11'
+    Example UTC:   '2025-11-04 07:09:11'
+    """
+    if local:
+        try:
+            if isinstance(conf.tz, datetime.tzinfo):
+                tz = conf.tz
+            elif conf.tz:
+                tz = ZoneInfo(conf.tz)
+            else:
+                tz = None
+        except Exception:
+            tz = None
+        return datetime.datetime.now(tz).strftime(DATETIME_PATTERN)
+    else:
+        return datetime.datetime.now(datetime.UTC).strftime(DATETIME_PATTERN)
+
+
+#-------------------------------------------------------------------------------
+#  Date and time methods
+#-------------------------------------------------------------------------------
+
+def normalizeTimeStamp(inputTimeStamp):
+    """
+    Normalize various timestamp formats into a datetime.datetime object.
+
+    Supports:
+    - SQLite-style 'YYYY-MM-DD HH:MM:SS'
+    - ISO 8601 'YYYY-MM-DDTHH:MM:SSZ'
+    - Epoch timestamps (int or float)
+    - datetime.datetime objects (returned as-is)
+    - Empty or invalid values (returns None)
+    """
+    if inputTimeStamp is None:
+        return None
+
+    # Already a datetime
+    if isinstance(inputTimeStamp, datetime.datetime):
+        return inputTimeStamp
+
+    # Epoch timestamp (integer or float)
+    if isinstance(inputTimeStamp, (int, float)):
+        try:            
+            return datetime.datetime.fromtimestamp(inputTimeStamp)
+        except (OSError, OverflowError, ValueError):
+            return None
+
+    # String formats (SQLite / ISO8601)
+    if isinstance(inputTimeStamp, str):
+        inputTimeStamp = inputTimeStamp.strip()
+        if not inputTimeStamp:
+            return None
+        try:
+            # match the "2025-11-08 14:32:10" format
+            pattern = DATETIME_REGEX
+
+            if pattern.match(inputTimeStamp):
+                return datetime.datetime.strptime(inputTimeStamp, DATETIME_PATTERN)
+            else:
+                # Handles SQLite and ISO8601 automatically
+                return parser.parse(inputTimeStamp)
+        except Exception:
+            return None
+
+    # Unrecognized type
+    return None
+
+
+# -------------------------------------------------------------------------------------------
+def format_date_iso(date1: str) -> str:
+    """Return ISO 8601 string for a date or None if empty"""
+    if date1 is None:
+        return None
+    dt = datetime.datetime.fromisoformat(date1) if isinstance(date1, str) else date1
+    return dt.isoformat()
+
+# -------------------------------------------------------------------------------------------
+def format_event_date(date_str: str, event_type: str) -> str:
+    """Format event date with fallback rules."""
+    if date_str:
+        return format_date(date_str)
+    elif event_type == "<missing event>":
+        return "<missing event>"
+    else:
+        return "<still connected>"
+
+# -------------------------------------------------------------------------------------------
+def ensure_datetime(dt: Union[str, datetime.datetime, None]) -> datetime.datetime:
+    if dt is None:
+        return timeNowTZ()
+    if isinstance(dt, str):
+        return datetime.datetime.fromisoformat(dt)
+    return dt
+
+
+def parse_datetime(dt_str):
+    if not dt_str:
+        return None
+    try:
+        # Try ISO8601 first
+        return datetime.datetime.fromisoformat(dt_str)
+    except ValueError:
+        # Try RFC1123 / HTTP format
+        try:
+            return datetime.datetime.strptime(dt_str, '%a, %d %b %Y %H:%M:%S GMT')
+        except ValueError:
+            return None
+
+def format_date(date_str: str) -> str:
+    try:
+        dt = parse_datetime(date_str)
+        if dt.tzinfo is None:
+            # Set timezone if missing — change to timezone.utc if you prefer UTC
+            now = datetime.datetime.now(conf.tz)
+            dt = dt.replace(tzinfo=now.astimezone().tzinfo)
+        return dt.astimezone().isoformat()
+    except (ValueError, AttributeError, TypeError):
+        return "invalid"
+
+def format_date_diff(date1, date2, tz_name):
+    """
+    Return difference between two datetimes as 'Xd   HH:MM'.
+    Uses app timezone if datetime is naive.
+    date2 can be None (uses now).
+    """
+    # Get timezone from settings    
+    tz = pytz.timezone(tz_name)
+
+    def parse_dt(dt):
+        if dt is None:
+            return datetime.datetime.now(tz)
+        if isinstance(dt, str):
+            try:
+                dt_parsed = email.utils.parsedate_to_datetime(dt)
+            except (ValueError, TypeError):
+             # fallback: parse ISO string
+             dt_parsed = datetime.datetime.fromisoformat(dt)
+            # convert naive GMT/UTC to app timezone
+            if dt_parsed.tzinfo is None:
+                dt_parsed = tz.localize(dt_parsed)
+            else:
+                dt_parsed = dt_parsed.astimezone(tz)
+            return dt_parsed
+        return dt if dt.tzinfo else tz.localize(dt)
+
+    dt1 = parse_dt(date1)
+    dt2 = parse_dt(date2)
+
+    delta = dt2 - dt1
+    total_minutes = int(delta.total_seconds() // 60)
+    days, rem_minutes = divmod(total_minutes, 1440)  # 1440 mins in a day
+    hours, minutes = divmod(rem_minutes, 60)
+
+    return {
+        "text": f"{days}d {hours:02}:{minutes:02}",
+        "days": days,
+        "hours": hours,
+        "minutes": minutes,
+        "total_minutes": total_minutes
+    }

server/utils/plugin_utils.py

@@ -0,0 +1,399 @@
+import os
+import json
+
+import conf
+from logger import mylog
+from utils.crypto_utils import decrypt_data
+from const import pluginsPath, apiPath
+from helper import (
+    get_file_content,
+    get_setting_value,
+    setting_value_to_python_type,
+)
+
+module_name = "Plugin utils"
+
+
+# -------------------------------------------------------------------------------
+def logEventStatusCounts(objName, pluginEvents):
+    status_counts = {}  # Dictionary to store counts for each status
+
+    for event in pluginEvents:
+        status = event.status
+        if status in status_counts:
+            status_counts[status] += 1
+        else:
+            status_counts[status] = 1
+
+    for status, count in status_counts.items():
+        mylog(
+            "debug",
+            [
+                f'[{module_name}] In {objName} there are {count} events with the status "{status}" '
+            ],
+        )
+
+
+# -------------------------------------------------------------------------------
+def print_plugin_info(plugin, elements=["display_name"]):
+    mylog("verbose", [f"[{module_name}] ---------------------------------------------"])
+
+    for el in elements:
+        res = get_plugin_string(plugin, el)
+        mylog("verbose", [f"[{module_name}] ", el, ": ", res])
+
+
+# -------------------------------------------------------------------------------
+# Gets the whole setting object
+def get_plugin_setting_obj(plugin, function_key):
+    result = None
+
+    for set in plugin["settings"]:
+        if set["function"] == function_key:
+            result = set
+
+    # if result == None:
+    #     mylog('debug', [f'[{module_name}] Setting with "function":"', function_key, '" is missing in plugin: ', get_plugin_string(plugin, 'display_name')])
+
+    return result
+
+
+# -------------------------------------------------------------------------------
+# Gets the setting value for a plugin from the default JSON
+def get_plugin_setting_value(plugin, function_key):
+    result = None
+
+    for set in plugin["settings"]:
+        if set["function"] == function_key:
+            result = set
+
+    # if result == None:
+    #     mylog('debug', [f'[{module_name}] Setting with "function":"', function_key, '" is missing in plugin: ', get_plugin_string(plugin, 'display_name')])
+
+    return result
+
+
+# -------------------------------------------------------------------------------
+# Get localized string value on the top JSON depth, not recursive
+def get_plugin_string(props, el):
+    result = ""
+
+    if el in props["localized"]:
+        for val in props[el]:
+            if val["language_code"] == "en_us":
+                result = val["string"]
+
+        if result == "":
+            result = "en_us string missing"
+
+    else:
+        result = props[el]
+
+    return result
+
+
+# -------------------------------------------------------------------------------
+#  generates a comma separated list of values from a list (or a string representing a list)
+def list_to_csv(arr):
+    tmp = ""
+    arrayItemStr = ""
+
+    mylog("debug", f"[{module_name}] Flattening the below array")
+    mylog("debug", arr)
+    mylog(
+        "debug",
+        f"[{module_name}] isinstance(arr, list) : {isinstance(arr, list)} | isinstance(arr, str) : {isinstance(arr, str)}",
+    )
+
+    if isinstance(arr, str):
+        tmpStr = (
+            arr.replace("[", "").replace("]", "").replace("'", "")
+        )  # removing brackets and single quotes (not allowed)
+
+        if "," in tmpStr:
+            # Split the string into a list and trim whitespace
+            cleanedStr = [tmpSubStr.strip() for tmpSubStr in tmpStr.split(",")]
+
+            # Join the list elements using a comma
+            result_string = ",".join(cleanedStr)
+        else:
+            result_string = tmpStr
+
+        return result_string
+
+    elif isinstance(arr, list):
+        for arrayItem in arr:
+            # only one column flattening is supported
+            if isinstance(arrayItem, list):
+                arrayItemStr = (
+                    str(arrayItem[0]).replace("'", "").strip()
+                )  # removing single quotes - not allowed
+            else:
+                # is string already
+                arrayItemStr = arrayItem
+
+            tmp += f"{arrayItemStr},"
+
+        tmp = tmp[:-1]  # Remove last comma ','
+
+        mylog("debug", f"[{module_name}] Flattened array: {tmp}")
+
+        return tmp
+
+    else:
+        mylog("none", f"[{module_name}] ⚠ ERROR Could not convert array: {arr}")
+
+
+# -------------------------------------------------------------------------------
+# Combine plugin objects, keep user-defined values, created time, changed time if nothing changed and the index
+def combine_plugin_objects(old, new):
+    new.userData = old.userData
+    new.index = old.index
+    new.created = old.created
+
+    # Keep changed time if nothing changed
+    if new.status in ["watched-not-changed"]:
+        new.changed = old.changed
+
+    #  return the new object, with some of the old values
+    return new
+
+
+# -------------------------------------------------------------------------------
+# Replace {wildcars} with parameters
+def resolve_wildcards_arr(commandArr, params):
+    mylog("debug", [f"[{module_name}] Pre-Resolved CMD: "] + commandArr)
+
+    for param in params:
+        # mylog('debug', ['[Plugins] key     : {', param[0], '}'])
+        # mylog('debug', ['[Plugins] resolved: ', param[1]])
+
+        i = 0
+
+        for comPart in commandArr:
+            commandArr[i] = comPart.replace(
+                "{" + str(param[0]) + "}", str(param[1])
+            ).replace("{s-quote}", "'")
+
+            i += 1
+
+    return commandArr
+
+
+# -------------------------------------------------------------------------------
+# Function to extract layer number from "execution_order"
+def get_layer(plugin):
+    order = plugin.get("execution_order", "Layer_N")
+    if order == "Layer_N":
+        return float("inf")  # Treat as the last layer if "execution_order" is missing
+    return int(order.split("_")[1])
+
+
+# -------------------------------------------------------------------------------
+def get_plugins_configs(loadAll):
+    pluginsList = []  # Create an empty list to store plugin configurations
+    pluginsListSorted = []  # Sorted by "execution_order" : "Layer_0" first, Layer_N last
+
+    # Get a list of top-level directories in the specified pluginsPath
+    dirs = next(os.walk(pluginsPath))[1]
+
+    # Sort the directories list if needed
+    dirs.sort()  # This will sort the directories alphabetically
+
+    # Loop through each directory (plugin folder) in dirs
+    for d in dirs:
+        # Check if the directory name does not start with "__" to skip python cache
+        if not d.startswith("__"):
+            # Check if the 'ignore_plugin' file exists in the plugin folder
+            ignore_plugin_path = os.path.join(pluginsPath, d, "ignore_plugin")
+            if not os.path.isfile(ignore_plugin_path):
+                # Construct the path to the config.json file within the plugin folder
+                config_path = os.path.join(pluginsPath, d, "config.json")
+
+                try:
+                    plugJson = json.loads(get_file_content(config_path))
+
+                    # Only load plugin if needed
+                    # Fetch the list of enabled plugins from the config, default to an empty list if not set
+                    enabledPlugins = getattr(conf, "LOADED_PLUGINS", [])
+
+                    # Load all plugins if `loadAll` is True, the plugin is in the enabled list,
+                    # or no specific plugins are enabled (enabledPlugins is empty)
+                    if (
+                        loadAll
+                        or plugJson["unique_prefix"] in enabledPlugins
+                        or enabledPlugins == []
+                    ):
+                        # Load the contents of the config.json file as a JSON object and append it to pluginsList
+                        pluginsList.append(plugJson)
+
+                except (FileNotFoundError, json.JSONDecodeError):
+                    # Handle the case when the file is not found or JSON decoding fails
+                    mylog(
+                        "none",
+                        [
+                            f"[{module_name}] ⚠ ERROR - JSONDecodeError or FileNotFoundError for file {config_path}"
+                        ],
+                    )
+                except Exception as e:
+                    mylog(
+                        "none",
+                        [
+                            f"[{module_name}] ⚠ ERROR - Exception for file {config_path}: {str(e)}"
+                        ],
+                    )
+
+    # Sort pluginsList based on "execution_order"
+    pluginsListSorted = sorted(pluginsList, key=get_layer)
+
+    return pluginsListSorted  # Return the sorted list of plugin configurations
+
+
+# -------------------------------------------------------------------------------
+def custom_plugin_decoder(pluginDict):
+    return namedtuple("X", pluginDict.keys())(*pluginDict.values())
+
+
+# -------------------------------------------------------------------------------
+# Handle empty value
+def handle_empty(value):
+    if value == "" or value is None:
+        value = "null"
+
+    return value
+
+
+# -------------------------------------------------------------------------------
+# Get and return a plugin object based on key-value pairs
+# keyValues example: getPluginObject({"Plugin":"MQTT", "Watched_Value4":"someValue"})
+def getPluginObject(keyValues):
+    plugins_objects = apiPath + "table_plugins_objects.json"
+
+    try:
+        with open(plugins_objects, "r") as json_file:
+            data = json.load(json_file)
+
+            objectEntries = data.get("data", [])
+
+            for item in objectEntries:
+                # Initialize a flag to check if all key-value pairs match
+                all_match = True
+
+                for key, value in keyValues.items():
+                    if item.get(key) != value:
+                        all_match = False
+                        break  # No need to continue checking if one pair doesn't match
+
+                if all_match:
+                    return item
+
+            mylog(
+                "verbose",
+                [
+                    f"[{module_name}] 💬 INFO - Object not found {json.dumps(keyValues)} "
+                ],
+            )
+
+            return {}
+
+    except (FileNotFoundError, json.JSONDecodeError, ValueError):
+        # Handle the case when the file is not found, JSON decoding fails, or data is not in the expected format
+        mylog(
+            "verbose",
+            [
+                f"[{module_name}] ⚠ ERROR - JSONDecodeError or FileNotFoundError for file {plugins_objects}"
+            ],
+        )
+
+        return {}
+
+
+# ------------------------------------------------------------------
+# decode any encoded last_result files
+def decode_and_rename_files(file_dir, file_prefix):
+    """
+    Decodes and renames files in the specified directory if they are encrypted.
+    Returns a list of files to be processed and the Sync Hub Node name.
+    """
+    # Initialize the list of files to be processed and Sync Hub Node name
+    files_to_process = []
+
+    # key to decrypt data if SYNC loaded and key available
+    encryption_key = None
+    if "SYNC" in get_setting_value("LOADED_PLUGINS"):
+        encryption_key = get_setting_value("SYNC_encryption_key")
+
+    # Check for files starting with the specified prefix
+    matching_files = [f for f in os.listdir(file_dir) if f.startswith(file_prefix)]
+
+    for filename in matching_files:
+        # Create the full file path
+        file_path = os.path.join(file_dir, filename)
+
+        # Check if the file exists
+        if os.path.exists(file_path):
+            # Check if the file name contains "encoded"
+            if ".encoded." in filename and encryption_key:
+                # Decrypt the entire file
+                with open(file_path, "r+") as f:
+                    encrypted_data = f.read()
+                    decrypted_data = decrypt_data(encrypted_data, encryption_key)
+
+                    # Write the decrypted data back to the file
+                    f.seek(0)
+                    f.write(decrypted_data)
+                    f.truncate()
+
+                    # Rename the file e.g. from last_result.encoded.Node_1.1.log to last_result.decoded.Node_1.1.log
+                    new_filename = filename.replace(".encoded.", ".decoded.")
+                    os.rename(file_path, os.path.join(file_dir, new_filename))
+
+                    files_to_process.append(new_filename)
+
+            else:
+                files_to_process.append(filename)
+        else:
+            mylog("debug", [f"[Plugins] The file {file_path} does not exist"])
+
+    return files_to_process
+
+
+# ------------------------------------------------------------------
+# Retrieve the value for a plugin's setting, prioritizing user-defined values over defaults.
+def get_set_value_for_init(plugin, c_d, setting_key):
+    """
+    Retrieve the value for a plugin's setting, prioritizing user-defined values over defaults.
+
+    Args:
+        plugin (str): The name or identifier of the plugin.
+        pref (str): Prefix for user-defined settings (e.g., plugin identifier prefix).
+        c_d (dict): Dictionary containing user-defined settings.
+        setting_key (str): The key for the setting to fetch (default is 'RUN').
+
+    Returns:
+        Any: The value for the specified setting, converted to an appropriate Python type.
+    """
+
+    pref = plugin["unique_prefix"]
+
+    # Step 1: Initialize the setting value as an empty string
+    setting_value = ""
+
+    # Step 2: Get the default setting object for the plugin's specified key
+    setting_obj = get_plugin_setting_obj(plugin, setting_key)
+
+    if setting_obj is not None:
+        # Retrieve the type and default value from the setting object
+        set_type = setting_obj.get("type")  # Lowercase 'type'
+        set_value = setting_obj.get("default_value")
+
+        # Convert the value to the appropriate Python type
+        setting_value = setting_value_to_python_type(set_type, set_value)
+
+    # Step 3: Check for user-defined setting value in the dictionary
+    user_key = f"{pref}_{setting_key}"
+    if user_key in c_d:
+        setting_value = c_d[user_key]
+
+    # Return the final setting value
+    return setting_value