Fix debounce of api points to address Disk IO #914 + NMAPDEV_FAKE_MAC

server/__main__.py

@@ -105,7 +105,7 @@ def main ():
         pluginsState = check_and_run_user_event(db, all_plugins, pluginsState)
 
         # Update API endpoints              
-        update_api(db, all_plugins)
+        update_api(db, all_plugins, False)
 
         # proceed if 1 minute passed
         if conf.last_scan_run + datetime.timedelta(minutes=1) < conf.loop_start_time :

server/api.py

@@ -24,7 +24,7 @@ stop_event = threading.Event()  # Event to signal thread termination
 #===============================================================================
 # API
 #===============================================================================
-def update_api(db, all_plugins, updateOnlyDataSources=[], is_ad_hoc_user_event=False):
+def update_api(db, all_plugins, forceUpdate, updateOnlyDataSources=[], is_ad_hoc_user_event=False):
     mylog('debug', ['[API] Update API starting'])
 
     # Start periodic write if not running
@@ -57,7 +57,7 @@ def update_api(db, all_plugins, updateOnlyDataSources=[], is_ad_hoc_user_event=F
     # Save selected database tables
     for dsSQL in dataSourcesSQLs:
         if not updateOnlyDataSources or dsSQL[0] in updateOnlyDataSources:
-            api_endpoint_class(db, dsSQL[1], folder + 'table_' + dsSQL[0] + '.json', is_ad_hoc_user_event)
+            api_endpoint_class(db, forceUpdate, dsSQL[1], folder + 'table_' + dsSQL[0] + '.json', is_ad_hoc_user_event)
             
     # Start the GraphQL server
     graphql_port_value = get_setting_value("GRAPHQL_PORT")
@@ -76,7 +76,7 @@ def update_api(db, all_plugins, updateOnlyDataSources=[], is_ad_hoc_user_event=F
 
 #-------------------------------------------------------------------------------
 class api_endpoint_class:
-    def __init__(self, db, query, path, is_ad_hoc_user_event=False):        
+    def __init__(self, db, forceUpdate, query, path, is_ad_hoc_user_event=False):        
         global apiEndpoints
 
         current_time = timeNowTZ()
@@ -125,17 +125,17 @@ class api_endpoint_class:
             apiEndpoints.append(self)
 
         # Needs to be called for initial updates
-        self.try_write()
+        self.try_write(forceUpdate)
 
     #----------------------------------------
-    def try_write(self):
+    def try_write(self, forceUpdate):
         current_time = timeNowTZ()
 
         # Debugging info to understand the issue 
         # mylog('debug', [f'[API] api_endpoint_class: {self.fileName} is_ad_hoc_user_event {self.is_ad_hoc_user_event} last_update_time={self.last_update_time}, debounce time={self.last_update_time + datetime.timedelta(seconds=self.debounce_interval)}.'])
 
         # Only attempt to write if the debounce time has passed
-        if self.needsUpdate and (self.changeDetectedWhen is None or current_time > (self.changeDetectedWhen + datetime.timedelta(seconds=self.debounce_interval))):
+        if forceUpdate == True or (self.needsUpdate and (self.changeDetectedWhen is None or current_time > (self.changeDetectedWhen + datetime.timedelta(seconds=self.debounce_interval)))):
 
             mylog('debug', [f'[API] api_endpoint_class: Writing {self.fileName} after debounce.'])
 

server/initialise.py

@@ -332,7 +332,7 @@ def importConfigs (db, all_plugins):
     
     # setup execution schedules AFTER OVERRIDE handling
 
-    mylog('verbose', [f"[Config] c_d {c_d}"])
+    # mylog('verbose', [f"[Config] c_d {c_d}"])
 
     for plugin in all_plugins:        
         # Setup schedules
@@ -383,7 +383,7 @@ def importConfigs (db, all_plugins):
     db.commitDB()
 
     #  update only the settings datasource
-    update_api(db, all_plugins, ["settings"])  
+    update_api(db, all_plugins, True, ["settings"])  
     
     # run plugins that are modifying the config   
     run_plugin_scripts(db, all_plugins, 'before_config_save' )

server/plugin.py

@@ -467,7 +467,7 @@ def execute_plugin(db, all_plugins, plugin, pluginsState = plugins_state() ):
         pluginsState = process_plugin_events(db, plugin, pluginsState, sqlParams)
 
         # update API endpoints
-        update_api(db, all_plugins, ["plugins_events","plugins_objects", "plugins_history", "appevents"])  
+        update_api(db, all_plugins, False, ["plugins_events","plugins_objects", "plugins_history", "appevents"])  
     
     return pluginsState
 
@@ -873,7 +873,7 @@ def check_and_run_user_event(db, all_plugins, pluginsState):
             execution_log.finalize_event("run")               
         elif event == 'update_api':
             # async handling
-            update_api(db, all_plugins, param.split(','), True)
+            update_api(db, all_plugins, False, param.split(','), True)
             
         else:
             mylog('minimal', ['[check_and_run_user_event] WARNING: Unhandled event in execution queue: ', event, ' | ', param])