Updates to @antgonza comments

Author: Gossty

qiita_db/meta_util.py

@@ -555,58 +555,42 @@ def generate_plugin_releases():
         f(redis_key, v)
 
 
-def _build_software_commands_col_names_object(active):
-    '''
-    Helper function for update_resource_allocation_redis
+def update_resource_allocation_redis(active=True, verbose=False):
+    """Updates redis with plots and information about current software.
 
-    Constructs a nested dictionary containing software commands and associated
-    column names from the database.
-    '''
+    Parameters
+    ----------
+    active: boolean, optional
+        Defaults to True. Should only be False when testing.
 
-    col_name_list = []
-    with qdb.sql_connection.TRN:
-        sql = ''' SELECT * FROM qiita.resource_allocation_column_names; '''
-        qdb.sql_connection.TRN.add(sql)
-        res = qdb.sql_connection.TRN.execute_fetchindex()
+    verbose: boolean, optional
+        Defaults to False. Prints status on what functin
 
-    for col_name in res:
-        col_name_list.append(col_name[1])
+    """
+    time = datetime.now().strftime('%m-%d-%y')
 
-    software_list = [s for s in qdb.software.Software.iter(active=active)]
-    software_commands = defaultdict(lambda: defaultdict(
-        lambda: defaultdict(list)))
+    # Retreave available col_name for commands
+    with qdb.sql_connection.TRN:
+        sql = 'SELECT col_name FROM qiita.resource_allocation_column_names;'
+        qdb.sql_connection.TRN.add(sql)
+        col_names_list = qdb.sql_connection.TRN.execute_fetchflatten()
 
+    # Retreave available software
+    software_list = list(qdb.software.Software.iter(active=active))
+    scommands = {}
     for software in software_list:
         sname = software.name
         sversion = software.version
 
-        for command in software.commands:
-            cmd_name = command.name
-            for col in col_name_list:
-                software_commands[sname][sversion][cmd_name].append(col)
-
-    final_obj = {
-        sname: {
-            sversion: dict(commands)
-            for sversion, commands in dict(versions).items()
-        }
-        for sname, versions in dict(software_commands).items()
-    }
-    return final_obj
-
-
-def update_resource_allocation_redis(active=True):
-    """Updates redis with plots and information about current software.
-
-    Parameters
-    ----------
-    active: boolean, optional
-        Defaults to True. Should only be False when testing.
+        if sname not in scommands:
+            scommands[sname] = {}
 
-    """
-    time = datetime.now().strftime('%m-%d-%y')
+        if sversion not in scommands[sname]:
+            scommands[sname][sversion] = {}
 
-    scommands = _build_software_commands_col_names_object(active)
+        for command in software.commands:
+            cmd_name = command.name
+            scommands[sname][sversion][cmd_name] = col_names_list
 
     redis_key = 'resources:commands'
     r_client.set(redis_key, str(scommands))
@@ -615,38 +599,41 @@ def update_resource_allocation_redis(active=True):
         for version, commands in versions.items():
             for cname, col_name_list in commands.items():
                 df = retrieve_resource_data(cname, sname, version, COLUMNS)
-                print(("Retrieving allocation resources for " +
-                       f" software: {sname}" +
-                       f" version: {version}" +
-                       f" command: {cname}"))
-                if len(df) == 0:
-                    print(("No allocation resources available for" +
+                if verbose:
+                    print(("Retrieving allocation resources for " +
                            f" software: {sname}" +
                            f" version: {version}" +
                            f" command: {cname}"))
+                if len(df) == 0:
+                    if verbose:
+                        print(("No allocation resources available for" +
+                               f" software: {sname}" +
+                               f" version: {version}" +
+                               f" command: {cname}"))
                     continue
                 # column_name_str looks like col1*col2*col3, etc
                 for col_name_str in col_name_list:
-                    df_copy = df.copy()
                     new_column = None
                     col_name_split = col_name_str.split('*')
-                    df_copy.dropna(subset=col_name_split, inplace=True)
+                    df_copy = df.dropna(subset=col_name_split)
 
                     # Create a column with the desired columns
                     for curr_column in col_name_split:
                         if new_column is None:
                             new_column = df_copy[curr_column]
                         else:
                             new_column *= df_copy[curr_column]
-                    print(("Building resource allocation plot for " +
-                           f" software: {sname}" +
-                           f" version: {version}" +
-                           f" command: {cname}" +
-                           f" column name: {col_name_str}"))
+                    if verbose:
+                        print(("Building resource allocation plot for " +
+                               f" software: {sname}" +
+                               f" version: {version}" +
+                               f" command: {cname}" +
+                               f" column name: {col_name_str}"))
 
                     fig, axs = resource_allocation_plot(df_copy,
                                                         col_name_str,
-                                                        new_column)
+                                                        new_column,
+                                                        verbose=verbose)
                     titles = [0, 0]
                     images = [0, 0]
 
@@ -697,11 +684,12 @@ def update_resource_allocation_redis(active=True):
                         ("title_mem", titles[0], r_client.set),
                         ("title_time", titles[1], r_client.set)
                     ]
-                    print(("Saving resource allocation image for " +
-                           f" software: {sname}" +
-                           f" version: {version}" +
-                           f" command: {cname}" +
-                           f" column name: {col_name_str}"))
+                    if verbose:
+                        print(("Saving resource allocation image for " +
+                               f" software: {sname}" +
+                               f" version: {version}" +
+                               f" command: {cname}" +
+                               f" column name: {col_name_str}"))
 
                     for k, v, f in values:
                         redis_key = 'resources$#%s$#%s$#%s$#%s:%s' % (

qiita_db/util.py

@@ -2326,7 +2326,7 @@ def send_email(to, subject, body):
         smtp.close()
 
 
-def resource_allocation_plot(df, col_name_str, curr_column):
+def resource_allocation_plot(df, col_name_str, curr_column, verbose=False):
     """Builds resource allocation plot for given filename and jobs
 
     Parameters
@@ -2350,22 +2350,27 @@ def resource_allocation_plot(df, col_name_str, curr_column):
     df[col_name_str] = curr_column
 
     # models for memory
-    print("\tCalculating best model for memory")
+    if verbose:
+        print("\tCalculating best model for memory")
     _resource_allocation_plot_helper(
-        df, ax, "MaxRSSRaw",  mem_models, col_name_str)
+        df, ax, "MaxRSSRaw",  mem_models, col_name_str, verbose=verbose)
     ax = axs[1]
 
     # models for time
-    print("\tCalculating best model for time")
+    if verbose:
+        print("\tCalculating best model for time")
     _resource_allocation_plot_helper(
-        df, ax, "ElapsedRaw",  time_models, col_name_str)
+        df, ax, "ElapsedRaw",  time_models, col_name_str, verbose=verbose)
     return fig, axs
 
 
 def _retrieve_equations():
     '''
     Helper function for resource_allocation_plot.
     Retrieves equations from db. Creates dictionary for memory and time models.
+    This function is needed because it utilizes np as a part of eval() below.
+    In test_util.py we need to retrieve equations without importing np to
+    comply with PEP8 styling standard.
 
     Returns
     -------
@@ -2379,19 +2384,20 @@ def _retrieve_equations():
     time_models = {}
     res = []
     with qdb.sql_connection.TRN:
-        sql = ''' SELECT * FROM qiita.resource_allocation_equations; '''
+        sql = '''SELECT equation_name, expression
+                 FROM qiita.resource_allocation_equations;'''
         qdb.sql_connection.TRN.add(sql)
         res = qdb.sql_connection.TRN.execute_fetchindex()
     for models in res:
-        if 'mem' in models[1]:
-            memory_models[models[1]] = {
-                "equation_name": models[2],
-                "equation": lambda x, k, a, b: eval(models[2])
+        if 'mem' in models[0]:
+            memory_models[models[0]] = {
+                "equation_name": models[1],
+                "equation": lambda x, k, a, b: eval(models[1])
             }
         else:
-            time_models[models[1]] = {
-                "equation_name": models[2],
-                "equation": lambda x, k, a, b: eval(models[2])
+            time_models[models[0]] = {
+                "equation_name": models[1],
+                "equation": lambda x, k, a, b: eval(models[1])
             }
     return (memory_models, time_models)
 
@@ -2452,7 +2458,7 @@ def retrieve_resource_data(cname, sname, version, columns):
 
 
 def _resource_allocation_plot_helper(
-        df, ax, curr, models, col_name):
+        df, ax, curr, models, col_name, verbose=False):
     """Helper function for resource allocation plot. Builds plot for MaxRSSRaw
     and ElapsedRaw
 
@@ -2517,15 +2523,14 @@ def _resource_allocation_plot_helper(
     ax.set_ylabel(curr)
     ax.set_xlabel(col_name)
 
-    print(f"\t\tFitting best model for {curr}; column {col_name}")
+    if verbose:
+        print(f"\t\tFitting best model for {curr}; column {col_name}")
     # 50 - number of maximum iterations, 3 - number of failures we tolerate
     best_model_name, best_model, options = _resource_allocation_calculate(
         df, x_data, y_data, models, curr, col_name, 50, 3)
-    if options is None:
-        print(df)
-        print(x_data)
-        print(y_data)
-    print(f"\t\tSuccessfully chose best model for {curr}; column {col_name}")
+    if verbose:
+        print(
+            f"\t\tSuccessfully chose best model for {curr}; column {col_name}")
     k, a, b = options.x
     x_plot = np.array(sorted(df[col_name].unique()))
     y_plot = best_model(x_plot, k, a, b)