Merge pull request #3208 from antgonza/slurm

mv to slurm

Author: charles-cowart

.github/workflows/qiita-ci.yml

@@ -184,6 +184,8 @@ jobs:
           conda activate qiita
           export QIITA_SERVER_CERT=`pwd`/qiita_core/support_files/server.crt
           export QIITA_CONFIG_FP=`pwd`/qiita_core/support_files/config_test.cfg
+          # for testing we only need to have this set, not actually exist
+          export QIITA_JOB_SCHEDULER_EPILOGUE=`/path/to/epilogue/file`
           export REDBIOM_HOST="http://localhost:7379"
 
           nosetests $COVER_PACKAGE --with-doctest --with-coverage --with-timer -v --cover-package=${COVER_PACKAGE// / --cover-package=} -e 'test_submit_EBI_parse_EBI_reply_failure' -e 'test_full_submission'

qiita_core/configuration_manager.py

@@ -42,11 +42,11 @@ class ConfigurationManager(object):
         Max upload size
     valid_upload_extension : str
         The extensions that are valid to upload, comma separated
-    trq_owner : str
-        Email address of submitter of Torque jobs
-    trq_poll_val : int
-        Interval (in seconds) to wait between calls to Torque's qstat program
-    trq_dependency_q_cnt : int
+    job_scheduler_owner : str
+        Email address of submitter of jobs
+    job_scheduler_poll_val : int
+        Interval (in seconds) to wait between calls to job_scheduler program
+    job_scheduler_dependency_q_cnt : int
         Hard upper-limit on the number of an artifact's concurrent validation
         processes.
     user : str
@@ -145,7 +145,7 @@ def __init__(self):
 
         self._get_main(config)
         self._get_smtp(config)
-        self._get_torque(config)
+        self._get_job_scheduler(config)
         self._get_postgres(config)
         self._get_redis(config)
         self._get_ebi(config)
@@ -234,22 +234,22 @@ def _get_main(self, config):
             self.key_file = join(install_dir, 'qiita_core', 'support_files',
                                  'server.key')
 
-    def _get_torque(self, config):
-        """Get the configuration of the torque section"""
-        self.trq_owner = config.get('torque', 'TORQUE_JOB_OWNER')
-        self.trq_poll_val = int(config.get('torque', 'TORQUE_POLLING_VALUE'))
-        self.trq_dependency_q_cnt = config.get('torque',
-                                               'TORQUE_PROCESSING_QUEUE_COUNT')
-        self.trq_dependency_q_cnt = int(self.trq_dependency_q_cnt)
-
-        if not self.trq_owner:
-            self.trq_owner = None
-
-        if not self.trq_poll_val:
-            self.trq_poll_val = None
-
-        if not self.trq_dependency_q_cnt:
-            self.trq_dependency_q_cnt = None
+    def _get_job_scheduler(self, config):
+        """Get the configuration of the job_scheduler section"""
+        self.job_scheduler_owner = config.get(
+            'job_scheduler', 'JOB_SCHEDULER_JOB_OWNER', fallback=None)
+        self.job_scheduler_poll_val = config.get(
+            'job_scheduler', 'JOB_SCHEDULER_POLLING_VALUE', fallback=None)
+        self.job_scheduler_dependency_q_cnt = config.get(
+            'job_scheduler', 'JOB_SCHEDULER_PROCESSING_QUEUE_COUNT',
+            fallback=None)
+
+        if self.job_scheduler_poll_val is not None:
+            self.job_scheduler_poll_val = int(self.job_scheduler_poll_val)
+
+        if self.job_scheduler_dependency_q_cnt is not None:
+            self.job_scheduler_dependency_q_cnt = int(
+                self.job_scheduler_dependency_q_cnt)
 
     def _get_postgres(self, config):
         """Get the configuration of the postgres section"""

qiita_core/support_files/config_test.cfg

@@ -123,16 +123,16 @@ PASSWORD = postgres
 # The postgres password for the admin_user
 ADMIN_PASSWORD = postgres
 
-# ----------------------------- Torque settings -----------------------------
-[torque]
-# The email address of the submitter of Torque jobs
-TORQUE_JOB_OWNER = torque_user@somewhere.org
+# ----------------------------- Job Scheduler Settings -----------------------------
+[job_scheduler]
+# The email address of the submitter of jobs
+JOB_SCHEDULER_JOB_OWNER = user@somewhere.org
 
-# The number of seconds to wait between successive qstat calls
-TORQUE_POLLING_VALUE = 15
+# The number of seconds to wait between successive calls
+JOB_SCHEDULER__POLLING_VALUE = 15
 
 # Hard upper-limit on concurrently running validator jobs
-TORQUE_PROCESSING_QUEUE_COUNT = 2
+JOB_SCHEDULER_PROCESSING_QUEUE_COUNT = 2
 
 # ----------------------------- EBI settings -----------------------------
 [ebi]

qiita_core/tests/test_configuration_manager.py

@@ -59,10 +59,10 @@ def test_init(self):
         self.assertEqual(obs.cookie_secret, "SECRET")
         self.assertEqual(obs.key_file, "/tmp/server.key")
 
-        # Torque section
-        self.assertEqual(obs.trq_owner, "torque_user@somewhere.org")
-        self.assertEqual(obs.trq_poll_val, 15)
-        self.assertEqual(obs.trq_dependency_q_cnt, 2)
+        # job_scheduler section
+        self.assertEqual(obs.job_scheduler_owner, "user@somewhere.org")
+        self.assertEqual(obs.job_scheduler_poll_val, 15)
+        self.assertEqual(obs.job_scheduler_dependency_q_cnt, 2)
 
         # Postgres section
         self.assertEqual(obs.user, "postgres")
@@ -180,13 +180,13 @@ def test_get_main(self):
 
         self.assertEqual(obs.qiita_env, "")
 
-    def test_get_torque(self):
+    def test_get_job_scheduler(self):
         obs = ConfigurationManager()
 
-        conf_setter = partial(self.conf.set, 'torque')
-        conf_setter('TORQUE_JOB_OWNER', '')
-        obs._get_torque(self.conf)
-        self.assertIsNone(obs.trq_owner)
+        conf_setter = partial(self.conf.set, 'job_scheduler')
+        conf_setter('JOB_SCHEDULER_JOB_OWNER', '')
+        obs._get_job_scheduler(self.conf)
+        self.assertEqual('', obs.job_scheduler_owner)
 
     def test_get_postgres(self):
         obs = ConfigurationManager()
@@ -329,16 +329,16 @@ def test_get_portal(self):
 # The postgres password for the admin_user
 ADMIN_PASSWORD = thishastobesecure
 
-# ----------------------------- Torque settings -----------------------------
-[torque]
-# The email address of the submitter of Torque jobs
-TORQUE_JOB_OWNER = torque_user@somewhere.org
+# ------------------------- job_scheduler settings -------------------------
+[job_scheduler]
+# The email address of the submitter of jobs
+JOB_SCHEDULER_JOB_OWNER = user@somewhere.org
 
-# The number of seconds to wait between successive qstat calls
-TORQUE_POLLING_VALUE = 15
+# The number of seconds to wait between successive calls
+JOB_SCHEDULER_POLLING_VALUE = 15
 
 # Hard upper-limit on concurrently running validator jobs
-TORQUE_PROCESSING_QUEUE_COUNT = 2
+JOB_SCHEDULER_PROCESSING_QUEUE_COUNT = 2
 
 # ----------------------------- EBI settings -----------------------------
 [ebi]

qiita_db/environment_manager.py

@@ -203,16 +203,12 @@ def make_environment(load_ontologies, download_reference, add_demo_user):
 
             # Insert the settings values to the database
             sql = """INSERT INTO settings
-                     (test, base_data_dir, base_work_dir, trq_owner,
-                     trq_poll_val, trq_dependency_q_cnt)
-                     VALUES (%s, %s, %s, %s, %s, %s)"""
+                     (test, base_data_dir, base_work_dir)
+                     VALUES (%s, %s, %s)"""
             qdb.sql_connection.TRN.add(
                 sql, [test,
                       qiita_config.base_data_dir,
-                      qiita_config.working_dir,
-                      qiita_config.trq_owner,
-                      qiita_config.trq_poll_val,
-                      qiita_config.trq_dependency_q_cnt])
+                      qiita_config.working_dir])
             qdb.sql_connection.TRN.execute()
             create_layout(test=test, verbose=verbose)