Merge pull request #9 from jlab/enable_study_creation

a novel mechanism to create mountpoint sub-directories in qiita's BASE_DIR

Author: sjanssen2

Images/nginx/nginx_qiita.conf

@@ -57,7 +57,7 @@ http {
 
           # CHANGE ME: This should match the WORKING_DIR in your qiita
           # config. E.g.,
-          alias /qiita/qiita_db/support_files/test_data/working_dir/;
+          alias /qiita_data/working_dir/;
         }
 
         # protected location
@@ -66,7 +66,7 @@ http {
 
             # CHANGE ME: This should match the BASE_DATA_DIR in your qiita
             # config. E.g.,
-            alias /qiita/qiita_db/support_files/test_data/;
+            alias /qiita_data/;
         }
 
         # enables communiction through websockets.

Images/plugin_collector/fix_test_db.py

@@ -9,14 +9,19 @@
 is_test = qiita_config['main']['TEST_ENVIRONMENT'].upper() == 'TRUE'
 print("qiita is in %s mode." % ('TEST' if is_test else 'PRODUCTIVE'))
 
-if is_test:
-    conn = psycopg2.connect(database=qiita_config['postgres']['DATABASE'],
-                            host=qiita_config['postgres']['HOST'],
-                            user=qiita_config['postgres']['ADMIN_USER'],
-                            password=qiita_config['postgres']['ADMIN_PASSWORD'],
-                            port=qiita_config['postgres']['PORT'])
-    cursor = conn.cursor()
+conn = psycopg2.connect(database=qiita_config['postgres']['DATABASE'],
+                        host=qiita_config['postgres']['HOST'],
+                        user=qiita_config['postgres']['ADMIN_USER'],
+                        password=qiita_config['postgres']['ADMIN_PASSWORD'],
+                        port=qiita_config['postgres']['PORT'])
+cursor = conn.cursor()
+
+# update conda env for qiita private plugins
+sql = "UPDATE qiita.software SET environment_script = 'source /opt/conda/etc/profile.d/conda.sh; conda activate /opt/conda/envs/qiita' WHERE description = 'Internal Qiita jobs';"
+cursor.execute(sql)
+conn.commit()
 
+if is_test:
     fps_plugin_configs = glob('/qiita_plugins/*.conf')
     print("Updating plugin credentials in dummy test DB with actual values from %i plugins." % len(fps_plugin_configs))
     for i, fp_plugin_config in enumerate(fps_plugin_configs):

Images/plugin_collector/startup_plugin_collector.sh

@@ -8,5 +8,7 @@ done
 # it seems to be necessary to give the plugin container some lead time
 # TODO: this might be more appropriately be addressed with healthchecks in the compose file
 sleep 3
+# create WORKING_DIR, UPLOAD_DATA_DIR and BASE_DATA_DIR in shared volume
+mkdir -p /qiita_data/working_dir/ /qiita_data/uploads/
 python3 /collect_configs.py
 python3 /fix_test_db.py

Images/qiita/config_portal.cfg

@@ -0,0 +1,43 @@
+# ---------- Base information for the website ----------
+[sitebase]
+# Logo should be 100px by 40px
+LOGO = /static/img/logo-clear.png
+# Full path to portal custom CSS styling file
+CSS_FP =
+TITLE = Qiita
+
+# ---------- Welcome text on index page ----------
+[index]
+HEADER = Qiita Spots Patterns
+TEXT = <p align="justify">
+  Qiita (<i>canonically pronounced cheetah</i>) is an entirely
+  <strong>open-source</strong> microbial study management platform. It allows
+  users to keep track of multiple studies with multiple ‘omics data.
+  Additionally, Qiita is capable of supporting multiple analytical pipelines
+  through a 3rd-party plugin system, allowing the user to have a single entry
+  point for all of their analyses.
+  </p>
+  <p align="justify">
+  Qiita provides database and compute resources to the global community,
+  alleviating the technical burdens that are typically limiting for
+  researchers studying microbial ecology (e.g. familiarity with the command
+  line or access to compute power).
+  </p>
+  <p align="justify">
+  Qiita’s platform allows for quick reanalysis of the datasets that have been
+  deposited using the latest analytical technologies. This means that Qiita’s
+  internal datasets are living data that is periodically re-annotated
+  according to current best practices.
+  </p>
+  <p align="justify">
+  For more information about how to use Qiita, visit the
+  <a href="static/doc/html/index.html">documentation</a>.
+  </p>
+  <p align="justify">
+  Note that you should be logged into the system to access any studies and
+  files available.
+  </p>
+
+# ---------- Study listing page ----------
+[study_list]
+EXAMPLE_SEARCH = env_matter = soil

Images/qiita/config_qiita_oidc.cfg

@@ -26,16 +26,16 @@ REQUIRE_APPROVAL = True
 BASE_URL = https://qiita-container-anna-nginx-1:8383
 
 # Download path files
-UPLOAD_DATA_DIR = /qiita/qiita_db/support_files/test_data/uploads/
+UPLOAD_DATA_DIR = /qiita_data/uploads/
 
 # Working directory path
-WORKING_DIR = /qiita/qiita_db/support_files/test_data/working_dir/
+WORKING_DIR = /qiita_data/working_dir/
 
 # Maximum upload size (in Gb)
 MAX_UPLOAD_SIZE = 100
 
 # Path to the base directory where the data files are going to be stored
-BASE_DATA_DIR = /qiita/qiita_db/support_files/test_data/
+BASE_DATA_DIR = /qiita_data/
 
 # Valid upload extension, comma separated. Empty for no uploads
 VALID_UPLOAD_EXTENSION = fastq,fastq.gz,txt,tsv,sff,fna,qual
@@ -181,7 +181,7 @@ PORTAL = QIITA
 PORTAL_DIR =
 
 # Full path to portal styling config file
-PORTAL_FP =
+PORTAL_FP = /qiita_configurations/config_portal.cfg
 
 # The center latitude of the world map, shown on the Stats map.
 # Defaults to 40.01027 (Boulder, CO, USA)

Images/qiita/qiita.dockerfile

@@ -62,7 +62,8 @@ RUN pip install -e qiita --no-binary redbiom
 
 # Copy Bash Script to run Qiita to the container. start_qiita differentiates between one "master" and multiple workers
 COPY start_qiita.sh .
-RUN chmod 755 start_qiita.sh
+COPY start_qiita-initDB.sh .
+RUN chmod 755 start_qiita.sh start_qiita-initDB.sh
 
 RUN apt-get install -y curl
 COPY start_plugin.py /start_plugin.py
@@ -71,4 +72,7 @@ RUN chmod a+x /start_plugin.py
 # hide certificate and server configuration copy from source code
 RUN rm -rf /qiita/qiita_core/support_files
 
+# hide default configurations from github sources
+RUN rm -f /qiita/qiita_pet/nginx_example.conf /qiita/qiita_pet/supervisor_example.conf /qiita/qiita_pet/support_files/config_portal.cfg
+
 # CMD ["conda", "run", "-n", "qiita"]

Images/qiita/start_qiita-initDB.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+CONDA_DIR=/opt/conda
+ENV_NAME=qiita
+#PORT=21174
+
+# We execute qiita-env make every time. We expect that it will fail always but the very first time, as the qiita DB should exist from then on
+source $CONDA_DIR/etc/profile.d/conda.sh; conda activate $CONDA_DIR/envs/$ENV_NAME; cd /qiita; qiita-env make --no-load-ontologies 2> .env-make.err || true
+# To avoid confusing the user, STDERR is written into a file and only reported if it does not contain the text that we expect to see if it just reports the existing DB
+grep 'already present on the system. You can drop it by running' .env-make.err > /dev/null || cat .env-make.err

Images/qiita/start_qiita.sh

@@ -10,10 +10,10 @@ if [ -n "${MASTER}" ] && [ ! -d /qiita/qiita_db/__pycache__ ]; then
 	source $CONDA_DIR/etc/profile.d/conda.sh; conda activate $CONDA_DIR/envs/$ENV_NAME; cd /qiita; pip install -e . --no-binary redbiom;
 fi
 
-# We execute qiita-env make every time. We expect that it will fail always but the very first time, as the qiita DB should exist from then on
-source $CONDA_DIR/etc/profile.d/conda.sh; conda activate $CONDA_DIR/envs/$ENV_NAME; cd /qiita; qiita-env make --no-load-ontologies 2> .env-make.err || true
-# To avoid confusing the user, STDERR is written into a file and only reported if it does not contain the text that we expect to see if it just reports the existing DB
-grep 'already present on the system. You can drop it by running' .env-make.err > /dev/null || cat .env-make.err
+# # We execute qiita-env make every time. We expect that it will fail always but the very first time, as the qiita DB should exist from then on
+# source $CONDA_DIR/etc/profile.d/conda.sh; conda activate $CONDA_DIR/envs/$ENV_NAME; cd /qiita; qiita-env make --no-load-ontologies 2> .env-make.err || true
+# # To avoid confusing the user, STDERR is written into a file and only reported if it does not contain the text that we expect to see if it just reports the existing DB
+# grep 'already present on the system. You can drop it by running' .env-make.err > /dev/null || cat .env-make.err
 
 # This was commented out bc it stopped working anymore and i was focusing on fixing something else, if you create the database for the first
 # time you will have to pick the appropriate options.

Makefile

@@ -84,7 +84,7 @@ plugin: Images/qtp-biom/trigger.py Certificates/
 	chmod a+rw ./logs/nginx_access.log ./logs/nginx_error.log
 	touch .built_image_nginx
 
-.built_image_qiita: Images/qiita/qiita.dockerfile Images/qiita/config_qiita_oidc.cfg Images/qiita/start_qiita.sh Images/qiita/supervisor_foreground.conf Images/qiita/start_plugin.py
+.built_image_qiita: Images/qiita/qiita.dockerfile Images/qiita/config_qiita_oidc.cfg Images/qiita/start_qiita.sh Images/qiita/start_qiita-initDB.sh Images/qiita/supervisor_foreground.conf Images/qiita/start_plugin.py Images/qiita/config_portal.cfg
 	test -d src/qiita || git clone -b auth_oidc https://github.com/jlab/qiita.git src/qiita
 	# remove configuration and certificate files from upstream qiita repo
 	rm -rf src/qiita/qiita_core/support_files