From c53b76e2f81a54952ccb7db3db76cb52a578a4f7 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 12 Sep 2025 14:33:49 +0200 Subject: [PATCH 01/14] switch to https file transfer for qp-deblur --- Images/qp-deblur/qp-deblur.dockerfile | 4 ++-- compose.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Images/qp-deblur/qp-deblur.dockerfile b/Images/qp-deblur/qp-deblur.dockerfile index 067fe51..40b5ba4 100644 --- a/Images/qp-deblur/qp-deblur.dockerfile +++ b/Images/qp-deblur/qp-deblur.dockerfile @@ -1,4 +1,4 @@ -# VERSION: 2025.08.22 +# VERSION: 2025.09.12 # ========================== # Stage 1: Build wheels @@ -130,7 +130,7 @@ RUN cd qiita_client && pip install . RUN mkdir -p /qiita_server_certificates/ COPY qiita_server_certificates/*_server.* /qiita_server_certificates/ -RUN /usr/local/bin/configure_deblur --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` filesystem +RUN /usr/local/bin/configure_deblur --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` https RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qp-deblur/" /unshared_plugins/*.conf # remove conda command from tigger.py diff --git a/compose.yaml b/compose.yaml index 53a4b4d..bffee2b 100644 --- a/compose.yaml +++ b/compose.yaml @@ -281,7 +281,7 @@ services: # tty: true restart: no volumes: - - qiita-data:/qiita_data + #- qiita-data:/qiita_data - ./references/qiita_server_certificates:/qiita_server_certificates - ./references/qp-deblur:/opt/conda/envs/deblur/share/fragment-insertion/ref #- ./src/qtp-biom:/qtp-biom:U From d623b358d70852404146e10512dfc0d41dc40bc3 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sat, 13 Sep 2025 20:37:42 +0200 Subject: [PATCH 02/14] Update compose_github.yaml --- compose_github.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose_github.yaml b/compose_github.yaml index 443a6f9..b9417a6 100644 --- a/compose_github.yaml +++ b/compose_github.yaml @@ -261,7 +261,7 @@ services: command: ['./start_qp-deblur.sh'] restart: no volumes: - - qiita-data:/qiita_data + # - qiita-data:/qiita_data - ./references/qiita_server_certificates:/qiita_server_certificates - ./references/qp-deblur:/opt/conda/envs/deblur/share/fragment-insertion/ref - test_tmp_dir:/tmp @@ -349,4 +349,4 @@ volumes: server-plugin-configs: # a shared directory on qiita server side (master and workers) to collect qiita plugin configuration files qiita-logs: test_tmp_dir: # many of plugin tests are written under the assumption that they are executed on same machine as qiita main and thus share /tmp - which is not the case in our docker compose scenario! - \ No newline at end of file + From c490477b86c70af1dc6e0ef1749d9b8123907934 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 15 Sep 2025 07:47:55 +0200 Subject: [PATCH 03/14] add env https protocol --- Images/test_plugin.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Images/test_plugin.sh b/Images/test_plugin.sh index 2f03968..87d5210 100644 --- a/Images/test_plugin.sh +++ b/Images/test_plugin.sh @@ -43,6 +43,9 @@ done export QIITA_PORT=21174 export QIITA_ROOTCA_CERT=$SSL_CERT_FILE +# either (old) plugins ignore this at all OR adapted plugins switch to https file transfer +export QIITA_PLUGINCOUPING=https + # change into plugin source directory and execute actual tests if [ "qp-qiime2" == "$PLUGIN" ]; then source /opt/conda/etc/profile.d/conda.sh; conda activate /opt/conda/envs/qiime2; cd ${PLUGIN} && pytest; From 4e4f6b364535c856b3c4dd4f0f651b97710015e2 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 16 Sep 2025 12:27:01 +0200 Subject: [PATCH 04/14] export protocol env --- Images/test_plugin.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Images/test_plugin.sh b/Images/test_plugin.sh index 2f03968..87d5210 100644 --- a/Images/test_plugin.sh +++ b/Images/test_plugin.sh @@ -43,6 +43,9 @@ done export QIITA_PORT=21174 export QIITA_ROOTCA_CERT=$SSL_CERT_FILE +# either (old) plugins ignore this at all OR adapted plugins switch to https file transfer +export QIITA_PLUGINCOUPING=https + # change into plugin source directory and execute actual tests if [ "qp-qiime2" == "$PLUGIN" ]; then source /opt/conda/etc/profile.d/conda.sh; conda activate /opt/conda/envs/qiime2; cd ${PLUGIN} && pytest; From 0e1fb4b0378c28a179828d43e4cdb274bf61aae1 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 16 Sep 2025 13:05:16 +0200 Subject: [PATCH 05/14] fix typo in env name --- Images/test_plugin.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Images/test_plugin.sh b/Images/test_plugin.sh index 87d5210..586afb7 100644 --- a/Images/test_plugin.sh +++ b/Images/test_plugin.sh @@ -44,7 +44,7 @@ export QIITA_PORT=21174 export QIITA_ROOTCA_CERT=$SSL_CERT_FILE # either (old) plugins ignore this at all OR adapted plugins switch to https file transfer -export QIITA_PLUGINCOUPING=https +export QIITA_PLUGINCOUPLING=https # change into plugin source directory and execute actual tests if [ "qp-qiime2" == "$PLUGIN" ]; then From 68e9ad1190d9a7f2a2f3e907e5e186a7e51ea14d Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 16 Sep 2025 13:58:55 +0200 Subject: [PATCH 06/14] clone specific fork/branch for plugin --- .github/workflows/buildContainer.yaml | 5 +++-- Images/test_plugin.sh | 8 +++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/buildContainer.yaml b/.github/workflows/buildContainer.yaml index 737c167..e551329 100644 --- a/.github/workflows/buildContainer.yaml +++ b/.github/workflows/buildContainer.yaml @@ -104,7 +104,8 @@ jobs: - make_references strategy: matrix: - plugin: ["qp-deblur", "qtp-biom", "qtp-sequencing", "qtp-visualization", "qtp-diversity", "qp-target-gene", "qtp-job-output-folder"] + plugin: ["qp-deblur"] + #, "qtp-biom", "qtp-sequencing", "qtp-visualization", "qtp-diversity", "qp-target-gene", "qtp-job-output-folder"] runs-on: ubuntu-latest steps: - name: Checkout tinqiita repo @@ -162,7 +163,7 @@ jobs: - name: Execute tests in the running services run: | sleep 5 - docker compose exec ${{ matrix.plugin }} /bin/bash -c "bash /test_plugin.sh" + docker compose exec ${{ matrix.plugin }} /bin/bash -c "PLUGIN_FORK=jlab PLUGIN_BRANCH=uncouplePlugins bash /test_plugin.sh" - name: Push image to ghcr (only if tests passed) if: success() diff --git a/Images/test_plugin.sh b/Images/test_plugin.sh index 586afb7..82c31c3 100644 --- a/Images/test_plugin.sh +++ b/Images/test_plugin.sh @@ -2,6 +2,11 @@ echo "plugin to be tested is: '$PLUGIN'" +# by default, clone from master. But can also clone different branches +branch="${PLUGIN_BRANCH:-master}" +# by default, clone from qiita-spots branch. But can also use alternative forks +fork="${PLUGIN_FORK:-qiita-spots}" + # install dependencies apt-get update apt-get -y --fix-missing install git @@ -13,7 +18,8 @@ fi; if [ "qp-qiime2" != "$PLUGIN" ]; then # clone plugin repository - git clone https://github.com/qiita-spots/${PLUGIN} + git clone -b ${branch} https://github.com/${fork}/${PLUGIN}; + echo "Clone from '${fork}', branch '${branch}'" 1>&2; fi; # NOTE: client api reset only works when communicating with Qitta Master, From b34c0286958975d117147c757e51ca35fdc7b7c1 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 17 Sep 2025 10:08:03 +0200 Subject: [PATCH 07/14] pull changes in qiita (send file through tornado) and qiita_client (read plugincoupling env) --- Images/qiita/qiita.dockerfile | 9 +++++++-- Images/qp-deblur/qp-deblur.dockerfile | 3 ++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Images/qiita/qiita.dockerfile b/Images/qiita/qiita.dockerfile index 0a1ab71..2f7aab2 100644 --- a/Images/qiita/qiita.dockerfile +++ b/Images/qiita/qiita.dockerfile @@ -47,9 +47,14 @@ RUN pip install \ psycopg2-binary -# Clone the Qiita Repo +# Clone the Qiita Repo: currently we need the oidc changes from our jlab fork + changes in the tornado_FetchFileFromCentralHandler branch, which send files if requested directly from tornado instead of nginx (happens in testing) # RUN git clone -b master https://github.com/qiita-spots/qiita.git -RUN git clone -b auth_oidc https://github.com/jlab/qiita.git +RUN git clone -b auth_oidc https://github.com/jlab/qiita.git \ + && cd qiita \ + && git config pull.rebase false \ + && git config --global user.email "jlab@uni-giessen.de" \ + && git config --global user.name "Stefan" && \ + git pull origin tornado_FetchFileFromCentralHandler # should tests re-populate the DB, ensure private plugin, qtp-biom and qp-target-gene use the correct conda env RUN sed -i "s|'source /home/runner/.profile; conda activate qiita'|'source /opt/conda/etc/profile.d/conda.sh; conda activate /opt/conda/envs/qiita'|" /qiita/qiita_db/support_files/populate_test_db.sql diff --git a/Images/qp-deblur/qp-deblur.dockerfile b/Images/qp-deblur/qp-deblur.dockerfile index 40b5ba4..186e67e 100644 --- a/Images/qp-deblur/qp-deblur.dockerfile +++ b/Images/qp-deblur/qp-deblur.dockerfile @@ -37,7 +37,8 @@ SHELL ["conda", "run", "-p", "/opt/conda/envs/deblur", "/bin/bash", "-c"] ENV LC_ALL=C.UTF-8 ENV LANG=C.UTF-8 -RUN git clone -b master https://github.com/qiita-spots/qiita_client.git +# RUN git clone -b master https://github.com/qiita-spots/qiita_client.git +RUN git clone -b enable_pluginprotocol_change https://github.com/jlab/qiita_client.git RUN sed -i "s/f'Entered BaseQiitaPlugin._register_command({command.name})'/'Entered BaseQiitaPlugin._register_command(%s)' % command.name/" qiita_client/qiita_client/plugin.py RUN cd qiita_client && pip install --no-cache-dir . From d5639687acbe01442fe11b8bc3138a5b1edf776c Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 17 Sep 2025 10:09:34 +0200 Subject: [PATCH 08/14] reactivate build of all plugins --- .github/workflows/buildContainer.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/buildContainer.yaml b/.github/workflows/buildContainer.yaml index e551329..3e31b11 100644 --- a/.github/workflows/buildContainer.yaml +++ b/.github/workflows/buildContainer.yaml @@ -104,8 +104,7 @@ jobs: - make_references strategy: matrix: - plugin: ["qp-deblur"] - #, "qtp-biom", "qtp-sequencing", "qtp-visualization", "qtp-diversity", "qp-target-gene", "qtp-job-output-folder"] + plugin: ["qp-deblur", "qtp-biom", "qtp-sequencing", "qtp-visualization", "qtp-diversity", "qp-target-gene", "qtp-job-output-folder"] runs-on: ubuntu-latest steps: - name: Checkout tinqiita repo From 915e3268d400c1d93c5110489944a3b890caa317 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 17 Sep 2025 10:45:20 +0200 Subject: [PATCH 09/14] select fork / branch dependent of plugin name --- .github/workflows/buildContainer.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/buildContainer.yaml b/.github/workflows/buildContainer.yaml index 3e31b11..b7ee352 100644 --- a/.github/workflows/buildContainer.yaml +++ b/.github/workflows/buildContainer.yaml @@ -162,7 +162,9 @@ jobs: - name: Execute tests in the running services run: | sleep 5 - docker compose exec ${{ matrix.plugin }} /bin/bash -c "PLUGIN_FORK=jlab PLUGIN_BRANCH=uncouplePlugins bash /test_plugin.sh" + fork=`if [[ "${{ matrix.plugin }}" == "qp-deblur" ]]; then echo "jlab"; else echo "qiita-spots"; fi` + branch=`if [[ "${{ matrix.plugin }}" == "qp-deblur" ]]; then echo "uncouplePlugins"; else echo "master"; fi` + docker compose exec ${{ matrix.plugin }} /bin/bash -c "PLUGIN_FORK=$fork PLUGIN_BRANCH=$branch bash /test_plugin.sh" - name: Push image to ghcr (only if tests passed) if: success() From 1dc4787f42b113865b8a7c4c1f900b2cab24c83d Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 17 Sep 2025 11:04:00 +0200 Subject: [PATCH 10/14] qtp-job-output-folder's branch is names "main" not "master" --- .github/workflows/buildContainer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/buildContainer.yaml b/.github/workflows/buildContainer.yaml index b7ee352..71a49d9 100644 --- a/.github/workflows/buildContainer.yaml +++ b/.github/workflows/buildContainer.yaml @@ -163,7 +163,7 @@ jobs: run: | sleep 5 fork=`if [[ "${{ matrix.plugin }}" == "qp-deblur" ]]; then echo "jlab"; else echo "qiita-spots"; fi` - branch=`if [[ "${{ matrix.plugin }}" == "qp-deblur" ]]; then echo "uncouplePlugins"; else echo "master"; fi` + branch=`if [[ "${{ matrix.plugin }}" == "qp-deblur" ]]; then echo "uncouplePlugins"; elif [[ "${{ matrix.plugin }}" == "qtp-job-output-folder" ]]; then echo "main"; else echo "master"; fi` docker compose exec ${{ matrix.plugin }} /bin/bash -c "PLUGIN_FORK=$fork PLUGIN_BRANCH=$branch bash /test_plugin.sh" - name: Push image to ghcr (only if tests passed) From 2a51ad69dae3c97480013145d4a06629076f6747 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 18 Sep 2025 13:47:36 +0200 Subject: [PATCH 11/14] use qiita_client branch + change to https --- Images/qp-target-gene/qp-target-gene.dockerfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Images/qp-target-gene/qp-target-gene.dockerfile b/Images/qp-target-gene/qp-target-gene.dockerfile index dc3b9f7..5482552 100644 --- a/Images/qp-target-gene/qp-target-gene.dockerfile +++ b/Images/qp-target-gene/qp-target-gene.dockerfile @@ -1,4 +1,4 @@ -# VERSION: 2025.08.28 +# VERSION: 2025.09.17 # ========================== # Stage 1: Build wheels @@ -44,11 +44,13 @@ RUN python2.7 get-pip2.7.py --force-reinstall RUN pip install -U pip #RUN pip install https://github.com/qiita-spots/qiita_client/archive/master.zip -RUN git clone -b master https://github.com/qiita-spots/qiita_client.git +#RUN git clone -b master https://github.com/qiita-spots/qiita_client.git +RUN git clone -b enable_pluginprotocol_change https://github.com/jlab/qiita_client.git + RUN cd /qiita_client && pip install --no-cache-dir . RUN pip install https://github.com/qiita-spots/qiita-files/archive/master.zip -RUN git clone https://github.com/qiita-spots/qp-target-gene.git +RUN git clone -b uncouplePlugin https://github.com/jlab/qp-target-gene.git WORKDIR /qp-target-gene RUN pip install biom-format RUN pip install -e . @@ -127,7 +129,7 @@ RUN export QIITA_ROOTCA_CERT=/unshared_certificates/ci_rootca.crt COPY qiita_server_certificates/*_server.* /qiita_server_certificates/ RUN sed -i "s|^#\!.*|#\!/usr/bin/python2|" /usr/local/bin/configure_target_gene RUN sed -i "s|^#\!.*|#\!/usr/bin/python2|" /usr/local/bin/start_target_gene -RUN configure_target_gene --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` +RUN configure_target_gene --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` https RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qp-target-gene/" /unshared_plugins/*.conf # for testing From faf076c2d1ce8f68389524ac0cb0ae352d040bc0 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 18 Sep 2025 13:53:58 +0200 Subject: [PATCH 12/14] don't mount qiita_data to plugin --- compose_github.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose_github.yaml b/compose_github.yaml index b9417a6..6a8e3c9 100644 --- a/compose_github.yaml +++ b/compose_github.yaml @@ -215,7 +215,7 @@ services: # tty: true restart: no volumes: - - qiita-data:/qiita_data + #- qiita-data:/qiita_data - ./references/qiita_server_certificates:/qiita_server_certificates - test_tmp_dir:/tmp - ./references/qp-target-gene:/databases/gg/13_8/rep_set From 62f6079e8d0969013eaf337f380ef982a940bda7 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 18 Sep 2025 13:54:22 +0200 Subject: [PATCH 13/14] clone correct branches --- .github/workflows/buildContainer.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/buildContainer.yaml b/.github/workflows/buildContainer.yaml index 71a49d9..6a7065e 100644 --- a/.github/workflows/buildContainer.yaml +++ b/.github/workflows/buildContainer.yaml @@ -162,8 +162,8 @@ jobs: - name: Execute tests in the running services run: | sleep 5 - fork=`if [[ "${{ matrix.plugin }}" == "qp-deblur" ]]; then echo "jlab"; else echo "qiita-spots"; fi` - branch=`if [[ "${{ matrix.plugin }}" == "qp-deblur" ]]; then echo "uncouplePlugins"; elif [[ "${{ matrix.plugin }}" == "qtp-job-output-folder" ]]; then echo "main"; else echo "master"; fi` + fork=`if [[ "${{ matrix.plugin }}" == "qp-deblur" || "${{ matrix.plugin }}" == "qp-target-gene" ]]; then echo "jlab"; else echo "qiita-spots"; fi` + branch=`if [[ "${{ matrix.plugin }}" == "qp-deblur" ]]; then echo "uncouplePlugins"; elif [[ "${{ matrix.plugin }}" == "qp-target-gene" ]]; then echo "uncouplePlugin"; elif [[ "${{ matrix.plugin }}" == "qtp-job-output-folder" ]]; then echo "main"; else echo "master"; fi` docker compose exec ${{ matrix.plugin }} /bin/bash -c "PLUGIN_FORK=$fork PLUGIN_BRANCH=$branch bash /test_plugin.sh" - name: Push image to ghcr (only if tests passed) From 2518662fa16c26057675862b8c713563b608a592 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 19 Sep 2025 12:17:49 +0200 Subject: [PATCH 14/14] abuse to re-trigger tests --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ec93ee9..86a3d82 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -## Howto start-up qiita through docker compose +## Howto start-up qiita through docker compose Note: this does currently **not** work with podman :-( So strictly stick to docker here. 1. We assume you operate on your local computer, i.e. not within the BCF cluster as you won't have docker, on a Ubuntu/Mint like OS. You will need approx. 55 GB free disk space.