Test plugin filesystem coupling

.github/workflows/buildContainer.yaml

@@ -103,6 +103,7 @@ jobs:
       - build_main
       - make_references
     strategy:
+      fail-fast: false
       matrix:
         plugin: ["qp-deblur", "qtp-biom", "qtp-sequencing", "qtp-visualization", "qtp-diversity", "qp-target-gene", "qtp-job-output-folder"]
     runs-on: ubuntu-latest
@@ -162,7 +163,9 @@ jobs:
       - name: Execute tests in the running services
         run: |
           sleep 5
-          docker compose exec ${{ matrix.plugin }} /bin/bash -c "bash /test_plugin.sh"
+          fork=`if [[ "${{ matrix.plugin }}" == "qp-deblur" || "${{ matrix.plugin }}" == "qp-target-gene" || "${{ matrix.plugin }}" == "qtp-sequencing" || "${{ matrix.plugin }}" == "qtp-biom" || "${{ matrix.plugin }}" == "qtp-visualization" || "${{ matrix.plugin }}" == "qtp-diversity" || "${{ matrix.plugin }}" == "qtp-job-output-folder" ]]; then echo "jlab"; else echo "qiita-spots"; fi`
+          branch=`if [[ "${{ matrix.plugin }}" == "qp-target-gene" || "${{ matrix.plugin }}" == "qp-deblur" || "${{ matrix.plugin }}" == "qtp-sequencing" || "${{ matrix.plugin }}" == "qtp-biom" || "${{ matrix.plugin }}" == "qtp-visualization" || "${{ matrix.plugin }}" == "qtp-diversity" || "${{ matrix.plugin }}" == "qtp-job-output-folder" ]]; then echo "uncouple_clientpush"; else echo "master"; fi`
+          docker compose exec ${{ matrix.plugin }} /bin/bash -c "PLUGIN_FORK=$fork PLUGIN_BRANCH=$branch bash /test_plugin.sh"
 
       - name: Push image to ghcr (only if tests passed)
         if: success()
@@ -223,6 +226,12 @@ jobs:
           cache-from: type=gha,scope=tinqiita-${{ github.ref_name }}
           cache-to: type=gha,scope=tinqiita-${{ github.ref_name }},mode=max
 
+      - name: Prune build cache after build
+        run: |
+          df -h /
+          docker buildx prune -af
+          df -h /
+
       - name: adapt compose file to select specific plugin
         run: |
           if [[ "${{ matrix.multiplugin }}" == "qp-qiime2" ]]; then sed -i 's|- QIITA_PLUGINS="MATRIXPLUGIN:"|- QIITA_PLUGINS="${{ matrix.multiplugin }}:qtp-diversity:qtp-visualization:"|' compose_github.yaml; sed -i 's|MATRIXPLUGIN:|${{ matrix.multiplugin }}:\n        condition: service_started\n      qtp-diversity:\n        condition: service_started\n      qtp-visualization:|' compose_github.yaml; fi;
@@ -237,61 +246,63 @@ jobs:
       - name: Execute tests in the running services
         run: |
           sleep 5
-          docker compose exec ${{ matrix.multiplugin }} /bin/bash -c "bash /test_plugin.sh"
+          docker compose exec ${{ matrix.multiplugin }} /bin/bash -c "PLUGIN_FORK=jlab PLUGIN_BRANCH=uncouple_clientpush bash /test_plugin.sh"
 
       - name: Push image to ghcr (only if tests passed)
         if: success()
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          push: true
-          file: Images/${{ matrix.multiplugin }}/${{ matrix.multiplugin }}.dockerfile
-          tags: ghcr.io/${{ github.repository }}/${{ matrix.multiplugin }}:testcandidate
-
-  publish_images:
-    needs:
-      - build_plugins
-      - build_main
-      - build_mulit_plugins
-    strategy:
-      matrix:
-        image: ["qp-deblur", "qtp-biom", "qtp-sequencing", "qtp-visualization", "qtp-diversity", "qp-target-gene", "qtp-job-output-folder",
-                 "nginx", "qiita", "plugin_collector",
-                 "qp-qiime2"]
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout tinqiita repo
-        uses: actions/checkout@v4
-
-      - name: Log in to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Login to computational.bio registry
-        uses: docker/login-action@v3
-        with:
-          registry: harbor.computational.bio.uni-giessen.de
-          username: ${{ vars.HARBOR_CB_USERNAME }}
-          password: ${{ secrets.HARBOR_CB_SECRET }}
-
-      - name: Pull image from GHCR
-        run: docker pull ghcr.io/${{ github.repository }}/${{ matrix.image }}:testcandidate
-
-      - name: Read image version from dockerfile
-        id: vars
-        run: |
-          VERSION=$(head -n 1 Images/${{ matrix.image }}/${{ matrix.image }}.dockerfile | cut -d ":" -f 2- | tr -d " ")
-          echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV
-
-      - name: Retag image for Docker Hub
-        run: |
-          docker tag ghcr.io/${{ github.repository }}/${{ matrix.image }}:testcandidate harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.image }}:${{ env.IMAGE_TAG }}
-          docker tag ghcr.io/${{ github.repository }}/${{ matrix.image }}:testcandidate harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.image }}:latest
-
-      - name: Push image to Docker Hub
         run: |
-          docker push harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.image }}:${{ env.IMAGE_TAG }}
-          docker push harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.image }}:latest
+          docker push ghcr.io/${{ github.repository }}/${{ matrix.multiplugin }}:testcandidate
+        #uses: docker/build-push-action@v6
+        #with:
+        #  context: .
+        #  push: true
+        #  file: Images/${{ matrix.multiplugin }}/${{ matrix.multiplugin }}.dockerfile
+        #  tags: ghcr.io/${{ github.repository }}/${{ matrix.multiplugin }}:testcandidate
+
+  # publish_images:
+  #   needs:
+  #     - build_plugins
+  #     - build_main
+  #     - build_mulit_plugins
+  #   strategy:
+  #     matrix:
+  #       image: ["qp-deblur", "qtp-biom", "qtp-sequencing", "qtp-visualization", "qtp-diversity", "qp-target-gene", "qtp-job-output-folder",
+  #                "nginx", "qiita", "plugin_collector",
+  #                "qp-qiime2"]
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Checkout tinqiita repo
+  #       uses: actions/checkout@v4
+
+  #     - name: Log in to GitHub Container Registry
+  #       uses: docker/login-action@v3
+  #       with:
+  #         registry: ghcr.io
+  #         username: ${{ github.actor }}
+  #         password: ${{ secrets.GITHUB_TOKEN }}
+
+  #     - name: Login to computational.bio registry
+  #       uses: docker/login-action@v3
+  #       with:
+  #         registry: harbor.computational.bio.uni-giessen.de
+  #         username: ${{ vars.HARBOR_CB_USERNAME }}
+  #         password: ${{ secrets.HARBOR_CB_SECRET }}
+
+  #     - name: Pull image from GHCR
+  #       run: docker pull ghcr.io/${{ github.repository }}/${{ matrix.image }}:testcandidate
+
+  #     - name: Read image version from dockerfile
+  #       id: vars
+  #       run: |
+  #         VERSION=$(head -n 1 Images/${{ matrix.image }}/${{ matrix.image }}.dockerfile | cut -d ":" -f 2- | tr -d " ")
+  #         echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV
+
+  #     - name: Retag image for Docker Hub
+  #       run: |
+  #         docker tag ghcr.io/${{ github.repository }}/${{ matrix.image }}:testcandidate harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.image }}:${{ env.IMAGE_TAG }}
+  #         docker tag ghcr.io/${{ github.repository }}/${{ matrix.image }}:testcandidate harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.image }}:latest
+
+  #     - name: Push image to Docker Hub
+  #       run: |
+  #         docker push harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.image }}:${{ env.IMAGE_TAG }}
+  #         docker push harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.image }}:latest

Images/qiita/qiita.dockerfile

@@ -1,4 +1,4 @@
-# VERSION: 2025.09.09
+# VERSION: 2025.11.12
 
 FROM ubuntu:24.04
 
@@ -47,9 +47,14 @@ RUN pip install \
 	psycopg2-binary
 
 
-# Clone the Qiita Repo
+# Clone the Qiita Repo: currently we need the oidc changes from our jlab fork + changes in the tornado_FetchFileFromCentralHandler branch, which send files if requested directly from tornado instead of nginx (happens in testing)
 # RUN git clone -b master https://github.com/qiita-spots/qiita.git
-RUN git clone -b auth_oidc https://github.com/jlab/qiita.git
+RUN git clone -b auth_oidc https://github.com/jlab/qiita.git \
+	&& cd qiita \
+	&& git config pull.rebase false \
+	&& git config --global user.email "jlab@uni-giessen.de" \
+	&& git config --global user.name "Stefan" && \
+	git pull origin  tornado_FetchFileFromCentralHandler
 
 # should tests re-populate the DB, ensure private plugin, qtp-biom and qp-target-gene use the correct conda env
 RUN sed -i "s|'source /home/runner/.profile; conda activate qiita'|'source /opt/conda/etc/profile.d/conda.sh; conda activate /opt/conda/envs/qiita'|" /qiita/qiita_db/support_files/populate_test_db.sql

Images/qp-deblur/qp-deblur.dockerfile

@@ -1,4 +1,4 @@
-# VERSION: 2025.08.22
+# VERSION: 2025.11.14
 
 # ==========================
 # Stage 1: Build wheels
@@ -37,7 +37,8 @@ SHELL ["conda", "run", "-p", "/opt/conda/envs/deblur", "/bin/bash", "-c"]
 ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
 
-RUN git clone -b master https://github.com/qiita-spots/qiita_client.git
+# RUN git clone -b master https://github.com/qiita-spots/qiita_client.git
+RUN git clone -b refactor_exposeBaseDataDir https://github.com/jlab/qiita_client.git
 RUN sed -i "s/f'Entered BaseQiitaPlugin._register_command({command.name})'/'Entered BaseQiitaPlugin._register_command(%s)' % command.name/"  qiita_client/qiita_client/plugin.py
 RUN cd qiita_client && pip install --no-cache-dir .
 
@@ -48,7 +49,7 @@ RUN pip install scikit-bio==0.5.5
 
 RUN pip install -U pip pip-system-certs
 
-RUN git clone -b uncouplePlugins https://github.com/jlab/qp-deblur.git
+RUN git clone -b uncouple_clientpush  https://github.com/jlab/qp-deblur.git
 RUN cd qp-deblur && pip install .
 
 COPY requirements.txt ./requirements.txt
@@ -130,7 +131,7 @@ RUN cd qiita_client && pip install .
 
 RUN mkdir -p /qiita_server_certificates/
 COPY qiita_server_certificates/*_server.* /qiita_server_certificates/
-RUN /usr/local/bin/configure_deblur --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` filesystem
+RUN /usr/local/bin/configure_deblur --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f`
 RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qp-deblur/" /unshared_plugins/*.conf
 
 # remove conda command from tigger.py

Images/qp-qiime2/qp-qiime2.dockerfile

@@ -1,4 +1,4 @@
-# VERSION: 2025.09.12
+# VERSION: 2025.11.14
 
 FROM ubuntu:24.04
 
@@ -7,16 +7,20 @@ ARG MINIFORGE_VERSION=24.1.2-0
 ENV CONDA_DIR=/opt/conda
 ENV PATH=${CONDA_DIR}/bin:${PATH}
 
-RUN apt-get -y  update
-RUN apt-get -y --fix-missing install \
-	git \
-	wget \
-	libpq-dev \
-	python3-dev \
-	gcc \
-	build-essential \
-	zip \
-	tzdata
+RUN apt-get -y  update \
+	&& apt-get install -y --no-install-recommends \
+		git \
+		wget \
+		libpq-dev \
+		python3-dev \
+		gcc \
+		build-essential \
+		zip \
+		unzip \
+		tzdata \
+		ca-certificates \
+	&& apt-get clean \
+	&& rm -rf /var/lib/apt/lists/*
 
 # install miniforge3 for "conda"
 # see https://github.com/conda-forge/miniforge-images/blob/master/ubuntu/Dockerfile
@@ -28,46 +32,49 @@ RUN wget https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_
 	rm -f /tmp/miniforge3.sh
 
 # install tornado based trigger layer in base environment
-RUN pip install -U pip
-RUN pip install tornado
+RUN pip install -U pip \
+	&& pip install tornado
 COPY trigger.py /trigger.py
 
-# Download qiime2 yaml
-RUN wget --quiet https://data.qiime2.org/distro/core/qiime2-2023.5-py38-linux-conda.yml
-
-# Create conda env
-RUN conda env create --name qiime2 -y --file qiime2-2023.5-py38-linux-conda.yml \
-  && conda clean --all -y \
-  && rm -rf /opt/conda/pkgs
+# Download qiime2 yaml and  Create conda env
+RUN wget --quiet https://data.qiime2.org/distro/core/qiime2-2023.5-py38-linux-conda.yml \
+	&& conda env create --name qiime2 -y --file qiime2-2023.5-py38-linux-conda.yml \
+  	&& conda clean --all -y \
+  	&& rm -rf /opt/conda/pkgs
 # Make RUN commands use the new environment:
 # append --format docker to the build command, see https://github.com/containers/podman/issues/8477
 SHELL ["conda", "run", "-p", "/opt/conda/envs/qiime2", "/bin/bash", "-c"]
 
 RUN pip install -U pip
-RUN pip install https://github.com/qiita-spots/qiita_client/archive/master.zip
-RUN pip install https://github.com/qiita-spots/qiita-files/archive/master.zip
-RUN pip install https://github.com/biocore/q2-mislabeled/archive/refs/heads/main.zip
-RUN pip install q2-umap q2-greengenes2
-RUN git clone https://github.com/qiita-spots/qp-qiime2.git
+#RUN pip install https://github.com/qiita-spots/qiita_client/archive/master.zip
+RUN git clone --depth 1 -b refactor_exposeBaseDataDir https://github.com/jlab/qiita_client.git
+RUN cd qiita_client && pip install --no-cache-dir .
+
+RUN pip install https://github.com/qiita-spots/qiita-files/archive/master.zip \
+	&& pip install https://github.com/biocore/q2-mislabeled/archive/refs/heads/main.zip \
+	&& pip install q2-umap q2-greengenes2
+
+	#RUN git clone https://github.com/qiita-spots/qp-qiime2.git
+RUN git clone --depth 1 -b uncouple_clientpush  https://github.com/jlab/qp-qiime2.git
 WORKDIR /qp-qiime2
 
 RUN sed -i "s|self.basedir, '..', '..', '|'/|g" /qp-qiime2/qp_qiime2/tests/test_qiime2.py
 
-RUN pip install -e .
-RUN pip install --upgrade certifi
-RUN pip install pip-system-certs
+RUN pip install -e . \
+	&& pip install --upgrade certifi \
+	&& pip install pip-system-certs
 
 # configuring the databases available for QIIME 2
-RUN mkdir /databases
-RUN wget --quiet -O "/databases/gg-13-8-99-515-806-nb-classifier.qza" "https://data.qiime2.org/2021.4/common/gg-13-8-99-515-806-nb-classifier.qza"
-RUN export QP_QIIME2_DBS=/databases
+RUN mkdir /databases \
+	&& wget --no-check-certificate --quiet -O "/databases/gg-13-8-99-515-806-nb-classifier.qza" "https://data.qiime2.org/2021.4/common/gg-13-8-99-515-806-nb-classifier.qza" \
+	&& export QP_QIIME2_DBS=/databases
 
 # configuring the filtering QZAs available for QIIME 2
-RUN mkdir /filtering
-RUN wget -O /filtering/bloom-analyses.zip https://github.com/knightlab-analyses/bloom-analyses/archive/refs/heads/master.zip \
-  && unzip -j /filtering/bloom-analyses.zip bloom-analyses-master/data/qiime2-artifacts-for-qiita/*.qza -d /filtering/ \
-  && rm -f /filtering/bloom-analyses.zip
-RUN export QP_QIIME2_FILTER_QZA=/filtering/
+RUN mkdir /filtering \
+	&& wget --no-check-certificate -O /filtering/bloom-analyses.zip https://github.com/knightlab-analyses/bloom-analyses/archive/refs/heads/master.zip \
+  	&& unzip -j /filtering/bloom-analyses.zip bloom-analyses-master/data/qiime2-artifacts-for-qiita/*.qza -d /filtering/ \
+  	&& rm -f /filtering/bloom-analyses.zip \
+	&& export QP_QIIME2_FILTER_QZA=/filtering/
 
 # TODO: should the plugin get the server configuration?!
 RUN export QIITA_CONFIG_FP=/qiita/config_qiita_oidc.cfg
@@ -80,8 +87,8 @@ ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
 
 # configure timezone
-RUN ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime
-RUN dpkg-reconfigure -f noninteractive tzdata
+RUN ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime \
+	&& dpkg-reconfigure -f noninteractive tzdata
 
 
 WORKDIR /
@@ -102,8 +109,8 @@ RUN chmod u+x /qp-qiime2/scripts/configure_qiime2 /qp-qiime2/scripts/start_qiime
 ENV QP_QIIME2_DBS=/databases
 ENV QP_QIIME2_FILTER_QZA=/filtering/
 COPY qiita_server_certificates/*_server.* /qiita_server_certificates/
-RUN /qp-qiime2/scripts/configure_qiime2 --env-script 'true' --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f`
-RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qp-qiime2/" /unshared_plugins/*.conf
+RUN /qp-qiime2/scripts/configure_qiime2 --env-script 'true' --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` \
+	&& sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qp-qiime2/" /unshared_plugins/*.conf
 
 # for testing
 COPY test_plugin.sh /test_plugin.sh

Images/qp-target-gene/qp-target-gene.dockerfile

@@ -1,4 +1,4 @@
-# VERSION: 2025.08.28
+# VERSION: 2025.09.17
 
 # ==========================
 # Stage 1: Build wheels
@@ -44,11 +44,13 @@ RUN python2.7 get-pip2.7.py --force-reinstall
 
 RUN pip install -U pip
 #RUN pip install https://github.com/qiita-spots/qiita_client/archive/master.zip
-RUN git clone -b master https://github.com/qiita-spots/qiita_client.git
+#RUN git clone -b master https://github.com/qiita-spots/qiita_client.git
+RUN git clone -b refactor_exposeBaseDataDir  https://github.com/jlab/qiita_client.git
+
 RUN cd /qiita_client && pip install --no-cache-dir .
 
 RUN pip install https://github.com/qiita-spots/qiita-files/archive/master.zip
-RUN git clone https://github.com/qiita-spots/qp-target-gene.git
+RUN git clone -b uncouple_clientpush  https://github.com/jlab/qp-target-gene.git
 WORKDIR /qp-target-gene
 RUN pip install biom-format
 RUN pip install -e .

Images/qtp-biom/qtp-biom.dockerfile

@@ -1,4 +1,4 @@
-# VERSION: 2025.08.29
+# VERSION: 2025.11.13
 
 FROM ubuntu:24.04 AS builder
 
@@ -57,14 +57,17 @@ SHELL ["conda", "run", "-p", "/opt/conda/envs/qtp-biom", "/bin/bash", "-c"]
 
 RUN pip install -U pip
 # RUN pip install https://github.com/qiita-spots/qiita_client/archive/master.zip
-RUN git clone -b master https://github.com/qiita-spots/qiita_client.git
+# RUN git clone -b master https://github.com/qiita-spots/qiita_client.git
+RUN git clone -b refactor_exposeBaseDataDir https://github.com/jlab/qiita_client.git
 RUN cd qiita_client && pip install --no-cache-dir .
 
 # RUN pip install https://github.com/qiita-spots/qiita-files/archive/master.zip
 RUN git clone -b master https://github.com/qiita-spots/qiita-files.git
 # COPY ./qiita-files /qiita-files
 RUN cd /qiita-files && pip install -e . -v
-RUN git clone https://github.com/qiita-spots/qtp-biom.git
+#RUN git clone https://github.com/qiita-spots/qtp-biom.git
+RUN git clone -b uncouple_clientpush  https://github.com/jlab/qtp-biom.git
+
 # COPY ./qtp-biom /qtp-biom
 WORKDIR /qtp-biom
 RUN sed -i "s|'qiita-files @ https://github.com/qiita-spots/'||" setup.py