Uncouple clientpush

.github/workflows/buildContainer.yaml

@@ -163,7 +163,9 @@ jobs:
       - name: Execute tests in the running services
         run: |
           sleep 5
-          docker compose exec ${{ matrix.plugin }} /bin/bash -c "bash /test_plugin.sh"
+          fork=`if [[ "${{ matrix.plugin }}" == "qp-deblur" || "${{ matrix.plugin }}" == "qp-target-gene" || "${{ matrix.plugin }}" == "qtp-sequencing" || "${{ matrix.plugin }}" == "qtp-biom" || "${{ matrix.plugin }}" == "qtp-visualization" || "${{ matrix.plugin }}" == "qtp-diversity" || "${{ matrix.plugin }}" == "qtp-job-output-folder" ]]; then echo "jlab"; else echo "qiita-spots"; fi`
+          branch=`if [[ "${{ matrix.plugin }}" == "qp-target-gene" || "${{ matrix.plugin }}" == "qp-deblur" || "${{ matrix.plugin }}" == "qtp-sequencing" || "${{ matrix.plugin }}" == "qtp-biom" || "${{ matrix.plugin }}" == "qtp-visualization" || "${{ matrix.plugin }}" == "qtp-diversity" || "${{ matrix.plugin }}" == "qtp-job-output-folder" ]]; then echo "uncouple_clientpush"; else echo "master"; fi`
+          docker compose exec ${{ matrix.plugin }} /bin/bash -c "PLUGIN_FORK=$fork PLUGIN_BRANCH=$branch bash /test_plugin.sh"
 
       - name: Push image to ghcr (only if tests passed)
         if: success()
@@ -244,7 +246,7 @@ jobs:
       - name: Execute tests in the running services
         run: |
           sleep 5
-          docker compose exec ${{ matrix.multiplugin }} /bin/bash -c "bash /test_plugin.sh"
+          docker compose exec ${{ matrix.multiplugin }} /bin/bash -c "PLUGIN_FORK=jlab PLUGIN_BRANCH=uncouple_clientpush bash /test_plugin.sh"
 
       - name: Push image to ghcr (only if tests passed)
         if: success()

Images/qiita/qiita.dockerfile

@@ -1,4 +1,4 @@
-# VERSION: 2025.09.09
+# VERSION: 2025.11.20
 
 FROM ubuntu:24.04
 
@@ -47,9 +47,14 @@ RUN pip install \
 	psycopg2-binary
 
 
-# Clone the Qiita Repo
+# Clone the Qiita Repo: currently we need the oidc changes from our jlab fork + changes in the tornado_FetchFileFromCentralHandler branch, which send files if requested directly from tornado instead of nginx (happens in testing)
 # RUN git clone -b master https://github.com/qiita-spots/qiita.git
-RUN git clone -b auth_oidc https://github.com/jlab/qiita.git
+RUN git clone -b auth_oidc https://github.com/jlab/qiita.git \
+	&& cd qiita \
+	&& git config pull.rebase false \
+	&& git config --global user.email "jlab@uni-giessen.de" \
+	&& git config --global user.name "Stefan" && \
+	git pull origin  tornado_FetchFileFromCentralHandler
 
 # should tests re-populate the DB, ensure private plugin, qtp-biom and qp-target-gene use the correct conda env
 RUN sed -i "s|'source /home/runner/.profile; conda activate qiita'|'source /opt/conda/etc/profile.d/conda.sh; conda activate /opt/conda/envs/qiita'|" /qiita/qiita_db/support_files/populate_test_db.sql

Images/qp-deblur/qp-deblur.dockerfile

@@ -1,4 +1,4 @@
-# VERSION: 2025.08.22
+# VERSION: 2025.11.20
 
 # ==========================
 # Stage 1: Build wheels
@@ -37,7 +37,8 @@ SHELL ["conda", "run", "-p", "/opt/conda/envs/deblur", "/bin/bash", "-c"]
 ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
 
-RUN git clone -b master https://github.com/qiita-spots/qiita_client.git
+# RUN git clone -b master https://github.com/qiita-spots/qiita_client.git
+RUN git clone -b refactor_exposeBaseDataDir https://github.com/jlab/qiita_client.git
 RUN sed -i "s/f'Entered BaseQiitaPlugin._register_command({command.name})'/'Entered BaseQiitaPlugin._register_command(%s)' % command.name/"  qiita_client/qiita_client/plugin.py
 RUN cd qiita_client && pip install --no-cache-dir .
 
@@ -48,7 +49,7 @@ RUN pip install scikit-bio==0.5.5
 
 RUN pip install -U pip pip-system-certs
 
-RUN git clone -b uncouplePlugins https://github.com/jlab/qp-deblur.git
+RUN git clone -b uncouple_clientpush  https://github.com/jlab/qp-deblur.git
 RUN cd qp-deblur && pip install .
 
 COPY requirements.txt ./requirements.txt
@@ -130,7 +131,7 @@ RUN cd qiita_client && pip install .
 
 RUN mkdir -p /qiita_server_certificates/
 COPY qiita_server_certificates/*_server.* /qiita_server_certificates/
-RUN /usr/local/bin/configure_deblur --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` filesystem
+RUN /usr/local/bin/configure_deblur --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` https
 RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qp-deblur/" /unshared_plugins/*.conf
 
 # remove conda command from tigger.py

Images/qp-qiime2/determine_spare_conda_dependencies.ipynb

@@ -0,0 +1,291 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 293,
+   "id": "a3a121f0-fe2d-43c3-9de2-27c954f744a2",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 23/23 [00:04<00:00,  4.81it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "import yaml\n",
+    "from tempfile import mkdtemp\n",
+    "from os.path import join, basename\n",
+    "from jinja2 import Template\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "#fp_workdir = mkdtemp()\n",
+    "\n",
+    "# this list of qiime2 plugins to be NOT installed is a result of:\n",
+    "# 1) create a full qiime2 environment with installed qp-qiime2\n",
+    "# 2) ipython\n",
+    "#    a) the same functions as in qp-qiime2/__init__.py\n",
+    "#    b) modify function qp-qiime2/util.py::register_qiime2_commands in a way that it stores plugins+functions in a dict, which\n",
+    "#       would have been registered via: plugin.register_command(qiime_cmd), line 328\n",
+    "#    c) subtract these white-listed plugins from what is listed if \"qiime\" is executed\n",
+    "blacklist = [\n",
+    "    'alignment',\n",
+    "    #'composition',\n",
+    "    'cutadapt',\n",
+    "    'dada2',\n",
+    "    'deblur',\n",
+    "    'demux',\n",
+    "    'fragment-insertion',\n",
+    "    'gneiss',\n",
+    "    'greengenes2',\n",
+    "    'metadata',\n",
+    "    'quality-control',\n",
+    "    'quality-filter',\n",
+    "    'sourcetracker2',\n",
+    "    'vsearch',\n",
+    "]\n",
+    "\n",
+    "# dependencies most likely needed, but flagges by this naive mechanism as superfluous.\n",
+    "# Note that we do NOT follow dependencies of dependencies of plugins!\n",
+    "whitelist = [\n",
+    "    'click',\n",
+    "    'decorator',\n",
+    "    'yaml',\n",
+    "    'pyyaml',\n",
+    "    'ca-certificates',\n",
+    "    'certifi',\n",
+    "    'openssl'\n",
+    "    'libgcc',   \n",
+    "    'libgfortran',\n",
+    "    'libllvm15',  \n",
+    "    'tomlkit',\n",
+    "    'formulaic',\n",
+    "    'typing-extensions',\n",
+    "    'interface_meta',\n",
+    "    'wrapt',  \n",
+    "    'altair',\n",
+    "    'narwhals',\n",
+    "    'attrs',\n",
+    "    'zipp',\n",
+    "    'importlib-metadata',\n",
+    "    'rpds-py',\n",
+    "    'referencing',\n",
+    "    'jsonschema-specifications',\n",
+    "    'jsonschema',\n",
+    "    'nose',\n",
+    "]\n",
+    "\n",
+    "release = \"2023.5\"  # this is the used Qiime2 release\n",
+    "\n",
+    "# download full dependency list of official qiime2 release\n",
+    "fp_conda = join(fp_workdir, \"qiime2-%s-py38-linux-conda.yml\" % release)\n",
+    "cmd = 'if [ ! -f %s ]; then export https_proxy=\"http://proxy.computational.bio.uni-giessen.de:3128\"; wget \"https://data.qiime2.org/distro/core/%s\" -O %s; fi' % (fp_conda, basename(fp_conda), fp_conda)\n",
+    "!$cmd\n",
+    "with open(fp_conda, \"r\") as file:\n",
+    "    qiime2 = yaml.safe_load(file)\n",
+    "\n",
+    "# iterate individual qiime2 plugins and retriev their individual dependency lists\n",
+    "plugin_dependencies = dict()\n",
+    "qiime_plugins = [ d.split('=')[0] for d in qiime2['dependencies'] if d.startswith('q2-')]\n",
+    "for plugin in tqdm(qiime_plugins):\n",
+    "    fp_conda_plugin = join(fp_workdir, \"%s-%s.0.yaml\" % (plugin, qrelease))\n",
+    "    cmd = 'if [ ! -f %s ]; then export https_proxy=\"http://proxy.computational.bio.uni-giessen.de:3128\"; wget \"https://raw.githubusercontent.com/qiime2/%s/refs/tags/%s.0/ci/recipe/meta.yaml\" -O - | grep -v \"^{%%\" > %s; fi' % (fp_conda_plugin, plugin, release, fp_conda_plugin)\n",
+    "    #print(cmd)\n",
+    "    !$cmd\n",
+    "\n",
+    "    with open(fp_conda_plugin) as f:\n",
+    "        template = Template(f.read())\n",
+    "        rendered = template.render(\n",
+    "            version=\"1.2.3\",\n",
+    "            python=\"3.9\",\n",
+    "            scikit_bio=\"0.5.8\",\n",
+    "            qiime2_epoch=\"2025\",\n",
+    "            qiime2=\"2025.1\",\n",
+    "            q2_types=\"2025.1\",\n",
+    "        )\n",
+    "        plugin_yaml = yaml.safe_load(rendered)\n",
+    "        plugin_dependencies[plugin] = {package.split()[0] for _, packages in plugin_yaml['requirements'].items() for package in packages}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 294,
+   "id": "3b472afd-c5a0-4419-8329-e106e73411db",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 55/55 [00:10<00:00,  5.22it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "def read_conda_out(fp):\n",
+    "    packages = set()\n",
+    "    with open(fp) as f:\n",
+    "        for line in f.readlines():\n",
+    "            if line.startswith('  + '):\n",
+    "                plus, package, version, channel, _, _ = line.split()\n",
+    "                assert plus == '+'\n",
+    "                packages |= set([package])\n",
+    "    return packages\n",
+    "\n",
+    "nonq2dependencies = {d for ds in plugin_dependencies.values() for d in ds if not d.startswith('q2-') and d not in ['qiime2', 'q2templates']}\n",
+    "channels = ' '.join(list(map(lambda x: '-c %s' % x, [c for c in qiime2['channels'] if not c.startswith('qiime2')])))\n",
+    "full_dependency_list = dict()\n",
+    "for dependency in tqdm(sorted(list(nonq2dependencies))):\n",
+    "    fp_conda_dependency = join(fp_workdir, \"%s.out\" % dependency)\n",
+    "    cmd = ('if [ ! -f %s ]; '\n",
+    "           'then export https_proxy=\"http://proxy.computational.bio.uni-giessen.de:3128\"; '\n",
+    "                 '/homes/sjanssen/miniconda3/bin/mamba create --name fake %s %s --dry-run -vvv > %s 2> %s; '\n",
+    "           'fi' % (fp_conda_dependency, dependency, channels, fp_conda_dependency, fp_conda_dependency.replace('.out', '.err')))\n",
+    "    #print(cmd)\n",
+    "    !$cmd\n",
+    "\n",
+    "    full_dependency_list[dependency] = read_conda_out(fp_conda_dependency)\n",
+    "    #break"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 295,
+   "id": "8a46a5de-7f83-4fe7-b3ed-c943564a17a1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# check which dependencies are used from black-listed plugins, BUT not in plugins used in qiita\n",
+    "deps = {'black': set(), 'white': set()}\n",
+    "for plugin in plugin_dependencies.keys():\n",
+    "    color = 'white'    \n",
+    "    if plugin.split('q2-')[-1] in blacklist:\n",
+    "        color = 'black'\n",
+    "    deps[color] |= set(plugin_dependencies[plugin])\n",
+    "\n",
+    "deps_full = dict()\n",
+    "for color in deps.keys():\n",
+    "    deps_full[color] = set()\n",
+    "    for level1_dependency in deps[color]:\n",
+    "        # print(color, level1_dependency, full_dependency_list[level1_dependency])\n",
+    "        if level1_dependency in full_dependency_list.keys():\n",
+    "            deps_full[color] |= full_dependency_list[level1_dependency]\n",
+    "        deps_full[color] |= set([level1_dependency])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 296,
+   "id": "c775ff3b-b4b4-48e8-add9-083b7ff01b60",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "spare_dependency_candidates = (deps_full['black'] - deps_full['white']) - set(whitelist)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 297,
+   "id": "6474f422-7874-45fb-acde-4ce1ae4c49e5",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'conda remove -n qiime2 --force htslib r-bh pigz bioconductor-summarizedexperiment q2-demux python-isal q2-dada2 q2-alignment gneiss r-futile.logger r-rcppparallel q2-fragment-insertion dnaio mafft pbzip2 cutadapt q2-quality-filter q2-quality-control gawk pcre xopen q2-vsearch q2-deblur r-futile.options r-hwriter sniffio xyzservices sortmerna bioconductor-biocparallel q2-metadata bioconductor-genomicalignments hmmer bioconductor-decontam bioconductor-rhtslib bioconductor-delayedarray bioconductor-shortread bioconductor-dada2 deblur samtools r-matrixstats r-lambda.r r-bitops isa-l bowtie2 sepp r-snow bioconductor-genomicranges openjdk q2-cutadapt bioconductor-matrixgenerics r-formatr q2-gneiss blast bokeh dendropy giflib bioconductor-rsamtools'"
+      ]
+     },
+     "execution_count": 297,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# list of packages, which cannot be removed, as they are not present in qiime env\n",
+    "missing = set([\n",
+    "    'openmpi',\n",
+    "    'narwhals',\n",
+    "    'dnspython',\n",
+    "    'pymongo',\n",
+    "    'r-multcomp',\n",
+    "    'clustalw',\n",
+    "    'jsonschema-specifications',\n",
+    "    'mpi',\n",
+    "    'bioconductor-ucsc.utils',\n",
+    "    'prank',\n",
+    "    'libgettextpo',\n",
+    "    'r-gtools',\n",
+    "    'pasta',\n",
+    "    'r-zoo',\n",
+    "    'tbb-devel',\n",
+    "    'r-reformulas',\n",
+    "    'python-zlib-ng',\n",
+    "    'rpds-py',\n",
+    "    'bioconductor-s4arrays',\n",
+    "    'q2-sourcetracker2',\n",
+    "    'pplacer',\n",
+    "    'r-s7',\n",
+    "    'bioconductor-sparsearray',\n",
+    "    'libasprintf',\n",
+    "    'r-abind',\n",
+    "    'r-th.data',\n",
+    "    'referencing',\n",
+    "    'r-sandwich',\n",
+    "    'muscle',\n",
+    "    'bioconductor-pwalign',\n",
+    "    'q2-greengenes2',\n",
+    "])\n",
+    "# \n",
+    "addremove = set([\n",
+    "    'blast'  # as \n",
+    "])\n",
+    "#missing = set()\n",
+    "'conda remove -n qiime2 --force %s' % ' '.join(list((spare_dependency_candidates | set(map(lambda x: 'q2-%s' % x, blacklist))) - missing | addremove))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "452f5745-d2b1-4d02-9949-b0e9a08e5763",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "qiime2-amplicon-2024.5",
+   "language": "python",
+   "name": "qiime2-amplicon-2024.5"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.19"
+  },
+  "toc": {
+   "base_numbering": 1,
+   "nav_menu": {},
+   "number_sections": true,
+   "sideBar": true,
+   "skip_h1_title": false,
+   "title_cell": "Table of Contents",
+   "title_sidebar": "Contents",
+   "toc_cell": false,
+   "toc_position": {},
+   "toc_section_display": true,
+   "toc_window_display": false
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}