Merge pull request #3132 from antgonza/rm-pt-generate-files-from-sample-update

rm pt.generate_files from sample update

Author: charles-cowart

qiita_db/metadata_template/sample_template.py

@@ -268,17 +268,6 @@ def generate_files(self, samples=None, columns=None):
             fp_id = qdb.util.convert_to_id("sample_template", "filepath_type")
             self.add_filepath(fp, fp_id=fp_id)
 
-            # generating all new QIIME mapping files
-            for pt in qdb.study.Study(self._id).prep_templates():
-                if samples is not None and samples and (
-                        columns is None or not columns):
-                    overlapping = set(samples) & set(pt.keys())
-                    # if the prep has no overlapping sample ids, we can skip
-                    # generationg the prep
-                    if not overlapping:
-                        continue
-                pt.generate_files(samples, columns)
-
     @property
     def ebi_sample_accessions(self):
         """The EBI sample accessions for the samples in the sample template

qiita_db/metadata_template/test/test_sample_template.py

@@ -1365,9 +1365,9 @@ def test_generate_files(self):
         fp_count = qdb.util.get_count("qiita.filepath")
         self.tester.generate_files()
         obs = qdb.util.get_count("qiita.filepath")
-        # We just make sure that the count has been increased by 3, since
+        # We just make sure that the count has been increased by 1, since
         # the contents of the files have been tested elsewhere.
-        self.assertEqual(obs, fp_count + 3)
+        self.assertEqual(obs, fp_count + 1)
 
     def test_to_file(self):
         """to file writes a tab delimited file with all the metadata"""

qiita_db/test/test_setup.py

@@ -33,7 +33,7 @@ def test_investigation_study(self):
         self.assertEqual(get_count("qiita.investigation_study"), 1)
 
     def test_filepath(self):
-        self.assertEqual(get_count("qiita.filepath"), 25)
+        self.assertEqual(get_count("qiita.filepath"), 23)
 
     def test_filepath_type(self):
         self.assertEqual(get_count("qiita.filepath_type"), 24)

qiita_pet/handlers/api_proxy/tests/test_prep_template.py

@@ -75,9 +75,8 @@ def test_prep_template_ajax_get_req(self):
                'message': '',
                'name': "Prep information 1",
                'files': ["uploaded_file.txt"],
-               'download_prep_id': 24,
-               'other_filepaths': ['1_prep_1_qiime_19700101-000000.txt',
-                                   '1_prep_1_19700101-000000.txt',
+               'download_prep_id': 21,
+               'other_filepaths': ['1_prep_1_19700101-000000.txt',
                                    '1_prep_1_qiime_19700101-000000.txt',
                                    '1_prep_1_19700101-000000.txt'],
                'num_samples': 27,
@@ -200,7 +199,7 @@ def test_prep_template_filepaths_get_req(self):
         self.assertEqual(obs['message'], '')
         # [0] the fp_id is the first element, that should change
         fp_ids = [fp[0] for fp in obs['filepaths']]
-        self.assertCountEqual(fp_ids, [18, 19, 20, 21, 24])
+        self.assertCountEqual(fp_ids, [18, 19, 20, 21])
 
     def test_prep_template_filepaths_get_req_no_access(self):
         obs = prep_template_filepaths_get_req(1, 'demo@microbio.me')

qiita_pet/handlers/download.py

@@ -124,8 +124,9 @@ def _list_artifact_files_nginx(self, artifact):
         for pt in artifact.prep_templates:
             # the latest prep template file is always the first [0] tuple and
             # we need the filepath [1]
-            pt_fp = pt.get_filepaths()[0][1]
-            if pt_fp is not None:
+            pt_fp = pt.get_filepaths()
+            if pt_fp:
+                pt_fp = pt_fp[0][1]
                 spt_fp = pt_fp
                 if pt_fp.startswith(basedir):
                     spt_fp = pt_fp[basedir_len:]

qiita_pet/test/test_download.py

@@ -148,31 +148,27 @@ def test_download_study(self):
         response = self.get('/download_study_bioms/1')
         self.assertEqual(response.code, 200)
         exp = (
-            '- 1256812 /protected/processed_data/'
-            '1_study_1001_closed_reference_otu_table.biom processed_data/'
-            '1_study_1001_closed_reference_otu_table.biom\n'
-            '- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt '
+            '- \\d+ /protected/processed_data/1_study_1001_closed_reference_'
+            'otu_table.biom processed_data/1_study_1001_closed_reference_otu'
+            '_table.biom\n'
+            '- \\d+ /protected/templates/1_prep_1_qiime_19700101-000000.txt '
             'mapping_files/4_mapping_file.txt\n'
-            '- 1256812 /protected/processed_data/'
-            '1_study_1001_closed_reference_otu_table.biom processed_data/'
-            '1_study_1001_closed_reference_otu_table.biom\n'
-            '- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt '
+            '- \\d+ /protected/processed_data/1_study_1001_closed_reference_'
+            'otu_table.biom processed_data/1_study_1001_closed_reference_otu'
+            '_table.biom\n'
+            '- \\d+ /protected/templates/1_prep_1_qiime_19700101-000000.txt '
             'mapping_files/5_mapping_file.txt\n'
-            '- 1256812 /protected/processed_data/1_study_1001_'
-            'closed_reference_otu_table_Silva.biom processed_data/'
-            '1_study_1001_closed_reference_otu_table_Silva.biom\n'
-            '- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt '
+            '- \\d+ /protected/processed_data/1_study_1001_closed_reference_'
+            'otu_table_Silva.biom processed_data/1_study_1001_closed_'
+            'reference_otu_table_Silva.biom\n'
+            '- \\d+ /protected/templates/1_prep_1_qiime_19700101-000000.txt '
             'mapping_files/6_mapping_file.txt\n'
-            '- 1093210 /protected/BIOM/7/biom_table.biom '
-            'BIOM/7/biom_table.biom\n'
-            '- [0-9]* /protected/templates/1_prep_2_[0-9]*-[0-9]*.txt '
-            'mapping_files/7_mapping_file.txt\n'
-            '- [0-9]* /protected/BIOM/{0}/otu_table.biom '
-            'BIOM/{0}/otu_table.biom\n'
-            '- 1 /protected/BIOM/10/sortmerna_picked_otus/seqs_otus.log '
-            'BIOM/{0}/sortmerna_picked_otus/seqs_otus.log\n'
-            '- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt '
-            'mapping_files/{0}_mapping_file.txt\n'.format(a.id))
+            '- \\d+ /protected/BIOM/7/biom_table.biom BIOM/7/biom_table.biom\n'
+            '- \\d+ /protected/BIOM/10/otu_table.biom BIOM/10/otu_table.biom\n'
+            '- \\d+ /protected/BIOM/10/sortmerna_picked_otus/seqs_otus.log '
+            'BIOM/10/sortmerna_picked_otus/seqs_otus.log\n'
+            '- \\d+ /protected/templates/1_prep_1_qiime_19700101-000000.txt '
+            'mapping_files/10_mapping_file.txt\n')
         self.assertRegex(response.body.decode('ascii'), exp)
 
         response = self.get('/download_study_bioms/200')
@@ -190,10 +186,9 @@ def test_download_study(self):
         a.visibility = 'private'
         self.assertEqual(response.code, 200)
         # we should have the same files than the previous test, except artifact
-        # and mapping file 7: position 6 and 7; thus removing 6 twice
+        # and mapping file 7: position 6; thus removing 6
         exp = exp.split('\n')
         exp.pop(6)
-        exp.pop(6)
         exp = '\n'.join(exp)
         self.assertRegex(response.body.decode('ascii'), exp)
 
@@ -244,7 +239,7 @@ def test_download_raw_data(self):
             '- 58 /protected/raw_data/'
             '1_s_G1_L001_sequences_barcodes.fastq.gz '
             'raw_data/1_s_G1_L001_sequences_barcodes.fastq.gz\n'
-            '- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt '
+            '- [0-9]* /protected/templates/1_prep_1_qiime_[0-9]*-[0-9]*.txt '
             'mapping_files/1_mapping_file.txt\n'
             '- 1093210 /protected/BIOM/7/biom_table.biom '
             'BIOM/7/biom_table.biom\n')
@@ -273,8 +268,8 @@ def test_download_raw_data(self):
             return_value=User("demo@microbio.me"))
         response = self.get('/download_study_bioms/1')
         self.assertEqual(response.code, 200)
-        exp = ('- [0-9]* /protected/templates/1_prep_2_[0-9]*-[0-9]*.txt '
-               'mapping_files/7_mapping_file.txt\n')
+        exp = ('- [0-9]* /protected/BIOM/7/biom_table.biom '
+               'BIOM/7/biom_table.biom\n')
         self.assertRegex(response.body.decode('ascii'), exp)
 
 
@@ -405,17 +400,16 @@ def test_download(self):
         # check success
         response = self.get('/public_download/?data=biom&study_id=1')
         self.assertEqual(response.code, 200)
-        exp = ('- [0-9]* /protected/templates/1_prep_2_[0-9]*-[0-9]*.txt '
-               'mapping_files/7_mapping_file.txt\n')
+        exp = ('- [0-9]* /protected/BIOM/7/biom_table.biom'
+               ' BIOM/7/biom_table.biom\n')
         self.assertRegex(response.body.decode('ascii'), exp)
 
         Study(1).public_raw_download = True
         # check success
         response = self.get('/public_download/?data=raw&study_id=1')
         self.assertEqual(response.code, 200)
-        exp = (
-            '- [0-9]* /protected/templates/1_prep_2_[0-9]*-[0-9]*.txt '
-            'mapping_files/7_mapping_file.txt\n')
+        exp = ('- [0-9]* /protected/BIOM/7/biom_table.biom'
+               ' BIOM/7/biom_table.biom\n')
         self.assertRegex(response.body.decode('ascii'), exp)
 
         # testing data_type
@@ -447,7 +441,7 @@ def test_download(self):
         exp = (
             '[0-9]* [0-9]* /protected/raw_data/1_s_G1_L001_sequences_barcodes'
             '.fastq.gz raw_data/1_s_G1_L001_sequences_barcodes.fastq.gz\n'
-            '- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt '
+            '- [0-9]* /protected/templates/1_prep_1_qiime_19700101-000000.txt '
             'mapping_files/1_mapping_file.txt')
         self.assertRegex(response.body.decode('ascii'), exp)
 
@@ -458,11 +452,12 @@ def test_download(self):
             '- [0-9]* /protected/processed_data/1_study_1001_closed_'
             'reference_otu_table.biom processed_data/1_study_1001_closed_'
             'reference_otu_table.biom\n- [0-9]* /protected/templates/1_prep_'
-            '1_[0-9]*-[0-9]*.txt mapping_files/4_mapping_file.txt\n'
+            '1_qiime_19700101-000000.txt mapping_files/4_mapping_file.txt\n'
             '- [0-9]* /protected/processed_data/1_study_1001_closed_'
             'reference_otu_table.biom processed_data/1_study_1001_closed_'
-            'reference_otu_table.biom\n- [0-9]* /protected/templates/1_prep_1'
-            '_[0-9]*-[0-9]*.txt mapping_files/5_mapping_file.txt\n')
+            'reference_otu_table.biom\n- [0-9]* /protected/templates/1_prep_'
+            '1_qiime_19700101-000000.txt mapping_files/5_mapping_file.txt\n')
+
         self.assertRegex(response.body.decode('ascii'), exp)
 
     def test_download_sample_information(self):
@@ -506,8 +501,8 @@ def test_download_sample_information(self):
         response = self.get(
             '/public_download/?data=prep_information&prep_id=1')
         self.assertEqual(response.code, 200)
-        exp = ('[0-9]* [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]'
-               '*.txt templates/1_prep_1_[0-9]*-[0-9]*.txt\n')
+        exp = ('- [0-9]* /protected/templates/1_prep_1_qiime_[0-9]*-[0-9]'
+               '*.txt templates/1_prep_1_qiime_[0-9]*-[0-9]*.txt\n')
         self.assertRegex(response.body.decode('ascii'), exp)
 
 
@@ -536,7 +531,7 @@ def test_download(self):
             '- [0-9]* /protected/processed_data/'
             '1_study_1001_closed_reference_otu_table.biom '
             'processed_data/1_study_1001_closed_reference_otu_table.biom\n'
-            '- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt '
+            '- [0-9]* /protected/templates/1_prep_1_qiime_19700101-000000.txt '
             'mapping_files/5_mapping_file.txt')
         self.assertRegex(response.body.decode('ascii'), exp)
 
@@ -557,13 +552,12 @@ def test_download(self):
         o = urlparse(resp_dict["url"])
         response_file = self.get(o.path)
         self.assertEqual(response_file.code, 200)
-
         exp = (
             '- 58 /protected/raw_data/1_s_G1_L001_sequences.fastq.gz '
             'raw_data/1_s_G1_L001_sequences.fastq.gz\n'
             '- 58 /protected/raw_data/1_s_G1_L001_sequences_barcodes.'
             'fastq.gz raw_data/1_s_G1_L001_sequences_barcodes.fastq.gz\n'
-            '- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt '
+            '- [0-9]* /protected/templates/1_prep_1_qiime_19700101-000000.txt '
             'mapping_files/1_mapping_file.txt\n'
         )
         self.assertRegex(response_file.body.decode('ascii'), exp)