adding test_db_sql (#3405)

* Update CHANGELOG.md

* clean plugins

* patch 54.sql

* fix Internal Qiita jobs

* sample_template.generate_files

* rm extra filepath

* qiita.filepath_filepath_id_seq

* basedir_len

* basedir_len * 2

* basedir_len * 3

* get_db_files_base_dir

* adding test_db_sql

* missing `

* echo $QIITA_CONFIG_FP

* -p 32768

* -l localhost

* pgport

* testing 1

* testing 2

* localhost

* testing 3

* -h localhost

* -W

* PGPASSWORD

* doc

* flake8

* mods after chatting with @charles-cowart

* exit 1

* "$row_counts"

* #

* `echo $row_counts`

* full tests

.github/workflows/qiita-ci.yml

@@ -201,7 +201,34 @@ jobs:
           QIITA_PID=`cat /tmp/supervisord.pid`
           kill $QIITA_PID
           sleep 10
-          if [[ "$COVER_PACKAGE" != *"qiita_db"* ]]; then test_data_studies/commands.sh; all-qiita-cron-job; fi
+          # due to qiita_db tests being more complex and taking longer than
+          # the other tests we will only add some extra tests to the run that is
+          # not testing qiita_db
+          if [[ "$COVER_PACKAGE" != *"qiita_db"* ]]; then
+              # 1. testing that we can add some "dummy" studies to the db via
+              #    CLI
+              test_data_studies/commands.sh;
+              # 2. making sure that all qiita cron jobs complete as expected
+              all-qiita-cron-job;
+              # 3. making sure than a production system has the expected rows
+              #    in all our tables; steps: a. drop test db, b. change $QIITA_CONFIG_FP
+              #    c. create new production system, c. count rows in the db.
+              qiita-env drop;
+              cp $QIITA_CONFIG_FP ${QIITA_CONFIG_FP}.bk
+              sed 's/TEST_ENVIRONMENT = TRUE/TEST_ENVIRONMENT = FALSE/g' ${QIITA_CONFIG_FP}.bk > $QIITA_CONFIG_FP;
+              qiita-env make --no-load-ontologies;
+
+              export PGPASSWORD=postgres
+              pgport=${{ job.services.postgres.ports[5432] }}
+              row_counts=`psql -h localhost -U postgres -d qiita_test -p $pgport -c "SELECT SUM(c.reltuples) FROM pg_class c JOIN pg_namespace n on n.oid = c.relnamespace WHERE n.nspname = 'qiita' AND c.relkind = 'r' AND n.nspname NOT IN ('information_schema', 'pg_catalog');"`
+              if [[ `echo $row_counts` != *" 0 "* ]]; then
+                 echo "***********";
+                 echo "The number of rows in a production system is not what's expected:";
+                 echo $row_counts;
+                 echo "***********";
+                 exit 1
+              fi
+          fi
 
       - name: Submit coveralls
         uses: AndreMiras/coveralls-python-action@develop

CONTRIBUTING.md

@@ -94,6 +94,17 @@ After the initial production release of Qiita, changes to the database schema wi
 2. We keep fully patched versions of the DBS and HTML files in the repository
 3. We keep a patch file for each patch as required in the `qiita_db/support_files/patches` directory. Note that **the patches will be applied in order based on the natural sort order of their filename** (e.g., `2.sql` will be applied before `10.sql`, and `10.sql` will be applied before `a.sql`)
 
+### Patch 91.sql
+
+In May 2024 we decided to:
+* Merge all patches into the main database schema, this means that there are no patches younger than 92.sql.
+* Added a new folder `patches/test_db_sql/` where we can store sql files that will only be applied for the test environment.
+* Added a test to the GitHub actions to test that the production database has an expected number of rows.
+
+Note that these changes mean:
+1. 92.sql is the current first sql file to patch the database.
+2. If you need to make changes (like INSERTS) _only_ to the tests database you need to add a patch to `patches/test_db_sql/`.
+
 ### Developer Workflow
 
 1. Load the fully patched DBS file (e.g., `qiita-db.dbs`) in [DBSchema](http://www.dbschema.com/)

qiita_db/environment_manager.py

@@ -381,6 +381,7 @@ def patch(patches_dir=PATCHES_DIR, verbose=False, test=False):
         current_patch = qdb.sql_connection.TRN.execute_fetchlast()
         current_sql_patch_fp = join(patches_dir, current_patch)
         corresponding_py_patch = partial(join, patches_dir, 'python_patches')
+        corresponding_test_sql = partial(join, patches_dir, 'test_db_sql')
 
         sql_glob = join(patches_dir, '*.sql')
         sql_patch_files = natsorted(glob(sql_glob))
@@ -402,15 +403,23 @@ def patch(patches_dir=PATCHES_DIR, verbose=False, test=False):
 
         patch_prefix = splitext(basename(sql_patch_fp))[0]
         py_patch_fp = corresponding_py_patch(f'{patch_prefix}.py')
+        test_sql_fp = corresponding_test_sql(f'{patch_prefix}.sql')
 
         with qdb.sql_connection.TRN:
             with open(sql_patch_fp, newline=None) as patch_file:
                 if verbose:
                     print('\tApplying patch %s...' % sql_patch_filename)
-
                 qdb.sql_connection.TRN.add(patch_file.read())
                 qdb.sql_connection.TRN.add(
                     patch_update_sql, [sql_patch_filename])
+
+            if test and exists(test_sql_fp):
+                if verbose:
+                    print('\t\tApplying test SQL %s...'
+                          % basename(test_sql_fp))
+                with open(test_sql_fp) as test_sql:
+                    qdb.sql_connection.TRN.add(test_sql.read())
+
             qdb.sql_connection.TRN.execute()
 
             if exists(py_patch_fp):

qiita_db/support_files/patches/test_db_sql/92.sql

@@ -0,0 +1,4 @@
+-- .. date .. 
+-- tmp file for @Gossty
+
+SELECT 1;