test: change way pytest is run in CI (#1251)

* fixes: #1159
* test: change way pytest is run in CI

Switch to using a full pytest gather for the async tests. Several test files are singled out to be run synchronously because they touch NVD and can cause us to get rate limited if they run in parallel.

This will ensure that no one needs to add new test files explicitly to CI unless they need to be run synchronously.

Several test files are (temporarily) disabled because they are not passing; we'll enable those as they are fixed.

Signed-off-by: Terri Oda <terri.oda@intel.com>

.github/workflows/pythonapp.yml

@@ -6,7 +6,7 @@ on:
   workflow_dispatch:
     inputs:
       longTests:
-        description: 'Set true to run long tests manually'     
+        description: 'Set true to run long tests manually'
         required: false
         default: 'false'
 
@@ -36,7 +36,7 @@ jobs:
         with:
             configuration: --profile black --check-only
             requirementsFiles: "requirements.txt doc/requirements.txt"
-      
+
   check:
     name: Black check
     runs-on: ubuntu-latest
@@ -131,23 +131,18 @@ jobs:
         run: |
           python -m pip install -e .
           NO_EXIT_CVE_NUM=1 python -m cve_bin_tool.cli test/assets
-      - name: Run Synchronous test
+      - name: Run async tests
+        run: >
+          pytest -n 4 -v
+          --ignore=test/test_cli.py
+          --ignore=test/test_cvedb.py
+          --ignore=test/test_requirements.py
+          --ignore=test/test_config.py
+      - name: Run synchronous tests
         run: >
           pytest -v
-          test/test_json.py
-          test/test_scanner.py
-          test/test_checkers.py
-          test/test_file.py
-          test/test_strings.py
-          test/test_input_engine.py
-          test/test_output_engine.py
-          test/test_util.py
-          test/test_cvedb.py
           test/test_cli.py
-          test/test_extractor.py
-          test/test_condensed_downloads.py
-          test/test_package_list_parser.py
-          test/test_merge.py
+          test/test_cvedb.py
 
   long_tests:
     name: Long tests on python3.8
@@ -201,26 +196,18 @@ jobs:
         run: |
           python -m pip install -e .
           NO_EXIT_CVE_NUM=1 python -m cve_bin_tool.cli test/assets
-      - name: Run parallel test
+      - name: Run async tests
         run: >
           pytest --cov --cov-append -n 4 -v
-          test/test_json.py
-          test/test_scanner.py
-          test/test_checkers.py
-          test/test_extractor.py
-          test/test_file.py
-          test/test_strings.py
-          test/test_input_engine.py
-          test/test_output_engine.py
-          test/test_util.py
-          test/test_condensed_downloads.py
-          test/test_package_list_parser.py
-          test/test_merge.py
-      - name: Run Synchronous test
+          --ignore=test/test_cli.py
+          --ignore=test/test_cvedb.py
+          --ignore=test/test_requirements.py
+          --ignore=test/test_config.py
+      - name: Run synchronous tests
         run: >
           pytest -v --cov --cov-append --cov-report=xml
-          test/test_cvedb.py
           test/test_cli.py
+          test/test_cvedb.py
       - name: upload code coverage to codecov
         uses: codecov/codecov-action@v1
         with:
@@ -269,27 +256,19 @@ jobs:
         run: |
           python -m pip install -e .
           python -m cve_bin_tool.cli test/assets
-      - name: Run parallel test
+      - name: Run async tests
         run: >
           pytest -n 4 -v
-          test/test_json.py
-          test/test_scanner.py
-          test/test_checkers.py
-          test/test_extractor.py
-          test/test_file.py
-          test/test_strings.py
-          test/test_input_engine.py
-          test/test_output_engine.py
-          test/test_util.py
-          test/test_condensed_downloads.py
-          test/test_async_utils.py
-          test/test_package_list_parser.py
-          test/test_merge.py
-      - name: Run Synchronous test
+          --ignore=test/test_cli.py
+          --ignore=test/test_cvedb.py
+          --ignore=test/test_requirements.py
+          --ignore=test/test_helper_script.py
+          --ignore=test/test_config.py
+      - name: Run synchronous tests
         run: >
           pytest -v
-          test/test_cvedb.py
           test/test_cli.py
+          test/test_cvedb.py
 
   cve_scan:
     name: CVE Scan on dependencies