Major changes in the documentation and schemas created

.github/workflows/integration-tests.yml

@@ -0,0 +1,67 @@
+name: Integration Tests
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      test_type:
+        description: 'Type of integration test to run'
+        required: true
+        default: 'all'
+        type: choice
+        options:
+        - all
+        - full-conversion-suite
+        - lfq-only
+        - tmt-only
+
+jobs:
+  integration-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60  # 1 hour timeout for the entire job
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.10"
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install pytest pytest-timeout
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install poetry
+        # Install package in development mode
+        poetry install
+
+    - name: Run Full Conversion Suite Tests
+      if: ${{ github.event.inputs.test_type == 'all' || github.event.inputs.test_type == 'full-conversion-suite' || github.event_name == 'release' }}
+      run: |
+        poetry run pytest -vv tests/test_full_conversion_suite.py --timeout=900
+      env:
+        PYTHONIOENCODING: utf-8
+
+    - name: Run LFQ Tests Only
+      if: ${{ github.event.inputs.test_type == 'lfq-only' }}
+      run: |
+        poetry run pytest -vv tests/test_full_conversion_suite.py -k "lfq" --timeout=900
+      env:
+        PYTHONIOENCODING: utf-8
+
+    - name: Run TMT Tests Only  
+      if: ${{ github.event.inputs.test_type == 'tmt-only' }}
+      run: |
+        poetry run pytest -vv tests/test_full_conversion_suite.py -k "tmt" --timeout=900
+      env:
+        PYTHONIOENCODING: utf-8
+
+    - name: Run All Integration Tests
+      if: ${{ github.event.inputs.test_type == 'all' || github.event_name == 'release' }}
+      run: |
+        poetry run pytest -vv -m "integration" --timeout=900
+      env:
+        PYTHONIOENCODING: utf-8 

.github/workflows/python-app.yml

@@ -8,6 +8,11 @@ on:
     branches: [ "main", "dev" ]
   pull_request:
     branches: [ "main" ]
+  schedule:
+    # Run full test suite daily at 2 AM UTC
+    - cron: '0 2 * * *'
+  workflow_dispatch:
+    # Allow manual triggering
 
 permissions:
   contents: read
@@ -17,36 +22,85 @@ jobs:
   PythonBlack:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Check code lints with Black
         uses: psf/black@stable
         with:
           options: ". --check"
 
-  build:
-
+  unit-tests:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python 3.10
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: "3.10"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install flake8 pytest
+        python -m pip install flake8 pytest pytest-timeout
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
         pip install poetry
-        poetry build
-        pip install dist/*.whl
+        # Install package in development mode
+        poetry install
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
         flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 130 chars wide
         flake8 . --count --exit-zero --max-complexity=30 --max-line-length=130 --statistics --ignore F401,W503,W504
-    - name: Test with pytest
+    - name: Test with pytest (Unit Tests Only)
+      run: |
+        # Run all tests except integration tests
+        poetry run pytest -vv -m "not integration"
+
+  integration-tests:
+    runs-on: ubuntu-latest
+    needs: unit-tests  # Only run if unit tests pass
+    if: github.event_name == 'push' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-integration-tests'))
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.10"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest pytest-timeout
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install poetry
+        # Install package in development mode
+        poetry install
+    - name: Test with pytest (Integration Tests)
+      run: |
+        # Run only integration tests with longer timeout
+        poetry run pytest -vv -m "integration" --timeout=900
+      env:
+        PYTHONIOENCODING: utf-8
+
+  full-test-suite:
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/main' || github.event_name == 'schedule'
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.10"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest pytest-timeout
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install poetry
+        # Install package in development mode
+        poetry install
+    - name: Test with pytest (All Tests)
       run: |
-        poetry run pytest -vv
+        # Run all tests including integration tests
+        poetry run pytest -vv --timeout=900
+      env:
+        PYTHONIOENCODING: utf-8

.github/workflows/python-package.yml

@@ -10,34 +10,63 @@ on:
     branches: [ "dev" ]
 
 jobs:
-  build:
-
+  unit-tests:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         python-version: ["3.10", "3.11"]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install flake8 pytest
+        python -m pip install flake8 pytest pytest-timeout
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
         pip install poetry
-        poetry build
-        pip install dist/*.whl
+        # Install package in development mode
+        poetry install
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
         flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    - name: Test with pytest
+    - name: Test with pytest (Unit Tests Only)
+      run: |
+        # Run all tests except integration tests
+        poetry run pytest -vv -m "not integration"
+
+  integration-tests:
+    runs-on: ubuntu-latest
+    needs: unit-tests  # Only run if unit tests pass
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11"]
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest pytest-timeout
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install poetry
+        # Install package in development mode
+        poetry install
+    - name: Test with pytest (Integration Tests)
       run: |
-        poetry run pytest -vv
+        # Run only integration tests with longer timeout
+        poetry run pytest -vv -m "integration" --timeout=900
+      env:
+        PYTHONIOENCODING: utf-8

.gitignore

@@ -5,4 +5,66 @@ build
 data
 docs/.vscode
 __pycache__
-.vscode
+.vscode
+
+# DuckDB database files
+*duckdb*.db
+*.db
+*.db.wal
+*.db-shm
+
+# Temporary files
+*.tmp
+*.temp
+temp/
+tmp/
+.tmp/
+.temp/
+
+# Temporary mzTab files (created during processing)
+mztab_temp*.mzTab
+mztab_temp*.mzTab.gz
+mztab_temp*.mzTab.bz2
+
+# Test output and temporary data
+test_output*/
+output*/
+results*/
+*.parquet
+
+# Log files
+*.log
+logs/
+
+# OS generated files
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# Python cache and compiled files
+*.pyc
+*.pyo
+*.pyd
+.Python
+*.so
+__pycache__/
+*.egg-info/
+.pytest_cache/
+.coverage
+htmlcov/
+
+# Jupyter Notebook checkpoints
+.ipynb_checkpoints/
+
+# Environment files
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/