[ENH] Use a BIDSLayout to index TemplateFlow

Uses pybids to access the data (fixes #4).

Additionally, resorts to datalad iff there are missing
files (rel. #5). I will not close that one because this is not
really a datalad-free fallback.

.circleci/config.yml

@@ -25,7 +25,7 @@ jobs:
           command: |
             pyenv global 3.5.2
             virtualenv venv
-            pip install -r /tmp/src/templateflow/requirements.txt
+            pip install pytest
             pytest -vsx --doctest-modules /tmp/src/templateflow/templateflow
 
       - run:

requirements.txt

@@ -1,2 +0,0 @@
-datalad
-pytest

setup.py

@@ -26,7 +26,7 @@ def main():
         EXTRA_REQUIRES,
     )
 
-    pkg_data = {'templateflow': []}
+    pkg_data = {'templateflow': ['conf/config.json']}
 
     root_dir = op.dirname(op.abspath(getfile(currentframe())))
     version = None

templateflow/__about__.py

@@ -44,7 +44,7 @@
 ]
 
 REQUIRES = [
-    'datalad',
+    'pybids>=0.7.0,<0.8.0a0',
 ]
 
 SETUP_REQUIRES = []
@@ -59,6 +59,7 @@
 EXTRA_REQUIRES = {
     'doc': [],
     'tests': TESTS_REQUIRES,
+    'datalad': 'datalad',
 }
 
 # Enable a handle to install all extra dependencies at once

templateflow/__init__.py

@@ -8,6 +8,7 @@
     __version__, __packagename__, __author__, __copyright__,
     __credits__, __license__, __maintainer__, __email__, __status__,
     __description__, __longdesc__)
+from .conf import TF_LAYOUT
 
 __all__ = [
     '__version__',
@@ -21,4 +22,5 @@
     '__status__',
     '__description__',
     '__longdesc__',
+    'TF_LAYOUT',
 ]

templateflow/api.py

@@ -1,36 +1,33 @@
 """
 TemplateFlow's Python Client
 """
+from pathlib import Path
 from json import loads
-from datalad import api
-from datalad.support.exceptions import IncompleteResultsError
-from .conf import TF_HOME
+from .conf import TF_LAYOUT
 
 
-def get(template_id, suffix):
+def get(template, **kwargs):
     """
     Fetch one file from one template
 
-    >>> get('MNI152Lin', 'res-01_T1w.nii.gz')  # doctest: +ELLIPSIS
+    >>> str(get('MNI152Lin', resolution=1, suffix='T1w'))  # doctest: +ELLIPSIS
     '.../tpl-MNI152Lin/tpl-MNI152Lin_res-01_T1w.nii.gz'
-    """
-    if suffix.startswith('_'):
-        suffix = suffix[1:]
 
-    filename = 'tpl-%s_%s' % (template_id, suffix)
-    filepath = str(TF_HOME / ('tpl-%s' % template_id) / filename)
+    >>> str(get('MNI152Lin', resolution=2, suffix='T1w'))  # doctest: +ELLIPSIS
+    '.../tpl-MNI152Lin/tpl-MNI152Lin_res-02_T1w.nii.gz'
 
-    try:
-        out_file = api.get(filepath)
-    except IncompleteResultsError as exc:
-        if exc.failed[0]['message'] == 'path not associated with any dataset':
-            from .conf import TF_GITHUB_SOURCE
-            api.install(path=str(TF_HOME), source=TF_GITHUB_SOURCE, recursive=True)
-            out_file = api.get(filepath)
-        else:
-            raise
+    >>> [str(p) for p in get(
+    ...     'MNI152Lin', suffix='T1w')]  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+    ['.../tpl-MNI152Lin/tpl-MNI152Lin_res-01_T1w.nii.gz',
+     '.../tpl-MNI152Lin/tpl-MNI152Lin_res-02_T1w.nii.gz']
+
+    """
+    out_file = [Path(p) for p in TF_LAYOUT.get(
+        template=template, return_type='file', **kwargs)]
+
+    for filepath in [p for p in out_file if not p.is_file()]:
+        _datalad_get(filepath)
 
-    out_file = [p['path'] for p in out_file]
     if len(out_file) == 1:
         return out_file[0]
     return out_file
@@ -45,16 +42,10 @@ def templates():
 'PNC', 'fMRIPrep', 'fsLR', 'fsaverage']
 
     """
-    templates = [str(p.name)[4:] for p in TF_HOME.glob('tpl-*')]
-    if not templates:
-        from .conf import TF_GITHUB_SOURCE
-        api.install(path=str(TF_HOME), source=TF_GITHUB_SOURCE, recursive=True)
-        templates = [str(p.name)[4:] for p in TF_HOME.glob('tpl-*')]
+    return sorted(TF_LAYOUT.get_templates())
 
-    return sorted(templates)
 
-
-def get_metadata(template_id):
+def get_metadata(template):
     """
     Fetch one file from one template
 
@@ -63,17 +54,31 @@ def get_metadata(template_id):
 
     """
 
-    filepath = TF_HOME / ('tpl-%s' % template_id) / 'template_description.json'
+    tf_home = Path(TF_LAYOUT.root)
+    filepath = tf_home / ('tpl-%s' % template) / 'template_description.json'
 
     # Ensure that template is installed and file is available
+    if not filepath.is_file():
+        _datalad_get(filepath)
+    return loads(filepath.read_text())
+
+
+def _datalad_get(filepath):
+    if not filepath:
+        return
+
+    try:
+        from datalad import api
+        from datalad.support.exceptions import IncompleteResultsError
+    except ImportError:
+        pass
+
     try:
         api.get(str(filepath))
     except IncompleteResultsError as exc:
         if exc.failed[0]['message'] == 'path not associated with any dataset':
             from .conf import TF_GITHUB_SOURCE
-            api.install(path=str(TF_HOME), source=TF_GITHUB_SOURCE, recursive=True)
+            api.install(path=TF_LAYOUT.root, source=TF_GITHUB_SOURCE, recursive=True)
             api.get(str(filepath))
         else:
             raise
-
-    return loads(filepath.read_text())

templateflow/conf.py → templateflow/conf/__init__.py

@@ -3,7 +3,12 @@
 """
 from os import getenv
 from pathlib import Path
+from .bids import Layout
 
 TF_DEFAULT_HOME = Path.home() / '.cache' / 'templateflow'
 TF_HOME = Path(getenv('TEMPLATEFLOW_HOME', str(TF_DEFAULT_HOME)))
 TF_GITHUB_SOURCE = 'https://github.com/templateflow/templateflow.git'
+
+TF_LAYOUT = Layout(
+    TF_HOME, validate=False, config='templateflow',
+    exclude=['.git', '.datalad', '.gitannex', '.gitattributes', 'scripts'])

templateflow/conf/bids.py

@@ -0,0 +1,17 @@
+"""
+Extending pyBIDS
+"""
+from pkg_resources import resource_filename
+from bids.layout import BIDSLayout, add_config_paths
+
+add_config_paths(templateflow=resource_filename('templateflow', 'conf/config.json'))
+
+
+class Layout(BIDSLayout):
+    def __repr__(self):
+        # A tidy summary of key properties
+        s = """\
+TemplateFlow Layout
+ - Home: {}
+ - Templates: {}.""".format(self.root, ', '.join(sorted(self.get_templates())))
+        return s

templateflow/conf/config.json

@@ -0,0 +1,63 @@
+{
+    "name": "templateflow",
+    "entities": [
+        {
+            "name": "template",
+            "pattern": "[/\\\\]tpl-([a-zA-Z0-9]+)",
+            "directory": "{{root}}{template}"
+        },
+        {
+            "name": "resolution",
+            "pattern": "[_/\\\\]+res-0*(\\d+)",
+            "dtype": "int"
+        },
+        {
+            "name": "atlas",
+            "pattern": "[_/\\\\]atlas-([a-zA-Z0-9]+)"
+        },
+        {
+            "name": "roi",
+            "pattern": "[_/\\\\]roi-([a-zA-Z0-9]+)"
+        },
+        {
+            "name": "label",
+            "pattern": "[_/\\\\]label-([a-zA-Z0-9]+)"
+        },
+        {
+            "name": "desc",
+            "pattern": "[_/\\\\]desc-([a-zA-Z0-9]+)"
+        },
+        {
+            "name": "space",
+            "pattern": "[_/\\\\]space-([a-zA-Z0-9]+)"
+        },
+        {
+            "name": "from",
+            "pattern": "(?:^|_)from-([a-zA-Z0-9]+).*xfm"
+        },
+        {
+            "name": "to",
+            "pattern": "(?:^|_)to-([a-zA-Z0-9]+).*xfm"
+        },
+        {
+            "name": "mode",
+            "pattern": "(?:^|_)mode-([a-zA-Z0-9]+).*xfm"
+        },
+        {
+            "name": "hemi",
+            "pattern": "[_/\\\\]hemi-(L|R)"
+        },
+        {
+            "name": "suffix",
+            "pattern": "[._]*([a-zA-Z0-9]*?)\\.[^/\\\\]+$"
+        },
+        {
+            "name": "description",
+            "pattern": "(.*\\template_description.json)$"
+        }
+    ],
+    "default_path_patterns": [
+        "tpl-{template}/template_description.json",
+        "tpl-{template}/tpl-{template}_res-{resolution}[_space-{space}][_hemi-{hemi}][_desc-{desc}]_{suffix<T1w|T2w|T1rho|T1map|T2map|T2star|FLAIR|FLASH|PDmap|PD|PDT2|inplaneT[12]|angio>}.{extension<nii|nii\\.gz>}"
+    ]
+}