[ENH] Datalad-free alternative for TemplateFlow

.circleci/config.yml

@@ -61,17 +61,39 @@ jobs:
             python setup.py sdist
             twine check dist/*
 
+      - run:
+          name: Update TemplateFlow skeleton
+          command: |
+            cd $HOME/.cache/templateflow/
+            find . -type d -not -path "*/\.*" -exec mkdir -p /tmp/skel/{} \;
+            for i in $( find . -type f -not -path "*/\.*" ); do cp $i /tmp/skel/${i:2}; done
+            find . -type l -not -path "*/\.*" -exec touch /tmp/skel/{} \;
+            cd /tmp/skel
+            mkdir -p /tmp/resources
+            zip -r /tmp/resources/templateflow-skel.zip */
+
       - save_cache:
           key: deps-v2-{{ checksum "/tmp/src/templateflow/requirements.txt"}}-{{ epoch }}
           paths:
             - "/opt/circleci/.pyenv/versions/3.5.2"
             - "/tmp/data/templateflow"
 
+      - persist_to_workspace:
+          root: /tmp
+          paths:
+            - resources
+
+      - store_artifacts:
+          path: /tmp/resources/templateflow-skel.zip
+
   deploy_pypi:
     machine:
       image: circleci/classic:201711-01
     working_directory: /tmp/src/templateflow
     steps:
+      - attach_workspace:
+          at: /tmp
+
       - checkout:
           path: /tmp/src/templateflow
 
@@ -84,6 +106,7 @@ jobs:
             pyenv global 3.5.2
             virtualenv venv
             pip install "setuptools>=27.0" twine
+            cp /tmp/resources/templateflow-skel.zip templateflow/conf/
             echo "${CIRCLE_TAG}" > templateflow/VERSION
             echo "include templateflow/VERSION" >> MANIFEST.in
             python setup.py sdist

MANIFEST.in

@@ -1,4 +1,6 @@
 # data
+include templateflow/conf/config.json
+include templateflow/conf/templateflow-skel.zip
 
 # misc
 include requirements.txt

setup.py

@@ -26,7 +26,10 @@ def main():
         EXTRA_REQUIRES,
     )
 
-    pkg_data = {'templateflow': ['conf/config.json']}
+    pkg_data = {'templateflow': [
+        'conf/config.json',
+        'conf/templateflow-skel.zip'
+    ]}
 
     root_dir = op.dirname(op.abspath(getfile(currentframe())))
     version = None

templateflow/api.py

@@ -3,7 +3,7 @@
 """
 from pathlib import Path
 from json import loads
-from .conf import TF_LAYOUT
+from .conf import TF_LAYOUT, TF_S3_ROOT
 
 
 def get(template, **kwargs):
@@ -25,6 +25,9 @@ def get(template, **kwargs):
     out_file = [Path(p) for p in TF_LAYOUT.get(
         template=template, return_type='file', **kwargs)]
 
+    for filepath in [p for p in out_file if p.stat().st_size == 0]:
+        _s3_get(filepath)
+
     for filepath in [p for p in out_file if not p.is_file()]:
         _datalad_get(filepath)
 
@@ -83,3 +86,29 @@ def _datalad_get(filepath):
             api.get(str(filepath))
         else:
             raise
+
+
+def _s3_get(filepath):
+    from math import ceil
+    from tqdm import tqdm
+    import requests
+
+    path = str(filepath.relative_to(TF_LAYOUT.root))
+    url = '%s/%s' % (TF_S3_ROOT, path)
+
+    print('Downloading %s' % url)
+    # Streaming, so we can iterate over the response.
+    r = requests.get(url, stream=True)
+
+    # Total size in bytes.
+    total_size = int(r.headers.get('content-length', 0))
+    block_size = 1024
+    wrote = 0
+    with filepath.open('wb') as f:
+        for data in tqdm(r.iter_content(block_size),
+                         total=ceil(total_size // block_size),
+                         unit='KB', unit_scale=True):
+            wrote = wrote + len(data)
+            f.write(data)
+    if total_size != 0 and wrote != total_size:
+        raise RuntimeError("ERROR, something went wrong")

templateflow/conf/__init__.py

@@ -2,23 +2,32 @@
 Settings
 """
 from os import getenv
+from warnings import warn
 from pathlib import Path
+from pkg_resources import resource_filename
 from .bids import Layout
 
 TF_DEFAULT_HOME = Path.home() / '.cache' / 'templateflow'
 TF_HOME = Path(getenv('TEMPLATEFLOW_HOME', str(TF_DEFAULT_HOME)))
 TF_GITHUB_SOURCE = 'https://github.com/templateflow/templateflow.git'
+TF_S3_ROOT = 'https://templateflow.s3.amazonaws.com'
 
-if not TF_HOME.exists():
+_msg = """\
+TemplateFlow: repository not found at %s. Populating a TemplateFlow stub.
+If the path reported above is not the desired location for Templateflow, \
+please set the TEMPLATEFLOW_HOME environment variable.
+""" % TF_HOME
+
+if not TF_HOME.exists() or not list(TF_HOME.iterdir()):
+    warn(_msg, ResourceWarning)
     try:
         from datalad.api import install
     except ImportError:
-        raise RuntimeError("""\
-'TemplateFlow repository not found at path %s, and DataLad/git-annex are not \
-installed.
-Please download TemplateFlow manually from \
-https://files.osf.io/v1/resources/ue5gx/providers/osfstorage/?zip= \
-and place all templates under the path indicated above.""" % TF_HOME)
+        from zipfile import ZipFile
+        TF_HOME.mkdir(exist_ok=True, parents=True)
+        with ZipFile(resource_filename('templateflow',
+                                       'conf/templateflow-skel.zip'), 'r') as zipref:
+            zipref.extractall(str(TF_HOME))
     else:
         TF_HOME.parent.mkdir(exist_ok=True, parents=True)
         install(path=str(TF_HOME), source=TF_GITHUB_SOURCE, recursive=True)