Operational updates

.github/workflows/deployment.yml

@@ -75,6 +75,58 @@ jobs:
           repository_url: https://test.pypi.org/legacy/
           verbose: true
 
+  docbuild:
+    needs: test-built-dist
+    if: github.event_name == 'release'
+    name: ubuntu-latest py3.9
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/download-artifact@v3
+        with:
+          name: releases
+          path: dist
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - name: Install dependencies
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          use-mamba: true
+          environment-file: conda-environments/docbuild.yml
+          python-version: 3.9
+          activate-environment: docbuild
+          auto-activate-base: false
+          auto-update-conda: false
+      - name: Install activitysim
+        run: |
+          python -m pip install dist/activitysim-*.whl
+      - name: Conda checkup
+        run: |
+          conda info -a
+          conda list
+          echo REPOSITORY ${{ github.repository }}
+          echo REF ${{ github.ref }}
+          echo REF_NAME ${{ github.ref_name }}
+      - name: Build the docs
+        run: |
+          cd docs
+          make clean
+          make html
+      - name: Push to GitHub Pages
+        uses: peaceiris/actions-gh-pages@v3.8.0
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          # Token is created automatically by Github Actions, no other config needed
+          publish_dir: ./docs/_build/html
+          destination_dir: ${{ github.ref_name }}
+
   upload-to-pypi:
     needs: test-built-dist
     if: github.event_name == 'release'
@@ -89,4 +141,4 @@ jobs:
         with:
           user: __token__
           password: ${{ secrets.PYPI_TOKEN }}
-          verbose: true
+          verbose: true

HOW_TO_RELEASE.md

@@ -5,9 +5,9 @@
 01. Check that the branch you intend to release is passing tests on Travis.
     If it's not passing there you should not release it.
 
-00. Start from a completely clean conda environment 
-    and git repository.  Assuming you have `conda` installed, you can do so 
-    by starting where ActivitySim is not yet cloned (e.g. in an empty 
+00. Start from a completely clean conda environment
+    and git repository.  Assuming you have `conda` installed, you can do so
+    by starting where ActivitySim is not yet cloned (e.g. in an empty
     directory) and running:
     ```sh
     conda create -n TEMP-ASIM-DEV python=3.9 git gh -c conda-forge --override-channels
@@ -19,7 +19,7 @@
 
 00. Per project policy, code on the main branch should have been released,
     but if you are *preparing* a release then the code should be on the `develop`
-    branch.  Switch to that branch now, and make sure it is synced to the 
+    branch.  Switch to that branch now, and make sure it is synced to the
     version on GitHub:
     ```sh
     git switch develop
@@ -28,45 +28,46 @@
 
 00. Update your Conda environment for testing.  We do not want to use an
     existing environment on your machine, as it may be out-of-date
-    and we want to make sure everything passes muster using the 
+    and we want to make sure everything passes muster using the
     most up-to-date dependencies available.  The following command
-    will update the active environment (we made this to be `TEMP-ASIM-DEV` 
+    will update the active environment (we made this to be `TEMP-ASIM-DEV`
     if you followed the directions above).
     ```sh
     conda env update --file=conda-environments/activitysim-dev.yml
     ```
-    If you add to the ActivitySim dependencies, make sure to also update 
-    the environments in `conda-environments`, which are used for testing 
-    and development.  If they are not updated, these environments will end 
+    If you add to the ActivitySim dependencies, make sure to also update
+    the environments in `conda-environments`, which are used for testing
+    and development.  If they are not updated, these environments will end
     up with dependencies loaded from *pip* instead of *conda-forge*.
 
-00. Run pycodestyle to ensure that the codebase passes all style checks.
+00. Run `black` to ensure that the codebase passes minimal style checks.
     This check should only take a few seconds.  These checks are also done on
-    Travis and are platform independent, so they should not be necessary to
+    GitHub Actions and are platform independent, so they should not be necessary to
     replicate locally, but are listed here for completeness.
     ```sh
-    pycodestyle .
+    black --check --diff .
     ```
 
-00. Run the regular test suite on Windows. Travis tests are done on Linux,
-    but most users are on Windows, and the test suite should also be run
-    on Windows to ensure that it works on that platform as well.  If you
+00. Run the regular test suite on Windows. Most GitHub Actions tests are done on Linux,
+    Linux (it's faster to start up and run a new clean VM for testing) but most
+    users are on Windows, and the test suite should also be run on Windows to
+    ensure that it works on that platform as well.  If you
     are not preparing this release on Windows, you should be sure to run
-    at least through this step on a Windows machine before finalizing a 
-    release.  
-    
-    A few of the tests require pre-created data that is not included in the 
-    repository directly, but rather recreated on the fly before testing. The 
-    regular test suite takes some time to run, between about half an hour and 
+    at least through this step on a Windows machine before finalizing a
+    release.
+
+    A few of the tests require pre-created data that is not included in the
+    repository directly, but rather recreated on the fly before testing. The
+    regular test suite takes some time to run, between about half an hour and
     two hours depending on the specs of your machine.
     ```sh
     python activitysim/examples/placeholder_multiple_zone/scripts/two_zone_example_data.py
     python activitysim/examples/placeholder_multiple_zone/scripts/three_zone_example_data.py
     pytest .
     ```
- 
+
 00. Test the full-scale regional examples. These examples are big, too
-    large to run on Travis, and will take a lot of time (many hours) to 
+    large to run on GitHub Actions, and will take a lot of time (many hours) to
     download and run.
     ```sh
     mkdir tmp-asim
@@ -76,68 +77,41 @@
     call run_all_examples.bat
     ```
     These tests will run through the gamut even if some of them crash, so
-    if you don't sit and watch them go (please don't do this) you'll need 
+    if you don't sit and watch them go (please don't do this) you'll need
     to scan through the results to make sure there are no errors after the
     fact.
     ```sh
     python ../activitysim/examples/scan_examples_for_errors.py .
     ```
 
 00. Test the notebooks in `activitysim/examples/prototype_mtc/notebooks`.
-    There are also demo notebooks for estimation, but their functionality  
+    There are also demo notebooks for estimation, but their functionality
     is completely tested in the unit tests run previously.
 
-00. Use bump2version to tag the release commit and update the 
-    version number.  The following code will generate a "patch" release,
-    incrementing the third value in the version number (i.e. "1.2.3" 
-    becomes "1.2.4").  Alternatively, make a "minor" or "major" release. 
-    The `--list` command will generate output to your console to confirm 
-    that the old and new version numbers are what you expect, before you 
-    push the commit (with the changed version in the code) and tags to 
-    GitHub.
-    ```sh
-    bump2version patch --list
-    ```
-
-    It is also possible to make a development pre-release. To do so, 
-    explicitly set the version number to the next patch plus a ".devN" 
-    suffix:  
-
-    ```sh
-    bump2version patch --new-version 1.2.3.dev0 --list
-    ```
-
-    Then, when ready to make a "final" release, set the version by 
-    explicitly removing the suffix:  
-    ```sh
-    bump2version patch --new-version 1.2.3 --list
-    ```
-
-00. Push the tagged commit to GitHub.
-    ```sh
-    git push --tags
-    ```
-
-00. For non-development releases, open a pull request to merge the proposed 
-    release into main. The following command will open a web browser for 
+00. For non-development releases, open a pull request to merge the proposed
+    release into main. The following command will open a web browser for
     you to create the pull request.
     ```sh
     gh pr create --web
     ```
     After creating the PR, confirm with the ActivitySim PMC that the release
     is ready before actually merging it.
-    
+
     Once final approval is granted, merge the PR into main.  The presence
     of the git tags added earlier will trigger automated build steps to
     prepare and deploy the release to pypi and conda-forge.
-    
+
 00. Create a "release" on GitHub.
     ```sh
     gh release create v1.2.3
     ```
+    The process of creating and tagging a release will automatically
+    trigger various GitHub Actions scripts to build, test, and publish the
+    new release to PyPI and conda forge, assuming there are no errors.
+
     For a development pre-release, include the `--prerelease` argument.
     As the project's policy is that only formally released code is merged
-    to the main branch, any pre-release should also be built against a 
+    to the main branch, any pre-release should also be built against a
     non-default branch.  For example, to pre-release from the `develop`
     branch:
     ```sh
@@ -147,11 +121,19 @@
         --notes "this pre-release is for a cool new feature" \
         --title "Development Pre-Release"
     ```
-    
-00. Clean up your workspace, including removing the Conda environment used for 
-    testing (which will prevent you from accidentally using an old 
+
+00. Clean up your workspace, including removing the Conda environment used for
+    testing (which will prevent you from accidentally using an old
     environment when you should have a fresh up-to-date one next time).
     ```sh
     conda deactivate
     conda env remove -n TEMP-ASIM-DEV
-    ```
+    ```
+
+00. Change the default redirect page for the ActivitySim documentation to point
+    to the newly released documentation.  The redirect page can be edited
+    [here](https://github.com/ActivitySim/activitysim/blob/gh-pages/index.html).
+
+00. Add the new release to the `switch.json` file. Don't delete the references
+    for existing old documentation.  The switcher can be edited
+    [here](https://github.com/ActivitySim/activitysim/blob/gh-pages/switcher.json).

activitysim/cli/run.py

@@ -1,6 +1,7 @@
 # ActivitySim
 # See full license in LICENSE.txt.
 import argparse
+import importlib
 import logging
 import os
 import sys
@@ -13,7 +14,13 @@
 logger = logging.getLogger(__name__)
 
 
-INJECTABLES = ["data_dir", "configs_dir", "output_dir", "settings_file_name"]
+INJECTABLES = [
+    "data_dir",
+    "configs_dir",
+    "output_dir",
+    "settings_file_name",
+    "imported_extensions",
+]
 
 
 def add_run_args(parser, multiprocess=True):
@@ -64,6 +71,15 @@ def add_run_args(parser, multiprocess=True):
     parser.add_argument(
         "--households_sample_size", type=int, metavar="N", help="households sample size"
     )
+    parser.add_argument(
+        "-e",
+        "--ext",
+        type=str,
+        action="append",
+        metavar="PATH",
+        help="Package of extension modules to load. Use of this option is not "
+        "generally secure.",
+    )
 
     if multiprocess:
         parser.add_argument(
@@ -111,6 +127,23 @@ def inject_arg(name, value, cache=False):
         # 'configs', 'data', and 'output' folders by default
         os.chdir(args.working_dir)
 
+    if args.ext:
+        for e in args.ext:
+            basepath, extpath = os.path.split(e)
+            if not basepath:
+                basepath = "."
+            sys.path.insert(0, os.path.abspath(basepath))
+            try:
+                importlib.import_module(extpath)
+            except ImportError as err:
+                logger.exception("ImportError")
+                raise
+            finally:
+                del sys.path[0]
+        inject_arg("imported_extensions", args.ext)
+    else:
+        inject_arg("imported_extensions", ())
+
     # settings_file_name should be cached or else it gets squashed by config.py
     if args.settings_file:
         inject_arg("settings_file_name", args.settings_file, cache=True)
@@ -183,13 +216,24 @@ def run(args):
     # other callers (e.g. populationsim) will have to arrange to register their own steps and injectables
     # (presumably) in a custom run_simulation.py instead of using the 'activitysim run' command
     if not inject.is_injectable("preload_injectables"):
-        from activitysim import (  # register abm steps and other abm-specific injectables
-            abm,
-        )
+        # register abm steps and other abm-specific injectables
+        from activitysim import abm  # noqa: F401
 
     tracing.config_logger(basic=True)
     handle_standard_args(args)  # possibly update injectables
 
+    if config.setting("memory_profile", False) and not config.setting(
+        "multiprocess", False
+    ):
+        # Memory sidecar is only useful for single process runs
+        # multiprocess runs log memory usage without blocking in the controlling process.
+        mem_prof_log = config.log_file_path("memory_profile.csv")
+        from ..core.memory_sidecar import MemorySidecar
+
+        memory_sidecar_process = MemorySidecar(mem_prof_log)
+    else:
+        memory_sidecar_process = None
+
     # legacy support for run_list setting nested 'models' and 'resume_after' settings
     if config.setting("run_list"):
         warnings.warn(
@@ -281,7 +325,11 @@ def run(args):
         else:
             logger.info("run single process simulation")
 
-            pipeline.run(models=config.setting("models"), resume_after=resume_after)
+            pipeline.run(
+                models=config.setting("models"),
+                resume_after=resume_after,
+                memory_sidecar_process=memory_sidecar_process,
+            )
 
             if config.setting("cleanup_pipeline_after_run", False):
                 pipeline.cleanup_pipeline()  # has side effect of closing open pipeline
@@ -300,12 +348,15 @@ def run(args):
 
     tracing.print_elapsed_time("all models", t0)
 
+    if memory_sidecar_process:
+        memory_sidecar_process.stop()
+
     return 0
 
 
 if __name__ == "__main__":
 
-    from activitysim import abm  # register injectables
+    from activitysim import abm  # register injectables  # noqa: F401
 
     parser = argparse.ArgumentParser()
     add_run_args(parser)