From 93065c88c9fcefb22f870f5af2b08398425718e7 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 27 Jan 2021 13:35:46 -1000 Subject: [PATCH 01/48] Updating docker files for both normal services and for tests. This will be followed by some additional cleanup. --- docker/Dockerfile | 78 ++++++--------------- docker/cloudbuild.yaml | 28 ++++---- env => docker/env | 8 +-- environment.yaml => docker/environment.yaml | 6 +- scripts/entrypoint.sh | 2 +- tests/Dockerfile | 38 ++++++++++ tests/docker-compose.yaml | 40 ++++------- tests/env | 8 +-- 8 files changed, 101 insertions(+), 107 deletions(-) rename env => docker/env (66%) rename environment.yaml => docker/environment.yaml (69%) create mode 100644 tests/Dockerfile diff --git a/docker/Dockerfile b/docker/Dockerfile index c46b8a9a8..c5757be27 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,78 +1,44 @@ -FROM ubuntu:latest +ARG image_url=gcr.io/panoptes-exp/panoptes-utils +ARG image_tag=develop +FROM ${image_url}:${image_tag} AS pocs-base -LABEL description="Installs the panoptes-pocs module from GitHub." +LABEL description="PANOPTES Observatory Control System Service" LABEL maintainers="developers@projectpanoptes.org" LABEL repo="github.com/panoptes/POCS" ENV DEBIAN_FRONTEND=noninteractive ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 +ENV POCS "/POCS" -ARG panuser=pocs-user ARG userid=1000 -ARG pan_dir=/var/panoptes -ARG pocs_dir="${pan_dir}/POCS" -ARG pip_install_extras="[google,testing]" - -ENV PANUSER $panuser ENV USERID $userid -ENV PANDIR $pan_dir -ENV POCS $pocs_dir - -# Install system dependencies. -RUN apt-get update && apt-get install --no-install-recommends --yes \ - bzip2 ca-certificates \ - wget gcc git pkg-config sudo gosu less udev \ - astrometry.net astrometry-data-tycho2-10-19 dcraw exiftool \ - libcfitsio-dev libcfitsio-bin \ - libfreetype6-dev libpng-dev libjpeg-dev libffi-dev \ - gphoto2 && \ - useradd -u ${USERID} -o -c "PANOPTES POCS" \ - -p panoptes -m -G plugdev,dialout,users,sudo ${PANUSER} && \ - # Allow sudo without password. - echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers && \ - # Setup SSH so localhost works without password - mkdir -p "/home/${panuser}/.ssh" && \ - echo "Host localhost\n\tStrictHostKeyChecking no\n" >> "/home/${panuser}/.ssh/config" USER "${userid}" -# Miniconda -WORKDIR /tmp -RUN echo "Installing conda via miniforge" && \ - sudo mkdir -p /conda && \ - sudo chown -R "${PANUSER}:${PANUSER}" /conda && \ - # Miniforge - wget "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-$(uname -m).sh" \ - -O install-miniforge.sh && \ - /bin/sh install-miniforge.sh -b -f -p /conda && \ - # Initialize conda for the shells. - /conda/bin/conda init bash - -ENV PATH "/home/${PANUSER}/.local/bin:$PATH" +COPY docker/environment.yaml . +RUN /conda/bin/conda env update -n base -f environment.yaml -RUN echo "Setting up filesystem and installing conda." && \ - # Create PANOPTES directories. - sudo mkdir -p ${PANDIR} && \ - sudo chown -R "${PANUSER}:${PANUSER}" ${PANDIR} && \ - mkdir -p ${PANDIR}/logs && \ - mkdir -p ${PANDIR}/images && \ - mkdir -p ${POCS} +# Set up some common directories +RUN echo "Building from ${image_name}:${image_tag}" && \ + sudo mkdir /images && sudo chown -R "${userid}:${userid}" /images && \ + sudo mkdir /logs && sudo chown -R "${userid}:${userid}" /logs && \ + sudo mkdir "${POCS}" && sudo chown -R "${userid}:${userid}" "${POCS}" -COPY environment.yaml . -RUN /conda/bin/conda env update -n base -f environment.yaml +ARG pip_install_extras="[google]" WORKDIR "${POCS}" -COPY --chown="${userid}:${userid}" . . -RUN echo "Installing editable module with ${pip_install_extras}" && \ - /conda/bin/pip install -e ".${pip_install_extras}" && \ - # Remove git folder - rm -rf "${POCS}/.git" && \ +RUN echo "Installing panoptes-pocs module with ${pip_install_extras}" && \ + /conda/bin/pip install -e "panoptes-pocs${pip_install_extras}" && \ # Cleanup - sudo apt-get autoremove --purge --yes && \ + /conda/bin/pip cache purge && \ + /conda/bin/conda clean -tipy && \ + sudo apt-get autoremove --purge --yes \ + gcc pkg-config git && \ sudo apt-get autoclean --yes && \ sudo apt-get --yes clean && \ sudo rm -rf /var/lib/apt/lists/* -USER root -ENTRYPOINT ["/bin/sh", "/var/panoptes/POCS/scripts/entrypoint.sh"] +ENTRYPOINT [ "/usr/bin/env", "bash", "-ic" ] + +# TODO Change to pocs cli. CMD [ "/bin/bash" ] diff --git a/docker/cloudbuild.yaml b/docker/cloudbuild.yaml index 39721473c..0ca8a2133 100644 --- a/docker/cloudbuild.yaml +++ b/docker/cloudbuild.yaml @@ -1,26 +1,29 @@ options: - machineType: "N1_HIGHCPU_8" substitutionOption: "ALLOW_LOOSE" timeout: 18000s # 5 hours substitutions: - _PLATFORMS: linux/amd64,linux/arm64 _IMAGE_NAME: panoptes-pocs - _REPO_URL: https://github.com/panoptes/POCS - _TAG: latest + _PLATFORM: linux/arm64,linux/amd64 steps: # Fetch the repo from github - name: gcr.io/cloud-builders/git id: "clone-repo" - args: [ "clone", "${_REPO_URL}" ] + args: [ "clone", "${REPO_NAME}" ] waitFor: [ "-" ] + # Fetch the repo from github + - name: gcr.io/cloud-builders/git + id: "checkout-branch" + args: [ "checkout", "${COMMIT_SHA}" ] + waitFor: [ "clone-repo" ] + # Pull the cached image. - name: 'gcr.io/cloud-builders/docker' id: "pull-cached-image" entrypoint: 'bash' - args: [ '-c', 'docker pull gcr.io/${PROJECT_ID}/${_IMAGE_NAME}:${_TAG} || exit 0' ] + args: [ '-c', 'docker pull gcr.io/${PROJECT_ID}/${_IMAGE_NAME}:${TAG_NAME} || exit 0' ] waitFor: [ "-" ] # Set up multiarch support @@ -43,6 +46,7 @@ steps: args: - "buildx" - "create" + - "--name=build" - "--use" - "--driver=docker-container" waitFor: [ "setup-buildx" ] @@ -55,10 +59,10 @@ steps: args: - "buildx" - "build" - - "--push" - - "--platform=${_PLATFORMS}" + - "--platform=${_PLATFORM}" - "-f=docker/Dockerfile" - - "--tag=gcr.io/${PROJECT_ID}/${_IMAGE_NAME}:${_TAG}" - - "--cache-from=gcr.io/${PROJECT_ID}/${_IMAGE_NAME}:${_TAG}" - - "POCS" - waitFor: [ "build-builder", "clone-repo" ] + - "--tag=gcr.io/${PROJECT_ID}/${_IMAGE_NAME}:${TAG_NAME}" + - "--cache-from=gcr.io/${PROJECT_ID}/${_IMAGE_NAME}:${TAG_NAME}" + - "--push" + - "." + waitFor: [ "build-builder", "checkout-branch" ] diff --git a/env b/docker/env similarity index 66% rename from env rename to docker/env index fb58cef63..a4dbb2699 100644 --- a/env +++ b/docker/env @@ -5,9 +5,9 @@ # # THESE ARE VALUES INSIDE A RUNNING DOCKER CONTAINER. # -PANDIR=/var/panoptes -POCS=/var/panoptes/POCS -PANLOG=/var/panoptes/logs -PANOPTES_CONFIG_FILE=/var/panoptes/POCS/conf_files/pocs.yaml +PANDIR=/POCS +POCS=/POCS +PANLOG=/logs +PANOPTES_CONFIG_FILE=/POCS/conf_files/pocs.yaml PANOPTES_CONFIG_HOST=0.0.0.0 PANOPTES_CONFIG_PORT=6563 diff --git a/environment.yaml b/docker/environment.yaml similarity index 69% rename from environment.yaml rename to docker/environment.yaml index c8daadb96..eeb5068af 100644 --- a/environment.yaml +++ b/docker/environment.yaml @@ -3,8 +3,10 @@ channels: dependencies: - matplotlib-base - numpy + - photutils + - pillow + - pip - readline # POCS shell - scipy - - pip - pip: - - panoptes-utils[config,images,social] >= 0.2.30 + - panoptes-utils[config] >= 0.2.30 diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index 1dd9a39b8..18836d6bf 100644 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -2,4 +2,4 @@ set -e # Pass arguments -exec gosu pocs-user /usr/bin/env bash -ic "$@" +exec /usr/bin/env bash -ic "$@" diff --git a/tests/Dockerfile b/tests/Dockerfile new file mode 100644 index 000000000..3e6fc163f --- /dev/null +++ b/tests/Dockerfile @@ -0,0 +1,38 @@ +ARG image_url=gcr.io/panoptes-exp/panoptes-utils +ARG image_tag=develop +FROM ${image_url}:${image_tag} AS pocs-base + +ENV DEBIAN_FRONTEND=noninteractive +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 +ENV SHELL /bin/bash + +ENV POCS "/POCS" + +ARG userid=1000 +ARG pip_extras="[focuser,google,testing]" + +USER "${userid}" + +WORKDIR /tmp +RUN sudo apt-get update && \ + sudo apt-get install -y --no-install-recommends \ + astrometry-data-2mass-08-19 && \ + sudo mkdir "${POCS}" && sudo chown -R "${userid}:${userid}" "${POCS}" + +COPY docker/environment.yaml . +RUN /conda/bin/conda env update -n base -f environment.yaml + +WORKDIR "${POCS}" +COPY --chown="${userid}:${userid}" . . +RUN /conda/bin/pip install -e ".${pip_extras}" && \ + # Cleanup + /conda/bin/pip cache purge && \ + /conda/bin/conda clean -tipy && \ + sudo apt-get autoremove --purge --yes \ + gcc pkg-config git && \ + sudo apt-get autoclean --yes && \ + sudo apt-get --yes clean && \ + sudo rm -rf /var/lib/apt/lists/* + +ENTRYPOINT [ "/usr/bin/env", "bash", "-ic" ] +CMD [ "pytest" ] diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index b2a5b9001..9a15d7f7e 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -1,41 +1,25 @@ version: '3.7' services: - config-server: - image: panoptes-pocs:latest + pocs-tester: + image: panoptes-pocs:testing build: context: ../ - dockerfile: docker/Dockerfile + dockerfile: tests/Dockerfile init: true tty: true - network_mode: host environment: - # These need to be defined in the shell on the host. - PANDIR: - PANLOG: - PANOPTES_CONFIG_HOST: - PANOPTES_CONFIG_PORT: - PANOPTES_CONFIG_FILE: - command: [ "panoptes-config-server --verbose run" ] - pocs: - image: panoptes-pocs:latest - build: - context: ../ - dockerfile: docker/Dockerfile - init: true + - PANDIR=/POCS + - POCS=/POCS + - PANLOG=/logs + - PANOPTES_CONFIG_FILE=/POCS/tests/testing.yaml + - PANOPTES_CONFIG_HOST=localhost + - PANOPTES_CONFIG_PORT=8765 privileged: true network_mode: host - environment: - # These need to be defined in the shell on the host. - PANDIR: - PANLOG: - PANOPTES_CONFIG_HOST: - PANOPTES_CONFIG_PORT: - PANOPTES_CONFIG_FILE: - # No-op to keep machine running, use $POCS/bin/pocs-shell to access - command: [ "wait-for-it ${PANOPTES_CONFIG_PORT}:${PANOPTES_CONFIG_PORT} -- pytest" ] + command: [ "pytest" ] volumes: - - logdir:/var/panoptes/logs - - builddir:/var/panoptes/POCS/build + - logdir:/logs + - builddir:/POCS/build volumes: logdir: driver: local diff --git a/tests/env b/tests/env index 159f5958b..ea10796df 100644 --- a/tests/env +++ b/tests/env @@ -5,9 +5,9 @@ # # THESE ARE VALUES INSIDE A RUNNING DOCKER CONTAINER. # -PANDIR=/var/panoptes -POCS=/var/panoptes/POCS -PANLOG=/var/panoptes/logs -PANOPTES_CONFIG_FILE=/var/panoptes/POCS/tests/testing.yaml +PANDIR=/POCS +POCS=/POCS +PANLOG=/logs +PANOPTES_CONFIG_FILE=/POCS/tests/testing.yaml PANOPTES_CONFIG_HOST=0.0.0.0 PANOPTES_CONFIG_PORT=8765 From e4ed10fe3ebe5d2173030579c066cfead77a7b6f Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 27 Jan 2021 13:36:12 -1000 Subject: [PATCH 02/48] * Remove unused env file. --- setup.cfg | 2 +- tests/env | 13 ------------- 2 files changed, 1 insertion(+), 14 deletions(-) delete mode 100644 tests/env diff --git a/setup.cfg b/setup.cfg index aff69ef8a..729da6f47 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,7 +42,7 @@ setup_requires = pyscaffold>=3.2a0,<3.3a0 install_requires = astroplan astropy - panoptes-utils>=0.2.30 + panoptes-utils[config]>=0.2.30 pyserial transitions # The usage of test_requires is discouraged, see `Dependency Management` docs diff --git a/tests/env b/tests/env deleted file mode 100644 index ea10796df..000000000 --- a/tests/env +++ /dev/null @@ -1,13 +0,0 @@ -# Envfile to be loaded by docker-compose for testing. -# -# Note this doesn't support full interpolation or quotes. -# See https://docs.docker.com/compose/compose-file/#env_file -# -# THESE ARE VALUES INSIDE A RUNNING DOCKER CONTAINER. -# -PANDIR=/POCS -POCS=/POCS -PANLOG=/logs -PANOPTES_CONFIG_FILE=/POCS/tests/testing.yaml -PANOPTES_CONFIG_HOST=0.0.0.0 -PANOPTES_CONFIG_PORT=8765 From 873738d94376567cda6af2396bbd0ca73507a263 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 27 Jan 2021 13:36:21 -1000 Subject: [PATCH 03/48] * Small cleanup to rpi file. --- resources/rpi/user-data | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/resources/rpi/user-data b/resources/rpi/user-data index 354af3a06..c5ae02fac 100644 --- a/resources/rpi/user-data +++ b/resources/rpi/user-data @@ -3,13 +3,6 @@ hostname: pocs-control -# If you have set up ssh on github you can pull down your -# key automatically so that you can log into the unit without -# a password. -ssh_import_id: - # - gh:your_github_id - - gh:panoptes - ##################################################################### # You shouldn't need to change anything below. ##################################################################### @@ -55,11 +48,11 @@ packages: - apt-transport-https - byobu - ca-certificates + - docker.io - git - htop - httpie - jq - - neovim - software-properties-common - speedometer - vim-nox From 37eb9a6d86bcceb89ce0a47a2a85a725219ed209 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 27 Jan 2021 13:39:31 -1000 Subject: [PATCH 04/48] * Updates to testing config. --- tests/testing.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/testing.yaml b/tests/testing.yaml index 8c2fa0c31..3f53eb395 100644 --- a/tests/testing.yaml +++ b/tests/testing.yaml @@ -31,12 +31,12 @@ location: gmt_offset: -600 # Offset in minutes from GMT during. # standard time (not daylight saving). directories: - base: /var/panoptes + base: /POCS images: images data: data - resources: POCS/resources/ - targets: POCS/conf_files/targets - mounts: POCS/resources/mounts + resources: resources/ + targets: conf_files/targets + mounts: resources/mounts db: name: panoptes_testing type: file From fb0fd471f9faa382d8cc6ff94f0462d21ed479a5 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 27 Jan 2021 13:48:03 -1000 Subject: [PATCH 05/48] * Remove `PANLOG` references and default to `./logs` for logger. --- conftest.py | 2 +- docker/docker-compose.yaml | 2 -- docker/env | 3 +-- src/panoptes/pocs/utils/logger.py | 13 ++----------- tests/docker-compose.yaml | 1 - 5 files changed, 4 insertions(+), 17 deletions(-) diff --git a/conftest.py b/conftest.py index 5f0e26cc6..b547310cf 100644 --- a/conftest.py +++ b/conftest.py @@ -30,7 +30,7 @@ "| {name} {function}:{line} | " \ "{message}" -log_file_path = os.path.expandvars('${PANLOG}/panoptes-testing.log') +log_file_path = os.path.expandvars('logs/panoptes-testing.log') startup_message = f' STARTING NEW PYTEST RUN - LOGS: {log_file_path} ' logger.add(log_file_path, enqueue=True, # multiprocessing diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index a9b54a9fc..c72fe3443 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -13,7 +13,6 @@ services: environment: # These need to be defined in the shell on the host. PANDIR: - PANLOG: PANOPTES_CONFIG_HOST: PANOPTES_CONFIG_PORT: PANOPTES_CONFIG_FILE: @@ -35,7 +34,6 @@ services: environment: # These need to be defined in the shell on the host. PANDIR: - PANLOG: PANOPTES_CONFIG_HOST: PANOPTES_CONFIG_PORT: PANOPTES_CONFIG_FILE: diff --git a/docker/env b/docker/env index a4dbb2699..044581177 100644 --- a/docker/env +++ b/docker/env @@ -7,7 +7,6 @@ # PANDIR=/POCS POCS=/POCS -PANLOG=/logs -PANOPTES_CONFIG_FILE=/POCS/conf_files/pocs.yaml +PANOPTES_CONFIG_FILE=conf_files/pocs.yaml PANOPTES_CONFIG_HOST=0.0.0.0 PANOPTES_CONFIG_PORT=6563 diff --git a/src/panoptes/pocs/utils/logger.py b/src/panoptes/pocs/utils/logger.py index 432d31231..070d7c63d 100644 --- a/src/panoptes/pocs/utils/logger.py +++ b/src/panoptes/pocs/utils/logger.py @@ -38,7 +38,7 @@ def format(self, record): def get_logger(console_log_file='panoptes.log', full_log_file='panoptes_{time:YYYYMMDD!UTC}.log', - log_dir=None, + log_dir='logs', console_log_level='DEBUG', stderr_log_level='INFO', ): @@ -50,9 +50,6 @@ def get_logger(console_log_file='panoptes.log', Note: This clobbers all existing loggers and forces the two files. - Note: The `log_dir` is determined first from `$PANLOG` if it exists, then - `$PANDIR/logs` if `$PANDIR` exists, otherwise defaults to `.`. - Args: console_log_file (str|None, optional): Filename for the file that is suitable for tailing in a shell (i.e., read by humans). This file is rotated daily however @@ -61,7 +58,7 @@ def get_logger(console_log_file='panoptes.log', and is serialized and rotated automatically. Useful for uploading to log service website. Defaults to `panoptes_{time:YYYYMMDD!UTC}.log.gz` with a daily rotation at 11:30am and a 7 day retention policy. If `None` then no file will be generated. - log_dir (str|None, optional): The directory to place the log file, see note. + log_dir (str|None, optional): The directory to place the log file, default local `logs`. stderr_log_level (str, optional): The log level to show on stderr, default INFO. console_log_level (str, optional): Log level for console file output, defaults to 'DEBUG'. Note that it should be a string that matches standard `logging` levels and @@ -71,12 +68,6 @@ def get_logger(console_log_file='panoptes.log', Returns: `loguru.logger`: A configured instance of the logger. """ - - if log_dir is None: - try: - log_dir = os.environ['PANLOG'] - except KeyError: - log_dir = os.path.join(os.getenv('PANDIR', '.'), 'logs') log_dir = os.path.normpath(log_dir) os.makedirs(log_dir, exist_ok=True) diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index 9a15d7f7e..9ded0fe14 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -10,7 +10,6 @@ services: environment: - PANDIR=/POCS - POCS=/POCS - - PANLOG=/logs - PANOPTES_CONFIG_FILE=/POCS/tests/testing.yaml - PANOPTES_CONFIG_HOST=localhost - PANOPTES_CONFIG_PORT=8765 From 15d884aca4a626db7f9dc35a5b9bd9f582f5eecd Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 27 Jan 2021 13:57:46 -1000 Subject: [PATCH 06/48] * docker-compose service cleaner with env file. * Default image has tycho2 and 2mass index files for solving. * Dockerfile installs from pip. --- docker/Dockerfile | 7 ++++- docker/README.md | 6 ---- docker/docker-compose.yaml | 60 +++++++++++++------------------------- 3 files changed, 27 insertions(+), 46 deletions(-) delete mode 100644 docker/README.md diff --git a/docker/Dockerfile b/docker/Dockerfile index c5757be27..e46fa90d4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,6 +20,11 @@ RUN /conda/bin/conda env update -n base -f environment.yaml # Set up some common directories RUN echo "Building from ${image_name}:${image_tag}" && \ + RUN sudo apt-get update && \ + sudo apt-get install -y --no-install-recommends \ + astrometry-data-tycho2 \ + astrometry-data-2mass && \ + sudo mkdir "${POCS}" && sudo chown -R "${userid}:${userid}" "${POCS}" sudo mkdir /images && sudo chown -R "${userid}:${userid}" /images && \ sudo mkdir /logs && sudo chown -R "${userid}:${userid}" /logs && \ sudo mkdir "${POCS}" && sudo chown -R "${userid}:${userid}" "${POCS}" @@ -28,7 +33,7 @@ ARG pip_install_extras="[google]" WORKDIR "${POCS}" RUN echo "Installing panoptes-pocs module with ${pip_install_extras}" && \ - /conda/bin/pip install -e "panoptes-pocs${pip_install_extras}" && \ + /conda/bin/pip install "panoptes-pocs${pip_install_extras}" && \ # Cleanup /conda/bin/pip cache purge && \ /conda/bin/conda clean -tipy && \ diff --git a/docker/README.md b/docker/README.md deleted file mode 100644 index afc4a3cef..000000000 --- a/docker/README.md +++ /dev/null @@ -1,6 +0,0 @@ -Docker Images -============= - -POCS is available as a docker image hosted on Google Cloud Registry (GCR): - -Image name: `gcr.io/panoptes-exp/panoptes-pocs:latest` diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index c72fe3443..cd91a1d23 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,57 +1,39 @@ version: '3.7' services: config-server: - image: panoptes-pocs:latest - build: - context: ../ - dockerfile: docker/Dockerfile + image: gcr.io/panoptes-exp/panoptes-utils:develop + deploy: + mode: global init: true tty: true container_name: config-server hostname: config-server network_mode: host - environment: - # These need to be defined in the shell on the host. - PANDIR: - PANOPTES_CONFIG_HOST: - PANOPTES_CONFIG_PORT: - PANOPTES_CONFIG_FILE: - restart: on-failure + env_file: + - env + command: [ "panoptes-config-server --verbose run --config-file /app/config.yaml" ] volumes: - - pocsdir:/var/panoptes/POCS - - confdir:/var/panoptes/config_files - command: [ "panoptes-config-server --verbose run" ] - pocs: - image: panoptes-pocs:latest - build: - context: ../ - dockerfile: docker/Dockerfile + - type: bind + source: "${PANOPTES_CONFIG_FILE}" + target: /app/config.yaml + pocs-control: + image: gcr.io/panoptes-exp/panoptes-pocs:develop + deploy: + mode: global init: true - container_name: pocs - hostname: pocs + container_name: pocs-control + hostname: pocs-control privileged: true network_mode: host - environment: - # These need to be defined in the shell on the host. - PANDIR: - PANOPTES_CONFIG_HOST: - PANOPTES_CONFIG_PORT: - PANOPTES_CONFIG_FILE: + env_file: + - env volumes: - - pocsdir:/var/panoptes/POCS - - /var/panoptes/logs:/var/panoptes/logs - # No-op to keep machine running, use $POCS/bin/pocs-shell to access - command: [ "wait-for-it ${PANOPTES_CONFIG_PORT}:${PANOPTES_CONFIG_PORT} -- echo POCS yo!" ] + - logdir:/logs + command: [ "wait-for-it ${PANOPTES_CONFIG_PORT}:${PANOPTES_CONFIG_PORT} -- /bin/bash" ] volumes: - pocsdir: + logdir: driver: local driver_opts: type: none - device: /var/panoptes/POCS - o: bind - confdir: - driver: local - driver_opts: - type: none - device: /var/panoptes/config_files + device: "${PANLOG:-${PWD}/logs}" o: bind From 3b8547ba167ff51a2d6ece18a9c5fdfca6b8df9e Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 27 Jan 2021 14:08:13 -1000 Subject: [PATCH 07/48] * Update tests index files. --- tests/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Dockerfile b/tests/Dockerfile index 3e6fc163f..95b3dc105 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -16,7 +16,7 @@ USER "${userid}" WORKDIR /tmp RUN sudo apt-get update && \ sudo apt-get install -y --no-install-recommends \ - astrometry-data-2mass-08-19 && \ + astrometry-data-tycho2-10-19 && \ sudo mkdir "${POCS}" && sudo chown -R "${userid}:${userid}" "${POCS}" COPY docker/environment.yaml . From 2bfff503f3ccdd6a55139394214a4ea1445ed300 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 27 Jan 2021 14:14:20 -1000 Subject: [PATCH 08/48] * Updates to GHA tests. --- .github/workflows/pythontest.yaml | 8 ++++---- tests/docker-compose.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pythontest.yaml b/.github/workflows/pythontest.yaml index 2e455e4ae..4980a59d1 100644 --- a/.github/workflows/pythontest.yaml +++ b/.github/workflows/pythontest.yaml @@ -31,12 +31,12 @@ jobs: uses: actions/checkout@v2 - name: Build panoptes-pocs image run: | - docker-compose -f tests/docker-compose.yaml --env-file tests/env build - - name: Test with pytest in panoptes-pocs container - run: | + docker-compose -f tests/docker-compose.yaml build mkdir -p logs && chmod -R 777 logs mkdir -p build && chmod -R 777 build - docker-compose -f tests/docker-compose.yaml --env-file tests/env run pocs pytest + - name: Test with pytest in panoptes-pocs container + run: | + docker-compose -f tests/docker-compose.yaml run pocs pytest - name: Upload coverage report to codecov.io uses: codecov/codecov-action@v1 if: success() diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index 9ded0fe14..0e69ba239 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -1,6 +1,6 @@ version: '3.7' services: - pocs-tester: + pocs: image: panoptes-pocs:testing build: context: ../ From b5edf4e56ac1422191d4e227114d31a8c55dd1fa Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 27 Jan 2021 15:00:02 -1000 Subject: [PATCH 09/48] * Remove old entrypoint script. --- CHANGELOG.rst | 21 ++++++++++++++++++++- docker/cloudbuild.yaml | 16 ++-------------- docker/env | 2 +- scripts/entrypoint.sh | 5 ----- 4 files changed, 23 insertions(+), 21 deletions(-) delete mode 100644 scripts/entrypoint.sh diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2592a1aaa..0201eb366 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,15 +2,33 @@ Changelog ========= -[0.7.dev8] +[0.7.8dev] ---------- +Generic +~~~~~~~ + +* **Breaking changes** #1068 + + * Docker services use the ``pip`` module, so don't include recent changes unless merged into ``develop``. + * Default service install does not include ``focuser`` dependencies. + * Default Docker command is a ``bash`` shell and will be changed with future PR to ``pocs`` cli. + * Directories inside the service image have been simplified for easier mapping onto desired targets on the host: + + * ``/POCS`` + * ``/logs`` + * ``/images`` + Docker ~~~~~~ * ``PANUSER`` owns ``conda``. #1068 * Dockerfile cleanup for better builds. #1068 * Docker image does not contain ``focuser`` extras by default. #1068 +* Images use ``gcr.io/panoptes-exp/panoptes-utils`` as base. #1074 +* Docker files are all contained within ``docker`` folder. #1074 +* Both tycho2 (wide-field) and 2mass (narrow-field) index files are included. #1074 + * Docker services (``config-server`` and ``pocs-control``) are started in ``global`` mode so tehre can be only one. # 1074 Testing ~~~~~~~ @@ -18,6 +36,7 @@ Testing * Fix the log level in conftest. #1068 * Move all tests into ``tests`` subdir from project root. #1068 * Cleanup of testing setup, especially for GHA. #1068 +* Simplify testing service by removing ``tests/env`` file. #1074 [0.7.7] - 2021-01-19 -------------------- diff --git a/docker/cloudbuild.yaml b/docker/cloudbuild.yaml index 0ca8a2133..782639673 100644 --- a/docker/cloudbuild.yaml +++ b/docker/cloudbuild.yaml @@ -7,18 +7,6 @@ substitutions: _PLATFORM: linux/arm64,linux/amd64 steps: - # Fetch the repo from github - - name: gcr.io/cloud-builders/git - id: "clone-repo" - args: [ "clone", "${REPO_NAME}" ] - waitFor: [ "-" ] - - # Fetch the repo from github - - name: gcr.io/cloud-builders/git - id: "checkout-branch" - args: [ "checkout", "${COMMIT_SHA}" ] - waitFor: [ "clone-repo" ] - # Pull the cached image. - name: 'gcr.io/cloud-builders/docker' id: "pull-cached-image" @@ -51,7 +39,7 @@ steps: - "--driver=docker-container" waitFor: [ "setup-buildx" ] - # Build with cloned panoptes-utils as source directory. + # Build the image. - name: "gcr.io/cloud-builders/docker" id: "build-images" env: @@ -65,4 +53,4 @@ steps: - "--cache-from=gcr.io/${PROJECT_ID}/${_IMAGE_NAME}:${TAG_NAME}" - "--push" - "." - waitFor: [ "build-builder", "checkout-branch" ] + waitFor: [ "build-builder" ] diff --git a/docker/env b/docker/env index 044581177..a8c926b3f 100644 --- a/docker/env +++ b/docker/env @@ -7,6 +7,6 @@ # PANDIR=/POCS POCS=/POCS -PANOPTES_CONFIG_FILE=conf_files/pocs.yaml +PANOPTES_CONFIG_FILE=/POCS/conf_files/pocs.yaml PANOPTES_CONFIG_HOST=0.0.0.0 PANOPTES_CONFIG_PORT=6563 diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh deleted file mode 100644 index 18836d6bf..000000000 --- a/scripts/entrypoint.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -set -e - -# Pass arguments -exec /usr/bin/env bash -ic "$@" From e6d196c7602f2f470024ac6936eceb0bf5b78983 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 27 Jan 2021 20:52:52 -1000 Subject: [PATCH 10/48] * Minor clean ups, especially some of the dependencies. * Copy specific files into the Docker image. * Add ipython to the docker build environment file. * Simplify `pocs-cmd` (might just replace later). --- .dockerignore | 1 - .gcloudignore | 9 ++++++--- README.md | 1 + bin/pocs-cmd | 17 ++++------------- docker/Dockerfile | 27 ++++++++++++++++----------- docker/docker-compose.yaml | 12 ++++++++---- docker/environment.yaml | 1 + src/panoptes/pocs/core.py | 6 ++---- src/panoptes/pocs/images.py | 8 ++++---- src/panoptes/pocs/mount/__init__.py | 4 ++-- src/panoptes/pocs/observatory.py | 4 +--- 11 files changed, 45 insertions(+), 45 deletions(-) diff --git a/.dockerignore b/.dockerignore index 91be5f110..6a9bb728d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -22,7 +22,6 @@ logs/ notebooks examples docs -build coverage.xml # Build and docs folder/files diff --git a/.gcloudignore b/.gcloudignore index 730ffb944..2d471a8c5 100644 --- a/.gcloudignore +++ b/.gcloudignore @@ -3,7 +3,7 @@ venv # See note in .dockerignore about the git folder. -.git +!.git .github *.md @@ -21,5 +21,8 @@ logs/ notebooks examples docs -build -coverage.xml + +# Build and docs folder/files +build/* +dist/* +sdist/* diff --git a/README.md b/README.md index be88185dc..ceaea90c4 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ about the project, including the science case and resources for interested indiv # POCS + POCS (PANOPTES Observatory Control System) is the main software driver for a PANOPTES unit, responsible for high-level control of the unit. diff --git a/bin/pocs-cmd b/bin/pocs-cmd index f15c131a1..84c959f48 100755 --- a/bin/pocs-cmd +++ b/bin/pocs-cmd @@ -1,15 +1,6 @@ -#!/bin/bash -e +#!/usr/bin/env bash +set -e -USER_ID=$(id -u) -DOCKER_NAME="pocs-shell" +DOCKER_NAME="pocs-control" -if [ ! "$(docker ps -q -f name=${DOCKER_NAME})" ]; then - echo "${DOCKER_NAME} not running. Start services with scripts/pocs-docker.sh" -else - if [ $# -eq 0 ]; then - echo "Starting shell on ${DOCKER_NAME}" - docker exec --user "${USER_ID}" -it "${DOCKER_NAME}" /bin/zsh -i - else - docker exec --user "${USER_ID}" -it "${DOCKER_NAME}" /bin/zsh -ic "$@" - fi -fi +docker exec -it "${DOCKER_NAME}" "$@" &>/dev/null || echo "${DOCKER_NAME} not running" diff --git a/docker/Dockerfile b/docker/Dockerfile index e46fa90d4..632e9655f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,35 +15,40 @@ ENV USERID $userid USER "${userid}" -COPY docker/environment.yaml . -RUN /conda/bin/conda env update -n base -f environment.yaml - # Set up some common directories RUN echo "Building from ${image_name}:${image_tag}" && \ - RUN sudo apt-get update && \ + sudo apt-get update && \ sudo apt-get install -y --no-install-recommends \ + gphoto2 \ astrometry-data-tycho2 \ astrometry-data-2mass && \ - sudo mkdir "${POCS}" && sudo chown -R "${userid}:${userid}" "${POCS}" sudo mkdir /images && sudo chown -R "${userid}:${userid}" /images && \ sudo mkdir /logs && sudo chown -R "${userid}:${userid}" /logs && \ sudo mkdir "${POCS}" && sudo chown -R "${userid}:${userid}" "${POCS}" +COPY docker/environment.yaml . +RUN /conda/bin/conda env update -n base -f environment.yaml + +ARG pip_install_name="." ARG pip_install_extras="[google]" -WORKDIR "${POCS}" -RUN echo "Installing panoptes-pocs module with ${pip_install_extras}" && \ - /conda/bin/pip install "panoptes-pocs${pip_install_extras}" && \ +COPY --chown="${userid}:${userid}" . . +RUN echo "Installing ${pip_install_name} module with ${pip_install_extras}" && \ + /conda/bin/pip install "${pip_install_name}${pip_install_extras}" && \ # Cleanup /conda/bin/pip cache purge && \ - /conda/bin/conda clean -tipy && \ + /conda/bin/conda clean -fay && \ sudo apt-get autoremove --purge --yes \ gcc pkg-config git && \ sudo apt-get autoclean --yes && \ sudo apt-get --yes clean && \ sudo rm -rf /var/lib/apt/lists/* +WORKDIR "${POCS}" +COPY --chown="${userid}:${userid}" docker/docker-compose.yaml . +COPY --chown="${userid}:${userid}" conf_files conf_files +COPY --chown="${userid}:${userid}" resources resources +COPY --chown="${userid}:${userid}" scripts scripts ENTRYPOINT [ "/usr/bin/env", "bash", "-ic" ] -# TODO Change to pocs cli. -CMD [ "/bin/bash" ] +CMD [ "ipython -i scripts/load-ipython.py" ] diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index cd91a1d23..155260338 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -9,8 +9,10 @@ services: container_name: config-server hostname: config-server network_mode: host - env_file: - - env + environment: + - PANOPTES_CONFIG_FILE=/POCS/conf_files/pocs.yaml + - PANOPTES_CONFIG_HOST=0.0.0.0 + - PANOPTES_CONFIG_PORT=6563 command: [ "panoptes-config-server --verbose run --config-file /app/config.yaml" ] volumes: - type: bind @@ -25,8 +27,10 @@ services: hostname: pocs-control privileged: true network_mode: host - env_file: - - env + environment: + - PANOPTES_CONFIG_FILE=/POCS/conf_files/pocs.yaml + - PANOPTES_CONFIG_HOST=0.0.0.0 + - PANOPTES_CONFIG_PORT=6563 volumes: - logdir:/logs command: [ "wait-for-it ${PANOPTES_CONFIG_PORT}:${PANOPTES_CONFIG_PORT} -- /bin/bash" ] diff --git a/docker/environment.yaml b/docker/environment.yaml index eeb5068af..02efafef9 100644 --- a/docker/environment.yaml +++ b/docker/environment.yaml @@ -1,6 +1,7 @@ channels: - https://conda.anaconda.org/conda-forge dependencies: + - ipython # Interactive - matplotlib-base - numpy - photutils diff --git a/src/panoptes/pocs/core.py b/src/panoptes/pocs/core.py index ff5210bde..79cfc237e 100644 --- a/src/panoptes/pocs/core.py +++ b/src/panoptes/pocs/core.py @@ -1,11 +1,8 @@ import os -import sys -import warnings from threading import Thread from contextlib import suppress from astropy import units as u - from panoptes.pocs.base import PanBase from panoptes.pocs.observatory import Observatory from panoptes.pocs.state.machine import PanStateMachine @@ -426,7 +423,8 @@ def is_weather_safe(self, stale=180): return is_safe - def has_free_space(self, directory=None, required_space=0.25 * u.gigabyte, low_space_percent=1.5): + def has_free_space(self, directory=None, required_space=0.25 * u.gigabyte, + low_space_percent=1.5): """Does hard drive have disk space (>= 0.5 GB). Args: diff --git a/src/panoptes/pocs/images.py b/src/panoptes/pocs/images.py index cff6dd449..0344f4703 100644 --- a/src/panoptes/pocs/images.py +++ b/src/panoptes/pocs/images.py @@ -1,5 +1,6 @@ import os from contextlib import suppress +from collections import namedtuple from astropy import units as u from astropy.coordinates import EarthLocation @@ -7,10 +8,8 @@ from astropy.coordinates import SkyCoord from astropy.io import fits from astropy.time import Time -from collections import namedtuple - -from .base import PanBase from panoptes.utils.images import fits as fits_utils +from panoptes.pocs.base import PanBase OffsetError = namedtuple('OffsetError', ['delta_ra', 'delta_dec', 'magnitude']) @@ -201,7 +200,8 @@ def solve_field(self, **kwargs): return solve_info def compute_offset(self, ref_image): - assert isinstance(ref_image, Image), self.logger.warning("Must pass an Image class for reference") + assert isinstance(ref_image, Image), self.logger.warning( + "Must pass an Image class for reference") mag = self.pointing.separation(ref_image.pointing) d_dec = self.pointing.dec - ref_image.pointing.dec diff --git a/src/panoptes/pocs/mount/__init__.py b/src/panoptes/pocs/mount/__init__.py index 2da0f085b..6d11c2ca8 100644 --- a/src/panoptes/pocs/mount/__init__.py +++ b/src/panoptes/pocs/mount/__init__.py @@ -1,7 +1,6 @@ from contextlib import suppress from glob import glob -from panoptes.pocs.mount.mount import AbstractMount # pragma: no flakes from panoptes.pocs.utils.location import create_location_from_config from panoptes.pocs.utils.logger import get_logger from panoptes.utils import error @@ -104,6 +103,7 @@ def create_mount_from_config(mount_info=None, def create_mount_simulator(mount_info=None, earth_location=None, + db_type='memory', *args, **kwargs): # Remove mount simulator current_simulators = get_config('simulator', default=[]) @@ -130,7 +130,7 @@ def create_mount_simulator(mount_info=None, except error.NotFound as e: raise error.MountNotFound(f'Error loading mount module: {e!r}') - mount = module.Mount(earth_location, *args, **kwargs) + mount = module.Mount(earth_location, db_type=db_type, *args, **kwargs) logger.success(f"{mount_config['driver'].title()} mount created") diff --git a/src/panoptes/pocs/observatory.py b/src/panoptes/pocs/observatory.py index 2a7b9bf70..ffc6cbaef 100644 --- a/src/panoptes/pocs/observatory.py +++ b/src/panoptes/pocs/observatory.py @@ -6,15 +6,13 @@ from astropy import units as u from astropy.coordinates import get_moon from astropy.coordinates import get_sun - from panoptes.pocs.base import PanBase from panoptes.pocs.camera import AbstractCamera from panoptes.pocs.dome import AbstractDome from panoptes.pocs.images import Image -from panoptes.pocs.mount import AbstractMount +from panoptes.pocs.mount.mount import AbstractMount from panoptes.pocs.scheduler import BaseScheduler from panoptes.pocs.utils.location import create_location_from_config - from panoptes.utils.time import current_time from panoptes.utils import error From 62e6edfadc66e18b47f77b5440af2301ea58e16a Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 27 Jan 2021 20:58:53 -1000 Subject: [PATCH 11/48] * Cleaning up help scripts. (will replace). * Removing unused bin script files. --- bin/panoptes-develop | 33 --------------------------------- bin/peas-shell | 11 ----------- bin/pocs | 27 +++++++++------------------ bin/pocs-shell | 14 -------------- docker/env | 12 ------------ setup.cfg | 4 +--- 6 files changed, 10 insertions(+), 91 deletions(-) delete mode 100755 bin/panoptes-develop delete mode 100755 bin/peas-shell delete mode 100755 bin/pocs-shell delete mode 100644 docker/env diff --git a/bin/panoptes-develop b/bin/panoptes-develop deleted file mode 100755 index 8958a938c..000000000 --- a/bin/panoptes-develop +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -set -e - -SUBCMD=$1 -PARAMS=${@:2} - -export PANDIR=${PANDIR:-/var/panoptes} -export IMAGE="${IMAGE:-panoptes-pocs}" -export TAG="${TAG:-developer}" - -cd "${PANDIR}" - -## Add the daemon option by default. -if [[ "${SUBCMD}" == "up" ]]; then - export CONTAINER_NAME="pocs-developer" - export COMPOSE_FILE="${PANDIR}/POCS/docker/docker-compose-developer.yaml" -fi - -# Pass any other cli args to the containers as an env var named CLI_ARGS -CLI_ARGS=("${@:2}") - -# We use a docker container for docker-compose, so we need to pass the env vars to -# that container so it can properly place them in the docker-compose file. -export DOCKER_RUN_OPTIONS="${DOCKER_RUN_OPTIONS:--e IMAGE=${IMAGE} -e TAG=${TAG} -e CONTAINER_NAME=${CONTAINER_NAME} -e CLI_ARGS=\"${CLI_ARGS}\"}" - -# Run the docker-compose command with user params. -eval "DOCKER_RUN_OPTIONS=\"${DOCKER_RUN_OPTIONS}\" \ - docker-compose \ - --project-directory ${PANDIR} \ - -f ${COMPOSE_FILE} \ - -p panoptes \ - ${SUBCMD} \ - ${PARAMS}" diff --git a/bin/peas-shell b/bin/peas-shell deleted file mode 100755 index 6c0d28364..000000000 --- a/bin/peas-shell +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -ie - -USER_ID=$(id -u) -DOCKER_NAME="peas-shell" - -if [ ! "$(docker ps -q -f name=${DOCKER_NAME})" ]; then - echo "${DOCKER_NAME} not running. Start services with scripts/pocs-docker.sh" -else - docker exec --user "${USER_ID}" -it peas-shell /bin/zsh -ic "python ${POCS}/scripts/${DOCKER_NAME}.py" -fi - diff --git a/bin/pocs b/bin/pocs index 9f6d40e81..115a11bd7 100755 --- a/bin/pocs +++ b/bin/pocs @@ -1,9 +1,11 @@ -#!/bin/bash -ie +#!/usr/bin/env bash + +set -e usage() { - echo -n "################################################## + echo -n " +################################################## # Start POCS via Docker. -# ################################################## $ $(basename $0) [COMMAND] @@ -35,27 +37,16 @@ usage() { START=${1:-help} if [ "${START}" = 'help' ] || [ "${START}" = '-h' ] || [ "${START}" = '--help' ]; then - usage - exit 1 + usage + exit 1 fi PARAMS="$@" - -cd "$PANDIR" -CMD="docker-compose \ - --project-directory ${PANDIR} \ - -f panoptes-utils/docker/docker-compose.yaml \ - -f POCS/docker/docker-compose-aag.yaml \ - -f POCS/docker/docker-compose.yaml \ - -p panoptes" +CMD="docker-compose -f docker/docker-compose.yaml" # If user only asked to start, check if already running and if so use "-d" option. if [[ "$PARAMS" == "up" ]]; then - if [[ ! -z $(eval "${CMD} top") ]]; then - echo "Some containers already running, using -d to only start non-running containers." - echo "For more info on managing docker containers manually, run bin/pocs --help". - PARAMS="up -d" - fi + PARAMS="up -d" fi # Run the docker-compose command with user params. diff --git a/bin/pocs-shell b/bin/pocs-shell deleted file mode 100755 index 48c5e4cf1..000000000 --- a/bin/pocs-shell +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -ie - -USER_ID=$(id -u) -DOCKER_NAME="pocs-shell" - -INDEX_DIR=${INDEX_DIR:-${PANDIR}/astrometry/data} -ASTROMETRY_URL=${ASTROMETRY_URL:-http://broiler.astrometry.net/~dstn/4200} - -if [ ! "$(docker ps -q -f name=${DOCKER_NAME})" ]; then - echo "${DOCKER_NAME} not running. Start services with scripts/pocs-docker.sh" -else - docker exec --user "${USER_ID}" -it pocs-shell /bin/zsh -ic "python ${POCS}/scripts/${DOCKER_NAME}.py" -fi - diff --git a/docker/env b/docker/env deleted file mode 100644 index a8c926b3f..000000000 --- a/docker/env +++ /dev/null @@ -1,12 +0,0 @@ -# Envfile for POCS operation. -# -# Note this doesn't support full interpolation or quotes. -# See https://docs.docker.com/compose/compose-file/#env_file -# -# THESE ARE VALUES INSIDE A RUNNING DOCKER CONTAINER. -# -PANDIR=/POCS -POCS=/POCS -PANOPTES_CONFIG_FILE=/POCS/conf_files/pocs.yaml -PANOPTES_CONFIG_HOST=0.0.0.0 -PANOPTES_CONFIG_PORT=6563 diff --git a/setup.cfg b/setup.cfg index 729da6f47..49163579a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,10 +30,8 @@ include_package_data = True package_dir = =src scripts = - bin/panoptes-develop bin/pocs - bin/pocs-shell - bin/peas-shell + bin/pocs-cmd # DON'T CHANGE THE FOLLOWING LINE! IT WILL BE UPDATED BY PYSCAFFOLD! From 9a39b63a4419c7852fdc7fdd9f4bdc100849cc33 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 27 Jan 2021 21:27:30 -1000 Subject: [PATCH 12/48] * Don't install astrometry indexes for now. * CMD is a simple ipython session. * Adding a basic script for loading POCS with all simulators. * Removing unused scripts. --- .style.yapf | 217 ----------------- docker/Dockerfile | 8 +- scripts/load-simulators.py | 12 + scripts/peas-shell.py | 471 ------------------------------------- sitecustomize.py | 6 - 5 files changed, 16 insertions(+), 698 deletions(-) delete mode 100644 .style.yapf create mode 100644 scripts/load-simulators.py delete mode 100755 scripts/peas-shell.py delete mode 100644 sitecustomize.py diff --git a/.style.yapf b/.style.yapf deleted file mode 100644 index eb4f05993..000000000 --- a/.style.yapf +++ /dev/null @@ -1,217 +0,0 @@ -[style] -# Align closing bracket with visual indentation. -align_closing_bracket_with_visual_indent=True - -# Allow dictionary keys to exist on multiple lines. For example: -# -# x = { -# ('this is the first element of a tuple', -# 'this is the second element of a tuple'): -# value, -# } -allow_multiline_dictionary_keys=False - -# Allow lambdas to be formatted on more than one line. -allow_multiline_lambdas=False - -# Allow splits before the dictionary value. -allow_split_before_dict_value=True - -# Insert a blank line before a class-level docstring. -blank_line_before_class_docstring=False - -# Insert a blank line before a 'def' or 'class' immediately nested -# within another 'def' or 'class'. For example: -# -# class Foo: -# # <------ this blank line -# def method(): -# ... -blank_line_before_nested_class_or_def=False - -# Do not split consecutive brackets. Only relevant when -# dedent_closing_brackets is set. For example: -# -# call_func_that_takes_a_dict( -# { -# 'key1': 'value1', -# 'key2': 'value2', -# } -# ) -# -# would reformat to: -# -# call_func_that_takes_a_dict({ -# 'key1': 'value1', -# 'key2': 'value2', -# }) -coalesce_brackets=False - -# The column limit. -column_limit=99 - -# Indent width used for line continuations. -continuation_indent_width=4 - -# Put closing brackets on a separate line, dedented, if the bracketed -# expression can't fit in a single line. Applies to all kinds of brackets, -# including function definitions and calls. For example: -# -# config = { -# 'key1': 'value1', -# 'key2': 'value2', -# } # <--- this bracket is dedented and on a separate line -# -# time_series = self.remote_client.query_entity_counters( -# entity='dev3246.region1', -# key='dns.query_latency_tcp', -# transform=Transformation.AVERAGE(window=timedelta(seconds=60)), -# start_ts=now()-timedelta(days=3), -# end_ts=now(), -# ) # <--- this bracket is dedented and on a separate line -dedent_closing_brackets=False - -# Place each dictionary entry onto its own line. -each_dict_entry_on_separate_line=True - -# The regex for an i18n comment. The presence of this comment stops -# reformatting of that line, because the comments are required to be -# next to the string they translate. -i18n_comment= - -# The i18n function call names. The presence of this function stops -# reformattting on that line, because the string it has cannot be moved -# away from the i18n comment. -i18n_function_call= - -# Indent the dictionary value if it cannot fit on the same line as the -# dictionary key. For example: -# -# config = { -# 'key1': -# 'value1', -# 'key2': value1 + -# value2, -# } -indent_dictionary_value=False - -# The number of columns to use for indentation. -indent_width=4 - -# Join short lines into one line. E.g., single line 'if' statements. -join_multiple_lines=True - -# Do not include spaces around selected binary operators. For example: -# -# 1 + 2 * 3 - 4 / 5 -# -# will be formatted as follows when configured with a value "*,/": -# -# 1 + 2*3 - 4/5 -# -no_spaces_around_selected_binary_operators=set() - -# Use spaces around default or named assigns. -spaces_around_default_or_named_assign=False - -# Use spaces around the power operator. -spaces_around_power_operator=False - -# The number of spaces required before a trailing comment. -spaces_before_comment=2 - -# Insert a space between the ending comma and closing bracket of a list, -# etc. -space_between_ending_comma_and_closing_bracket=True - -# Split before arguments if the argument list is terminated by a -# comma. -split_arguments_when_comma_terminated=False - -# Set to True to prefer splitting before '&', '|' or '^' rather than -# after. -split_before_bitwise_operator=True - -# Split before a dictionary or set generator (comp_for). For example, note -# the split before the 'for': -# -# foo = { -# variable: 'Hello world, have a nice day!' -# for variable in bar if variable != 42 -# } -split_before_dict_set_generator=True - -# Split after the opening paren which surrounds an expression if it doesn't -# fit on a single line. -split_before_expression_after_opening_paren=False - -# If an argument / parameter list is going to be split, then split before -# the first argument. -split_before_first_argument=False - -# Set to True to prefer splitting before 'and' or 'or' rather than -# after. -split_before_logical_operator=False - -# Split named assignments onto individual lines. -split_before_named_assigns=True - -# Set to True to split list comprehensions and generators that have -# non-trivial expressions and multiple clauses before each of these -# clauses. For example: -# -# result = [ -# a_long_var + 100 for a_long_var in xrange(1000) -# if a_long_var % 10] -# -# would reformat to something like: -# -# result = [ -# a_long_var + 100 -# for a_long_var in xrange(1000) -# if a_long_var % 10] -split_complex_comprehension=False - -# The penalty for splitting right after the opening bracket. -split_penalty_after_opening_bracket=30 - -# The penalty for splitting the line after a unary operator. -split_penalty_after_unary_operator=10000 - -# The penalty for splitting right before an if expression. -split_penalty_before_if_expr=0 - -# The penalty of splitting the line around the '&', '|', and '^' -# operators. -split_penalty_bitwise_operator=300 - -# The penalty for splitting a list comprehension or generator -# expression. -split_penalty_comprehension=80 - -# The penalty for characters over the column limit. -split_penalty_excess_character=4500 - -# The penalty incurred by adding a line split to the unwrapped line. The -# more line splits added the higher the penalty. -split_penalty_for_added_line_split=30 - -# The penalty of splitting a list of "import as" names. For example: -# -# from a_very_long_or_indented_module_name_yada_yad import (long_argument_1, -# long_argument_2, -# long_argument_3) -# -# would reformat to something like: -# -# from a_very_long_or_indented_module_name_yada_yad import ( -# long_argument_1, long_argument_2, long_argument_3) -split_penalty_import_names=0 - -# The penalty of splitting the line around the 'and' and 'or' -# operators. -split_penalty_logical_operator=300 - -# Use the Tab character for indentation. -use_tabs=False - diff --git a/docker/Dockerfile b/docker/Dockerfile index 632e9655f..e794a82ac 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -19,9 +19,9 @@ USER "${userid}" RUN echo "Building from ${image_name}:${image_tag}" && \ sudo apt-get update && \ sudo apt-get install -y --no-install-recommends \ - gphoto2 \ - astrometry-data-tycho2 \ - astrometry-data-2mass && \ + gphoto2 && \ +# astrometry-data-tycho2 \ +# astrometry-data-2mass && \ sudo mkdir /images && sudo chown -R "${userid}:${userid}" /images && \ sudo mkdir /logs && sudo chown -R "${userid}:${userid}" /logs && \ sudo mkdir "${POCS}" && sudo chown -R "${userid}:${userid}" "${POCS}" @@ -51,4 +51,4 @@ COPY --chown="${userid}:${userid}" resources resources COPY --chown="${userid}:${userid}" scripts scripts ENTRYPOINT [ "/usr/bin/env", "bash", "-ic" ] -CMD [ "ipython -i scripts/load-ipython.py" ] +CMD [ "ipython" ] diff --git a/scripts/load-simulators.py b/scripts/load-simulators.py new file mode 100644 index 000000000..579720e4f --- /dev/null +++ b/scripts/load-simulators.py @@ -0,0 +1,12 @@ +import time + +from panoptes.utils.config.server import config_server +from panoptes.pocs.observatory import Observatory +from panoptes.pocs.core import POCS + +print("Starting config-server") +conf_server = config_server('/POCS/conf_files/pocs.yaml') + +pocs = POCS(Observatory(db_type='memory'), simulators=['all']) +time.sleep(1) +print(f'POCS simulator instance created as "pocs"') diff --git a/scripts/peas-shell.py b/scripts/peas-shell.py deleted file mode 100755 index 2d7b6b827..000000000 --- a/scripts/peas-shell.py +++ /dev/null @@ -1,471 +0,0 @@ -#!/usr/bin/env python - -import os -import cmd -import datetime -import readline -import sys - -from pytz import utc -from astropy.utils import console -from threading import Timer -from pprint import pprint - -from panoptes.peas.sensors import ArduinoSerialMonitor -from panoptes.peas.remote_sensors import RemoteMonitor - -from panoptes.utils.config.client import get_config -from panoptes.utils import current_time -from panoptes.utils.database import PanDB - - -class PanSensorShell(cmd.Cmd): - - """ A simple command loop for the sensors. """ - intro = 'Welcome to PEAS Shell! Type ? for help' - prompt = 'PEAS > ' - weather = None - control_board = None - control_env_board = None - camera_board = None - camera_env_board = None - active_sensors = dict() - db = PanDB(db_type=get_config('db.type', default='file')) - _keep_looping = False - _loop_delay = 60 - _timer = None - captured_data = list() - - telemetry_relay_lookup = { - 'computer': {'pin': 8, 'board': 'telemetry_board'}, - 'fan': {'pin': 6, 'board': 'telemetry_board'}, - 'camera_box': {'pin': 7, 'board': 'telemetry_board'}, - 'weather': {'pin': 5, 'board': 'telemetry_board'}, - 'mount': {'pin': 4, 'board': 'telemetry_board'}, - 'cam_0': {'pin': 5, 'board': 'camera_board'}, - 'cam_1': {'pin': 6, 'board': 'camera_board'}, - } - - # NOTE: These are not pins but zero-based index numbers. - controlboard_relay_lookup = { - 'computer': {'pin': 0, 'board': 'control_board'}, - 'mount': {'pin': 1, 'board': 'control_board'}, - 'camera_box': {'pin': 2, 'board': 'control_board'}, - 'weather': {'pin': 3, 'board': 'control_board'}, - 'fan': {'pin': 4, 'board': 'control_board'}, - } - - -################################################################################################## -# Generic Methods -################################################################################################## - - def do_status(self, *arg): - """ Get the entire system status and print it pretty like! """ - if self._keep_looping: - console.color_print("{:>12s}: ".format('Loop Timer'), - "default", "active", "lightgreen") - else: - console.color_print("{:>12s}: ".format('Loop Timer'), - "default", "inactive", "yellow") - - for sensor_name in ['control_board', 'camera_board', 'weather']: - if sensor_name in self.active_sensors: - console.color_print("{:>12s}: ".format(sensor_name.title()), - "default", "active", "lightgreen") - else: - console.color_print("{:>12s}: ".format(sensor_name.title()), - "default", "inactive", "yellow") - - def do_last_reading(self, device): - """ Gets the last reading from the device. """ - if not device: - print_warning('Usage: last_reading ') - return - if not hasattr(self, device): - print_warning('No such sensor: {!r}'.format(device)) - return - - rec = self.db.get_current(device) - - if rec is None: - print_warning('No reading found for {!r}'.format(device)) - return - - print_info('*' * 80) - print("{}:".format(device.upper())) - pprint(rec) - print_info('*' * 80) - - # Display the age in seconds of the record - if isinstance(rec.get('date'), datetime.datetime): - now = current_time(datetime=True).astimezone(utc) - record_date = rec['date'].astimezone(utc) - age = (now - record_date).total_seconds() - if age < 120: - print_info('{:.1f} seconds old'.format(age)) - else: - print_info('{:.1f} minutes old'.format(age / 60.0)) - - def complete_last_reading(self, text, line, begidx, endidx): - """Provide completions for sensor names.""" - names = list(self.active_sensors.keys()) - return [name for name in names if name.startswith(text)] - - def do_enable_sensor(self, sensor, delay=None): - """ Enable the given sensor """ - if delay is None: - delay = self._loop_delay - - if hasattr(self, sensor) and sensor not in self.active_sensors: - self.active_sensors[sensor] = {'reader': sensor, 'delay': delay} - - def do_disable_sensor(self, sensor): - """ Disable the given sensor """ - if hasattr(self, sensor) and sensor in self.active_sensors: - del self.active_sensors[sensor] - - def do_toggle_debug(self, sensor): - """ Toggle DEBUG on/off for sensor - - Arguments: - sensor {str} -- environment, weather - """ - # TODO(jamessynge): We currently use a single logger, not one per module or sensor. - # Figure out whether to keep this code and make it work, or get rid of it. - import logging - get_level = { - logging.DEBUG: logging.INFO, - logging.INFO: logging.DEBUG, - } - - if hasattr(self, sensor): - try: - log = getattr(self, sensor).logger - log.setLevel(get_level[log.getEffectiveLevel()]) - except Exception: - print_error("Can't change log level for {}".format(sensor)) - - def complete_toggle_debug(self, text, line, begidx, endidx): - """Provide completions for toggling debug logging.""" - names = list(self.active_sensors.keys()) - return [name for name in names if name.startswith(text)] - -################################################################################################## -# Load Methods -################################################################################################## - - def do_load_all(self, *arg): - """Load the weather and environment sensors.""" - if self._keep_looping: - print_error('The timer loop is already running.') - return - self.do_load_weather() - self.do_load_control_board() - self.do_load_camera_board() - - def do_load_control_board(self, *arg): - """ Load the arduino control_board sensors """ - if self._keep_looping: - print_error('The timer loop is already running.') - return - print("Loading control board sensor") - self.control_board = ArduinoSerialMonitor( - sensor_name='control_board', db_type=get_config('db.type', default='file')) - self.do_enable_sensor('control_board', delay=10) - - def do_load_camera_board(self, *arg): - """ Load the arduino camera_board sensors """ - if self._keep_looping: - print_error('The timer loop is already running.') - return - print("Loading camera board sensor") - self.camera_board = ArduinoSerialMonitor( - sensor_name='camera_board', db_type=get_config('db.type', default='file')) - self.do_enable_sensor('camera_board', delay=10) - - def do_load_control_env_board(self, *arg): - """ Load the arduino control_board sensors """ - if self._keep_looping: - print_error('The timer loop is already running.') - return - print("Loading control box environment board sensor") - endpoint_url = get_config('environment.control_env_board.url') - self.control_env_board = RemoteMonitor(endpoint_url=endpoint_url, - sensor_name='control_env_board', - db_type=get_config('db.type', default='file') - ) - self.do_enable_sensor('control_env_board', delay=10) - - def do_load_camera_env_board(self, *arg): - """ Load the arduino control_board sensors """ - if self._keep_looping: - print_error('The timer loop is already running.') - return - print("Loading camera box environment board sensor") - endpoint_url = get_config('environment.camera_env_board.url') - self.camera_env_board = RemoteMonitor(endpoint_url=endpoint_url, - sensor_name='camera_env_board', - db_type=get_config('db.type', default='file') - ) - self.do_enable_sensor('camera_env_board', delay=10) - - def do_load_weather(self, *arg): - """ Load the weather reader """ - if self._keep_looping: - print_error('The timer loop is already running.') - return - - print("Loading weather reader endpoint") - endpoint_url = get_config('environment.weather.url') - self.weather = RemoteMonitor(endpoint_url=endpoint_url, - sensor_name='weather', - db_type=get_config('db.type', default='file') - ) - self.do_enable_sensor('weather', delay=60) - - -################################################################################################## -# Relay Methods -################################################################################################## - - def do_turn_off_relay(self, *arg): - """Turn on relay. - - The argument should be the name of the relay, i.e. on of: - - * fan - * mount - * weather - * camera_box - - The names must correspond to the entries in the lookup tables above. - """ - relay = arg[0] - - if hasattr(self, 'control_board'): - relay_lookup = self.controlboard_relay_lookup - else: - relay_lookup = self.telemetry_relay_lookup - - try: - relay_info = relay_lookup[relay] - serial_connection = self.control_board.serial_readers[relay_info['board']]['reader'] - - serial_connection.ser.reset_input_buffer() - serial_connection.write("{},0\n".format(relay_info['pin'])) - except Exception as e: - print_warning(f"Problem turning relay off {relay} {e!r}") - print_warning(e) - - def do_turn_on_relay(self, *arg): - """Turn off relay. - - The argument should be the name of the relay, i.e. on of: - - * fan - * mount - * weather - * camera_box - - The names must correspond to the entries in the lookup tables above. - """ - relay = arg[0] - - if hasattr(self, 'control_board'): - relay_lookup = self.controlboard_relay_lookup - else: - relay_lookup = self.telemetry_relay_lookup - - try: - relay_info = relay_lookup[relay] - serial_connection = self.control_board.serial_readers[relay_info['board']]['reader'] - - serial_connection.ser.reset_input_buffer() - serial_connection.write("{},1\n".format(relay_info['pin'])) - except Exception as e: - print_warning(f"Problem turning relay off {relay} {e!r}") - print_warning(e) - - def complete_turn_off_relay(self, text, line, begidx, endidx): - """Provide completions for relay names.""" - if hasattr(self, 'control_board'): - names = ['camera_box', 'fan', 'mount', 'weather'] - else: - names = ['cam_0', 'cam_1', 'camera_box', 'fan', 'mount', 'weather'] - return [name for name in names if name.startswith(text)] - - def complete_turn_on_relay(self, text, line, begidx, endidx): - """Provide completions for relay names.""" - if hasattr(self, 'control_board'): - names = ['camera_box', 'fan', 'mount', 'weather'] - else: - names = ['cam_0', 'cam_1', 'camera_box', 'fan', 'mount', 'weather'] - return [name for name in names if name.startswith(text)] - - def do_toggle_computer(self, *arg): - """Toggle the computer relay off and then on again after 30 seconds. - - Note: - - The time delay is set on the arduino and is blocking. - """ - relay = 'computer' - - if hasattr(self, 'control_board'): - relay_lookup = self.controlboard_relay_lookup - else: - relay_lookup = self.telemetry_relay_lookup - - try: - relay_info = relay_lookup[relay] - serial_connection = self.control_board.serial_readers[relay_info['board']]['reader'] - - serial_connection.ser.reset_input_buffer() - serial_connection.write("{},9\n".format(relay_info['pin'])) - except Exception as e: - print_warning(f"Problem toggling computer: {e!r}") - print_warning(e) - -################################################################################################## -# Start/Stop Methods -################################################################################################## - - def do_start(self, *arg): - """ Runs all the `active_sensors`. Blocking loop for now """ - if self._keep_looping: - print_error('The timer loop is already running.') - return - - self._keep_looping = True - - print_info("Starting sensors") - - self._loop() - - def do_stop(self, *arg): - """ Stop the loop and cancel next call """ - # NOTE: We don't yet have a way to clear _timer. - if not self._keep_looping and not self._timer: - print_error('The timer loop is not running.') - return - - print_info("Stopping loop") - - self._keep_looping = False - - if self._timer: - self._timer.cancel() - - def do_change_delay(self, *arg): - """Change the timing between reads from the named sensor.""" - # NOTE: For at least the Arduinos, we should not need a delay and a timer, but - # simply a separate thread, reading from the board as data is available. - # We might use a delay to deal with the case where the device is off-line - # but we want to periodically check if it becomes available. - parts = None - if len(arg) == 1: - parts = arg[0].split() - if parts is None or len(parts) != 2: - print_error('Expected a sensor name and a delay, not "{}"'.format(' '.join(arg))) - return - sensor_name, delay = parts - try: - delay = float(delay) - if delay <= 0: - raise ValueError() - except ValueError: - print_warning("Not a positive number: {!r}".format(delay)) - return - try: - print_info("Changing sensor {} to a {} second delay".format(sensor_name, delay)) - self.active_sensors[sensor_name]['delay'] = delay - except KeyError: - print_warning("Sensor not active: {!r}".format(sensor_name)) - -################################################################################################## -# Shell Methods -################################################################################################## - - def do_shell(self, line): - """ Run a raw shell command. Can also prepend '!'. """ - print("Shell command:", line) - - output = os.popen(line).read() - - print_info("Shell output: ", output) - - self.last_output = output - - def emptyline(self): - self.do_status() - - def do_exit(self, *arg): - """ Exits PEAS Shell """ - print("Shutting down") - if self._timer or self._keep_looping: - self.do_stop() - - print("Please be patient and allow for process to finish. Thanks! Bye!") - return True - -################################################################################################## -# Private Methods -################################################################################################## - - def _capture_data(self, sensor_name): - # We are missing a Mutex here for accessing these from active_sensors and - # self. - if sensor_name in self.active_sensors: - sensor = getattr(self, sensor_name) - try: - sensor.capture(store_result=True) - except Exception as e: - print_warning(f'Problem storing captured data: {e!r}') - - self._setup_timer(sensor_name, delay=self.active_sensors[sensor_name]['delay']) - - def _loop(self, *arg): - for sensor_name in self.active_sensors.keys(): - self._capture_data(sensor_name) - - def _setup_timer(self, sensor_name, delay=None): - if self._keep_looping and len(self.active_sensors) > 0: - - if not delay: - delay = self._loop_delay - - # WARNING: It appears we have a single _timer attribute, but we create - # one Timer for each active sensor (i.e. environment and weather). - self._timer = Timer(delay, self._capture_data, args=(sensor_name,)) - - self._timer.start() - -################################################################################################## -# Utility Methods -################################################################################################## - - -def print_info(msg): - console.color_print(msg, 'lightgreen') - - -def print_warning(msg): - console.color_print(msg, 'yellow') - - -def print_error(msg): - console.color_print(msg, 'red') - - -if __name__ == '__main__': - invoked_script = os.path.basename(sys.argv[0]) - histfile = os.path.expanduser('~/.{}_history'.format(invoked_script)) - histfile_size = 1000 - if os.path.exists(histfile): - readline.read_history_file(histfile) - - PanSensorShell().cmdloop() - - readline.set_history_length(histfile_size) - readline.write_history_file(histfile) diff --git a/sitecustomize.py b/sitecustomize.py deleted file mode 100644 index 6c70e25d1..000000000 --- a/sitecustomize.py +++ /dev/null @@ -1,6 +0,0 @@ -# Ensure coverage starts for all Python processes so that test coverage is calculated -# properly when using subprocesses (see https://coverage.readthedocs.io/en/latest/subprocess.html) -import coverage - -print("Starting coverage from sitecustomize") -coverage.process_startup() From c71eb4ce13e8f9cb3d6dee163021f7854f105eee Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 27 Jan 2021 21:36:04 -1000 Subject: [PATCH 13/48] * Fix imports for mount removed by auto-ide --- CHANGELOG.rst | 20 ++++++++++++++++---- src/panoptes/pocs/mount/__init__.py | 1 + 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0201eb366..00f31b1c2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,17 +8,24 @@ Changelog Generic ~~~~~~~ -* **Breaking changes** #1068 +* **Breaking changes** #1074 - * Docker services use the ``pip`` module, so don't include recent changes unless merged into ``develop``. * Default service install does not include ``focuser`` dependencies. - * Default Docker command is a ``bash`` shell and will be changed with future PR to ``pocs`` cli. + * Default Docker command is a ``ipython`` console and will be changed with future PR to ``pocs`` cli. + * Docker image only contains limited set of files. * Directories inside the service image have been simplified for easier mapping onto desired targets on the host: * ``/POCS`` * ``/logs`` * ``/images`` + * Removing ``peas`` scripts. + +Added +~~~~~ + +* Simple example script for creating a ``POCS`` instance with all simulators. #1074 + Docker ~~~~~~ @@ -28,7 +35,12 @@ Docker * Images use ``gcr.io/panoptes-exp/panoptes-utils`` as base. #1074 * Docker files are all contained within ``docker`` folder. #1074 * Both tycho2 (wide-field) and 2mass (narrow-field) index files are included. #1074 - * Docker services (``config-server`` and ``pocs-control``) are started in ``global`` mode so tehre can be only one. # 1074 +* Docker services (``config-server`` and ``pocs-control``) are started in ``global`` mode so tehre can be only one. # 1074 + +Removing +~~~~~~~~ + +* Old scripts and config files. #1074 Testing ~~~~~~~ diff --git a/src/panoptes/pocs/mount/__init__.py b/src/panoptes/pocs/mount/__init__.py index 6d11c2ca8..77f08b197 100644 --- a/src/panoptes/pocs/mount/__init__.py +++ b/src/panoptes/pocs/mount/__init__.py @@ -1,6 +1,7 @@ from contextlib import suppress from glob import glob +from panoptes.pocs.mount.mount import AbstractMount # noqa from panoptes.pocs.utils.location import create_location_from_config from panoptes.pocs.utils.logger import get_logger from panoptes.utils import error From 245791521a92572b691c256d155c6a2c0c244e93 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 27 Jan 2021 21:45:17 -1000 Subject: [PATCH 14/48] * Update README. --- README.md | 10 ++++++++-- tests/docker-compose.yaml | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ceaea90c4..2ecf8a755 100644 --- a/README.md +++ b/README.md @@ -129,9 +129,15 @@ To test the software, you can build a local [Docker](https://docs.docker.com/) i First clone the repository, then run the following from the project's root directory: ```bash -docker-compose -f tests/docker-compose.yaml --env-file tests/env build +docker-compose -f tests/docker-compose.yaml build -docker-compose -f tests/docker-compose.yaml --env-file tests/env run pocs pytest +docker-compose -f tests/docker-compose.yaml up +``` + +By default that will build and run all tests. If you want to run one specific test, run the `pytests test_file.py` on the `pocs` service: + +```bash +docker-compose -f tests/docker-compose.yaml run pocs "pytest tests/test_mount.py" ``` Links diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index 0e69ba239..f2012e94a 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -17,7 +17,7 @@ services: network_mode: host command: [ "pytest" ] volumes: - - logdir:/logs + - logdir:/POCS/logs - builddir:/POCS/build volumes: logdir: From c391a3db34e75cf77327ac0489e265dc28040598 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 27 Jan 2021 22:19:17 -1000 Subject: [PATCH 15/48] * Updated compose file before removal. --- docker/docker-compose.yaml | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 155260338..9288f3b03 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,7 +1,7 @@ version: '3.7' services: config-server: - image: gcr.io/panoptes-exp/panoptes-utils:develop + image: "${IMAGE_URL:-gcr.io/panoptes-exp/panoptes-pocs:develop}" deploy: mode: global init: true @@ -10,34 +10,37 @@ services: hostname: config-server network_mode: host environment: - - PANOPTES_CONFIG_FILE=/POCS/conf_files/pocs.yaml - PANOPTES_CONFIG_HOST=0.0.0.0 - PANOPTES_CONFIG_PORT=6563 - command: [ "panoptes-config-server --verbose run --config-file /app/config.yaml" ] volumes: - - type: bind - source: "${PANOPTES_CONFIG_FILE}" - target: /app/config.yaml + - confdir:/POCS/conf_files + command: [ "panoptes-config-server --verbose run --config-file /POCS/conf_files/pocs.yaml" ] pocs-control: - image: gcr.io/panoptes-exp/panoptes-pocs:develop + image: "${IMAGE_URL:-gcr.io/panoptes-exp/panoptes-pocs:develop}" deploy: mode: global init: true + tty: true container_name: pocs-control hostname: pocs-control privileged: true network_mode: host environment: - - PANOPTES_CONFIG_FILE=/POCS/conf_files/pocs.yaml - PANOPTES_CONFIG_HOST=0.0.0.0 - PANOPTES_CONFIG_PORT=6563 volumes: - logdir:/logs - command: [ "wait-for-it ${PANOPTES_CONFIG_PORT}:${PANOPTES_CONFIG_PORT} -- /bin/bash" ] + command: [ "wait-for-it $${PANOPTES_CONFIG_PORT}:$${PANOPTES_CONFIG_PORT} -- tail -f /dev/null" ] volumes: logdir: driver: local driver_opts: type: none - device: "${PANLOG:-${PWD}/logs}" + device: "${PANLOG:-logs}" + o: bind + confdir: + driver: local + driver_opts: + type: none + device: conf_files o: bind From a092064d358bac4df30818b3eb35479866db1bc7 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 27 Jan 2021 22:22:00 -1000 Subject: [PATCH 16/48] * Remove docker-compose file. --- docker/docker-compose.yaml | 46 -------------------------------------- 1 file changed, 46 deletions(-) delete mode 100644 docker/docker-compose.yaml diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml deleted file mode 100644 index 9288f3b03..000000000 --- a/docker/docker-compose.yaml +++ /dev/null @@ -1,46 +0,0 @@ -version: '3.7' -services: - config-server: - image: "${IMAGE_URL:-gcr.io/panoptes-exp/panoptes-pocs:develop}" - deploy: - mode: global - init: true - tty: true - container_name: config-server - hostname: config-server - network_mode: host - environment: - - PANOPTES_CONFIG_HOST=0.0.0.0 - - PANOPTES_CONFIG_PORT=6563 - volumes: - - confdir:/POCS/conf_files - command: [ "panoptes-config-server --verbose run --config-file /POCS/conf_files/pocs.yaml" ] - pocs-control: - image: "${IMAGE_URL:-gcr.io/panoptes-exp/panoptes-pocs:develop}" - deploy: - mode: global - init: true - tty: true - container_name: pocs-control - hostname: pocs-control - privileged: true - network_mode: host - environment: - - PANOPTES_CONFIG_HOST=0.0.0.0 - - PANOPTES_CONFIG_PORT=6563 - volumes: - - logdir:/logs - command: [ "wait-for-it $${PANOPTES_CONFIG_PORT}:$${PANOPTES_CONFIG_PORT} -- tail -f /dev/null" ] -volumes: - logdir: - driver: local - driver_opts: - type: none - device: "${PANLOG:-logs}" - o: bind - confdir: - driver: local - driver_opts: - type: none - device: conf_files - o: bind From 9fa52f6295b73117f6df5b3b4bc6e34c0688e2b7 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 27 Jan 2021 22:25:55 -1000 Subject: [PATCH 17/48] * Setting up GHA actions. --- tests/Dockerfile | 7 ++++++- tests/docker-compose.yaml | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/Dockerfile b/tests/Dockerfile index 95b3dc105..049b4541a 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -13,10 +13,15 @@ ARG pip_extras="[focuser,google,testing]" USER "${userid}" +# Set up some common directories WORKDIR /tmp -RUN sudo apt-get update && \ +RUN echo "Building from ${image_name}:${image_tag}" && \ + sudo apt-get update && \ sudo apt-get install -y --no-install-recommends \ + gphoto2 \ astrometry-data-tycho2-10-19 && \ + sudo mkdir /images && sudo chown -R "${userid}:${userid}" /images && \ + sudo mkdir /logs && sudo chown -R "${userid}:${userid}" /logs && \ sudo mkdir "${POCS}" && sudo chown -R "${userid}:${userid}" "${POCS}" COPY docker/environment.yaml . diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index f2012e94a..0e69ba239 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -17,7 +17,7 @@ services: network_mode: host command: [ "pytest" ] volumes: - - logdir:/POCS/logs + - logdir:/logs - builddir:/POCS/build volumes: logdir: From 222fc18cc5ad968808127a9cbbe2eb569408b9e9 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 27 Jan 2021 22:58:22 -1000 Subject: [PATCH 18/48] * More log dir fixing. * Removing `pocs` and `pocs-cmd` scripts. --- bin/pocs | 53 --------------------------------------- bin/pocs-cmd | 6 ----- conftest.py | 2 +- setup.cfg | 16 +++--------- tests/docker-compose.yaml | 2 +- 5 files changed, 6 insertions(+), 73 deletions(-) delete mode 100755 bin/pocs delete mode 100755 bin/pocs-cmd diff --git a/bin/pocs b/bin/pocs deleted file mode 100755 index 115a11bd7..000000000 --- a/bin/pocs +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash - -set -e - -usage() { - echo -n " -################################################## -# Start POCS via Docker. -################################################## - - $ $(basename $0) [COMMAND] - - Options: - COMMAND These options are passed at the end of the docker-compose command. - To start all service simply pass 'up'. - - Examples: - - # Start all services in the foreground. - $POCS/bin/pocs up - - #Start specific docker containers in the background with - $POCS/bin/pocs up --no-deps -d - - e.g. - - # Start config-server service in the background. - $POCS/bin/pocs up --no-deps -d config-server - - # Read the logs from the config-server - $POCS/bin/pocs logs config-server - - # Manually stop docker containers in the with - docker stop -" -} - -START=${1:-help} -if [ "${START}" = 'help' ] || [ "${START}" = '-h' ] || [ "${START}" = '--help' ]; then - usage - exit 1 -fi - -PARAMS="$@" -CMD="docker-compose -f docker/docker-compose.yaml" - -# If user only asked to start, check if already running and if so use "-d" option. -if [[ "$PARAMS" == "up" ]]; then - PARAMS="up -d" -fi - -# Run the docker-compose command with user params. -eval "${CMD} ${PARAMS}" diff --git a/bin/pocs-cmd b/bin/pocs-cmd deleted file mode 100755 index 84c959f48..000000000 --- a/bin/pocs-cmd +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -set -e - -DOCKER_NAME="pocs-control" - -docker exec -it "${DOCKER_NAME}" "$@" &>/dev/null || echo "${DOCKER_NAME} not running" diff --git a/conftest.py b/conftest.py index b547310cf..d3298497b 100644 --- a/conftest.py +++ b/conftest.py @@ -30,7 +30,7 @@ "| {name} {function}:{line} | " \ "{message}" -log_file_path = os.path.expandvars('logs/panoptes-testing.log') +log_file_path = os.path.expandvars('${PANLOG:-logs}/panoptes-testing.log') startup_message = f' STARTING NEW PYTEST RUN - LOGS: {log_file_path} ' logger.add(log_file_path, enqueue=True, # multiprocessing diff --git a/setup.cfg b/setup.cfg index 49163579a..e3d15a47a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,8 +30,7 @@ include_package_data = True package_dir = =src scripts = - bin/pocs - bin/pocs-cmd + scripts/take-pic.sh # DON'T CHANGE THE FOLLOWING LINE! IT WILL BE UPDATED BY PYSCAFFOLD! @@ -72,16 +71,9 @@ testing = pytest-remotedata>=0.3.1 responses -[options.entry_points] -# Add here console scripts like: -# console_scripts = -# script_name = panoptes.pocs.module:function -# For example: -# console_scripts = -# fibonacci = panoptes.pocs.skeleton:run -# And any other entry points, for example: -# pyscaffold.cli = -# awesome = pyscaffoldext.awesome.extension:AwesomeExtension +#[options.entry_points] +#console_scripts = +# pocs = panoptes.pocs.cli.main:app [test] # py.test options when running `python setup.py test` diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index 0e69ba239..f2012e94a 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -17,7 +17,7 @@ services: network_mode: host command: [ "pytest" ] volumes: - - logdir:/logs + - logdir:/POCS/logs - builddir:/POCS/build volumes: logdir: From 7f13e1f256aed487daf212cd53d4990c48c6fb42 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 27 Jan 2021 23:03:43 -1000 Subject: [PATCH 19/48] * Simplify the install script. --- scripts/install/install-pocs.sh | 33 +++++++-------------------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index 4068d3748..47d0650c1 100644 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -49,7 +49,7 @@ usage() { # removing zsh, etc. Removed Darwin options. # ############################################################# - $ $(basename $0) [--user panoptes] [--pandir /var/panoptes] + $ $(basename $0) [--user panoptes] [--pandir /panoptes] Options: USER The PANUSER environment variable, defaults to current user (i.e. PANUSER=$USER). @@ -63,19 +63,13 @@ PS3="Select: " # TODO should be checking to matching userid=1000 PANUSER=${PANUSER:-$USER} -PANDIR=${PANDIR:-/var/panoptes} -LOGFILE="${PANDIR}/install-pocs.log" +PANDIR=${PANDIR:-/panoptes} +TAG_NAME=${TAG_NAME:-develop} +LOGFILE="${PANDIR}/logs/install-pocs.log" OS="$(uname -s)" DOCKER_BASE=${DOCKER_BASE:-"gcr.io/panoptes-exp"} -function command_exists() { - # https://gist.github.com/gubatron/1eb077a1c5fcf510e8e5 - # this should be a very portable way of checking if something is on the path - # usage: "if command_exists foo; then echo it exists; fi" - type "$1" &>/dev/null -} - function make_directories() { sudo mkdir -p "${PANDIR}/logs" sudo mkdir -p "${PANDIR}/images" @@ -89,14 +83,12 @@ function system_deps() { byobu \ docker.io \ docker-compose \ - git \ htop \ httpie \ jq \ openssh-server \ - speedometer \ - vim-nox \ - wget | sudo tee -a "${LOGFILE}" 2>&1 + wget \ + zsh | sudo tee -a "${LOGFILE}" 2>&1 # Add an SSH key if one doesn't exist. if [[ ! -f "${HOME}/.ssh/id_rsa" ]]; then @@ -106,23 +98,12 @@ function system_deps() { # Add to docker group if not already. sudo usermod -aG docker "${PANUSER}" | sudo tee -a "${LOGFILE}" 2>&1 - - # Source the environment variables if available. - cat <>"/home/${PANUSER}/.bashrc" -export LANG="en_US.UTF-8" - -# Load POCS env file if it exists -if test -f "${PANDIR}/POCS/env"; then - source "${PANDIR}/POCS/env" -fi -EOF } function get_or_build_images() { echo "Pulling POCS docker images from Google Cloud Registry (GCR)." - sudo docker pull "${DOCKER_BASE}/panoptes-pocs:latest" - sudo docker pull "${DOCKER_BASE}/aag-weather:latest" + sudo docker pull "${DOCKER_BASE}/panoptes-pocs:${TAG_NAME}" } function do_install() { From a0c1343afa7cac6decd85e0890939adc23f4f0bc Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 27 Jan 2021 23:06:06 -1000 Subject: [PATCH 20/48] * GHA testing dirs. --- conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index d3298497b..f21207609 100644 --- a/conftest.py +++ b/conftest.py @@ -30,7 +30,8 @@ "| {name} {function}:{line} | " \ "{message}" -log_file_path = os.path.expandvars('${PANLOG:-logs}/panoptes-testing.log') +log_dir = os.getenv('PANLOG', 'logs') +log_file_path = os.path.join(log_dir, 'panoptes-testing.log') startup_message = f' STARTING NEW PYTEST RUN - LOGS: {log_file_path} ' logger.add(log_file_path, enqueue=True, # multiprocessing From fba87e9aa8731bc13ee7c9411b933e1e6adca092 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 27 Jan 2021 23:29:48 -1000 Subject: [PATCH 21/48] * Add index files. --- CHANGELOG.rst | 2 +- docker/Dockerfile | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 00f31b1c2..8d67921b2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -34,7 +34,7 @@ Docker * Docker image does not contain ``focuser`` extras by default. #1068 * Images use ``gcr.io/panoptes-exp/panoptes-utils`` as base. #1074 * Docker files are all contained within ``docker`` folder. #1074 -* Both tycho2 (wide-field) and 2mass (narrow-field) index files are included. #1074 +* Docker image has tycho2 10-19 index files for plate-solving. #1074 * Docker services (``config-server`` and ``pocs-control``) are started in ``global`` mode so tehre can be only one. # 1074 Removing diff --git a/docker/Dockerfile b/docker/Dockerfile index e794a82ac..61549c91b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -19,9 +19,8 @@ USER "${userid}" RUN echo "Building from ${image_name}:${image_tag}" && \ sudo apt-get update && \ sudo apt-get install -y --no-install-recommends \ - gphoto2 && \ -# astrometry-data-tycho2 \ -# astrometry-data-2mass && \ + gphoto2 \ + astrometry-data-tycho2-10-19 && \ sudo mkdir /images && sudo chown -R "${userid}:${userid}" /images && \ sudo mkdir /logs && sudo chown -R "${userid}:${userid}" /logs && \ sudo mkdir "${POCS}" && sudo chown -R "${userid}:${userid}" "${POCS}" From aad48a64f935b0c622c92aa2c10affc936ccab65 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Thu, 28 Jan 2021 08:12:33 -1000 Subject: [PATCH 22/48] * The testing docker image has the 2mass index files to solve the smaller files. --- tests/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Dockerfile b/tests/Dockerfile index 049b4541a..c656eb22d 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -19,7 +19,7 @@ RUN echo "Building from ${image_name}:${image_tag}" && \ sudo apt-get update && \ sudo apt-get install -y --no-install-recommends \ gphoto2 \ - astrometry-data-tycho2-10-19 && \ + astrometry-data-2mass-08-19 && \ sudo mkdir /images && sudo chown -R "${userid}:${userid}" /images && \ sudo mkdir /logs && sudo chown -R "${userid}:${userid}" /logs && \ sudo mkdir "${POCS}" && sudo chown -R "${userid}:${userid}" "${POCS}" From 3b60a89503be766dd326235350bc1e9b4af05f28 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Thu, 28 Jan 2021 08:14:35 -1000 Subject: [PATCH 23/48] * Slight formatting update. --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 61549c91b..cdfd70828 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -48,6 +48,6 @@ COPY --chown="${userid}:${userid}" docker/docker-compose.yaml . COPY --chown="${userid}:${userid}" conf_files conf_files COPY --chown="${userid}:${userid}" resources resources COPY --chown="${userid}:${userid}" scripts scripts -ENTRYPOINT [ "/usr/bin/env", "bash", "-ic" ] +ENTRYPOINT [ "/usr/bin/env", "bash", "-ic" ] CMD [ "ipython" ] From d712c92bacbb16b0f486601c71da7396b034521e Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Thu, 28 Jan 2021 11:08:38 -1000 Subject: [PATCH 24/48] * Simplify compose file for testing. --- tests/docker-compose.yaml | 16 +--------------- tests/test_images.py | 5 ++--- 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index f2012e94a..4435dbafa 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -17,18 +17,4 @@ services: network_mode: host command: [ "pytest" ] volumes: - - logdir:/POCS/logs - - builddir:/POCS/build -volumes: - logdir: - driver: local - driver_opts: - type: none - device: logs - o: bind - builddir: - driver: local - driver_opts: - type: none - device: build - o: bind + - "$PWD:/POCS" diff --git a/tests/test_images.py b/tests/test_images.py index f3a8dfcbe..35214b362 100644 --- a/tests/test_images.py +++ b/tests/test_images.py @@ -1,11 +1,10 @@ import os -import pytest import shutil import tempfile +import pytest from astropy import units as u from astropy.coordinates import SkyCoord - from panoptes.pocs.images import Image from panoptes.pocs.images import OffsetError from panoptes.utils.error import SolveError @@ -42,7 +41,7 @@ def test_solve_timeout(tiny_fits_file): im0 = Image(tiny_fits_file) assert str(im0) with pytest.raises(Timeout): - im0.solve_field(verbose=True, replace=False, radius=4, timeout=1) + im0.solve_field(verbose=True, replace=False, radius=4, timeout=0) def test_fail_solve(tiny_fits_file): From 389b14f68f9535eec554b03e2978f50d7a60dea1 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Thu, 28 Jan 2021 12:41:29 -1000 Subject: [PATCH 25/48] * Specific astrometry index file for tests. * Tests are run in global mode so don't conflict if run twice at same time. --- tests/Dockerfile | 3 ++- tests/docker-compose.yaml | 2 ++ tests/test_images.py | 1 - 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/Dockerfile b/tests/Dockerfile index c656eb22d..4c3af5aa3 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -17,9 +17,10 @@ USER "${userid}" WORKDIR /tmp RUN echo "Building from ${image_name}:${image_tag}" && \ sudo apt-get update && \ + sudo apt-get -y full-upgrade && \ sudo apt-get install -y --no-install-recommends \ gphoto2 \ - astrometry-data-2mass-08-19 && \ + astrometry-data-tycho2-08 && \ sudo mkdir /images && sudo chown -R "${userid}:${userid}" /images && \ sudo mkdir /logs && sudo chown -R "${userid}:${userid}" /logs && \ sudo mkdir "${POCS}" && sudo chown -R "${userid}:${userid}" "${POCS}" diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index 4435dbafa..862c10703 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -7,6 +7,8 @@ services: dockerfile: tests/Dockerfile init: true tty: true + deploy: + mode: global environment: - PANDIR=/POCS - POCS=/POCS diff --git a/tests/test_images.py b/tests/test_images.py index 35214b362..b3a62ef39 100644 --- a/tests/test_images.py +++ b/tests/test_images.py @@ -76,7 +76,6 @@ def test_solve_field_unsolved(unsolved_fits_file, # Compare it to another file of known offset. im1 = Image(copy_file_to_dir(tmpdir, solved_fits_file)) offset_info = im0.compute_offset(im1) - # print('offset_info:', offset_info) expected_offset = [10.1 * u.arcsec, 5.29 * u.arcsec, 8.77 * u.arcsec] assert u.allclose(offset_info, expected_offset, rtol=0.1) From a6473543d30536b6fe7c09d840b51a2083a16049 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Thu, 28 Jan 2021 12:42:26 -1000 Subject: [PATCH 26/48] * Require python>=3.8 (already building that inside docker images). --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index e3d15a47a..9e9ce8a45 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,7 +18,7 @@ classifiers = License :: OSI Approved :: MIT License Operating System :: POSIX Programming Language :: Python :: 3 - Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 Programming Language :: Python :: 3 :: Only Topic :: Scientific/Engineering :: Astronomy Topic :: Scientific/Engineering :: Physics @@ -45,7 +45,7 @@ install_requires = # The usage of test_requires is discouraged, see `Dependency Management` docs # tests_require = pytest; pytest-cov # Require a specific Python version, e.g. Python 2.7 or >= 3.4 -python_requires = >=3.7 +python_requires = >=3.8 [options.packages.find] where = src From 3dda4fac5c420df89e1022f88a17e0af5da5cfd7 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Thu, 28 Jan 2021 14:29:47 -1000 Subject: [PATCH 27/48] * Fix imags. --- tests/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Dockerfile b/tests/Dockerfile index 4c3af5aa3..5114517a2 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -20,7 +20,7 @@ RUN echo "Building from ${image_name}:${image_tag}" && \ sudo apt-get -y full-upgrade && \ sudo apt-get install -y --no-install-recommends \ gphoto2 \ - astrometry-data-tycho2-08 && \ + astrometry-data-tycho2 && \ sudo mkdir /images && sudo chown -R "${userid}:${userid}" /images && \ sudo mkdir /logs && sudo chown -R "${userid}:${userid}" /logs && \ sudo mkdir "${POCS}" && sudo chown -R "${userid}:${userid}" "${POCS}" From 5a6e02e3c8c8edcd0fe04c501562aae9ec9c406a Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Thu, 28 Jan 2021 15:06:43 -1000 Subject: [PATCH 28/48] * Fix tests. --- .github/workflows/pythontest.yaml | 2 +- tests/docker-compose.yaml | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythontest.yaml b/.github/workflows/pythontest.yaml index 4980a59d1..aa98cce64 100644 --- a/.github/workflows/pythontest.yaml +++ b/.github/workflows/pythontest.yaml @@ -36,7 +36,7 @@ jobs: mkdir -p build && chmod -R 777 build - name: Test with pytest in panoptes-pocs container run: | - docker-compose -f tests/docker-compose.yaml run pocs pytest + docker-compose -f tests/docker-compose.yaml up - name: Upload coverage report to codecov.io uses: codecov/codecov-action@v1 if: success() diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index 862c10703..dfc79060d 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -19,4 +19,7 @@ services: network_mode: host command: [ "pytest" ] volumes: - - "$PWD:/POCS" + - "$PWD/logs:/POCS/logs" + - "$PWD/build:/POCS/build" + - "$PWD/src:/POCS/src" + - "$PWD/tests:/POCS/tests" From e80c4d5fe8b59cae9e86e044d55fa75d2694aefe Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Fri, 29 Jan 2021 16:09:54 -1000 Subject: [PATCH 29/48] * Default (i.e. example) config improved. --- conf_files/pocs.yaml | 53 ++++++++++++++------------------------------ 1 file changed, 17 insertions(+), 36 deletions(-) diff --git a/conf_files/pocs.yaml b/conf_files/pocs.yaml index 350e54e71..bc6f2c9f3 100644 --- a/conf_files/pocs.yaml +++ b/conf_files/pocs.yaml @@ -27,17 +27,17 @@ location: gmt_offset: -600 # Offset in minutes from GMT during. directories: - base: /var/panoptes - images: images + base: /POCS data: data - resources: POCS/resources/ - targets: POCS/resources/targets - mounts: POCS/resources/mounts + images: images + mounts: resources/mounts + resources: resources/ + targets: conf_files/targets db: name: panoptes type: file - folder: metadata + folder: json_store wait_delay: 180 # time in seconds before checking safety/etc while waiting. max_transition_attempts: 5 # number of transitions attempts. @@ -72,39 +72,20 @@ cameras: defaults: primary: None auto_detect: False - file_extension: fits - compress_fits: False - make_pretty_images: False - keep_jpgs: False - readout_time: 0.5 # seconds + compress_fits: True + make_pretty_images: True + keep_jpgs: True + readout_time: 1.0 # seconds timeout: 10 # seconds filter_type: RGGB - cooling: - enabled: False - temperature: - target: 0 # celsius - tolerance: 0.1 # celsius - stable_time: 60 # seconds - check_interval: 5 # seconds - timeout: 300 # seconds - filterwheel: - model: panoptes.pocs.filterwheel.simulator.FilterWheel - filter_names: [ ] - move_time: 0.1 # seconds - timeout: 0.5 # seconds - focuser: - enabled: False - autofocus_seconds: 0.1 # seconds - autofocus_size: 500 # seconds - autofocus_keep_files: False - autofocus_make_plots: False devices: - - model: panoptes.pocs.camera.gphoto.canon.Camera - name: dslr.00 + - model: panoptes.pocs.camera.simulator.dslr.Camera + name: dslr.sim.00 file_extension: cr2 - - model: panoptes.pocs.camera.gphoto.canon.Camera - name: dslr.01 + - model: panoptes.pocs.camera.simulator.dslr.Camera + name: dslr.sim.01 file_extension: cr2 + primary: True ######################### Environmental Sensors ################################ # Configuration for the power distribution board and other environmental sensors. @@ -154,9 +135,9 @@ observations: panoptes_network: image_storage: False service_account_key: # Location of JSON account key - project_id: panoptes-survey + project_id: panoptes-exp buckets: - images: panoptes-survey + images: panoptes-exp ############################### pocs ################################## # POCS status flags. The values below represent initial values but From 4a44b0e1c3e916146e8b8eeed080a8ae4390f310 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Fri, 29 Jan 2021 16:10:06 -1000 Subject: [PATCH 30/48] * Adding back zsh to the install script. --- scripts/install/install-pocs.sh | 34 ++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index 47d0650c1..54caaffb7 100644 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -81,8 +81,8 @@ function system_deps() { sudo apt-get --yes install \ ack \ byobu \ - docker.io \ docker-compose \ + docker.io \ htop \ httpie \ jq \ @@ -106,6 +106,36 @@ function get_or_build_images() { sudo docker pull "${DOCKER_BASE}/panoptes-pocs:${TAG_NAME}" } +function install_zsh() { + echo "Setting up zsh for a better experience." + + # Oh my zsh + wget -q https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh -O /tmp/install-ohmyzsh.sh + bash /tmp/install-ohmyzsh.sh --unattended + + export ZSH_CUSTOM="$HOME/.oh-my-zsh" + + # Autosuggestions plugin + git clone https://github.com/zsh-users/zsh-autosuggestions ${ZSH_CUSTOM:-~/.oh-my-zsh/custom}/plugins/zsh-autosuggestions + + # Spaceship theme + git clone https://github.com/denysdovhan/spaceship-prompt.git "$ZSH_CUSTOM/themes/spaceship-prompt" --depth=1 + ln -s "$ZSH_CUSTOM/themes/spaceship-prompt/spaceship.zsh-theme" "$ZSH_CUSTOM/themes/spaceship.zsh-theme" + + write_zshrc +} + +function write_zshrc() { + cat >"${HOME}/.zshrc" <<'EOT' +export PATH=$HOME/bin:$HOME/.local/bin:/usr/local/bin:$PATH +export ZSH="/home/panoptes/.oh-my-zsh" +ZSH_THEME="spaceship" +plugins=(git sudo zsh-autosuggestions docker docker-compose python) +source $ZSH/oh-my-zsh.sh +unsetopt share_history +EOT +} + function do_install() { clear @@ -123,6 +153,8 @@ function do_install() { get_or_build_images + install_zsh + echo "Please reboot your machine before using POCS." read -p "Reboot now? [y/N]: " -r From c00a447b3adb0a7b002e03ec20e8f214756d4e9a Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Fri, 29 Jan 2021 16:10:55 -1000 Subject: [PATCH 31/48] * Ignore any local conf files. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e3d5fac4b..2fbdb6259 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,4 @@ MANIFEST **/.ipynb_checkpoints/** logs +conf_files/*_local.yaml From 86c4946b1c0cebca251be540d9129691684191ac Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Fri, 29 Jan 2021 16:11:13 -1000 Subject: [PATCH 32/48] * Simulator load script actually loads simulator. --- scripts/load-simulators.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/load-simulators.py b/scripts/load-simulators.py index 579720e4f..d56804fdd 100644 --- a/scripts/load-simulators.py +++ b/scripts/load-simulators.py @@ -1,12 +1,15 @@ import time -from panoptes.utils.config.server import config_server +from panoptes.pocs.mount import create_mount_simulator +from panoptes.pocs.scheduler import create_scheduler_from_config +from panoptes.pocs.camera import create_cameras_from_config from panoptes.pocs.observatory import Observatory from panoptes.pocs.core import POCS -print("Starting config-server") -conf_server = config_server('/POCS/conf_files/pocs.yaml') - -pocs = POCS(Observatory(db_type='memory'), simulators=['all']) +pocs = POCS(Observatory(mount=create_mount_simulator(), + scheduler=create_scheduler_from_config(), + cameras=create_cameras_from_config(), + db_type='memory'), + simulators=['dome', 'night', 'weather', 'power']) time.sleep(1) print(f'POCS simulator instance created as "pocs"') From b18f01abe5ac5ce86d5bab44ba8259d80cab4b66 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Fri, 29 Jan 2021 16:12:03 -1000 Subject: [PATCH 33/48] * Move items out of the rpi `user-data` cloud-init file and into install script. Other cleanup to script. --- resources/rpi/user-data | 50 ++++++++++------------------------------- 1 file changed, 12 insertions(+), 38 deletions(-) diff --git a/resources/rpi/user-data b/resources/rpi/user-data index c5ae02fac..04450b1b3 100644 --- a/resources/rpi/user-data +++ b/resources/rpi/user-data @@ -7,6 +7,12 @@ hostname: pocs-control # You shouldn't need to change anything below. ##################################################################### +# Setting "expire: true" will force a password change on first login. +chpasswd: + expire: true + list: + - panoptes:panoptes + ntp: enabled: true servers: @@ -15,70 +21,38 @@ ntp: - time3.google.com - time4.google.com -# Setting "expire: true" will force a password change on first login. -chpasswd: - expire: true - list: - - panoptes:panoptes - ssh_pwauth: yes # New groups to create. groups: - - panoptes - docker users: - name: panoptes gecos: PANOPTES User primary_group: panoptes - groups: users, admin, dialout, plugdev, docker, i2c, input, gpio, panoptes + groups: users, admin, dialout, plugdev, docker, i2c, input, gpio sudo: "ALL=(ALL) NOPASSWD:ALL" lock_passwd: false - shell: /bin/zsh + shell: /bin/bash ## Update apt database and upgrade packages on first boot -package_update: true -package_upgrade: true - -byobu: enable - -## Install additional packages on first boot. -packages: - - apt-transport-https - - byobu - - ca-certificates - - docker.io - - git - - htop - - httpie - - jq - - software-properties-common - - speedometer - - vim-nox - - watchdog - - zsh +package_update: false +package_upgrade: false write_files: # Allow panoptes user to mount via sshfs. - - content: | + - path: /etc/fuse.conf + content: | user_allow_other - path: /etc/fuse.conf append: true ## Get and run the install script upon first boot. runcmd: # Setup hardware watchdog -# - echo 'interface = eth0' >> /etc/watchdog.conf -# - echo 'interface = wlan0' >> /etc/watchdog.conf - echo 'watchdog-device = /dev/watchdog' >> /etc/watchdog.conf - echo 'watchdog-timeout = 15' >> /etc/watchdog.conf - echo 'max-load-1 = 24' >> /etc/watchdog.conf - # Get the install file. - - mkdir -p /var/panoptes/scripts - - chown -R panoptes:panoptes /var/panoptes - - wget https://install.projectpanoptes.org -O /var/panoptes/scripts/install-pocs.sh - - bash /var/panoptes/scripts/install-pocs.sh power_state: mode: reboot From d4048b7d868974bf66693bd2560e0b7e3df67932 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sat, 30 Jan 2021 06:53:39 -1000 Subject: [PATCH 34/48] * Pass the test data to the Docker image so can be used by the simulator. * Fixing incorrect `wait` call on `run` method. * Shortening some long debug statements. * Make example config file work with Docker images. * Skip pointing if `num_pointing_images==0` --- conf_files/pocs.yaml | 10 +++--- docker/Dockerfile | 1 + src/panoptes/pocs/camera/camera.py | 2 +- src/panoptes/pocs/state/machine.py | 32 +++++++------------ .../pocs/state/states/default/pointing.py | 7 +++- .../pocs/state/states/default/sleeping.py | 3 +- 6 files changed, 25 insertions(+), 30 deletions(-) diff --git a/conf_files/pocs.yaml b/conf_files/pocs.yaml index bc6f2c9f3..9a98bd0bd 100644 --- a/conf_files/pocs.yaml +++ b/conf_files/pocs.yaml @@ -28,11 +28,9 @@ location: directories: base: /POCS - data: data - images: images - mounts: resources/mounts - resources: resources/ - targets: conf_files/targets + images: /POCS/images + mounts: /POCS/resources/mounts + targets: /POCS/conf_files/targets db: name: panoptes @@ -63,10 +61,10 @@ mount: max_tracking_threshold: 99999 # ms pointing: + max_iterations: 5 # Set to 0 to disable auto_correct: True threshold: 100 # arcseconds ~ 10 pixels exptime: 30 # seconds - max_iterations: 5 cameras: defaults: diff --git a/docker/Dockerfile b/docker/Dockerfile index cdfd70828..b8d0c4f04 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -48,6 +48,7 @@ COPY --chown="${userid}:${userid}" docker/docker-compose.yaml . COPY --chown="${userid}:${userid}" conf_files conf_files COPY --chown="${userid}:${userid}" resources resources COPY --chown="${userid}:${userid}" scripts scripts +COPY --chown="${userid}:${userid}" tests/data tests/data ENTRYPOINT [ "/usr/bin/env", "bash", "-ic" ] CMD [ "ipython" ] diff --git a/src/panoptes/pocs/camera/camera.py b/src/panoptes/pocs/camera/camera.py index 0ebd2ea27..e3ce9bc02 100644 --- a/src/panoptes/pocs/camera/camera.py +++ b/src/panoptes/pocs/camera/camera.py @@ -581,7 +581,7 @@ def process_exposure(self, if not os.path.exists(file_path): observation_event.set() raise FileNotFoundError( - f"Expected image at file_path={file_path!r} does not exist or " + + f"Expected image at {file_path=!r} does not exist or " + "cannot be accessed, cannot process.") self.logger.debug(f'Starting FITS processing for {file_path}') diff --git a/src/panoptes/pocs/state/machine.py b/src/panoptes/pocs/state/machine.py index 8c67c018b..886e058fe 100644 --- a/src/panoptes/pocs/state/machine.py +++ b/src/panoptes/pocs/state/machine.py @@ -1,15 +1,13 @@ import os - from contextlib import suppress -from transitions.extensions.states import Tags as MachineState +from transitions.extensions.states import Tags as MachineState +from transitions import Machine from panoptes.utils import error from panoptes.utils.utils import listify from panoptes.utils.library import load_module from panoptes.utils.serializers import from_yaml -from transitions import Machine - class PanStateMachine(Machine): """ A finite state machine for PANOPTES. @@ -112,8 +110,7 @@ def run(self, exit_when_done=False, run_once=False, initial_next_state='ready'): while self.keep_running: # BEFORE TRANSITION TO STATE - self.logger.info(f'Run loop: self.state={self.state!r}' - f'self.next_state={self.next_state!r}') + self.logger.info(f'Run loop: {self.state!r} -> {self.next_state!r}') # Before moving to next state, wait for required horizon if necessary while True: @@ -130,21 +127,19 @@ def run(self, exit_when_done=False, run_once=False, initial_next_state='ready'): required_horizon = self._horizon_lookup.get(self.next_state, 'observe') if self.is_dark(horizon=required_horizon): break - self.logger.info(f"Waiting for required_horizon={required_horizon!r} for " - f"self.next_state={self.next_state!r}") + self.logger.info(f"Waiting for {required_horizon=!r} for {self.next_state=!r}") # Sleep before checking again self.wait(delay=check_delay) # TRANSITION TO STATE - self.logger.info(f'Going to self.next_state={self.next_state!r}') + self.logger.info(f'Going to {self.next_state!r}') try: # The state's `on_enter` logic will be performed here. state_changed = self.goto_next_state() except Exception as e: - self.logger.critical(f"Problem going from self.state={self.state!r} to " - f" self.next_state={self.next_state!r}" - f", exiting loop [{e!r}]") + self.logger.critical(f"Problem going from {self.state!r} to {self.next_state!r}, " + f"exiting loop [{e!r}]") # TODO should we automatically park here? self.stop_states() break @@ -153,37 +148,34 @@ def run(self, exit_when_done=False, run_once=False, initial_next_state='ready'): # If we didn't successfully transition, wait a while then try again if not state_changed: - self.logger.warning(f"Failed to move from self.state={self.state!r} to " - f"self.next_state={self.next_state!r}") + self.logger.warning(f"Failed to move from {self.state=!r} to {self.next_state=!r}") if self.is_safe() is False: self.logger.warning( "Conditions have become unsafe; setting next state to 'parking'") self.next_state = 'parking' elif _transition_iteration > max_transition_attempts: self.logger.warning( - f"Stuck in current state for " - f"max_transition_attempts={max_transition_attempts!r}, parking") + f"Stuck in current state for {max_transition_attempts=!r}, parking") self.next_state = 'parking' else: _transition_iteration = _transition_iteration + 1 self.logger.warning( f"Sleeping before trying again ({_transition_iteration}/" f"{max_transition_attempts})") - self.wait(with_status=False, delay=7) # wait 7 seconds (no good reason) + self.wait(delay=7) # wait 7 seconds (no good reason) else: _transition_iteration = 0 # Note that `self.state` below has changed from above - # We started in the sleeping state, so if we are back here we have - # done a full iteration. + # We started in the sleeping state, so if we are back here we have done a full loop. if self.state == 'sleeping': self._obs_run_retries -= 1 if run_once: self.stop_states() if exit_when_done: - self.logger.info(f'Leaving run loop exit_when_done={exit_when_done!r}') + self.logger.info(f'Leaving run loop {exit_when_done=!r}') break def goto_next_state(self): diff --git a/src/panoptes/pocs/state/states/default/pointing.py b/src/panoptes/pocs/state/states/default/pointing.py index 385617945..a127cee2a 100644 --- a/src/panoptes/pocs/state/states/default/pointing.py +++ b/src/panoptes/pocs/state/states/default/pointing.py @@ -17,6 +17,10 @@ def on_enter(event_data): # Get pointing parameters pointing_config = pocs.get_config('pointing') num_pointing_images = int(pointing_config.get('max_iterations', 3)) + if num_pointing_images == 0: + pocs.next_state = 'tracking' + return + should_correct = pointing_config.get('auto_correct', False) pointing_threshold = pointing_config.get('threshold', 0.05) # degrees exptime = pointing_config.get('exptime', 30) # seconds @@ -57,7 +61,8 @@ def waiting_cb(): pocs.logger.info(f'Waiting for pointing image {img_num + 1}/{num_pointing_images}') return pocs.is_safe() - wait_for_events(camera_event, timeout=maximum_duration, callback=waiting_cb, sleep_delay=wait_delay) + wait_for_events(camera_event, timeout=maximum_duration, callback=waiting_cb, + sleep_delay=wait_delay) # Analyze pointing if observation is not None: diff --git a/src/panoptes/pocs/state/states/default/sleeping.py b/src/panoptes/pocs/state/states/default/sleeping.py index 029f65a92..1d7fdff9b 100644 --- a/src/panoptes/pocs/state/states/default/sleeping.py +++ b/src/panoptes/pocs/state/states/default/sleeping.py @@ -3,8 +3,7 @@ def on_enter(event_data): pocs = event_data.model if pocs.is_safe() and pocs.should_retry is False: - pocs.say("Weather is good and it is dark. Something must have gone wrong. " + - "Stopping loop.") + pocs.say("Weather is good and it is dark. Something must have gone wrong. Stopping loop.") pocs.stop_states() else: # Note: Unit will "sleep" before transition until it is safe From 670b27d3f38726e4a72be23641d124fb6fe3b417 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sat, 30 Jan 2021 07:01:06 -1000 Subject: [PATCH 35/48] * GHA flake test bumped to python 3.8 --- .github/workflows/pythontest.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pythontest.yaml b/.github/workflows/pythontest.yaml index aa98cce64..5854fbf38 100644 --- a/.github/workflows/pythontest.yaml +++ b/.github/workflows/pythontest.yaml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ 3.7 ] + python-version: [ 3.8 ] steps: - name: Checkout code uses: actions/checkout@v2 @@ -19,13 +19,13 @@ jobs: pip install flake8 # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + # exit-zero treats all errors as warnings. flake8 . --count --exit-zero --max-complexity=10 --max-line-length=100 --statistics test: runs-on: ubuntu-latest strategy: matrix: - python-version: [ 3.7 ] + python-version: [ 3.8 ] steps: - name: Checkout code uses: actions/checkout@v2 From fb8094573b66271b0b7f73f128b130d6286c7663 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sat, 30 Jan 2021 07:01:26 -1000 Subject: [PATCH 36/48] * Adding docker compose file for starting services. --- docker/docker-compose.yaml | 46 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 docker/docker-compose.yaml diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 000000000..e48f7a6ac --- /dev/null +++ b/docker/docker-compose.yaml @@ -0,0 +1,46 @@ +version: '3.7' +services: + pocs-config-server: + image: "${IMAGE_NAME:-gcr.io/panoptes-exp/panoptes-pocs}:${TAG_NAME:-develop}" + build: + context: ../ + dockerfile: ./docker/Dockerfile + deploy: + mode: global + restart: on-failure + init: true + container_name: pocs-config-server + hostname: pocs-config-server + network_mode: host + environment: + PANOPTES_CONFIG_HOST: 0.0.0.0 + PANOPTES_CONFIG_PORT: 6563 + PANOPTES_CONFIG_FILE: + command: [ "panoptes-config-server --verbose run --no-save-local --no-load-local --config-file /POCS/conf_files/pocs.yaml" ] + volumes: + - "$PWD/logs:/POCS/logs" + - "$PWD/conf_files:/POCS/conf_files" + pocs-control: + image: "${IMAGE_NAME:-gcr.io/panoptes-exp/panoptes-pocs}:${TAG_NAME:-develop}" + depends_on: + - pocs-config-server + build: + context: ../ + dockerfile: ./docker/Dockerfile + deploy: + mode: global + init: true + tty: true + stdin_open: true + privileged: true + container_name: pocs-control + hostname: pocs-control + network_mode: host + environment: + PANOPTES_CONFIG_HOST: localhost + PANOPTES_CONFIG_PORT: 6563 + command: [ "wait-for-it localhost:6563 -- ipython -i /POCS/scripts/load-simulators.py" ] + volumes: + - "$PWD/logs:/POCS/logs" + - "/var/panoptes/images:/POCS/images" + - "/var/panoptes/json_store:/POCS/json_store" From 011fbc538c4faeed023c7118b3b2d42d50e1be5b Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sat, 30 Jan 2021 07:31:07 -1000 Subject: [PATCH 37/48] * Updated changelog. --- CHANGELOG.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8d67921b2..a0e93ff06 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,8 +10,9 @@ Generic * **Breaking changes** #1074 + * Python 3.8 * Default service install does not include ``focuser`` dependencies. - * Default Docker command is a ``ipython`` console and will be changed with future PR to ``pocs`` cli. + * Default Docker command is a ``ipython`` console with the simulators loaded. * Docker image only contains limited set of files. * Directories inside the service image have been simplified for easier mapping onto desired targets on the host: @@ -26,6 +27,12 @@ Added * Simple example script for creating a ``POCS`` instance with all simulators. #1074 +Changed +~~~~~~~ + +* Updated install script (includes ZSH again). #1074 +* Pointing state is skipped if ``num_pointing_images==0`. #1074 + Docker ~~~~~~ @@ -36,6 +43,7 @@ Docker * Docker files are all contained within ``docker`` folder. #1074 * Docker image has tycho2 10-19 index files for plate-solving. #1074 * Docker services (``config-server`` and ``pocs-control``) are started in ``global`` mode so tehre can be only one. # 1074 +* Config changed to run with simulators out of the box. #1074 Removing ~~~~~~~~ From 5ccf40d31084ec1bdf4e01d2b4e1438f6b013ea3 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sat, 30 Jan 2021 07:40:18 -1000 Subject: [PATCH 38/48] * Updated changelog. --- CHANGELOG.rst | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a0e93ff06..6c2f92d77 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,12 +14,7 @@ Generic * Default service install does not include ``focuser`` dependencies. * Default Docker command is a ``ipython`` console with the simulators loaded. * Docker image only contains limited set of files. - * Directories inside the service image have been simplified for easier mapping onto desired targets on the host: - - * ``/POCS`` - * ``/logs`` - * ``/images`` - + * Directories inside the service image have been simplified for easier mapping onto desired targets on the host. The main top-level directory (i.e. ``$PANDIR``) is now ``/POCS`` with other folders nested underneath. * Removing ``peas`` scripts. Added From 92f0a31999a1e623e977f8b9afe5d077545cba31 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sat, 30 Jan 2021 12:44:45 -1000 Subject: [PATCH 39/48] * Adding missing base class properties to simulator. * Adding `threading.excepthook` to log errors from camera threads. --- .gitignore | 27 +++++++++++----------- src/panoptes/pocs/camera/camera.py | 5 ++++ src/panoptes/pocs/camera/simulator/dslr.py | 4 ++++ 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index 2fbdb6259..7e5a459e9 100644 --- a/.gitignore +++ b/.gitignore @@ -24,32 +24,31 @@ tags # Package files *.egg -*.eggs/ +*.eggs .installed.cfg *.egg-info # Unittest and coverage -htmlcov/* -.coverage +htmlcov .tox junit.xml coverage.xml -.coverage.* -.pytest_cache/ +.coverage* +.pytest_cache +cover # Build and docs folder/files -build/* -dist/* -sdist/* -docs/api/* -docs/_rst/* -docs/_build/* -cover/* +build +dist +sdist +docs/api +docs/_rst +docs/_build MANIFEST # Per-project virtualenvs -.venv*/ -**/.ipynb_checkpoints/** +.venv* +.ipynb_checkpoints logs conf_files/*_local.yaml diff --git a/src/panoptes/pocs/camera/camera.py b/src/panoptes/pocs/camera/camera.py index e3ce9bc02..0aebe686e 100644 --- a/src/panoptes/pocs/camera/camera.py +++ b/src/panoptes/pocs/camera/camera.py @@ -505,6 +505,11 @@ def take_exposure(self, self._is_exposing_event.clear() raise err + def log_thread_error(exc_info): + self.logger.warning(f'{exc_info!r}') + + threading.excepthook = log_thread_error + # Start polling thread that will call camera type specific _readout method when done readout_thread = threading.Thread(target=self._poll_exposure, args=(readout_args, seconds), diff --git a/src/panoptes/pocs/camera/simulator/dslr.py b/src/panoptes/pocs/camera/simulator/dslr.py index 13f7008bf..914e410ee 100644 --- a/src/panoptes/pocs/camera/simulator/dslr.py +++ b/src/panoptes/pocs/camera/simulator/dslr.py @@ -13,6 +13,10 @@ class Camera(AbstractCamera): + @property + def egain(self): + return 1 + @property def bit_depth(self): return 12 * u.bit From fe132b9e1f191a10c6dc61bdc4c0f45b710fee01 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sat, 30 Jan 2021 12:57:41 -1000 Subject: [PATCH 40/48] * Adding `threading.excepthook` to try and catch threading errors. * Small cleanup. --- src/panoptes/pocs/observatory.py | 17 ++++++----------- tests/test_pocs.py | 7 ++----- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/src/panoptes/pocs/observatory.py b/src/panoptes/pocs/observatory.py index ffc6cbaef..88d226f05 100644 --- a/src/panoptes/pocs/observatory.py +++ b/src/panoptes/pocs/observatory.py @@ -644,34 +644,29 @@ def autofocus_cameras(self, camera_list=None, **kwargs): cameras = {cam_name: self.cameras[ cam_name] for cam_name in camera_list if cam_name in self.cameras.keys()} if cameras == {}: - self.logger.warning( - "Passed a list of camera names ({}) but no matches found".format(camera_list)) + self.logger.warning(f"No matching camera names in ({camera_list})") else: - # No cameras specified, will try to autofocus all cameras from - # self.cameras + # No cameras specified, will try to autofocus all cameras from self.cameras cameras = self.cameras autofocus_events = dict() # Start autofocus with each camera for cam_name, camera in cameras.items(): - self.logger.debug("Autofocusing camera: {}".format(cam_name)) + self.logger.debug(f"Autofocusing camera: {cam_name}") try: assert camera.focuser.is_connected except AttributeError: - self.logger.debug( - 'Camera {} has no focuser, skipping autofocus'.format(cam_name)) + self.logger.debug(f'Camera {cam_name} has no focuser, skipping autofocus') except AssertionError: - self.logger.debug( - 'Camera {} focuser not connected, skipping autofocus'.format(cam_name)) + self.logger.debug(f'Camera {cam_name} focuser not connected, skipping autofocus') else: try: # Start the autofocus autofocus_event = camera.autofocus(**kwargs) except Exception as e: - self.logger.error( - "Problem running autofocus: {}".format(e)) + self.logger.error(f"Problem running autofocus: {e!r}") else: autofocus_events[cam_name] = autofocus_event diff --git a/tests/test_pocs.py b/tests/test_pocs.py index 39f0853c5..799f55a46 100644 --- a/tests/test_pocs.py +++ b/tests/test_pocs.py @@ -4,16 +4,12 @@ import pytest import requests - from astropy import units as u - from panoptes.pocs import hardware - from panoptes.pocs.core import POCS from panoptes.pocs.observatory import Observatory from panoptes.utils.config.client import set_config from panoptes.utils.serializers import to_json, to_yaml - from panoptes.pocs.mount import create_mount_simulator from panoptes.pocs.dome import create_dome_simulator from panoptes.pocs.camera import create_cameras_from_config @@ -114,6 +110,7 @@ def valid_observation(): 'exp_set_size': 2, } + # An observation that is valid at night @pytest.fixture(scope='module') def valid_observation_day(): @@ -432,7 +429,7 @@ def start_pocs(): pocs.run(run_once=True, exit_when_done=True) # After done running. - assert pocs.is_weather_safe() is True + assert pocs.is_weather_safe() is False pocs.power_down() observatory.logger.info('start_pocs EXIT') From 4b4e4097e375aa5a5f95ddabaf6e836276576af3 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sat, 30 Jan 2021 13:16:50 -1000 Subject: [PATCH 41/48] * Adding `threading.excepthook` to log threading errors. * Small cleanup. --- src/panoptes/pocs/camera/camera.py | 2 +- tests/test_camera.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/panoptes/pocs/camera/camera.py b/src/panoptes/pocs/camera/camera.py index 0aebe686e..3f18cd86e 100644 --- a/src/panoptes/pocs/camera/camera.py +++ b/src/panoptes/pocs/camera/camera.py @@ -506,7 +506,7 @@ def take_exposure(self, raise err def log_thread_error(exc_info): - self.logger.warning(f'{exc_info!r}') + self.logger.error(f'{exc_info!r}') threading.excepthook = log_thread_error diff --git a/tests/test_camera.py b/tests/test_camera.py index 9041d66fa..05e0f22b5 100644 --- a/tests/test_camera.py +++ b/tests/test_camera.py @@ -1,11 +1,10 @@ -import pytest - import os import time import glob from ctypes.util import find_library from contextlib import suppress +import pytest import astropy.units as u from astropy.io import fits import requests @@ -13,7 +12,7 @@ from panoptes.pocs.camera.simulator.dslr import Camera as SimCamera from panoptes.pocs.camera.simulator.ccd import Camera as SimSDKCamera from panoptes.pocs.camera.sbig import Camera as SBIGCamera -from panoptes.pocs.camera.sbigudrv import SBIGDriver, INVALID_HANDLE_VALUE +from panoptes.pocs.camera.sbigudrv import INVALID_HANDLE_VALUE, SBIGDriver from panoptes.pocs.camera.fli import Camera as FLICamera from panoptes.pocs.camera.zwo import Camera as ZWOCamera From 585ac1e0439f8ee48a4249cb0324a3f5e1d8960c Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sat, 30 Jan 2021 14:30:32 -1000 Subject: [PATCH 42/48] * Adding `threading.excepthook` to log threading errors. * Small cleanup. --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6c2f92d77..0bead33c4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -21,6 +21,7 @@ Added ~~~~~ * Simple example script for creating a ``POCS`` instance with all simulators. #1074 +* Using ``threading.excepthook`` to log errors in camera exposure threads. #1074 Changed ~~~~~~~ From 6520b808a0ff077fa548a1e66e41a2e44af2989a Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sat, 30 Jan 2021 15:36:46 -1000 Subject: [PATCH 43/48] * Testing files need 2mass, not tycho2 index. --- tests/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Dockerfile b/tests/Dockerfile index 5114517a2..d717f4e2e 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -20,7 +20,7 @@ RUN echo "Building from ${image_name}:${image_tag}" && \ sudo apt-get -y full-upgrade && \ sudo apt-get install -y --no-install-recommends \ gphoto2 \ - astrometry-data-tycho2 && \ + astrometry-data-2mass && \ sudo mkdir /images && sudo chown -R "${userid}:${userid}" /images && \ sudo mkdir /logs && sudo chown -R "${userid}:${userid}" /logs && \ sudo mkdir "${POCS}" && sudo chown -R "${userid}:${userid}" "${POCS}" From bc4ddd74bc8b950a8cb205c16d04602bd008f3c7 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sat, 30 Jan 2021 17:18:51 -1000 Subject: [PATCH 44/48] * Pass the radius to `Images.solve_field` otherwise it was defaulting to `0`. * Only include index file that is needed for tests. * Pointing doesn't try to correct itself in test. --- src/panoptes/pocs/images.py | 1 + tests/Dockerfile | 6 ++++-- tests/testing.yaml | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/panoptes/pocs/images.py b/src/panoptes/pocs/images.py index 0344f4703..feecc2a45 100644 --- a/src/panoptes/pocs/images.py +++ b/src/panoptes/pocs/images.py @@ -187,6 +187,7 @@ def solve_field(self, **kwargs): solve_info = fits_utils.get_solve_field(self.fits_file, ra=self.header_pointing.ra.value, dec=self.header_pointing.dec.value, + radius=15, # degrees **kwargs) self.wcs_file = solve_info['solved_fits_file'] diff --git a/tests/Dockerfile b/tests/Dockerfile index d717f4e2e..80169b4f8 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -19,8 +19,10 @@ RUN echo "Building from ${image_name}:${image_tag}" && \ sudo apt-get update && \ sudo apt-get -y full-upgrade && \ sudo apt-get install -y --no-install-recommends \ - gphoto2 \ - astrometry-data-2mass && \ + gphoto2 && \ + # Index file used for tests. + sudo wget http://data.astrometry.net/4100/index-4111.fits -O /usr/share/astrometry/index-4111.fits + # Support directories. sudo mkdir /images && sudo chown -R "${userid}:${userid}" /images && \ sudo mkdir /logs && sudo chown -R "${userid}:${userid}" /logs && \ sudo mkdir "${POCS}" && sudo chown -R "${userid}:${userid}" "${POCS}" diff --git a/tests/testing.yaml b/tests/testing.yaml index 3f53eb395..3dcbc0cc6 100644 --- a/tests/testing.yaml +++ b/tests/testing.yaml @@ -54,10 +54,10 @@ mount: baudrate: 9600 non_sidereal_available: True pointing: - auto_correct: True + auto_correct: False threshold: 500 # arcseconds ~ 50 pixels exptime: 30 # seconds - max_iterations: 3 + max_iterations: 1 cameras: defaults: primary: None From be558eef0cca4dd42ff1d44368163f9273f7753d Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sat, 30 Jan 2021 17:21:01 -1000 Subject: [PATCH 45/48] =?UTF-8?q?*=20Default=20the=20solve=20`radius`=20to?= =?UTF-8?q?=2015=C2=B0.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.rst | 1 + src/panoptes/pocs/images.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0bead33c4..bde89632c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -28,6 +28,7 @@ Changed * Updated install script (includes ZSH again). #1074 * Pointing state is skipped if ``num_pointing_images==0`. #1074 +* The default ``radius`` for solving images is 15°. Docker ~~~~~~ diff --git a/src/panoptes/pocs/images.py b/src/panoptes/pocs/images.py index feecc2a45..fb7f432a3 100644 --- a/src/panoptes/pocs/images.py +++ b/src/panoptes/pocs/images.py @@ -178,16 +178,17 @@ def get_wcs_pointing(self): # Precess to the current equinox otherwise the RA - LST method will be off. self.ha = self.pointing.transform_to(self.FK5_Jnow).ra.to(u.degree) - self.sidereal - def solve_field(self, **kwargs): + def solve_field(self, radius=15, **kwargs): """ Solve field and populate WCS information. Args: + radius (scalar): The radius (in degrees) to search near RA-Dec. Defaults to 15°. **kwargs: Options to be passed to `get_solve_field`. """ solve_info = fits_utils.get_solve_field(self.fits_file, ra=self.header_pointing.ra.value, dec=self.header_pointing.dec.value, - radius=15, # degrees + radius=radius, **kwargs) self.wcs_file = solve_info['solved_fits_file'] From 209afe543198c55fb4be12d62f5c5e33edc037b9 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sat, 30 Jan 2021 18:08:46 -1000 Subject: [PATCH 46/48] Typo fix. --- tests/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/Dockerfile b/tests/Dockerfile index 80169b4f8..0775264d2 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -10,6 +10,7 @@ ENV POCS "/POCS" ARG userid=1000 ARG pip_extras="[focuser,google,testing]" +ARG index_url="http://data.astrometry.net/4100/index-4111.fits" USER "${userid}" @@ -21,7 +22,7 @@ RUN echo "Building from ${image_name}:${image_tag}" && \ sudo apt-get install -y --no-install-recommends \ gphoto2 && \ # Index file used for tests. - sudo wget http://data.astrometry.net/4100/index-4111.fits -O /usr/share/astrometry/index-4111.fits + sudo wget "${index_url}" -O /usr/share/astrometry/index-4111.fits && \ # Support directories. sudo mkdir /images && sudo chown -R "${userid}:${userid}" /images && \ sudo mkdir /logs && sudo chown -R "${userid}:${userid}" /logs && \ From 657884fd7a6ffd80e6c78d1f12c9bdb60db58580 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sat, 30 Jan 2021 21:54:30 -1000 Subject: [PATCH 47/48] Minor cleanup. --- src/panoptes/pocs/state/machine.py | 2 +- src/panoptes/pocs/state/states/default/sleeping.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/panoptes/pocs/state/machine.py b/src/panoptes/pocs/state/machine.py index 886e058fe..0340cff0c 100644 --- a/src/panoptes/pocs/state/machine.py +++ b/src/panoptes/pocs/state/machine.py @@ -148,7 +148,7 @@ def run(self, exit_when_done=False, run_once=False, initial_next_state='ready'): # If we didn't successfully transition, wait a while then try again if not state_changed: - self.logger.warning(f"Failed to move from {self.state=!r} to {self.next_state=!r}") + self.logger.warning(f"Failed to move from {self.state!r} to {self.next_state!r}") if self.is_safe() is False: self.logger.warning( "Conditions have become unsafe; setting next state to 'parking'") diff --git a/src/panoptes/pocs/state/states/default/sleeping.py b/src/panoptes/pocs/state/states/default/sleeping.py index 1d7fdff9b..3fcbc4fea 100644 --- a/src/panoptes/pocs/state/states/default/sleeping.py +++ b/src/panoptes/pocs/state/states/default/sleeping.py @@ -6,8 +6,7 @@ def on_enter(event_data): pocs.say("Weather is good and it is dark. Something must have gone wrong. Stopping loop.") pocs.stop_states() else: - # Note: Unit will "sleep" before transition until it is safe - # to observe again. + # Note: Unit will "sleep" before transition until it is safe to observe again. pocs.next_state = 'ready' pocs.reset_observing_run() From 2f86d640ac77ad14c0196f6890cdf0ef3e08d439 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sun, 31 Jan 2021 05:58:19 -1000 Subject: [PATCH 48/48] * With autocorrection of pointing. --- tests/testing.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testing.yaml b/tests/testing.yaml index 3dcbc0cc6..208997e24 100644 --- a/tests/testing.yaml +++ b/tests/testing.yaml @@ -54,7 +54,7 @@ mount: baudrate: 9600 non_sidereal_available: True pointing: - auto_correct: False + auto_correct: True threshold: 500 # arcseconds ~ 50 pixels exptime: 30 # seconds max_iterations: 1