Send USGS Data to Kafka

.github/workflows/ci.yaml

@@ -1,4 +1,4 @@
-name: Python CI
+name: CI
 
 "on":
   merge_group: {}
@@ -14,12 +14,13 @@ name: Python CI
       - "renovate/**"
       - "tickets/**"
       - "u/**"
+    tags:
+      - "*"
   release:
     types: [published]
 
 jobs:
   lint:
-
     runs-on: ubuntu-latest
     timeout-minutes: 5
 
@@ -29,10 +30,10 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       - name: Run pre-commit
-        uses: pre-commit/action@v3.0.0
+        uses: pre-commit/action@v3.0.1
 
   test:
 
@@ -42,7 +43,7 @@ jobs:
     strategy:
       matrix:
         python:
-          - "3.10"
+          - "3.12"
 
     steps:
       - uses: actions/checkout@v4
@@ -51,7 +52,8 @@ jobs:
         uses: lsst-sqre/run-tox@v1
         with:
           python-version: ${{ matrix.python }}
-          tox-envs: "py,typing"
+          tox-envs: "py,coverage-report,typing"
+          tox-requirements: "requirements/tox.txt"
 
   docs:
 
@@ -69,7 +71,7 @@ jobs:
       - name: Run tox
         uses: lsst-sqre/run-tox@v1
         with:
-          python-version: "3.10"
+          python-version: "3.12"
           tox-envs: "docs"
           # Add docs-linkcheck when the docs and PyPI package are published
           # tox-envs: "docs,docs-linkcheck"
@@ -103,7 +105,7 @@ jobs:
       - name: Build and publish
         uses: lsst-sqre/build-and-publish-to-pypi@v2
         with:
-          python-version: "3.11"
+          python-version: "3.12"
           upload: false
 
   pypi:
@@ -130,4 +132,4 @@ jobs:
       - name: Build and publish
         uses: lsst-sqre/build-and-publish-to-pypi@v2
         with:
-          python-version: "3.11"
+          python-version: "3.12"

.github/workflows/periodic-ci.yaml

@@ -1,12 +1,14 @@
-# This is a separate run of the Python test suite that doesn't cache the tox
-# environment and runs from a schedule.  The purpose is to test compatibility
-# with the latest versions of dependencies.
+# This is a separate run of the Python test suite that runs from a schedule,
+# doesn't cache the tox environment, and updates pinned dependencies first.
+# The purpose is to test compatibility with the latest versions of
+# dependencies.
 
 name: Periodic CI
 
 "on":
   schedule:
     - cron: "0 12 * * 1"
+  workflow_dispatch: {}
 
 jobs:
   test:
@@ -16,43 +18,34 @@ jobs:
     strategy:
       matrix:
         python:
-          - "3.11"
+          - "3.12"
 
     steps:
       - uses: actions/checkout@v4
 
+      # Use the oldest supported version of Python to update dependencies,
+      # not the matrixed Python version, since this accurately reflects
+      # how dependencies should later be updated.
+      - name: Run neophile
+        uses: lsst-sqre/run-neophile@v1
+        with:
+          python-version: "3.12"
+          mode: update
+
       - name: Run tests in tox
         uses: lsst-sqre/run-tox@v1
         with:
           python-version: ${{ matrix.python }}
           tox-envs: "lint,typing,py"
+          tox-requirements: "requirements/tox.txt"
           use-cache: false
 
-  docs:
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Build docs in tox
-        uses: lsst-sqre/run-tox@v1
-        with:
-          python-version: "3.11"
-          tox-envs: "docs,docs-linkcheck"
-          use-cache: false
-
-  test-packaging:
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0 # full history for setuptools_scm
-
-      - name: Build and publish
-        uses: lsst-sqre/build-and-publish-to-pypi@v2
+      - name: Report status
+        if: always()
+        uses: ravsamhq/notify-slack-action@v2
         with:
-          python-version: "3.11"
-          upload: false
+          status: ${{ job.status }}
+          notify_when: "failure"
+          notification_title: "Periodic test for {repo} failed"
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_ALERT_WEBHOOK }}

.gitignore

@@ -83,7 +83,9 @@ profile_default/
 ipython_config.py
 
 # pyenv
-.python-version
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
 
 # pipenv
 #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
@@ -128,3 +130,9 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# macOS
+.DS_Store

.pre-commit-config.yaml

@@ -1,36 +1,14 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
-      - id: trailing-whitespace
-      - id: check-yaml
       - id: check-toml
+      - id: check-yaml
+      - id: trailing-whitespace
 
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.13.2
-    hooks:
-      - id: isort
-        additional_dependencies: [toml]
-
-  - repo: https://github.com/psf/black
-    rev: 24.1.1
-    hooks:
-      - id: black
-
-  - repo: https://github.com/asottile/blacken-docs
-    rev: v1.12.1
-    hooks:
-      - id: blacken-docs
-        additional_dependencies: [black==23.3.0]
-        args: [-l, '79', -t, py38]
-
-  - repo: https://github.com/PyCQA/flake8
-    rev: 7.0.0
-    hooks:
-      - id: flake8
-
-  - repo: https://github.com/PyCQA/pydocstyle
-    rev: 6.3.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.4.7
     hooks:
-      - id: pydocstyle
-        additional_dependencies: [tomli]
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+      - id: ruff-format

Makefile

@@ -1,29 +1,48 @@
 .PHONY: help
 help:
-	@echo "Make targets for sasquatch-backpack:"
-	@echo "make clean - Remove generated files"
-	@echo "make init - Set up dev environment (install pre-commit hooks)"
-	@echo "make linkcheck - Check for broken links in documentation"
-
-.PHONY: clean
-clean:
-	rm -rf .tox
-	rm -rf docs/_build
-	rm -rf docs/api
+	@echo "Make targets for example"
+	@echo "make init - Set up dev environment"
+	@echo "make run - Start a local development instance"
+	@echo "make update - Update pinned dependencies and run make init"
+	@echo "make update-deps - Update pinned dependencies"
+	@echo "make update-deps-no-hashes - Pin dependencies without hashes"
 
 .PHONY: init
 init:
-	pip install --upgrade pip tox pre-commit
-	pip install --upgrade -e ".[dev]"
-	pre-commit install
+	pip install --upgrade uv
+	uv pip install -r requirements/main.txt -r requirements/dev.txt \
+	    -r requirements/tox.txt
+	uv pip install --editable .
 	rm -rf .tox
+	uv pip install --upgrade pre-commit
+	pre-commit install
+
+.PHONY: run
+run:
+	tox run -e run
+
+.PHONY: update
+update: update-deps init
+
+.PHONY: update-deps
+update-deps:
+	pip install --upgrade uv
+	uv pip install --upgrade pre-commit
+	pre-commit autoupdate
+	uv pip compile --upgrade --generate-hashes			\
+	    --output-file requirements/main.txt requirements/main.in
+	uv pip compile --upgrade --generate-hashes			\
+	    --output-file requirements/dev.txt requirements/dev.in
+	uv pip compile --upgrade --generate-hashes			\
+	    --output-file requirements/tox.txt requirements/tox.in
 
-# This is defined as a Makefile target instead of only a tox command because
-# if the command fails we want to cat output.txt, which contains the
-# actually useful linkcheck output. tox unfortunately doesn't support this
-# level of shell trickery after failed commands.
-.PHONY: linkcheck
-linkcheck:
-	sphinx-build --keep-going -n -W -T -b linkcheck docs	\
-	    docs/_build/linkcheck				\
-	    || (cat docs/_build/linkcheck/output.txt; exit 1)
+# Useful for testing against a Git version of Safir.
+.PHONY: update-deps-no-hashes
+update-deps-no-hashes:
+	pip install --upgrade uv
+	uv pip compile --upgrade					\
+	    --output-file requirements/main.txt requirements/main.in
+	uv pip compile --upgrade					\
+	    --output-file requirements/dev.txt requirements/dev.in
+	uv pip compile --upgrade					\
+	    --output-file requirements/tox.txt requirements/tox.in