Update raw.pixls.us masterset cache management #9
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update raw.pixls.us masterset cache management | |
| on: | |
| workflow_call: | |
| outputs: | |
| rpuu-cache-key: | |
| value: ${{ jobs.linux.outputs.rpuu-cache-key }} | |
| schedule: | |
| - cron: "30 0 * * SUN" | |
| workflow_dispatch: | |
| concurrency: | |
| group: rpuu-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: false | |
| env: | |
| RPUU_DST: ${{ github.workspace }}/raw-camera-samples/raw.pixls.us-unique | |
| CACHE_WARMUP_ONLY: ${{ github.workflow != 'Update raw.pixls.us masterset cache management' }} | |
| jobs: | |
| linux: | |
| runs-on: ubuntu-latest | |
| name: Cache RPU masterset | |
| timeout-minutes: 3 | |
| container: | |
| image: "debian:bookworm-slim" | |
| options: --tmpfs /tmp:exec | |
| steps: | |
| - name: Configure APT | |
| timeout-minutes: 1 | |
| run: | | |
| set -xe | |
| tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io > /dev/null <<EOT | |
| force-unsafe-io | |
| EOT | |
| tee /etc/apt/apt.conf.d/tmpfs > /dev/null <<EOT | |
| Dir::Cache::Archives "/tmp/apt/archives"; | |
| APT::ExtractTemplates::TempDir "/tmp/apt/temp"; | |
| EOT | |
| mkdir -p /tmp/apt/archives | |
| tee /etc/apt/apt.conf.d/80retry > /dev/null <<EOT | |
| Acquire::Retries "10"; | |
| EOT | |
| tee /etc/apt/apt.conf.d/80recommends > /dev/null <<EOT | |
| APT::Install-Recommends "false"; | |
| EOT | |
| tee /etc/apt/apt.conf.d/80suggests > /dev/null <<EOT | |
| APT::Install-Suggests "false"; | |
| EOT | |
| tee /etc/apt/apt.conf.d/80forceyes > /dev/null <<EOT | |
| APT::Get::Assume-Yes "true"; | |
| EOT | |
| tee /etc/apt/apt.conf.d/80fixmissing > /dev/null <<EOT | |
| APT::Get::Fix-Missing "true"; | |
| EOT | |
| rm -rf /etc/apt/sources.list* | |
| tee /etc/apt/sources.list > /dev/null <<EOT | |
| deb http://debian-archive.trafficmanager.net/debian bookworm main | |
| deb http://debian-archive.trafficmanager.net/debian bookworm-updates main | |
| deb http://debian-archive.trafficmanager.net/debian-security bookworm-security main | |
| deb http://debian-archive.trafficmanager.net/debian bookworm-backports main | |
| EOT | |
| - name: Update base packages | |
| timeout-minutes: 1 | |
| env: | |
| DEBIAN_FRONTEND: noninteractive | |
| run: | | |
| set -xe | |
| rm -rf /var/lib/apt/lists/* | |
| apt update | |
| apt install eatmydata | |
| eatmydata apt upgrade | |
| - name: Install necessary packages | |
| timeout-minutes: 1 | |
| env: | |
| DEBIAN_FRONTEND: noninteractive | |
| run: | | |
| set -xe | |
| eatmydata apt install ca-certificates wget git git-annex zstd | |
| git config --global user.email "bot@local" | |
| - name: Check raw.pixls.us masterset cache availability | |
| timeout-minutes: 1 | |
| id: cache-restore | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: ${{ env.RPUU_DST }} | |
| key: "raw.pixls.us-masterset" | |
| fail-on-cache-miss: false | |
| lookup-only: ${{ env.CACHE_WARMUP_ONLY == 'true' }} | |
| - name: Fetch raw.pixls.us masterset | |
| timeout-minutes: 3 | |
| id: cache-get | |
| if: steps.cache-restore.outputs.cache-matched-key == '' | |
| env: | |
| RPUU_DST: ${{ env.RPUU_DST }} | |
| run: | | |
| set -xe | |
| git clone https://raw.pixls.us/data-unique.annex.git ${RPUU_DST} | |
| cd ${RPUU_DST} | |
| git annex get --jobs cpus . | |
| sha256sum -c --strict filelist.sha256 | |
| - name: Update raw.pixls.us masterset | |
| timeout-minutes: 3 | |
| id: cache-update | |
| if: env.CACHE_WARMUP_ONLY != 'true' && steps.cache-restore.outputs.cache-matched-key != '' | |
| env: | |
| RPUU_DST: ${{ env.RPUU_DST }} | |
| run: | | |
| set -xe | |
| cd ${RPUU_DST} | |
| git fetch --all --prune | |
| git branch -f git-annex origin/git-annex | |
| git reset --hard origin/master | |
| git reflog expire --all --expire all | |
| git annex get --jobs cpus . | |
| git annex unused | |
| git-annex dropunused all --force | |
| git repack -a -d -f -F --depth=4095 --window=1000000 | |
| sha256sum -c --strict filelist.sha256 | |
| - name: Get the raw.pixls.us masterset cache key | |
| timeout-minutes: 1 | |
| env: | |
| RPUU_DST: ${{ env.RPUU_DST }} | |
| id: memoize-key | |
| run: | | |
| set -xe | |
| if [ -f "${RPUU_DST}/timestamp.txt" ]; then | |
| echo "rpuu-cache-key=raw.pixls.us-masterset-$(cat ${RPUU_DST}/timestamp.txt)" >> $GITHUB_OUTPUT | |
| else | |
| echo "rpuu-cache-key=${{ steps.cache-restore.outputs.cache-matched-key }}" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Save updated raw.pixls.us masterset cache | |
| if: ((steps.cache-get.outcome == 'success') || (steps.cache-update.outcome == 'success')) && (steps.memoize-key.outputs.rpuu-cache-key != steps.cache-restore.outputs.cache-matched-key) | |
| timeout-minutes: 1 | |
| id: cache-save | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: ${{ env.RPUU_DST }} | |
| key: ${{ steps.memoize-key.outputs.rpuu-cache-key }} | |
| outputs: | |
| rpuu-cache-key: ${{ steps.memoize-key.outputs.rpuu-cache-key }} |