From 121131035fa1d2933c768947c46a483472c53ecb Mon Sep 17 00:00:00 2001 From: Takuya Iwanaga Date: Mon, 14 Jul 2025 12:42:45 +1000 Subject: [PATCH 01/12] Clean up dockerfile for container without sysimage --- Dockerfile | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8ea05e1..418d3cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,16 @@ +# Worker container without sysimage + # See https://hub.docker.com/_/julia for valid versions. ARG JULIA_VERSION="1.11.5" +ARG BASE_IMAGE="julia:${JULIA_VERSION}-bookworm" #------------------------------------------------------------------------------ # internal-base build target: julia with OS updates and an empty @app # Julia environment prepared for use. NOT intended for standalone use. #------------------------------------------------------------------------------ -FROM julia:${JULIA_VERSION}-bookworm AS internal-base +FROM ${BASE_IMAGE} AS internal-base # Record the actual base image used from the FROM command as label in the compiled image -ARG BASE_IMAGE="julia:${JULIA_VERSION}-bookworm" LABEL org.opencontainers.image.base.name=${BASE_IMAGE} # Update all pre-installed OS packages (to get security updates) @@ -36,8 +38,8 @@ RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \ # See https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH # This allows apps derived from this image to drop privileges and run as non-root # user accounts, but still activate environments configured by this dockerfile. -ENV JULIA_DEPOT_PATH="/usr/local/share/julia" -ENV PRJ_PATH="/usr/local/share/julia/environments/app" +ENV JULIA_DEPOT_PATH=/usr/local/share/julia +ENV PRJ_PATH=/usr/local/share/julia/environments/app ENV JULIA_PKG_USE_CLI_GIT=true # Coerce Julia to build across multiple targets @@ -51,8 +53,7 @@ ENV JULIA_CPU_TARGET=generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base # Prepare an empty @app Julia environment for derived images to use - this is created in the shared depot path RUN mkdir -p "${JULIA_DEPOT_PATH}" && \ - chmod 0755 "${JULIA_DEPOT_PATH}" && \ - julia -e 'using Pkg; Pkg.activate("app", shared=true)' + chmod 0755 "${JULIA_DEPOT_PATH}" # Ensure the @app environment is in the load path for Julia, so that apps derived # from this image can access any packages installed to there. @@ -67,12 +68,14 @@ COPY Project.toml Manifest*.toml ./ # Should be v speedy if the .toml file is unchanged, because all the # dependencies *should* already be installed. COPY ./src src -RUN julia --project=@app \ - -e 'using Pkg; \ - Pkg.add("MKL"); \ +RUN julia --project=@app -e \ + 'using Pkg; \ + Pkg.add("PackageCompiler"); \ Pkg.develop(PackageSpec(path=pwd())); \ - Pkg.precompile(); \ - using ReefGuideWorker;' + Pkg.instantiate(); \ + Pkg.precompile();' + +# Pkg.add(name="MKL_jll", version="2024.2.0"); \ # Run Julia commands by default as the container launches. # Derived applications should override the command. @@ -82,9 +85,10 @@ ENTRYPOINT ["julia", "--project=@app"] # app-src build target: installs directly from source files in this repo. #------------------------------------------------------------------------------ FROM internal-base AS app-src +ENV JULIA_DEPOT_PATH=/usr/local/share/julia -ENV APP_ENV_DIR="${JULIA_DEPOT_PATH}/environments/app" \ - APP_SRC_DIR="/usr/local/src/app" \ +ENV APP_ENV_DIR=${JULIA_DEPOT_PATH}/environments/app \ + APP_SRC_DIR=/usr/local/src/app \ JULIA_PKG_USE_CLI_GIT=true # Expect to include the prepped data at /data/app and the config at @@ -92,7 +96,7 @@ ENV APP_ENV_DIR="${JULIA_DEPOT_PATH}/environments/app" \ VOLUME ["/data/app"] # By default, drops the user into a julia shell with ReefGuideWorker activated -ENTRYPOINT ["julia", "--project=@app", "-t", "auto,1", "-e"] +ENTRYPOINT ["julia", "--project=@app", "-e"] # Derived applications should override the command e.g. to start CMD ["using ReefGuideWorker; ReefGuideWorker.start_worker()"] \ No newline at end of file From 49ce6222967f2b127e09eacbf94ca3b6be86695c Mon Sep 17 00:00:00 2001 From: Takuya Iwanaga Date: Mon, 14 Jul 2025 12:42:54 +1000 Subject: [PATCH 02/12] Add dockerfile with sysimage --- Dockerfile_sysimage | 74 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 Dockerfile_sysimage diff --git a/Dockerfile_sysimage b/Dockerfile_sysimage new file mode 100644 index 0000000..8e76d1b --- /dev/null +++ b/Dockerfile_sysimage @@ -0,0 +1,74 @@ +# Worker container with sysimage + +# Following advice found in this Discourse thread: +# https://discourse.julialang.org/t/creating-a-docker-base-image-for-faster-deployments/121165/2 + +ARG JULIA_VERSION="1.11.5" +FROM julia:${JULIA_VERSION}-bookworm AS internal-base + +# Since 1.9.0 Julia, the CPU target is set to "native" by default. +# This settings avoids the need to compile the Julia packages for the specific CPU architecture of the host machine +# Make sure the image can be used on any x86_64 machine by setting JULIA_CPU_TARGET +# to the same value used by the generic julia binaries, see +# https://github.com/JuliaCI/julia-buildkite/blob/4b6932992f7985af71fc3f73af77abf4d25bd146/utilities/build_envs.sh#L23-L31 +ENV JULIA_CPU_TARGET="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1);x86-64-v4,-rdrnd,base(1);znver4,-rdrnd,base(1)" + +ENV JULIA_VERSION=1.11.5 +ENV JULIA_DIR=/usr/local/julia +ENV JULIA_PATH=${JULIA_DIR} +ENV JULIA_DEPOT_PATH=usr/local/share/julia +ENV APP_ENV_PATH=${JULIA_DEPOT_PATH}/environments/app +ENV APP_SRC_DIR=/usr/local/src/app + +# Update all pre-installed OS packages (to get security updates) +# and add a few extra utilities +RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \ + --mount=target=/var/cache/apt,type=cache,sharing=locked \ + apt-get update \ + && apt-get -y upgrade \ + && apt-get install --no-install-recommends -y \ + git \ + openssl \ + libssl-dev \ + g++ \ + curl \ + ca-certificates \ + && apt-get clean \ + && apt-get autoremove --purge \ + && rm -rf /var/lib/apt/lists/* + +# Setup shared environment and add packages +RUN mkdir -p "${JULIA_DEPOT_PATH}" \ + && chmod 0755 "${JULIA_DEPOT_PATH}" + +WORKDIR "${APP_SRC_DIR}" + +COPY Project.toml Manifest*.toml ./ +COPY src/ src/ + +# Expect to include the prepped data at /data/app and the config at +# /data/.config.toml +VOLUME ["/data", "/data/app"] + +# # By default, drops the user into a julia shell with ReefGuideWorker activated +ENV JULIA_DIR=/usr/local/julia +ENV JULIA_PATH=${JULIA_DIR} +ENV JULIA_DEPOT_PATH=usr/local/share/julia + +# If dependencies draw in specific Artifacts (additional and lazily downloaded +# dependencies), then the environment using sysimages must also contain those Artifacts. +# So we recreate the conditions here. +# https://github.com/JuliaLang/PackageCompiler.jl/issues/743 +RUN julia --project=@app -e \ + 'using Pkg; \ + Pkg.develop(PackageSpec(path=pwd())); \ + Pkg.instantiate(); \ + Pkg.precompile();' + +ENTRYPOINT [ \ + "julia", "--project=@app", \ + "-J", "/data/reefguide_img.so", \ + "--sysimage-native-code=yes", \ + "-e"] + +CMD ["using ReefGuideWorker; ReefGuideWorker.start_worker()"] From e43d2bf81d12f0fcbb62823b1ef769757b2e83fe Mon Sep 17 00:00:00 2001 From: Takuya Iwanaga Date: Mon, 14 Jul 2025 12:44:38 +1000 Subject: [PATCH 03/12] Indicate usage --- Dockerfile_sysimage | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile_sysimage b/Dockerfile_sysimage index 8e76d1b..329ec6d 100644 --- a/Dockerfile_sysimage +++ b/Dockerfile_sysimage @@ -1,4 +1,5 @@ # Worker container with sysimage +# docker build -t reefguide-img-worker -f Dockerfile_sysimage . # Following advice found in this Discourse thread: # https://discourse.julialang.org/t/creating-a-docker-base-image-for-faster-deployments/121165/2 From 296c6ad5d7f96177d16e71fc3abe4d902c9ed64c Mon Sep 17 00:00:00 2001 From: Takuya Iwanaga Date: Mon, 14 Jul 2025 12:53:25 +1000 Subject: [PATCH 04/12] Add sysimage build file and rename to match docker conventions --- Dockerfile.build_sysimage | 77 ++++++++++++++++++++++ Dockerfile_sysimage => Dockerfile.sysimage | 2 +- 2 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 Dockerfile.build_sysimage rename Dockerfile_sysimage => Dockerfile.sysimage (97%) diff --git a/Dockerfile.build_sysimage b/Dockerfile.build_sysimage new file mode 100644 index 0000000..cda3954 --- /dev/null +++ b/Dockerfile.build_sysimage @@ -0,0 +1,77 @@ +# Build process to generate sysimage for ReefGuideWorker. +# docker build -t reefguide-img-worker -f Dockerfile.build_sysimage . + +# Following advice found in this Discourse thread: +# https://discourse.julialang.org/t/creating-a-docker-base-image-for-faster-deployments/121165/2 + +# Also found this relevant issue: +# https://github.com/JuliaLang/PackageCompiler.jl/issues/743 + +# docker build --target export-sysimage -t my-sysimage . +# docker run --target app-src --env-file .env -it -v [src dir]:/data/app reefguide-worker + +ARG JULIA_VERSION="1.11.5" +FROM julia:${JULIA_VERSION}-bookworm AS internal-base + +# Since 1.9.0 Julia, the CPU target is set to "native" by default. +# This settings avoids the need to compile the Julia packages for the specific CPU architecture of the host machine +# Make sure the image can be used on any x86_64 machine by setting JULIA_CPU_TARGET +# to the same value used by the generic julia binaries, see +# https://github.com/JuliaCI/julia-buildkite/blob/4b6932992f7985af71fc3f73af77abf4d25bd146/utilities/build_envs.sh#L23-L31 +ENV JULIA_CPU_TARGET="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1);x86-64-v4,-rdrnd,base(1);znver4,-rdrnd,base(1)" + +ENV JULIA_VERSION=1.11.5 +ENV JULIA_DIR=/usr/local/julia +ENV JULIA_PATH=${JULIA_DIR} +ENV JULIA_DEPOT_PATH=usr/local/share/julia +ENV APP_ENV_PATH=${JULIA_DEPOT_PATH}/environments/app +ENV APP_SRC_DIR=/usr/local/src/app +ENV SYSIMAGE_SRC_DIR=/usr/local/src/app_sysimage +ENV JULIA_PKG_USE_CLI_GIT=true + +# Update all pre-installed OS packages (to get security updates) +# and add a few extra utilities +RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \ + --mount=target=/var/cache/apt,type=cache,sharing=locked \ + apt-get update \ + && apt-get -y upgrade \ + && apt-get install --no-install-recommends -y \ + git \ + openssl \ + libssl-dev \ + g++ \ + curl \ + ca-certificates \ + && apt-get clean \ + && apt-get autoremove --purge \ + && rm -rf /var/lib/apt/lists/* + +# Setup shared environment and add packages +RUN mkdir -p "${JULIA_DEPOT_PATH}" \ + && chmod 0755 "${JULIA_DEPOT_PATH}" + +WORKDIR "${APP_SRC_DIR}" + +COPY Project.toml Manifest*.toml ./ +COPY src/ src/ + +# Build sysimage +RUN julia --project=@app -e \ + 'using Pkg; \ + Pkg.add("PackageCompiler"); \ + Pkg.develop(PackageSpec(path=pwd())); \ + Pkg.instantiate(); ' + +# Reduce number of tasks/threads to avoid heavy memory use during sysimage compilation +# https://github.com/JuliaLang/PackageCompiler.jl/issues/1031#issuecomment-2823054267 +ENV JULIA_NUM_PRECOMPILE_TAKS=2 +RUN julia --project=@app -t 1,1 -e 'include("src/sysimage.jl")' + +# Export Julia sysimage to host filesystem +# From project root +# docker build --target export-sysimage -t reefguide-sysimage -f sandbox/smaller_sysimage/Dockerfile . +FROM scratch AS export-sysimage +ENV APP_SRC_DIR=/usr/local/src/app +COPY --from=internal-base ${APP_SRC_DIR}/reefguide_img.so /reefguide_img.so + +ENTRYPOINT ["bash"] diff --git a/Dockerfile_sysimage b/Dockerfile.sysimage similarity index 97% rename from Dockerfile_sysimage rename to Dockerfile.sysimage index 329ec6d..b15f5b3 100644 --- a/Dockerfile_sysimage +++ b/Dockerfile.sysimage @@ -1,5 +1,5 @@ # Worker container with sysimage -# docker build -t reefguide-img-worker -f Dockerfile_sysimage . +# docker build -t reefguide-img-worker -f Dockerfile.sysimage . # Following advice found in this Discourse thread: # https://discourse.julialang.org/t/creating-a-docker-base-image-for-faster-deployments/121165/2 From 28c44e2acf410ad4a999b4c69dbcd58b2238112e Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Tue, 15 Jul 2025 12:58:44 +1000 Subject: [PATCH 05/12] Documentation, consolidation, single stage build Signed-off-by: Peter Baker --- .dockerignore | 1 + .github/workflows/PublishDockerImage.yml | 1 - .gitignore | 1 + Dockerfile | 94 +++++++++++------------- Dockerfile.build_sysimage | 41 ++++++----- Dockerfile.sysimage | 75 ------------------- README.md | 44 ++++++++++- launch_sysimage.sh | 49 ++++++++++++ 8 files changed, 160 insertions(+), 146 deletions(-) delete mode 100644 Dockerfile.sysimage create mode 100755 launch_sysimage.sh diff --git a/.dockerignore b/.dockerignore index 88bc099..d295451 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,3 +6,4 @@ docs/ Dockerfile .gitignore data +launch_sysimage.sh diff --git a/.github/workflows/PublishDockerImage.yml b/.github/workflows/PublishDockerImage.yml index 0a2dfe4..56c7759 100644 --- a/.github/workflows/PublishDockerImage.yml +++ b/.github/workflows/PublishDockerImage.yml @@ -60,7 +60,6 @@ jobs: uses: docker/build-push-action@v6 with: context: . - target: app-src # Specifies which stage of the Dockerfile to build push: true # Pushes the image to the registry tags: ${{ steps.meta.outputs.tags }} # Uses the tags generated in the metadata step labels: ${{ steps.meta.outputs.labels }} # Uses the labels generated in the metadata step diff --git a/.gitignore b/.gitignore index 8effa53..e1642aa 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ sandbox/ data .env +sysimages diff --git a/Dockerfile b/Dockerfile index 418d3cf..3625408 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,31 @@ -# Worker container without sysimage +#============================================================================== +# ReefGuide Worker Base Image +#============================================================================== +# This Dockerfile creates a Julia-based container image for the ReefGuide Worker +# application. It provides a complete Julia environment with the ReefGuideWorker +# package installed from source as a development dependency. +# +# Usage: +# Build> docker build -t reefguide-worker . +# Run> docker build -t reefguide-worker . +#============================================================================== # See https://hub.docker.com/_/julia for valid versions. ARG JULIA_VERSION="1.11.5" ARG BASE_IMAGE="julia:${JULIA_VERSION}-bookworm" -#------------------------------------------------------------------------------ -# internal-base build target: julia with OS updates and an empty @app -# Julia environment prepared for use. NOT intended for standalone use. -#------------------------------------------------------------------------------ -FROM ${BASE_IMAGE} AS internal-base +FROM ${BASE_IMAGE} -# Record the actual base image used from the FROM command as label in the compiled image +# Redeclare +ARG BASE_IMAGE + +# Record the actual base image used from the FROM command as label in the +# compiled image LABEL org.opencontainers.image.base.name=${BASE_IMAGE} -# Update all pre-installed OS packages (to get security updates) -# and add a few extra utilities + +# Update all pre-installed OS packages (to get security updates) and add a few +# extra utilities RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \ --mount=target=/var/cache/apt,type=cache,sharing=locked \ apt-get update \ @@ -34,69 +45,50 @@ RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \ && rm -rf /var/lib/apt/lists/* # Tweak the JULIA_DEPOT_PATH setting so that our shared environments will end up -# in a user-agnostic location, not in ~/.julia => /root/.julia which is the default. -# See https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH -# This allows apps derived from this image to drop privileges and run as non-root -# user accounts, but still activate environments configured by this dockerfile. +# in a user-agnostic location, not in ~/.julia => /root/.julia which is the +# default. See +# https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH +# This allows apps derived from this image to drop privileges and run as +# non-root user accounts, but still activate environments configured by this +# dockerfile. ENV JULIA_DEPOT_PATH=/usr/local/share/julia -ENV PRJ_PATH=/usr/local/share/julia/environments/app + +# Ensure the @app environment is in the load path for Julia, so that apps +# derived from this image can access any packages installed to there. (See +# https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_LOAD_PATH) +ENV JULIA_LOAD_PATH="@:@app:@v#.#:@stdlib" + +# This tells Julia's package manager to use the CLI installation of Git rather +# than an internal lib version - this works better with auth for example in +# CI/CD environments ENV JULIA_PKG_USE_CLI_GIT=true -# Coerce Julia to build across multiple targets -# Generic targets taken from: cpu_targets taken from: +# Coerce Julia to build across multiple targets. See: # https://docs.julialang.org/en/v1/devdocs/sysimg/#Specifying-multiple-system-image-targets -ENV JULIA_CPU_TARGET=generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1) - -# Alternate set that was found to initially alleviate issues on AWS at expense of very -# long build times. -# ENV JULIA_CPU_TARGET=x86_64;haswell;skylake;skylake-avx512;tigerlake +# Alternate set that was found to initially alleviate excessive +# (re)precompilation issues on AWS at expense of very long build times... +# x86_64;haswell;skylake;skylake-avx512;tigerlake +ENV JULIA_CPU_TARGET="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1);x86-64-v4,-rdrnd,base(1);znver4,-rdrnd,base(1)" -# Prepare an empty @app Julia environment for derived images to use - this is created in the shared depot path +# Prepare an empty @app Julia environment for derived images to use - this is +# created in the shared depot path RUN mkdir -p "${JULIA_DEPOT_PATH}" && \ chmod 0755 "${JULIA_DEPOT_PATH}" -# Ensure the @app environment is in the load path for Julia, so that apps derived -# from this image can access any packages installed to there. -# (See https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_LOAD_PATH) -ENV JULIA_LOAD_PATH="@:@app:@v#.#:@stdlib" - # Copy project and manifest - includes Manifest-v1.11 etc COPY Project.toml Manifest*.toml ./ # Install ReefGuideWorker from source and configure it as a development # package in the @app shared environment. -# Should be v speedy if the .toml file is unchanged, because all the -# dependencies *should* already be installed. COPY ./src src RUN julia --project=@app -e \ 'using Pkg; \ - Pkg.add("PackageCompiler"); \ Pkg.develop(PackageSpec(path=pwd())); \ Pkg.instantiate(); \ Pkg.precompile();' -# Pkg.add(name="MKL_jll", version="2024.2.0"); \ - -# Run Julia commands by default as the container launches. -# Derived applications should override the command. -ENTRYPOINT ["julia", "--project=@app"] - -#------------------------------------------------------------------------------ -# app-src build target: installs directly from source files in this repo. -#------------------------------------------------------------------------------ -FROM internal-base AS app-src -ENV JULIA_DEPOT_PATH=/usr/local/share/julia - -ENV APP_ENV_DIR=${JULIA_DEPOT_PATH}/environments/app \ - APP_SRC_DIR=/usr/local/src/app \ - JULIA_PKG_USE_CLI_GIT=true - -# Expect to include the prepped data at /data/app and the config at -# /data/.config.toml -VOLUME ["/data/app"] - # By default, drops the user into a julia shell with ReefGuideWorker activated ENTRYPOINT ["julia", "--project=@app", "-e"] # Derived applications should override the command e.g. to start -CMD ["using ReefGuideWorker; ReefGuideWorker.start_worker()"] \ No newline at end of file +CMD ["using ReefGuideWorker; ReefGuideWorker.start_worker()"] diff --git a/Dockerfile.build_sysimage b/Dockerfile.build_sysimage index cda3954..c394edc 100644 --- a/Dockerfile.build_sysimage +++ b/Dockerfile.build_sysimage @@ -1,22 +1,33 @@ -# Build process to generate sysimage for ReefGuideWorker. -# docker build -t reefguide-img-worker -f Dockerfile.build_sysimage . - +#============================================================================== +# ReefGuide Worker: Sysimage Builder +#============================================================================== +# Builds a Julia system image for faster ReefGuide Worker startup times. +# The system image is compiled using PackageCompiler.jl and can be extracted +# from the resulting container. +# # Following advice found in this Discourse thread: # https://discourse.julialang.org/t/creating-a-docker-base-image-for-faster-deployments/121165/2 - +# # Also found this relevant issue: # https://github.com/JuliaLang/PackageCompiler.jl/issues/743 - -# docker build --target export-sysimage -t my-sysimage . -# docker run --target app-src --env-file .env -it -v [src dir]:/data/app reefguide-worker +# +# Build: +# docker build --target export-sysimage -f Dockerfile.build_sysimage -t reefguide-sysimage . +# +# Extract sysimage: +# docker create --name temp-sysimage reefguide-sysimage +# docker cp temp-sysimage:/reefguide_img.so ./reefguide_img.so +# docker rm temp-sysimage +#============================================================================== ARG JULIA_VERSION="1.11.5" FROM julia:${JULIA_VERSION}-bookworm AS internal-base -# Since 1.9.0 Julia, the CPU target is set to "native" by default. -# This settings avoids the need to compile the Julia packages for the specific CPU architecture of the host machine -# Make sure the image can be used on any x86_64 machine by setting JULIA_CPU_TARGET -# to the same value used by the generic julia binaries, see +# Since 1.9.0 Julia, the CPU target is set to "native" by default. This settings +# avoids the need to compile the Julia packages for the specific CPU +# architecture of the host machine Make sure the image can be used on any x86_64 +# machine by setting JULIA_CPU_TARGET to the same value used by the generic +# julia binaries, see # https://github.com/JuliaCI/julia-buildkite/blob/4b6932992f7985af71fc3f73af77abf4d25bd146/utilities/build_envs.sh#L23-L31 ENV JULIA_CPU_TARGET="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1);x86-64-v4,-rdrnd,base(1);znver4,-rdrnd,base(1)" @@ -26,7 +37,6 @@ ENV JULIA_PATH=${JULIA_DIR} ENV JULIA_DEPOT_PATH=usr/local/share/julia ENV APP_ENV_PATH=${JULIA_DEPOT_PATH}/environments/app ENV APP_SRC_DIR=/usr/local/src/app -ENV SYSIMAGE_SRC_DIR=/usr/local/src/app_sysimage ENV JULIA_PKG_USE_CLI_GIT=true # Update all pre-installed OS packages (to get security updates) @@ -64,14 +74,11 @@ RUN julia --project=@app -e \ # Reduce number of tasks/threads to avoid heavy memory use during sysimage compilation # https://github.com/JuliaLang/PackageCompiler.jl/issues/1031#issuecomment-2823054267 -ENV JULIA_NUM_PRECOMPILE_TAKS=2 +ENV JULIA_NUM_PRECOMPILE_TASKS=1 RUN julia --project=@app -t 1,1 -e 'include("src/sysimage.jl")' # Export Julia sysimage to host filesystem # From project root # docker build --target export-sysimage -t reefguide-sysimage -f sandbox/smaller_sysimage/Dockerfile . FROM scratch AS export-sysimage -ENV APP_SRC_DIR=/usr/local/src/app -COPY --from=internal-base ${APP_SRC_DIR}/reefguide_img.so /reefguide_img.so - -ENTRYPOINT ["bash"] +COPY --from=internal-base /usr/local/src/app/reefguide_img.so /reefguide_img.so diff --git a/Dockerfile.sysimage b/Dockerfile.sysimage deleted file mode 100644 index b15f5b3..0000000 --- a/Dockerfile.sysimage +++ /dev/null @@ -1,75 +0,0 @@ -# Worker container with sysimage -# docker build -t reefguide-img-worker -f Dockerfile.sysimage . - -# Following advice found in this Discourse thread: -# https://discourse.julialang.org/t/creating-a-docker-base-image-for-faster-deployments/121165/2 - -ARG JULIA_VERSION="1.11.5" -FROM julia:${JULIA_VERSION}-bookworm AS internal-base - -# Since 1.9.0 Julia, the CPU target is set to "native" by default. -# This settings avoids the need to compile the Julia packages for the specific CPU architecture of the host machine -# Make sure the image can be used on any x86_64 machine by setting JULIA_CPU_TARGET -# to the same value used by the generic julia binaries, see -# https://github.com/JuliaCI/julia-buildkite/blob/4b6932992f7985af71fc3f73af77abf4d25bd146/utilities/build_envs.sh#L23-L31 -ENV JULIA_CPU_TARGET="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1);x86-64-v4,-rdrnd,base(1);znver4,-rdrnd,base(1)" - -ENV JULIA_VERSION=1.11.5 -ENV JULIA_DIR=/usr/local/julia -ENV JULIA_PATH=${JULIA_DIR} -ENV JULIA_DEPOT_PATH=usr/local/share/julia -ENV APP_ENV_PATH=${JULIA_DEPOT_PATH}/environments/app -ENV APP_SRC_DIR=/usr/local/src/app - -# Update all pre-installed OS packages (to get security updates) -# and add a few extra utilities -RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \ - --mount=target=/var/cache/apt,type=cache,sharing=locked \ - apt-get update \ - && apt-get -y upgrade \ - && apt-get install --no-install-recommends -y \ - git \ - openssl \ - libssl-dev \ - g++ \ - curl \ - ca-certificates \ - && apt-get clean \ - && apt-get autoremove --purge \ - && rm -rf /var/lib/apt/lists/* - -# Setup shared environment and add packages -RUN mkdir -p "${JULIA_DEPOT_PATH}" \ - && chmod 0755 "${JULIA_DEPOT_PATH}" - -WORKDIR "${APP_SRC_DIR}" - -COPY Project.toml Manifest*.toml ./ -COPY src/ src/ - -# Expect to include the prepped data at /data/app and the config at -# /data/.config.toml -VOLUME ["/data", "/data/app"] - -# # By default, drops the user into a julia shell with ReefGuideWorker activated -ENV JULIA_DIR=/usr/local/julia -ENV JULIA_PATH=${JULIA_DIR} -ENV JULIA_DEPOT_PATH=usr/local/share/julia - -# If dependencies draw in specific Artifacts (additional and lazily downloaded -# dependencies), then the environment using sysimages must also contain those Artifacts. -# So we recreate the conditions here. -# https://github.com/JuliaLang/PackageCompiler.jl/issues/743 -RUN julia --project=@app -e \ - 'using Pkg; \ - Pkg.develop(PackageSpec(path=pwd())); \ - Pkg.instantiate(); \ - Pkg.precompile();' - -ENTRYPOINT [ \ - "julia", "--project=@app", \ - "-J", "/data/reefguide_img.so", \ - "--sysimage-native-code=yes", \ - "-e"] - -CMD ["using ReefGuideWorker; ReefGuideWorker.start_worker()"] diff --git a/README.md b/README.md index 3127238..112ba0b 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ A Julia-based worker template for the [open-AIMS/reefguide](https://github.com/o ## Where does it fit in? -ReefGuideWorker.jl is a worker node processing Site Selection and Regional Assessment jobs in the [reefguide](https://github.com/open-AIMS/reefguide) system. +ReefGuideWorker.jl is a worker node processing Site Selection and Regional Assessment jobs in the [reefguide](https://github.com/open-AIMS/reefguide) system. ReefGuide deploys `ReefGuideWorker.jl` and `ADRIAReefGuideWorker.jl` as Docker containers on ECS to consume jobs generated by users in the frontend. They each use their respective Julia library, and fork the template repository. Shown below. @@ -35,7 +35,6 @@ Related repos include - [ADRIA.jl](https://github.com/open-AIMS/ADRIA.jl) - The ADRIA model Julia library code, used by ADRIAReefGuideWorker.jl - [ADRIAReefGuideWorker.jl](https://github.com/open-AIMS/ADRIAReefGuideWorker.jl) - Julia Job Worker to run ADRIA algorithms for ReefGuide - ## Context This worker template integrates with the ReefGuide ecosystem, connecting to the ReefGuide web API to receive and process distributed computing tasks. The `API_ENDPOINT` environment variable should point to your ReefGuide web API instance. @@ -86,6 +85,47 @@ This worker template integrates with the ReefGuide ecosystem, connecting to the docker run --env-file .env worker ``` +#### Building the sysimage + +To build, using docker, a current sysimage + +``` +docker build --target export-sysimage -t sysimage . +``` + +Then copy the file out: + +``` +docker create --name temp-sysimage sysimage +docker cp temp-sysimage:/reefguide_img.so ./reefguide_img.so +docker rm temp-sysimage +``` + +#### Running the worker with the sysimage + +The main Dockerfile, once built, can be run in a sysimage mode. + +You need to + +1. mount a volume containing the sysimage file (as above) +2. override the entrypoint and cmd to run using the sysimage + +The script `./launch_sysimage.sh` does this for you + +``` +./launch_sysimage.sh +``` + +So for example, if you have a file `sysimage.so` in the folder `sysimages`, you would run + +``` +./run-worker.sh ./sysimages sysimage.so worker +``` + +**NOTE**: the `./` relative path is important in the first argument. + +Where worker was the `-t` argument when building the main dockerfile. + ## How It Works ### Core Architecture diff --git a/launch_sysimage.sh b/launch_sysimage.sh new file mode 100755 index 0000000..aea2fdd --- /dev/null +++ b/launch_sysimage.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Script to run the ReefGuide worker container +# Usage: ./run-worker.sh + +set -e + +# Check if required arguments are provided +if [ $# -lt 3 ]; then + echo "Usage: $0 " + echo "Example: $0 ./sysimages reefguide_img.so worker" + exit 1 +fi + +SYSIMAGE_DIR="$1" +SYSIMAGE_NAME="$2" +CONTAINER_NAME="$3" + +# Check if sysimage directory exists +if [ ! -d "$SYSIMAGE_DIR" ]; then + echo "Error: System image directory not found at $SYSIMAGE_DIR" + exit 1 +fi + +# Check if sysimage file exists in the directory +if [ ! -f "$SYSIMAGE_DIR/$SYSIMAGE_NAME" ]; then + echo "Error: System image file $SYSIMAGE_NAME not found in $SYSIMAGE_DIR" + exit 1 +fi + +echo "Starting ReefGuide worker container..." +echo "System image directory: $SYSIMAGE_DIR" +echo "System image file: $SYSIMAGE_NAME" +echo "Worker container name: $CONTAINER_NAME" + +# Run the container +docker run \ + -v "$SYSIMAGE_DIR:/data" \ + --entrypoint julia \ + ${CONTAINER_NAME} \ + --project=@app \ + -J "/data/$SYSIMAGE_NAME" \ + --sysimage-native-code=yes \ + -e "using ReefGuideWorker; ReefGuideWorker.start_worker()" + +echo "Container $CONTAINER_NAME started successfully!" +echo "To view logs: docker logs $CONTAINER_NAME" +echo "To stop: docker stop $CONTAINER_NAME" +echo "To remove: docker rm $CONTAINER_NAME" From e2656cb99a3559c19c377e6ba3ba4c44c04ceb38 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Tue, 15 Jul 2025 14:31:05 +1000 Subject: [PATCH 06/12] Fixing path and adding gdal dep Signed-off-by: Peter Baker --- Dockerfile.build_sysimage | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile.build_sysimage b/Dockerfile.build_sysimage index c394edc..bc6a7d6 100644 --- a/Dockerfile.build_sysimage +++ b/Dockerfile.build_sysimage @@ -34,7 +34,7 @@ ENV JULIA_CPU_TARGET="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,bas ENV JULIA_VERSION=1.11.5 ENV JULIA_DIR=/usr/local/julia ENV JULIA_PATH=${JULIA_DIR} -ENV JULIA_DEPOT_PATH=usr/local/share/julia +ENV JULIA_DEPOT_PATH=/usr/local/share/julia ENV APP_ENV_PATH=${JULIA_DEPOT_PATH}/environments/app ENV APP_SRC_DIR=/usr/local/src/app ENV JULIA_PKG_USE_CLI_GIT=true @@ -52,6 +52,9 @@ RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \ g++ \ curl \ ca-certificates \ + gdal-bin \ + libgdal-dev \ + libfftw3-dev \ && apt-get clean \ && apt-get autoremove --purge \ && rm -rf /var/lib/apt/lists/* From 30fa03a35c30d4c99b007e340dc6fdb2e9a63c2f Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Wed, 16 Jul 2025 09:21:05 +1000 Subject: [PATCH 07/12] Removing cache mounting - adding .env file to launch script (for now) Signed-off-by: Peter Baker --- Dockerfile | 4 +--- Dockerfile.build_sysimage | 4 +--- launch_sysimage.sh | 1 + 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3625408..c7f1f57 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,9 +26,7 @@ LABEL org.opencontainers.image.base.name=${BASE_IMAGE} # Update all pre-installed OS packages (to get security updates) and add a few # extra utilities -RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \ - --mount=target=/var/cache/apt,type=cache,sharing=locked \ - apt-get update \ +RUN apt-get update \ && apt-get -y upgrade \ && apt-get install --no-install-recommends -y \ git \ diff --git a/Dockerfile.build_sysimage b/Dockerfile.build_sysimage index bc6a7d6..eb99c75 100644 --- a/Dockerfile.build_sysimage +++ b/Dockerfile.build_sysimage @@ -41,9 +41,7 @@ ENV JULIA_PKG_USE_CLI_GIT=true # Update all pre-installed OS packages (to get security updates) # and add a few extra utilities -RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \ - --mount=target=/var/cache/apt,type=cache,sharing=locked \ - apt-get update \ +RUN apt-get update \ && apt-get -y upgrade \ && apt-get install --no-install-recommends -y \ git \ diff --git a/launch_sysimage.sh b/launch_sysimage.sh index aea2fdd..24066fb 100755 --- a/launch_sysimage.sh +++ b/launch_sysimage.sh @@ -36,6 +36,7 @@ echo "Worker container name: $CONTAINER_NAME" # Run the container docker run \ -v "$SYSIMAGE_DIR:/data" \ + --env-file=.env \ --entrypoint julia \ ${CONTAINER_NAME} \ --project=@app \ From 1d27782b1909c70c0a0e3ca3b36c8349c331a1ab Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Thu, 17 Jul 2025 12:03:31 +1000 Subject: [PATCH 08/12] Quick hack on launch sysimage scripts - and bumping version of ReefGuide to v0.1.5 Signed-off-by: Peter Baker --- Manifest.toml | 407 +-------------------------------------------- Project.toml | 2 +- launch_sysimage.sh | 5 +- 3 files changed, 8 insertions(+), 406 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index a2beb31..2d17be8 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.11.5" manifest_format = "2.0" -project_hash = "ba3217a02c7f1941986e65ab7d0d83ee973a9f81" +project_hash = "81da2104ce8a0102d0ead581f8b6618ffc125f56" [[deps.AWS]] deps = ["Base64", "Compat", "Dates", "Downloads", "GitHub", "HTTP", "IniFile", "JSON", "MbedTLS", "Mocking", "OrderedCollections", "Random", "SHA", "Sockets", "URIs", "UUIDs", "XMLDict"] @@ -72,12 +72,6 @@ git-tree-sha1 = "7e651ea8d262d2d74ce75fdf47c4d63c07dba7a6" uuid = "35492f91-a3bd-45ad-95db-fcad7dcfedb7" version = "1.2.0" -[[deps.AliasTables]] -deps = ["PtrArrays", "Random"] -git-tree-sha1 = "9876e1e164b144ca45e9e3198d0b689cadfed9ff" -uuid = "66dad0bd-aa9a-41b7-9441-69ab47430ed8" -version = "1.1.3" - [[deps.ArchGDAL]] deps = ["CEnum", "ColorTypes", "Dates", "DiskArrays", "Extents", "GDAL", "GeoFormatTypes", "GeoInterface", "GeoInterfaceMakie", "GeoInterfaceRecipes", "ImageCore", "Tables"] git-tree-sha1 = "706bc126751ce5468cb2b31b5620eb765a3275cf" @@ -101,12 +95,6 @@ version = "2.5.0" uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" version = "1.1.2" -[[deps.ArnoldiMethod]] -deps = ["LinearAlgebra", "Random", "StaticArrays"] -git-tree-sha1 = "d57bd3762d308bded22c3b82d033bff85f6195c6" -uuid = "ec485272-7323-5ecc-a04f-4719b315124d" -version = "0.4.0" - [[deps.ArrayInterface]] deps = ["Adapt", "LinearAlgebra"] git-tree-sha1 = "9606d7832795cbef89e06a550475be300364a8aa" @@ -219,12 +207,6 @@ git-tree-sha1 = "f98cfeaba814d9746617822032d50a31c9926604" uuid = "c3b6d118-76ef-56ca-8cc7-ebb389d030a1" version = "0.3.5" -[[deps.BitTwiddlingConvenienceFunctions]] -deps = ["Static"] -git-tree-sha1 = "f21cfd4950cb9f0587d5067e69405ad2acd27b87" -uuid = "62783981-4cbd-42fc-bca8-16325de8dc4b" -version = "0.1.6" - [[deps.Blosc_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Lz4_jll", "Zlib_jll", "Zstd_jll"] git-tree-sha1 = "535c80f1c0847a4c967ea945fca21becc9de1522" @@ -248,12 +230,6 @@ git-tree-sha1 = "937628bf8b377208ac359f57314fd85d3e0165d9" uuid = "179af706-886a-5703-950a-314cd64e0468" version = "0.1.4" -[[deps.CPUSummary]] -deps = ["CpuId", "IfElse", "PrecompileTools", "Static"] -git-tree-sha1 = "5a97e67919535d6841172016c9530fd69494e5ec" -uuid = "2a0fbf3d-bb9c-48f3-b0a9-814d99fd7ab9" -version = "0.2.6" - [[deps.CRlibm]] deps = ["CRlibm_jll"] git-tree-sha1 = "66188d9d103b92b6cd705214242e27f5737a1e5e" @@ -266,18 +242,6 @@ git-tree-sha1 = "e329286945d0cfc04456972ea732551869af1cfc" uuid = "4e9b3aee-d8a1-5a3d-ad8b-7d824db253f0" version = "1.0.1+0" -[[deps.CSV]] -deps = ["CodecZlib", "Dates", "FilePathsBase", "InlineStrings", "Mmap", "Parsers", "PooledArrays", "PrecompileTools", "SentinelArrays", "Tables", "Unicode", "WeakRefStrings", "WorkerUtilities"] -git-tree-sha1 = "deddd8725e5e1cc49ee205a1964256043720a6c3" -uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" -version = "0.10.15" - -[[deps.CatIndices]] -deps = ["CustomUnitRanges", "OffsetArrays"] -git-tree-sha1 = "a0f80a09780eed9b1d106a1bf62041c2efc995bc" -uuid = "aafaddc9-749c-510e-ac4f-586e18779b91" -version = "0.2.2" - [[deps.ChainRulesCore]] deps = ["Compat", "LinearAlgebra"] git-tree-sha1 = "06ee8d1aa558d2833aa799f6f0b31b30cada405f" @@ -323,30 +287,12 @@ git-tree-sha1 = "e851a6d1a53b6246820c3ee49a703bd54e4a8ad5" uuid = "55437552-ac27-4d47-9aa3-63184e8fd398" version = "0.2.0" -[[deps.CloseOpenIntervals]] -deps = ["Static", "StaticArrayInterface"] -git-tree-sha1 = "05ba0d07cd4fd8b7a39541e31a7b0254704ea581" -uuid = "fb6a15b2-703c-40df-9091-08a04967cfa9" -version = "0.1.13" - -[[deps.Clustering]] -deps = ["Distances", "LinearAlgebra", "NearestNeighbors", "Printf", "Random", "SparseArrays", "Statistics", "StatsBase"] -git-tree-sha1 = "3e22db924e2945282e70c33b75d4dde8bfa44c94" -uuid = "aaaa29a8-35af-508c-8bc3-b662a17a0fe5" -version = "0.15.8" - [[deps.CodecZlib]] deps = ["TranscodingStreams", "Zlib_jll"] git-tree-sha1 = "962834c22b66e32aa10f7611c08c8ca4e20749a9" uuid = "944b1d66-785c-5afd-91f1-9de20f533193" version = "0.7.8" -[[deps.ColorSchemes]] -deps = ["ColorTypes", "ColorVectorSpace", "Colors", "FixedPointNumbers", "PrecompileTools", "Random"] -git-tree-sha1 = "b5278586822443594ff615963b0c09755771b3e0" -uuid = "35d6a980-a343-548e-a6ea-1d62b119f2f4" -version = "3.26.0" - [[deps.ColorTypes]] deps = ["FixedPointNumbers", "Random"] git-tree-sha1 = "b10d0b65641d57b8b4d5e234446582de5047050d" @@ -371,11 +317,6 @@ git-tree-sha1 = "358bf5a7d5c1387b995a43577673290c5d344758" uuid = "1fbeeb36-5f17-413c-809b-666fb144f157" version = "0.3.8" -[[deps.CommonWorldInvalidations]] -git-tree-sha1 = "ae52d1c52048455e85a387fbee9be553ec2b68d0" -uuid = "f70d9fcc-98c5-4d4a-abd7-e4cdeebd8ca8" -version = "1.0.0" - [[deps.Compat]] deps = ["TOML", "UUIDs"] git-tree-sha1 = "8ae8d32e09f0dcf42a36b90d4e17f5dd2e4c4215" @@ -400,11 +341,6 @@ weakdeps = ["InverseFunctions"] [deps.CompositionsBase.extensions] CompositionsBaseInverseFunctionsExt = "InverseFunctions" -[[deps.ComputationalResources]] -git-tree-sha1 = "52cb3ec90e8a8bea0e62e275ba577ad0f74821f7" -uuid = "ed09eef8-17a6-5b46-8889-db040fac31e3" -version = "0.3.2" - [[deps.ConcurrentUtilities]] deps = ["Serialization", "Sockets"] git-tree-sha1 = "d9d26935a0bcffc87d2613ce14c527c99fc543fd" @@ -434,22 +370,11 @@ git-tree-sha1 = "a692f5e257d332de1e554e4566a4e5a8a72de2b2" uuid = "150eb455-5306-5404-9cee-2592286d6298" version = "0.6.4" -[[deps.CpuId]] -deps = ["Markdown"] -git-tree-sha1 = "fcbb72b032692610bfbdb15018ac16a36cf2e406" -uuid = "adafc99b-e345-5852-983c-f28acb93d879" -version = "0.3.1" - [[deps.Crayons]] git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" version = "4.1.1" -[[deps.CustomUnitRanges]] -git-tree-sha1 = "1a3f97f907e6dd8983b744d2642651bb162a3f7a" -uuid = "dc8bdbbb-1ca9-579f-8c36-e416f6a65cce" -version = "1.0.2" - [[deps.DataAPI]] git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe" uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" @@ -627,24 +552,6 @@ git-tree-sha1 = "f6f44ab51d253f851d2084c1ac761bb679798408" uuid = "8f5d6c58-4d21-5cfd-889c-e3ad7ee6a615" version = "1.2.1" -[[deps.FFTViews]] -deps = ["CustomUnitRanges", "FFTW"] -git-tree-sha1 = "cbdf14d1e8c7c8aacbe8b19862e0179fd08321c2" -uuid = "4f61f5a4-77b1-5117-aa51-3ab5ef4ef0cd" -version = "0.3.2" - -[[deps.FFTW]] -deps = ["AbstractFFTs", "FFTW_jll", "LinearAlgebra", "MKL_jll", "Preferences", "Reexport"] -git-tree-sha1 = "797762812ed063b9b94f6cc7742bc8883bb5e69e" -uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" -version = "1.9.0" - -[[deps.FFTW_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "6d6219a004b8cf1e0b4dbe27a2860b8e04eba0be" -uuid = "f5851436-0d7a-5f13-b9de-f02708fd171a" -version = "3.3.11+0" - [[deps.FLoops]] deps = ["BangBang", "Compat", "FLoopsBase", "InitialValues", "JuliaVariables", "MLStyle", "Serialization", "Setfield", "Transducers"] git-tree-sha1 = "0a2e5873e9a5f54abb06418d57a8df689336a660" @@ -829,12 +736,6 @@ git-tree-sha1 = "69fc98947b06f8ac4279cf5bf8810373fe042be4" uuid = "05efe853-fabf-41c8-927e-7063c8b9f013" version = "0.1.7" -[[deps.Ghostscript_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "43ba3d3c82c18d88471cfd2924931658838c9d8f" -uuid = "61579ee1-b43e-5ca0-a5da-69d92c66a64b" -version = "9.55.0+4" - [[deps.Giflib_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] git-tree-sha1 = "6570366d757b50fabae9f4315ad74d2e40c0560a" @@ -858,12 +759,6 @@ git-tree-sha1 = "a641238db938fff9b2f60d08ed9030387daf428c" uuid = "a2bd30eb-e257-5431-a919-1863eab51364" version = "1.1.3" -[[deps.Graphs]] -deps = ["ArnoldiMethod", "DataStructures", "Distributed", "Inflate", "LinearAlgebra", "Random", "SharedArrays", "SimpleTraits", "SparseArrays", "Statistics"] -git-tree-sha1 = "c5abfa0ae0aaee162a3fbb053c13ecda39be545b" -uuid = "86223c79-3864-5bf0-83f7-82e725a168b6" -version = "1.13.0" - [[deps.HDF4_jll]] deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Zlib_jll", "libaec_jll"] git-tree-sha1 = "b4936dcc929748932ae40ebb3762255654602bc6" @@ -882,18 +777,6 @@ git-tree-sha1 = "f93655dc73d7a0b4a368e3c0bce296ae035ad76e" uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" version = "1.10.16" -[[deps.HistogramThresholding]] -deps = ["ImageBase", "LinearAlgebra", "MappedArrays"] -git-tree-sha1 = "7194dfbb2f8d945abdaf68fa9480a965d6661e69" -uuid = "2c695a8d-9458-5d45-9878-1b8a99cf7853" -version = "0.3.1" - -[[deps.HostCPUFeatures]] -deps = ["BitTwiddlingConvenienceFunctions", "IfElse", "Libdl", "Static"] -git-tree-sha1 = "8e070b599339d622e9a081d17230d74a5c473293" -uuid = "3e5b6fbb-0976-4d2c-9146-d79de83f2fb0" -version = "0.1.17" - [[deps.Hwloc_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] git-tree-sha1 = "92f65c4d78ce8cdbb6b68daf88889950b0a99d11" @@ -912,11 +795,6 @@ git-tree-sha1 = "b007cfc7f9bee9a958992d2301e9c5b63f332a90" uuid = "88f59080-6952-5380-9ea5-54057fb9a43f" version = "0.2.0" -[[deps.IfElse]] -git-tree-sha1 = "debdd00ffef04665ccbb3e150747a77560e8fad1" -uuid = "615f187c-cbe4-4ef1-ba3b-2fcf58d6d173" -version = "0.1.1" - [[deps.ImageAxes]] deps = ["AxisArrays", "ImageBase", "ImageCore", "Reexport", "SimpleTraits"] git-tree-sha1 = "2e4520d67b0cef90865b3ef727594d2a58e0e1f8" @@ -929,102 +807,24 @@ git-tree-sha1 = "b51bb8cae22c66d0f6357e3bcb6363145ef20835" uuid = "c817782e-172a-44cc-b673-b171935fbb9e" version = "0.1.5" -[[deps.ImageBinarization]] -deps = ["HistogramThresholding", "ImageCore", "LinearAlgebra", "Polynomials", "Reexport", "Statistics"] -git-tree-sha1 = "33485b4e40d1df46c806498c73ea32dc17475c59" -uuid = "cbc4b850-ae4b-5111-9e64-df94c024a13d" -version = "0.3.1" - -[[deps.ImageContrastAdjustment]] -deps = ["ImageBase", "ImageCore", "ImageTransformations", "Parameters"] -git-tree-sha1 = "eb3d4365a10e3f3ecb3b115e9d12db131d28a386" -uuid = "f332f351-ec65-5f6a-b3d1-319c6670881a" -version = "0.3.12" - [[deps.ImageCore]] deps = ["AbstractFFTs", "ColorVectorSpace", "Colors", "FixedPointNumbers", "Graphics", "MappedArrays", "MosaicViews", "OffsetArrays", "PaddedViews", "Reexport"] git-tree-sha1 = "acf614720ef026d38400b3817614c45882d75500" uuid = "a09fc81d-aa75-5fe9-8630-4744c3626534" version = "0.9.4" -[[deps.ImageCorners]] -deps = ["ImageCore", "ImageFiltering", "PrecompileTools", "StaticArrays", "StatsBase"] -git-tree-sha1 = "24c52de051293745a9bad7d73497708954562b79" -uuid = "89d5987c-236e-4e32-acd0-25bd6bd87b70" -version = "0.1.3" - -[[deps.ImageDistances]] -deps = ["Distances", "ImageCore", "ImageMorphology", "LinearAlgebra", "Statistics"] -git-tree-sha1 = "08b0e6354b21ef5dd5e49026028e41831401aca8" -uuid = "51556ac3-7006-55f5-8cb3-34580c88182d" -version = "0.2.17" - -[[deps.ImageFiltering]] -deps = ["CatIndices", "ComputationalResources", "DataStructures", "FFTViews", "FFTW", "ImageBase", "ImageCore", "LinearAlgebra", "OffsetArrays", "PrecompileTools", "Reexport", "SparseArrays", "StaticArrays", "Statistics", "TiledIteration"] -git-tree-sha1 = "3447781d4c80dbe6d71d239f7cfb1f8049d4c84f" -uuid = "6a3955dd-da59-5b1f-98d4-e7296123deb5" -version = "0.7.6" - [[deps.ImageIO]] deps = ["FileIO", "IndirectArrays", "JpegTurbo", "LazyModules", "Netpbm", "OpenEXR", "PNGFiles", "QOI", "Sixel", "TiffImages", "UUIDs"] git-tree-sha1 = "437abb322a41d527c197fa800455f79d414f0a3c" uuid = "82e4d734-157c-48bb-816b-45c225c6df19" version = "0.6.8" -[[deps.ImageMagick]] -deps = ["FileIO", "ImageCore", "ImageMagick_jll", "InteractiveUtils", "Libdl", "Pkg", "Random"] -git-tree-sha1 = "5bc1cb62e0c5f1005868358db0692c994c3a13c6" -uuid = "6218d12a-5da1-5696-b52f-db25d2ecc6d1" -version = "1.2.1" - -[[deps.ImageMagick_jll]] -deps = ["Artifacts", "Ghostscript_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "OpenJpeg_jll", "Zlib_jll", "libpng_jll"] -git-tree-sha1 = "0a65144bab7228fa338c224314335238b171e8df" -uuid = "c73af94c-d91f-53ed-93a7-00f77d67a9d7" -version = "6.9.13025+0" - [[deps.ImageMetadata]] deps = ["AxisArrays", "ImageAxes", "ImageBase", "ImageCore"] git-tree-sha1 = "355e2b974f2e3212a75dfb60519de21361ad3cb7" uuid = "bc367c6b-8a6b-528e-b4bd-a4b897500b49" version = "0.9.9" -[[deps.ImageMorphology]] -deps = ["DataStructures", "ImageCore", "LinearAlgebra", "LoopVectorization", "OffsetArrays", "Requires", "TiledIteration"] -git-tree-sha1 = "cffa21df12f00ca1a365eb8ed107614b40e8c6da" -uuid = "787d08f9-d448-5407-9aad-5290dd7ab264" -version = "0.4.6" - -[[deps.ImageQualityIndexes]] -deps = ["ImageContrastAdjustment", "ImageCore", "ImageDistances", "ImageFiltering", "LazyModules", "OffsetArrays", "PrecompileTools", "Statistics"] -git-tree-sha1 = "783b70725ed326340adf225be4889906c96b8fd1" -uuid = "2996bd0c-7a13-11e9-2da2-2f5ce47296a9" -version = "0.3.7" - -[[deps.ImageSegmentation]] -deps = ["Clustering", "DataStructures", "Distances", "Graphs", "ImageCore", "ImageFiltering", "ImageMorphology", "LinearAlgebra", "MetaGraphs", "RegionTrees", "SimpleWeightedGraphs", "StaticArrays", "Statistics"] -git-tree-sha1 = "44664eea5408828c03e5addb84fa4f916132fc26" -uuid = "80713f31-8817-5129-9cf8-209ff8fb23e1" -version = "1.8.1" - -[[deps.ImageShow]] -deps = ["Base64", "ColorSchemes", "FileIO", "ImageBase", "ImageCore", "OffsetArrays", "StackViews"] -git-tree-sha1 = "3b5344bcdbdc11ad58f3b1956709b5b9345355de" -uuid = "4e3cecfd-b093-5904-9786-8bbb286a6a31" -version = "0.3.8" - -[[deps.ImageTransformations]] -deps = ["AxisAlgorithms", "CoordinateTransformations", "ImageBase", "ImageCore", "Interpolations", "OffsetArrays", "Rotations", "StaticArrays"] -git-tree-sha1 = "dfde81fafbe5d6516fb864dc79362c5c6b973c82" -uuid = "02fcd773-0e25-5acc-982a-7f6622650795" -version = "0.10.2" - -[[deps.Images]] -deps = ["Base64", "FileIO", "Graphics", "ImageAxes", "ImageBase", "ImageBinarization", "ImageContrastAdjustment", "ImageCore", "ImageCorners", "ImageDistances", "ImageFiltering", "ImageIO", "ImageMagick", "ImageMetadata", "ImageMorphology", "ImageQualityIndexes", "ImageSegmentation", "ImageShow", "ImageTransformations", "IndirectArrays", "IntegralArrays", "Random", "Reexport", "SparseArrays", "StaticArrays", "Statistics", "StatsBase", "TiledIteration"] -git-tree-sha1 = "12fdd617c7fe25dc4a6cc804d657cc4b2230302b" -uuid = "916415d5-f1e6-5110-898d-aaa5f9f070e0" -version = "0.26.1" - [[deps.Imath_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] git-tree-sha1 = "0936ba688c6d201805a83da835b55c61a180db52" @@ -1061,18 +861,6 @@ weakdeps = ["ArrowTypes", "Parsers"] ArrowTypesExt = "ArrowTypes" ParsersExt = "Parsers" -[[deps.IntegralArrays]] -deps = ["ColorTypes", "FixedPointNumbers", "IntervalSets"] -git-tree-sha1 = "b842cbff3f44804a84fda409745cc8f04c029a20" -uuid = "1d092043-8f09-5a30-832f-7509e371ab51" -version = "0.1.6" - -[[deps.IntelOpenMP_jll]] -deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl"] -git-tree-sha1 = "10bd689145d2c3b2a9844005d01087cc1194e79e" -uuid = "1d5cc7b8-4909-519e-a0f8-d0f5ad9712d0" -version = "2024.2.1+0" - [[deps.InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" @@ -1160,12 +948,6 @@ git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" uuid = "82899510-4779-5014-852e-03e436cf321d" version = "1.0.0" -[[deps.JLD2]] -deps = ["FileIO", "MacroTools", "Mmap", "OrderedCollections", "PrecompileTools", "Requires", "TranscodingStreams"] -git-tree-sha1 = "89e1e5c3d43078d42eed2306cab2a11b13e5c6ae" -uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819" -version = "0.4.54" - [[deps.JLLWrappers]] deps = ["Artifacts", "Preferences"] git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6" @@ -1238,12 +1020,6 @@ git-tree-sha1 = "dda21b8cbd6a6c40d9d02a73230f9d70fed6918c" uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" version = "1.4.0" -[[deps.LayoutPointers]] -deps = ["ArrayInterface", "LinearAlgebra", "ManualMemory", "SIMDTypes", "Static", "StaticArrayInterface"] -git-tree-sha1 = "a9eaadb366f5493a5654e843864c13d8b107548c" -uuid = "10f19ff3-798f-405d-979b-55457f8fc047" -version = "0.1.17" - [[deps.LazyArrays]] deps = ["ArrayLayouts", "FillArrays", "LinearAlgebra", "MacroTools", "SparseArrays"] git-tree-sha1 = "866ce84b15e54d758c11946aacd4e5df0e60b7a3" @@ -1380,33 +1156,12 @@ git-tree-sha1 = "f02b56007b064fbfddb4c9cd60161b6dd0f40df3" uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36" version = "1.1.0" -[[deps.LoopVectorization]] -deps = ["ArrayInterface", "CPUSummary", "CloseOpenIntervals", "DocStringExtensions", "HostCPUFeatures", "IfElse", "LayoutPointers", "LinearAlgebra", "OffsetArrays", "PolyesterWeave", "PrecompileTools", "SIMDTypes", "SLEEFPirates", "Static", "StaticArrayInterface", "ThreadingUtilities", "UnPack", "VectorizationBase"] -git-tree-sha1 = "e5afce7eaf5b5ca0d444bcb4dc4fd78c54cbbac0" -uuid = "bdcacae8-1622-11e9-2a5c-532679323890" -version = "0.12.172" - - [deps.LoopVectorization.extensions] - ForwardDiffExt = ["ChainRulesCore", "ForwardDiff"] - SpecialFunctionsExt = "SpecialFunctions" - - [deps.LoopVectorization.weakdeps] - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" - SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b" - [[deps.Lz4_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] git-tree-sha1 = "191686b1ac1ea9c89fc52e996ad15d1d241d1e33" uuid = "5ced341a-0733-55b8-9ab6-a4889d929147" version = "1.10.1+0" -[[deps.MKL_jll]] -deps = ["Artifacts", "IntelOpenMP_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "oneTBB_jll"] -git-tree-sha1 = "f046ccd0c6db2832a9f639e2c669c6fe867e5f4f" -uuid = "856f044c-d86e-5d09-b602-aeab76dc8ba7" -version = "2024.2.0+0" - [[deps.MLStyle]] git-tree-sha1 = "bc38dff0548128765760c79eb7388a4b37fae2c8" uuid = "d8e11817-5142-5d16-987a-aa16d5891078" @@ -1441,11 +1196,6 @@ git-tree-sha1 = "c3159eb1e3aa3e409edbb71f4035ed8b1fc16e23" uuid = "20f20a25-4f0e-4fdf-b5d1-57303727442b" version = "0.9.5" -[[deps.ManualMemory]] -git-tree-sha1 = "bcaef4fc7a0cfe2cba636d84cda54b5e4e4ca3cd" -uuid = "d125e4d3-2237-4719-b19c-fa641b8a4667" -version = "0.1.8" - [[deps.MappedArrays]] git-tree-sha1 = "2dab0221fe2b0f2cb6754eaa743cc266339f527e" uuid = "dbb5928d-eab1-5f90-85c2-b9b0edb7c900" @@ -1467,18 +1217,6 @@ deps = ["Artifacts", "Libdl"] uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" version = "2.28.6+0" -[[deps.Memoization]] -deps = ["MacroTools"] -git-tree-sha1 = "7dbf904fa6c4447bd1f1d316886bfbe29feacf45" -uuid = "6fafb56a-5788-4b4e-91ca-c0cea6611c73" -version = "0.2.2" - -[[deps.MetaGraphs]] -deps = ["Graphs", "JLD2", "Random"] -git-tree-sha1 = "1130dbe1d5276cb656f6e1094ce97466ed700e5a" -uuid = "626554b9-1ddb-594c-aa3c-2596fe9399a5" -version = "0.7.2" - [[deps.MicroCollections]] deps = ["Accessors", "BangBang", "InitialValues"] git-tree-sha1 = "44d32db644e84c75dab479f1bc15ee76a1a3618f" @@ -1657,12 +1395,6 @@ git-tree-sha1 = "0fac6313486baae819364c52b4f483450a9d793f" uuid = "5432bcbf-9aad-5242-b902-cca2824c8663" version = "0.5.12" -[[deps.Parameters]] -deps = ["OrderedCollections", "UnPack"] -git-tree-sha1 = "34c0e9ad262e5f7fc75b10a9952ca7692cfc5fbe" -uuid = "d96e819e-fc66-5662-9728-84c9c7592b0a" -version = "0.12.3" - [[deps.Parquet2]] deps = ["AbstractTrees", "BitIntegers", "ChunkCodecCore", "ChunkCodecLibBrotli", "ChunkCodecLibLz4", "ChunkCodecLibSnappy", "ChunkCodecLibZlib", "ChunkCodecLibZstd", "DataAPI", "Dates", "DecFP", "FilePathsBase", "FillArrays", "JSON3", "LazyArrays", "LightBSON", "Mmap", "OrderedCollections", "PooledArrays", "PrecompileTools", "SentinelArrays", "StaticArrays", "TableOperations", "Tables", "Thrift2", "Transducers", "UUIDs", "WeakRefStrings"] git-tree-sha1 = "70ec79ef9c8058ca58c546ea462458885438ba4f" @@ -1690,30 +1422,6 @@ git-tree-sha1 = "f9501cc0430a26bc3d156ae1b5b0c1b47af4d6da" uuid = "eebad327-c553-4316-9ea0-9fa01ccd7688" version = "0.3.3" -[[deps.PolyesterWeave]] -deps = ["BitTwiddlingConvenienceFunctions", "CPUSummary", "IfElse", "Static", "ThreadingUtilities"] -git-tree-sha1 = "645bed98cd47f72f67316fd42fc47dee771aefcd" -uuid = "1d0040c9-8b98-4ee7-8388-3f51789ca0ad" -version = "0.2.2" - -[[deps.Polynomials]] -deps = ["LinearAlgebra", "OrderedCollections", "RecipesBase", "Requires", "Setfield", "SparseArrays"] -git-tree-sha1 = "972089912ba299fba87671b025cd0da74f5f54f7" -uuid = "f27b6e38-b328-58d1-80ce-0feddd5e7a45" -version = "4.1.0" - - [deps.Polynomials.extensions] - PolynomialsChainRulesCoreExt = "ChainRulesCore" - PolynomialsFFTWExt = "FFTW" - PolynomialsMakieExt = "Makie" - PolynomialsMutableArithmeticsExt = "MutableArithmetics" - - [deps.Polynomials.weakdeps] - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" - Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" - MutableArithmetics = "d8a4904e-b15c-11e9-3269-09a3773c0cb0" - [[deps.PooledArrays]] deps = ["DataAPI", "Future"] git-tree-sha1 = "36d8b4b899628fb92c2749eb488d884a926614d3" @@ -1765,11 +1473,6 @@ git-tree-sha1 = "7fbdd2a507a06cf577ee465c4a6ec8d951b4bcee" uuid = "c94c279d-25a6-4763-9509-64d165bea63e" version = "1.8.1" -[[deps.PtrArrays]] -git-tree-sha1 = "1d36ef11a9aaf1e8b74dacc6a731dd1de8fd493d" -uuid = "43287f4e-b6f4-7ad1-bb20-aadabca52c3d" -version = "1.3.0" - [[deps.QOI]] deps = ["ColorTypes", "FileIO", "FixedPointNumbers"] git-tree-sha1 = "8b3fc30bc0390abdce15f8822c889f669baed73d" @@ -1782,12 +1485,6 @@ git-tree-sha1 = "c69da20496799bbdd56c15ecf5d80a5e6cbcc904" uuid = "784f63db-0788-585a-bace-daefebcd302b" version = "10008.0.1004+0" -[[deps.Quaternions]] -deps = ["LinearAlgebra", "Random", "RealDot"] -git-tree-sha1 = "994cc27cdacca10e68feb291673ec3a76aa2fae9" -uuid = "94ee1d12-ae83-5a48-8b1c-48b8ff168ae0" -version = "0.7.6" - [[deps.REPL]] deps = ["InteractiveUtils", "Markdown", "Sockets", "StyledStrings", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" @@ -1841,12 +1538,6 @@ weakdeps = ["FixedPointNumbers"] [deps.Ratios.extensions] RatiosFixedPointNumbersExt = "FixedPointNumbers" -[[deps.RealDot]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "9f0a1b71baaf7650f4fa8a1d168c7fb6ee41f0c9" -uuid = "c1ae055f-0cd5-4b69-90a6-9a35b1a98df9" -version = "0.1.0" - [[deps.RecipesBase]] deps = ["PrecompileTools"] git-tree-sha1 = "5c3d09cc4f31f5fc6af001c250bf1278733100ff" @@ -1854,10 +1545,10 @@ uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" version = "1.3.4" [[deps.ReefGuide]] -deps = ["ArchGDAL", "CSV", "CoordinateTransformations", "DataFrames", "Dates", "DiskArrays", "Distances", "ExtendableSparse", "FLoops", "GeoDataFrames", "GeoFormatTypes", "GeoInterface", "GeoJSON", "GeoParquet", "GeometryBasics", "GeometryOps", "Glob", "ImageIO", "Images", "Interpolations", "LibGEOS", "LinearAlgebra", "Logging", "MKL_jll", "Memoization", "Mmap", "NearestNeighbors", "OrderedCollections", "Proj", "Rasters", "Serialization", "SortTileRecursiveTree", "SparseArrays", "StaticArrays", "Statistics", "StatsBase", "StructTypes", "Test", "ThreadsX"] -git-tree-sha1 = "8f939465e3b9d38bb0796e1ea5fc392cddc0bee7" +deps = ["ArchGDAL", "CoordinateTransformations", "DataFrames", "Dates", "DiskArrays", "Distances", "ExtendableSparse", "FLoops", "GeoDataFrames", "GeoFormatTypes", "GeoInterface", "GeoJSON", "GeoParquet", "GeometryBasics", "GeometryOps", "Glob", "ImageIO", "Interpolations", "LibGEOS", "LinearAlgebra", "Logging", "Mmap", "NearestNeighbors", "OrderedCollections", "PrecompileSignatures", "Proj", "Rasters", "Serialization", "SortTileRecursiveTree", "SparseArrays", "StaticArrays", "Statistics", "StructTypes", "ThreadsX"] +git-tree-sha1 = "aac2a281d16c36eeb5dfed6fc1fa799a4db03052" uuid = "cee0a5fb-9790-4c4c-b8b7-e40ecbebeb72" -version = "0.1.2" +version = "0.1.5" [[deps.Reexport]] git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" @@ -1870,12 +1561,6 @@ git-tree-sha1 = "02d31ad62838181c1a3a5fd23a1ce5914a643601" uuid = "42d2dcc6-99eb-4e98-b66c-637b7d73030e" version = "0.1.3" -[[deps.RegionTrees]] -deps = ["IterTools", "LinearAlgebra", "StaticArrays"] -git-tree-sha1 = "4618ed0da7a251c7f92e869ae1a19c74a7d2a7f9" -uuid = "dee08c22-ab7f-5625-9660-a9af2021b33f" -version = "0.3.2" - [[deps.Requires]] deps = ["UUIDs"] git-tree-sha1 = "62389eeff14780bfe55195b7204c0d8738436d64" @@ -1887,16 +1572,6 @@ git-tree-sha1 = "41ac127cd281bb33e42aba46a9d3b25cd35fc6d5" uuid = "20febd7b-183b-5ae2-ac4a-720e7ce64774" version = "0.4.1" -[[deps.Rotations]] -deps = ["LinearAlgebra", "Quaternions", "Random", "StaticArrays"] -git-tree-sha1 = "5680a9276685d392c87407df00d57c9924d9f11e" -uuid = "6038ab10-8711-5258-84ad-4b1120ba62dc" -version = "1.7.1" -weakdeps = ["RecipesBase"] - - [deps.Rotations.extensions] - RotationsRecipesBaseExt = "RecipesBase" - [[deps.RoundingEmulator]] git-tree-sha1 = "40b9edad2e5287e05bd413a38f61a8ff55b9557b" uuid = "5eaf0fd0-dfba-4ccb-bf02-d820a40db705" @@ -1912,17 +1587,6 @@ git-tree-sha1 = "fea870727142270bdf7624ad675901a1ee3b4c87" uuid = "fdea26ae-647d-5447-a871-4b548cad5224" version = "3.7.1" -[[deps.SIMDTypes]] -git-tree-sha1 = "330289636fb8107c5f32088d2741e9fd7a061a5c" -uuid = "94e857df-77ce-4151-89e5-788b33177be4" -version = "0.1.0" - -[[deps.SLEEFPirates]] -deps = ["IfElse", "Static", "VectorizationBase"] -git-tree-sha1 = "456f610ca2fbd1c14f5fcf31c6bfadc55e7d66e0" -uuid = "476501e8-09a2-5ece-8869-fb82de89a1fa" -version = "0.6.43" - [[deps.SQLite_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Zlib_jll"] git-tree-sha1 = "9a325057cdb9b066f1f96dc77218df60fe3007cb" @@ -1961,12 +1625,6 @@ git-tree-sha1 = "5d7e3f4e11935503d3ecaf7186eac40602e7d231" uuid = "699a6c99-e7fa-54fc-8d76-47d257e15c1d" version = "0.9.4" -[[deps.SimpleWeightedGraphs]] -deps = ["Graphs", "LinearAlgebra", "Markdown", "SparseArrays"] -git-tree-sha1 = "3e5f165e58b18204aed03158664c4982d691f454" -uuid = "47aef6b3-ad0c-573a-a1e2-d07658019622" -version = "1.5.0" - [[deps.Sixel]] deps = ["Dates", "FileIO", "ImageCore", "IndirectArrays", "OffsetArrays", "REPL", "libsixel_jll"] git-tree-sha1 = "2da10356e31327c7096832eb9cd86307a50b1eb6" @@ -2033,23 +1691,6 @@ git-tree-sha1 = "be1cf4eb0ac528d96f5115b4ed80c26a8d8ae621" uuid = "cae243ae-269e-4f55-b966-ac2d0dc13c15" version = "0.1.2" -[[deps.Static]] -deps = ["CommonWorldInvalidations", "IfElse", "PrecompileTools"] -git-tree-sha1 = "f737d444cb0ad07e61b3c1bef8eb91203c321eff" -uuid = "aedffcd0-7271-4cad-89d0-dc628f76c6d3" -version = "1.2.0" - -[[deps.StaticArrayInterface]] -deps = ["ArrayInterface", "Compat", "IfElse", "LinearAlgebra", "PrecompileTools", "Static"] -git-tree-sha1 = "96381d50f1ce85f2663584c8e886a6ca97e60554" -uuid = "0d7ed370-da01-4f52-bd93-41d350b8b718" -version = "1.8.0" -weakdeps = ["OffsetArrays", "StaticArrays"] - - [deps.StaticArrayInterface.extensions] - StaticArrayInterfaceOffsetArraysExt = "OffsetArrays" - StaticArrayInterfaceStaticArraysExt = "StaticArrays" - [[deps.StaticArrays]] deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"] git-tree-sha1 = "0feb6b9031bd5c51f9072393eb5ab3efd31bf9e4" @@ -2082,12 +1723,6 @@ git-tree-sha1 = "9d72a13a3f4dd3795a195ac5a44d7d6ff5f552ff" uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" version = "1.7.1" -[[deps.StatsBase]] -deps = ["AliasTables", "DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] -git-tree-sha1 = "b81c5035922cc89c2d9523afc6c54be512411466" -uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" -version = "0.34.5" - [[deps.StringManipulation]] deps = ["PrecompileTools"] git-tree-sha1 = "725421ae8e530ec29bcbdddbe91ff8053421d023" @@ -2158,12 +1793,6 @@ deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" version = "1.11.0" -[[deps.ThreadingUtilities]] -deps = ["ManualMemory"] -git-tree-sha1 = "d969183d3d244b6c33796b5ed01ab97328f2db85" -uuid = "8290d209-cae3-49c0-8002-c8c24d57dab5" -version = "0.5.5" - [[deps.ThreadsX]] deps = ["Accessors", "ArgCheck", "BangBang", "ConstructionBase", "InitialValues", "MicroCollections", "Referenceables", "SplittablesBase", "Transducers"] git-tree-sha1 = "70bd8244f4834d46c3d68bd09e7792d8f571ef04" @@ -2188,12 +1817,6 @@ git-tree-sha1 = "38f139cc4abf345dd4f22286ec000728d5e8e097" uuid = "731e570b-9d59-4bfa-96dc-6df516fadf69" version = "0.10.2" -[[deps.TiledIteration]] -deps = ["OffsetArrays", "StaticArrayInterface"] -git-tree-sha1 = "1176cc31e867217b06928e2f140c90bd1bc88283" -uuid = "06e1c1a7-607b-532d-9fad-de7d9aa2abac" -version = "0.5.0" - [[deps.TranscodingStreams]] git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742" uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" @@ -2231,11 +1854,6 @@ deps = ["Random", "SHA"] uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" version = "1.11.0" -[[deps.UnPack]] -git-tree-sha1 = "387c1f73762231e86e0c9c5443ce3b4a0a9a0c2b" -uuid = "3a884ed6-31ef-47d7-9d2a-63182c4928ed" -version = "1.0.2" - [[deps.Unicode]] uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" version = "1.11.0" @@ -2245,12 +1863,6 @@ git-tree-sha1 = "efcbb6828697ffffc245a9de7300552535abe005" uuid = "c4a57d5a-5b31-53a6-b365-19f8c011fbd6" version = "1.0.8" -[[deps.VectorizationBase]] -deps = ["ArrayInterface", "CPUSummary", "HostCPUFeatures", "IfElse", "LayoutPointers", "Libdl", "LinearAlgebra", "SIMDTypes", "Static", "StaticArrayInterface"] -git-tree-sha1 = "4ab62a49f1d8d9548a1c8d1a75e5f55cf196f64e" -uuid = "3d5dd08c-fd9d-11e8-17fa-ed2836048c2f" -version = "0.21.71" - [[deps.WeakRefStrings]] deps = ["DataAPI", "InlineStrings", "Parsers"] git-tree-sha1 = "b1be2855ed9ed8eac54e5caff2afcdb442d52c23" @@ -2269,11 +1881,6 @@ git-tree-sha1 = "c1a7aa6219628fcd757dede0ca95e245c5cd9511" uuid = "efce3f68-66dc-5838-9240-27a6d6f5f9b6" version = "1.0.0" -[[deps.WorkerUtilities]] -git-tree-sha1 = "cd1659ba0d57b71a464a29e64dbc67cfe83d54e7" -uuid = "76eceee3-57b5-4d4a-8e66-0e911cebbf60" -version = "1.6.1" - [[deps.XML2_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Zlib_jll"] git-tree-sha1 = "b8b243e47228b4a3877f1dd6aee0c5d56db7fcf4" @@ -2409,12 +2016,6 @@ deps = ["Artifacts", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" version = "1.59.0+0" -[[deps.oneTBB_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "d5a767a3bb77135a99e433afe0eb14cd7f6914c3" -uuid = "1317d2d5-d96f-522e-a858-c73665f53c3e" -version = "2022.0.0+0" - [[deps.p7zip_jll]] deps = ["Artifacts", "Libdl"] uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" diff --git a/Project.toml b/Project.toml index 6afc92b..ddfa98d 100644 --- a/Project.toml +++ b/Project.toml @@ -32,5 +32,5 @@ Logging = "1.11.0" Minio = "0.2.2" PrecompileSignatures = "3.0.3" Random = "1.11.0" -ReefGuide = "0.1.2" +ReefGuide = "0.1.5" Serialization = "1.11.0" diff --git a/launch_sysimage.sh b/launch_sysimage.sh index 24066fb..c6e3e1f 100755 --- a/launch_sysimage.sh +++ b/launch_sysimage.sh @@ -35,12 +35,13 @@ echo "Worker container name: $CONTAINER_NAME" # Run the container docker run \ - -v "$SYSIMAGE_DIR:/data" \ + -v "$SYSIMAGE_DIR:/sysimage" \ + -v "./data:/data" \ --env-file=.env \ --entrypoint julia \ ${CONTAINER_NAME} \ --project=@app \ - -J "/data/$SYSIMAGE_NAME" \ + -J "/sysimage/$SYSIMAGE_NAME" \ --sysimage-native-code=yes \ -e "using ReefGuideWorker; ReefGuideWorker.start_worker()" From eaf31beb55685bacc7b31a5384c9cb40a6ea9ade Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Fri, 18 Jul 2025 11:48:48 +1000 Subject: [PATCH 09/12] Some minor changes Signed-off-by: Peter Baker --- Dockerfile.build_sysimage | 5 ++--- README.md | 4 ++-- launch_sysimage.sh | 1 + 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile.build_sysimage b/Dockerfile.build_sysimage index eb99c75..82438b4 100644 --- a/Dockerfile.build_sysimage +++ b/Dockerfile.build_sysimage @@ -67,7 +67,7 @@ COPY Project.toml Manifest*.toml ./ COPY src/ src/ # Build sysimage -RUN julia --project=@app -e \ +RUN julia -t auto --project=@app -e \ 'using Pkg; \ Pkg.add("PackageCompiler"); \ Pkg.develop(PackageSpec(path=pwd())); \ @@ -75,8 +75,7 @@ RUN julia --project=@app -e \ # Reduce number of tasks/threads to avoid heavy memory use during sysimage compilation # https://github.com/JuliaLang/PackageCompiler.jl/issues/1031#issuecomment-2823054267 -ENV JULIA_NUM_PRECOMPILE_TASKS=1 -RUN julia --project=@app -t 1,1 -e 'include("src/sysimage.jl")' +RUN julia --project=@app -t auto -e 'include("src/sysimage.jl")' # Export Julia sysimage to host filesystem # From project root diff --git a/README.md b/README.md index 112ba0b..f444019 100644 --- a/README.md +++ b/README.md @@ -90,13 +90,13 @@ This worker template integrates with the ReefGuide ecosystem, connecting to the To build, using docker, a current sysimage ``` -docker build --target export-sysimage -t sysimage . +docker build --target export-sysimage -f Dockerfile.build_sysimage -t sysimage . ``` Then copy the file out: ``` -docker create --name temp-sysimage sysimage +docker create --name temp-sysimage sysimage bash docker cp temp-sysimage:/reefguide_img.so ./reefguide_img.so docker rm temp-sysimage ``` diff --git a/launch_sysimage.sh b/launch_sysimage.sh index c6e3e1f..cfda47c 100755 --- a/launch_sysimage.sh +++ b/launch_sysimage.sh @@ -37,6 +37,7 @@ echo "Worker container name: $CONTAINER_NAME" docker run \ -v "$SYSIMAGE_DIR:/sysimage" \ -v "./data:/data" \ + --network host \ --env-file=.env \ --entrypoint julia \ ${CONTAINER_NAME} \ From adb22597eb8a62c9c412d6ed039904f07dfd964d Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Fri, 18 Jul 2025 11:54:19 +1000 Subject: [PATCH 10/12] Adding ReefGuide debug to dist Signed-off-by: Peter Baker --- .env.dist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.dist b/.env.dist index 1538836..12dc6a7 100644 --- a/.env.dist +++ b/.env.dist @@ -14,7 +14,7 @@ WORKER_USERNAME=worker@email.com WORKER_PASSWORD=password # Debug -JULIA_DEBUG=ReefGuideWorker +JULIA_DEBUG=ReefGuideWorker,ReefGuide # Configuration for the worker POLL_INTERVAL_MS=5000 From 1bcf4651c904032a58aa578484f838566d60a3ed Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Fri, 18 Jul 2025 12:12:02 +1000 Subject: [PATCH 11/12] Threads in launch script Signed-off-by: Peter Baker --- launch_sysimage.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/launch_sysimage.sh b/launch_sysimage.sh index cfda47c..90c5f13 100755 --- a/launch_sysimage.sh +++ b/launch_sysimage.sh @@ -42,6 +42,7 @@ docker run \ --entrypoint julia \ ${CONTAINER_NAME} \ --project=@app \ + -t auto \ -J "/sysimage/$SYSIMAGE_NAME" \ --sysimage-native-code=yes \ -e "using ReefGuideWorker; ReefGuideWorker.start_worker()" From dcc00543f64acf88d7fc44c1ae58ff63b6554b39 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Fri, 18 Jul 2025 12:15:12 +1000 Subject: [PATCH 12/12] Bump to v0.1.6 ReefGuide.jl Signed-off-by: Peter Baker --- Manifest.toml | 8 ++++---- Project.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index 2d17be8..2e66560 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.11.5" manifest_format = "2.0" -project_hash = "81da2104ce8a0102d0ead581f8b6618ffc125f56" +project_hash = "a366cf9eea592b242ee9090122479b4b8650da3d" [[deps.AWS]] deps = ["Base64", "Compat", "Dates", "Downloads", "GitHub", "HTTP", "IniFile", "JSON", "MbedTLS", "Mocking", "OrderedCollections", "Random", "SHA", "Sockets", "URIs", "UUIDs", "XMLDict"] @@ -1545,10 +1545,10 @@ uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" version = "1.3.4" [[deps.ReefGuide]] -deps = ["ArchGDAL", "CoordinateTransformations", "DataFrames", "Dates", "DiskArrays", "Distances", "ExtendableSparse", "FLoops", "GeoDataFrames", "GeoFormatTypes", "GeoInterface", "GeoJSON", "GeoParquet", "GeometryBasics", "GeometryOps", "Glob", "ImageIO", "Interpolations", "LibGEOS", "LinearAlgebra", "Logging", "Mmap", "NearestNeighbors", "OrderedCollections", "PrecompileSignatures", "Proj", "Rasters", "Serialization", "SortTileRecursiveTree", "SparseArrays", "StaticArrays", "Statistics", "StructTypes", "ThreadsX"] -git-tree-sha1 = "aac2a281d16c36eeb5dfed6fc1fa799a4db03052" +deps = ["ArchGDAL", "CoordinateTransformations", "DataFrames", "Dates", "DiskArrays", "Distances", "ExtendableSparse", "FLoops", "GeoDataFrames", "GeoFormatTypes", "GeoInterface", "GeoJSON", "GeoParquet", "GeometryBasics", "GeometryOps", "Glob", "ImageIO", "Interpolations", "LibGEOS", "LinearAlgebra", "Logging", "Mmap", "NearestNeighbors", "OrderedCollections", "PrecompileSignatures", "PrecompileTools", "Proj", "Rasters", "Serialization", "SortTileRecursiveTree", "SparseArrays", "StaticArrays", "Statistics", "StructTypes", "ThreadsX"] +git-tree-sha1 = "e144888a4fe6663ad3721301825d2cf57267764c" uuid = "cee0a5fb-9790-4c4c-b8b7-e40ecbebeb72" -version = "0.1.5" +version = "0.1.6" [[deps.Reexport]] git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" diff --git a/Project.toml b/Project.toml index ddfa98d..029e4b6 100644 --- a/Project.toml +++ b/Project.toml @@ -32,5 +32,5 @@ Logging = "1.11.0" Minio = "0.2.2" PrecompileSignatures = "3.0.3" Random = "1.11.0" -ReefGuide = "0.1.5" +ReefGuide = "0.1.6" Serialization = "1.11.0"