Skip to content

Commit

Permalink
Merge branch 'develop' into cleanup-modformring
Browse files Browse the repository at this point in the history
  • Loading branch information
@DavidAyotte
DavidAyotte committed Aug 29, 2024
2 parents 9435547 + e042294 commit 31ed0d5
Show file tree
Hide file tree
Showing 3,556 changed files with 98,980 additions and 102,263 deletions.
The diff you're trying to view is too large. We only load the first 3000 changed files.
51 changes: 32 additions & 19 deletions .ci/create-changes-html.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@ fi
BASE_DOC_COMMIT="$1"
DOC_REPOSITORY="$2"

# Wipe out chronic diffs between old doc and new doc
(cd $DOC_REPOSITORY && find . -name "*.html" | xargs sed -i -e '\;<script type="application/vnd\.jupyter\.widget-state+json">;,\;</script>; d')
# Create CHANGES.html
echo '<html>' > CHANGES.html
echo '<head>' >> CHANGES.html
Expand All @@ -19,31 +17,34 @@ echo '<script>hljs.highlightAll();</script>' >> CHANGES.html
cat >> CHANGES.html << EOF
<script>
document.addEventListener('DOMContentLoaded', () => {
const baseDocURL = 'https://sagemath.netlify.app'
// This URL is hardcoded in the file .github/workflows/doc-publish.yml.
// See NETLIFY_ALIAS of the "Deploy to Netlify" step.
const baseDocURL = 'https://doc-develop--sagemath.netlify.app'
const diffSite = 'https://pianomister.github.io/diffsite'
const diffParagraphs = document.querySelectorAll('p.diff');
diffParagraphs.forEach(paragraph => {
const rootURL = window.location.origin;
const docAnchor = paragraph.querySelector('a'); // first "a" element
const docAnchor = paragraph.querySelector('a');
const url = new URL(docAnchor.href);
const path = url.pathname;
const anchor = document.createElement('a');
anchor.href = diffSite + '/?url1=' + rootURL + path + '&url2=' + baseDocURL + path;
anchor.textContent = 'compare with the base';
anchor.setAttribute('target', '_blank');
paragraph.innerHTML += '&nbsp;&nbsp;';
paragraph.appendChild(anchor);
paragraph.innerHTML += '&nbsp;';
const hunkAnchors = paragraph.querySelectorAll('a.hunk');
hunkAnchors.forEach(hunkAnchor => {
const hunks = paragraph.parentNode.querySelectorAll('p.hunk');
hunks.forEach(hunk => {
const hunkAnchor = hunk.querySelector('a');
const url = new URL(hunkAnchor.href);
const path = url.pathname;
const pathHash = path + url.hash.replace('#', '%23');
const anchor = document.createElement('a');
anchor.href = diffSite + '/?url1=' + rootURL + pathHash + '&url2=' + baseDocURL + path;
anchor.textContent = hunkAnchor.textContent;
anchor.textContent = 'compare with the base';
anchor.setAttribute('target', '_blank');
paragraph.appendChild(anchor);
paragraph.innerHTML += '&nbsp;';
hunk.innerHTML += '&nbsp;&nbsp;';
hunk.appendChild(anchor);
});
});
});
Expand All @@ -54,39 +55,51 @@ echo '<body>' >> CHANGES.html
(cd $DOC_REPOSITORY && git diff $BASE_DOC_COMMIT -- "*.html") > diff.txt
python3 - << EOF
import os, re, html
from itertools import chain
with open('diff.txt', 'r') as f:
diff_text = f.read()
diff_blocks = re.split(r'^(?=diff --git)', diff_text, flags=re.MULTILINE)
out_blocks = []
for block in diff_blocks:
match = re.search(r'^diff --git a/(.*) b/\1', block, flags=re.MULTILINE)
if match:
doc = match.group(1)
file_path = os.path.join('$DOC_REPOSITORY', doc)
path = match.group(1)
file_path = os.path.join('$DOC_REPOSITORY', path)
try:
with open(file_path, 'r') as file:
content = file.readlines()
except FileNotFoundError:
content = []
count = 0
hunks = []
hunk_lines = []
in_hunk = False
for line in block.splitlines():
if line.startswith('@@ -'):
if hunk_lines:
hunks.append('<pre><code class="language-diff">'
+ html.escape('\n'.join(hunk_lines)).strip()
+ '</code></pre>')
hunk_lines = []
search_result = re.search(r'@@ -(\d+),(\d+) \+(\d+),(\d+)', line)
if search_result:
line_number = int(search_result.group(3))
for i in range(line_number - 1, -1, -1):
if content[i].startswith('<'):
span = int(search_result.group(4))
for i in chain(range(line_number, line_number + span), range(line_number - 1, -1, -1)):
if content[i].startswith('<') and not content[i].startswith('</'):
count += 1
content[i] = f'<span id="hunk{count}" style="visibility: hidden;"></span>' + content[i]
hunks.append(f'<p class="hunk"><a href="{path}#hunk{count}" class="hunk" target="_blank">hunk #{count}</a></p>')
break
hunk_lines.append(line)
if hunk_lines:
hunks.append('<pre><code class="language-diff">'
+ html.escape('\n'.join(hunk_lines)).strip()
+ '</code></pre>')
if content:
with open(file_path, 'w') as file:
file.writelines(content)
path = doc
hunks = '&nbsp;'.join(f'<a href="{path}#hunk{i+1}" class="hunk" target="_blank">#{i + 1}</a>' for i in range(count))
out_blocks.append(f'<p class="diff"><a href="{path}">{doc}</a>&nbsp;' + hunks + '&emsp;</p>'
+ '\n<pre><code class="language-diff">'
+ html.escape(block).strip() + '</code></pre>')
out_blocks.append(f'<div class="diff"><p class="diff"><a href="{path}">{path}</a></p>\n' + '\n'.join(hunks) + '\n</div>')
output_text = '\n'.join(out_blocks)
with open('diff.html', 'w') as f:
f.write(output_text)
Expand Down
15 changes: 6 additions & 9 deletions .ci/retrofit-worktree.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,27 +23,24 @@ set -x
# If actions/checkout downloaded our source tree using the GitHub REST API
# instead of with git (because do not have git installed in our image),
# we first make the source tree a repo.
if [ ! -d .git ]; then git init && git add -A && git commit --quiet -m "new"; fi
if [ ! -d .git ]; then git init; fi

# Tag this state of the source tree "new". This is what we want to build and test.
# Commit and tag this state of the source tree "new". This is what we want to build and test.
git add -A && git commit --quiet --allow-empty -m "new"
git tag -f new

# Our container image contains a source tree in $WORKTREE_DIRECTORY with a full build of Sage.
# But $WORKTREE_DIRECTORY is not a git repository.
# We make $WORKTREE_DIRECTORY a worktree whose index is at tag "new".
# We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.)
# Then we update worktree and index with "git reset --hard new".
# Then we update worktree and index with "git checkout new".
# (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.)
# Finally we reset the index to "old". (This keeps all mtimes unchanged.)
# The changed files now show up as uncommitted changes.
# The final "git add -N" makes sure that files that were added in "new" do not show
# as untracked files, which would be removed by "git clean -fx".
if [ -L $WORKTREE_NAME ]; then
rm -f $WORKTREE_NAME
git worktree prune --verbose
fi
git worktree prune --verbose
git worktree add --detach $WORKTREE_NAME
rm -rf $WORKTREE_DIRECTORY/.git && mv $WORKTREE_NAME/.git $WORKTREE_DIRECTORY/
rm -rf $WORKTREE_NAME && ln -s $WORKTREE_DIRECTORY $WORKTREE_NAME
if [ ! -f $WORKTREE_NAME/.gitignore ]; then cp .gitignore $WORKTREE_NAME/; fi
(cd $WORKTREE_NAME && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status)
(cd $WORKTREE_NAME && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git checkout new && git status)
38 changes: 25 additions & 13 deletions .ci/write-dockerfile.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ ADD="ADD $__CHOWN"
RUN=RUN
cat <<EOF
ARG BASE_IMAGE=$(eval echo "${FULL_BASE_IMAGE_AND_TAG}")
FROM \${BASE_IMAGE} as with-system-packages
FROM \${BASE_IMAGE} AS with-system-packages
EOF
case $SYSTEM in
debian*|ubuntu*)
Expand Down Expand Up @@ -88,7 +88,7 @@ case $SYSTEM in
cat <<EOF
RUN if command -v unminimize > /dev/null; then \
(yes | unminimize) || echo "(ignored)"; \
rm -f "$(command -v unminimize)"; \
rm -f "\$(command -v unminimize)"; \
fi
EOF
if [ -n "$DIST_UPGRADE" ]; then
Expand Down Expand Up @@ -199,7 +199,7 @@ EOF
*)
cat <<EOF
ARG BASE_IMAGE
FROM \${BASE_IMAGE} as with-system-packages
FROM \${BASE_IMAGE} AS with-system-packages
EOF
INSTALL=$(sage-print-system-package-command $SYSTEM install " ")
;;
Expand Down Expand Up @@ -260,10 +260,17 @@ case ${DOCKER_BUILDKIT-0} in
CHECK_STATUS_THEN='STATUS=$(cat STATUS 2>/dev/null); case "$STATUS" in ""|0) ;; *) exit $STATUS;; esac; '
esac

if [ -n "$GITHUB_ACTIONS" ]; then
cat <<EOF
ENV GITHUB_ACTIONS=1
EOF
fi

cat <<EOF
FROM with-system-packages as bootstrapped
FROM with-system-packages AS bootstrapped
#:bootstrapping:
RUN rm -rf /new /sage/.git
$ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap bootstrap-conda configure.ac sage .homebrew-build-env tox.ini Pipfile.m4 .gitignore /new/
$ADD config/config.rpath /new/config/config.rpath
$ADD src/doc/bootstrap /new/src/doc/bootstrap
Expand All @@ -272,13 +279,14 @@ $ADD src/Pipfile.m4 src/pyproject.toml src/requirements.txt.m4 src/setup.cfg.m4
$ADD m4 /new/m4
$ADD pkgs /new/pkgs
$ADD build /new/build
$ADD .ci /new/.ci
$ADD .upstream.d /new/.upstream.d
ADD .ci /.ci
RUN if [ -d /sage ]; then \
echo "### Incremental build from \$(cat /sage/VERSION.txt)" && \
printf '/src\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/src/VERSION.txt\n' >> /sage/.gitignore && \
printf '/src\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/src/VERSION.txt\n' >> /new/.gitignore && \
if ! (cd /new && ./.ci/retrofit-worktree.sh worktree-image /sage); then \
if ! (cd /new && /.ci/retrofit-worktree.sh worktree-image /sage); then \
echo "retrofit-worktree.sh failed, falling back to replacing /sage"; \
for a in local logs; do \
if [ -d /sage/\$a ]; then mv /sage/\$a /new/; fi; \
done; \
Expand All @@ -293,7 +301,7 @@ WORKDIR /sage
ARG BOOTSTRAP="${BOOTSTRAP-./bootstrap}"
$RUN sh -x -c "\${BOOTSTRAP}" $ENDRUN $THEN_SAVE_STATUS
FROM bootstrapped as configured
FROM bootstrapped AS configured
#:configuring:
RUN $CHECK_STATUS_THEN mkdir -p logs/pkgs; rm -f config.log; ln -s logs/pkgs/config.log config.log
ARG CONFIGURE_ARGS="${CONFIGURE_ARGS:---enable-build-as-root}"
Expand All @@ -309,7 +317,7 @@ EOF
fi
cat <<EOF
FROM configured as with-base-toolchain
FROM configured AS with-base-toolchain
# We first compile base-toolchain because otherwise lots of packages are missing their dependency on 'patch'
ARG NUMPROC=8
ENV MAKE="make -j\${NUMPROC}"
Expand All @@ -319,7 +327,7 @@ ENV SAGE_CHECK_PACKAGES="!cython,!r,!python3,!gap,!cysignals,!linbox,!git,!ppl,!
#:toolchain:
$RUN $CHECK_STATUS_THEN make \${USE_MAKEFLAGS} base-toolchain $ENDRUN $THEN_SAVE_STATUS
FROM with-base-toolchain as with-targets-pre
FROM with-base-toolchain AS with-targets-pre
ARG NUMPROC=8
ENV MAKE="make -j\${NUMPROC}"
ARG USE_MAKEFLAGS="-k V=0"
Expand All @@ -329,7 +337,7 @@ ENV SAGE_CHECK_PACKAGES="!cython,!r,!python3,!gap,!cysignals,!linbox,!git,!ppl,!
ARG TARGETS_PRE="all-sage-local"
$RUN $CHECK_STATUS_THEN make SAGE_SPKG="sage-spkg -y -o" \${USE_MAKEFLAGS} \${TARGETS_PRE} $ENDRUN $THEN_SAVE_STATUS
FROM with-targets-pre as with-targets
FROM with-targets-pre AS with-targets
ARG NUMPROC=8
ENV MAKE="make -j\${NUMPROC}"
ARG USE_MAKEFLAGS="-k V=0"
Expand All @@ -338,16 +346,20 @@ ENV SAGE_CHECK_PACKAGES="!cython,!r,!python3,!gap,!cysignals,!linbox,!git,!ppl,!
$ADD .gitignore /new/.gitignore
$ADD src /new/src
RUN cd /new && rm -rf .git && \
if ! /sage/.ci/retrofit-worktree.sh worktree-pre /sage; then \
if /.ci/retrofit-worktree.sh worktree-pre /sage; then \
cd /sage && touch configure build/make/Makefile; \
else \
echo "retrofit-worktree.sh failed, falling back to replacing /sage/src"; \
rm -rf /sage/src; \
mv src /sage/src; \
cd /sage && ./bootstrap && ./config.status; \
fi
fi; \
cd /sage && rm -rf .git; rm -rf /new || echo "(error ignored)"
ARG TARGETS="build"
$RUN $CHECK_STATUS_THEN make SAGE_SPKG="sage-spkg -y -o" \${USE_MAKEFLAGS} \${TARGETS} $ENDRUN $THEN_SAVE_STATUS
FROM with-targets as with-targets-optional
FROM with-targets AS with-targets-optional
ARG NUMPROC=8
ENV MAKE="make -j\${NUMPROC}"
ARG USE_MAKEFLAGS="-k V=0"
Expand Down
25 changes: 0 additions & 25 deletions .devcontainer/portability-centos-7-devtoolset-gcc_11-minimal/devcontainer.json
View file

This file was deleted.

1 change: 0 additions & 1 deletion .devcontainer/portability-centos-7-devtoolset-gcc_11-minimal/portability-Dockerfile
View file

This file was deleted.

25 changes: 0 additions & 25 deletions .devcontainer/portability-centos-7-devtoolset-gcc_11-standard/devcontainer.json
View file

This file was deleted.

1 change: 0 additions & 1 deletion .devcontainer/portability-centos-7-devtoolset-gcc_11-standard/portability-Dockerfile
View file

This file was deleted.

25 changes: 0 additions & 25 deletions .devcontainer/portability-centos-stream-8-python3.9-minimal/devcontainer.json
View file

This file was deleted.

1 change: 0 additions & 1 deletion .devcontainer/portability-centos-stream-8-python3.9-minimal/portability-Dockerfile
View file

This file was deleted.

25 changes: 0 additions & 25 deletions .devcontainer/portability-centos-stream-8-python3.9-standard/devcontainer.json
View file

This file was deleted.

1 change: 0 additions & 1 deletion .devcontainer/portability-centos-stream-8-python3.9-standard/portability-Dockerfile
View file

This file was deleted.

Loading

0 comments on commit 31ed0d5

Please sign in to comment.