[sapling] run .t tests for getdeps sapling cli build

Summary:
Connect up the sapling *.t tests so that github CI has visible cli test status.

To get them to run:
 * some needed fbpython on path. I included a shim for that in the test makefile target.
 * tests using  testrepohg/testrepohgenv assumed a system hg was present since b71ac97, changed that to check if system hg present for internal runs.
 * test-rust-hooks: Command not found message in -added a glob
  * test-identity.t: add a glob for the sapling version
 * excluded a few remaining tests (see comments in Makefile)

NB  the tests run as "hg".  The expectations would need to be updated if we were to run as "sl"

This makes the tests runnable with:
`./build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. sapling --num-jobs=48 --retry 0`

Test Plan:

enter ubuntu 22.04 toolbox:
`toolbox enter ubuntu-toolbox-22.04 `

make sure system packages are installed
`./build/fbcode_builder/getdeps.py install-system-deps --recursive  sapling`

build sapling
`./build/fbcode_builder/getdeps.py build --allow-system-packages --no-facebook-internal --src-dir=. sapling`

run the tests.  48 was about max concurrency on my personal 64GB machine and took 4 mins.  Setting less concurrency for CI.
```
./build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. sapling --num-jobs=48
----------------------------------------------------------------------
Skipped 2 tests (missing feature: Mononoke server available):
  test-fb-ext-remotefilelog-log.t
  test-fb-ext-remotefilelog-worker.t

Skipped 3 tests (missing feature: running tests with fsmonitor):
  test-fsmonitor-filemerge.t
  test-fsmonitor-nonutf8-path.t
  test-rust-hgevents.t

Skipped 46 tests (skipped):
  test-atexit-epipe.t
  test-casefolding.t
  test-check-clang-format.t
  test-check-code.t
  test-check-fix-code.t
  test-check-help.t
  test-checkserverbookmark.t
  test-chg.t
  test-debugbacktrace.t
  test-debugshell-namespace.t
  test-eden-clone.t
  test-eden-commands.t
  test-eden-du.t
  test-eden-glob.t
  test-eden-sparse.t
  test-eden-watchman-edenapi-glob.t
  test-eden-watchman-noedenapi-glob.t
  test-fb-ext-fbconduit.t
  test-fb-ext-scm-prompt-compat.t
  test-fb-ext-scm-prompt-git.t
  test-fb-ext-tweakdefaults-grep.t
  test-fsmonitor-casefolding.t
  test-fsmonitor-fallback.t
  test-fsmonitor-metadata.t
  test-fsmonitor-refreshclock.t
  test-fsmonitor-status.t
  test-fsmonitor-warn-fresh.t
  test-histedit-reorder.t
  test-matcher-expand-globs.t
  test-mutation-infinitepush.t
  test-non-working-symlink.t
  test-progress-rust-renderer.t
  test-progressfile.t
  test-purge-fsmonitor.t
  test-remotenames-journal.t
  test-runlog.t
  test-rust-status-in-transaction.t
  test-sparse-track-ignored.t
  test-status-case-insensitive.t
  test-status-fresh-instance.t
  test-status-root-ignored-py.t
  test-status-watchman.t
  test-symlink-migration.t
  test-tracing-under-chg.t
  test-treestate-fresh-instance.t
  test-treestate-trackignore.t

# Ran 736 tests, 51 skipped, 0 failed.
passed on try 0
```

generate github actions CI
```
./build/fbcode_builder/getdeps.py --allow-system-packages generate-github-actions --free-up-disk --os-type=linux --src-dir=. --output-dir=.github/workflows --job-name "Sapling CLI Getdeps " --job-file-prefix=sapling-cli-getdeps_ sapling --num-jobs=16  --project-install-prefix sapling:/
```

.github/workflows/sapling-cli-getdeps_linux.yml

@@ -185,6 +185,6 @@ jobs:
         name: sapling
         path: _artifacts
     - name: Test sapling
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. sapling  --project-install-prefix sapling:/
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --num-jobs 16 --src-dir=. sapling  --project-install-prefix sapling:/
     - name: Show disk space at end
       run: df -h

eden/scm/Makefile

@@ -65,6 +65,9 @@ COMPILERFLAG_tmp_ =
 COMPILERFLAG_tmp_${COMPILER} ?= -c $(COMPILER)
 COMPILERFLAG=${COMPILERFLAG_tmp_${COMPILER}}
 
+MAKE_PID := $(shell echo $$PPID)
+JOBS := $(shell ps T | sed -n 's%.*$(MAKE_PID).*$(MAKE).* \(-j\|--jobs=\) *\([0-9][0-9]*\).*%\2%p')
+
 # Mac Big Sur doesn't find the standard library without this.
 export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk
 
@@ -177,12 +180,59 @@ install-getdeps: getdepsbuild
 	    GETDEPS_BUILD=1 $(shell $(PYTHON3) contrib/pick_python.py $(PYTHON3)) \
 		setup.py $(PURE) install --root="$(DESTDIR)/" --prefix="$(PREFIX)" --install-lib="$(PREFIX)/bin" --force
 
+# Exclusions for OSS getdeps cli tests.  Newline separated
+# test-cats.t: internal crpyto token test
+# test-config-precedence.t:  output mismatch: DEBUG configloader::hg: spawn ["false"] because * (glob)
+# test-dynamicconfig-unicode.t: output mismatch: cat: .hg/hgrc.dynamic: $ENOENT$
+# test-debugrefreshconfig.t: assumes an internal config location
+# test-fb-ext-fastlog.t: timesout, likely due to internal endpoint assumptions
+# test-fb-ext-smartlog.t: output mismatch
+# test-help.t: help is different vs internal build
+# test-network-doctor.t: times out
+# test-smartlog-interactive.t,test-smartlog-interactive-highlighting.t: smartlog format is different causing output mismatch 
+GETDEPS_TEST_EXCLUSION_LIST := test-cats.t \
+	test-config-precedence.t \
+	test-dynamicconfig-unicode.t \
+	test-debugrefreshconfig.t \
+	test-fb-ext-fastlog.t \
+	test-fb-ext-smartlog.t \
+	test-help.t \
+	test-network-doctor.t \
+	test-smartlog-interactive.t \
+	test-smartlog-interactive-highlighting.t
+
+# convert to a sed expression
+GETDEPS_TEST_EXCLUSIONS := $(subst $() $(),|,$(GETDEPS_TEST_EXCLUSION_LIST))
+
 .PHONY: test-getdeps
 test-getdeps:
-	# Run indicative tests to check the binary is minimally good as will be used later in Mononoke getdeps tests
-	# Running all the tests requires a bit of filtering to run the good set (or deleting flaky ones)
-	cd tests && PYTHON_SYS_EXECUTABLE=$(shell $(PYTHON3) contrib/pick_python.py $(PYTHON3)) \
-	    $(shell $(PYTHON3) contrib/pick_python.py $(PYTHON3)) run-tests.py -j2 --getdeps-build --with-hg="$(PREFIX)/bin/$(HGNAME)" test-status.t test-commit.t
+    # Remove the .testfailed and .testerrored files so that after this next
+	# step they are written clean
+	rm -f ./tests/.test*
+	# ensure that fbpython is present, as some tests depend on it being on PATH
+	if ! which fbpython >/dev/null 2>&1; then \
+		FBPYTHON="$(GETDEPS_INSTALL_DIR)/sapling/bin/fbpython"; \
+		PYTHON_SYS_EXECUTABLE=$(shell $(PYTHON3) contrib/pick_python.py $(PYTHON3)); \
+		printf "#!/bin/sh\nexec \"$$PYTHON_SYS_EXECUTABLE\" \"\$$@\"\n"  > $$FBPYTHON; \
+		chmod +x "$$FBPYTHON"; \
+	fi;
+	export GETDEPS_BUILD=1; \
+	export HGTEST_HG=$(GETDEPS_INSTALL_DIR)/sapling/bin/$(HG_BIN_NAME); \
+	cd tests && export PYTHON_SYS_EXECUTABLE=$(shell $(PYTHON3) contrib/pick_python.py $(PYTHON3)); \
+	for try in $$(seq 0 $(GETDEPS_TEST_RETRY)); do \
+		RERUN_ARG=""; \
+		GETDEPS_TEST_FILTER=$(GETDEPS_TEST_FILTER); \
+		if [ $$try -gt 0 ]; then \
+			GETDEPS_TEST_FILTER=$$(sort -u .testfailed | grep -v '^$$'); \
+		elif [ -z "$$GETDEPS_TEST_FILTER" ]; then \
+		    GETDEPS_TEST_FILTER=$$(echo *.t | sed -Ee 's/($(GETDEPS_TEST_EXCLUSIONS))//g'); \
+		fi; \
+		$$PYTHON_SYS_EXECUTABLE run-tests.py -j $(JOBS) --getdeps-build --with-hg="$(PREFIX)/bin/$(HGNAME)" $$GETDEPS_TEST_FILTER; \
+		status=$$?; \
+		# stop if all good \
+		if [ $$status = 0 ]; then echo "passed on try $$try"; exit 0; fi; \
+	done; \
+	exit $$status
 
 check: tests
 

eden/scm/README.md

@@ -4,7 +4,7 @@ Sapling is a fast, easy to use, distributed revision control tool for software
 developers.
 
 
-Basic install:
+# Basic install
 
 ```
 $ make install-oss
@@ -20,6 +20,23 @@ $ make oss        # build for inplace usage
 $ ./sl --version  # should show the latest version
 ```
 
-
 See <https://sapling-scm.com/> for detailed installation instructions,
 platform-specific notes, and Sapling user information.
+
+# Thrift enabled build for use by Mononoke or EdenFS
+
+Mononoke and EdenFS need the thrift enabled sapling CLI built via getdeps. Check github actions to see current OS version the Sapling CLI Getdeps CI runs with.
+
+This build also provides a way to run the sapling .t tests in an open source environment.
+
+make sure system packages are installed
+`./build/fbcode_builder/getdeps.py install-system-deps --recursive sapling`
+
+build sapling:
+`./build/fbcode_builder/getdeps.py build --allow-system-packages --no-facebook-internal --src-dir=. sapling`
+
+run the tests.  48 jobs was about max concurrency on a 64GB RAM personal linux machine,  CI runs specify less.
+`./build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. sapling --num-jobs=48`
+
+to iterate on one test run with --retry 0 --filter:
+`./build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. sapling --num-jobs=48 --retry 0 --filter test-check-execute.t`

eden/scm/tests/helpers-testrepo.sh

@@ -36,11 +36,24 @@ cat >> "$HGRCPATH" << EOF
 evolution = createmarkers
 EOF
 
-# Unconditionally use the system hg to avoid auto migration logic from
-# the in-repo hg.
-testrepohgenv () {
-    syshgenv
-}
+
+# check if system hg exists and has a chance of reading a sapling repo
+SYSTEM_HG=$(syshgenv; hg --version -q 2>/dev/null)
+case "$SYSTEM_HG" in
+    Sapling*)
+        # Unconditionally use the system hg to avoid auto migration logic from
+        # the in-repo hg
+        testrepohgenv () {
+            syshgenv
+        }
+        ;;
+    *)
+        testrepohgenv () {
+            # No need to do anything in this case.
+            :
+        }
+        ;;
+esac
 
 testrepohg () {
     (

eden/scm/tests/test-identity.t

@@ -97,7 +97,7 @@ Test we prefer ".sl" over ".hg"
 
 Can choose flavor of dot dir using REPO_IDENTITY override:
   $ SL_IDENTITY=sl SL_REPO_IDENTITY=hg hg version -q
-  Sapling 4.4.2_dev
+  Sapling 4.* (glob)
   $ SL_IDENTITY=sl SL_REPO_IDENTITY=hg newrepo
   $ ls .hg/requires
   .hg/requires

eden/scm/tests/test-rust-hooks.t

@@ -136,7 +136,7 @@ Warn about python hooks since we can't fall back to Python:
   [255]
 #else
   $ hg debugtestcommand --echo running
-  * command not found (glob)
+  * not found (glob)
   abort: pre-debugtestcommand hook exited with status 127
   [255]
 #endif