Merge remote-tracking branch 'stable-1.13' into master

.github/workflows/bandit.yml

@@ -1,7 +1,7 @@
 # Run the Bandit scanner for finding vulnerabilities in the python code.
 name: bandit
 
-on: [push, pull_request]
+on: [push, pull_request, workflow_dispatch]
 
 env:
   GITHUB_REPO:  pmem/pmdk

.github/workflows/codeql.yml

@@ -3,7 +3,7 @@
 # This workflow does not use docker containers.
 name: CodeQL
 
-on: [push, pull_request]
+on: [push, pull_request, workflow_dispatch]
 
 env:
   GITHUB_REPO:  pmem/pmdk

.github/workflows/gha.yml

@@ -2,7 +2,7 @@
 # checks basic builds with various compilers and executes all sets of tests.
 name: PMDK
 
-on: [push, pull_request]
+on: [push, pull_request, workflow_dispatch]
 
 env:
   GITHUB_REPO: pmem/pmdk
@@ -26,23 +26,22 @@ jobs:
       SRC_CHECKERS: 0
     strategy:
       matrix:
-        CONFIG: ["N=1 OS=ubuntu OS_VER=22.04 FAULT_INJECTION=1 TEST_BUILD=debug",
+        CONFIG: [
+                 "N=1 OS=ubuntu OS_VER=22.04 FAULT_INJECTION=1 TEST_BUILD=debug PUSH_IMAGE=1",
                  "N=2 OS=ubuntu OS_VER=22.04 FAULT_INJECTION=1 TEST_BUILD=nondebug UBSAN=1",
                  "N=3 OS=ubuntu OS_VER=22.04 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=debug SRC_CHECKERS=1",
-                 "N=4 OS=ubuntu OS_VER=22.04 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=nondebug",
-                 "N=5 OS=fedora OS_VER=37    PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=debug",
-                 "N=6 OS=fedora OS_VER=37    PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=nondebug AUTO_DOC_UPDATE=1",
-                 "N=7 OS=fedora OS_VER=37    MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 PUSH_IMAGE=1",
-                 "N=8 OS=ubuntu OS_VER=22.04 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 NDCTL_ENABLE=n PUSH_IMAGE=1",
-                 "N=9 OS=ubuntu OS_VER=22.04 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 NDCTL_ENABLE=n PMDK_CC=clang PMDK_CXX=clang++",
-                 "N=10 OS=ubuntu OS_VER=22.04 COVERAGE=1 FAULT_INJECTION=1 TEST_BUILD=debug"
+                 "N=4 OS=ubuntu OS_VER=22.04 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=nondebug AUTO_DOC_UPDATE=1",
+                 "N=5 OS=ubuntu OS_VER=22.04 COVERAGE=1 FAULT_INJECTION=1 TEST_BUILD=debug",
                 ]
     steps:
       - name: Clone the git repo
         uses: actions/checkout@v3
         with:
           fetch-depth: 50
 
+      - name: Get system information
+        run: ./$WORKDIR/get-system-info.sh
+
       - name: Pull or rebuild the image
         run: cd $WORKDIR && ${{ matrix.CONFIG }} ./pull-or-rebuild-image.sh
 

.github/workflows/nightly.yml

@@ -30,23 +30,32 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        CONFIG: ["OS=debian OS_VER=11 FAULT_INJECTION=1 TEST_BUILD=debug",
+        CONFIG: [
+                 "OS=ubuntu OS_VER=22.04 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 NDCTL_ENABLE=n",
+                 "OS=ubuntu OS_VER=22.04 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 NDCTL_ENABLE=n PMDK_CC=clang PMDK_CXX=clang++",
+                 "OS=debian OS_VER=11 FAULT_INJECTION=1 TEST_BUILD=debug",
                  "OS=debian OS_VER=11 FAULT_INJECTION=1 TEST_BUILD=nondebug PUSH_IMAGE=1",
                  "OS=opensuse-leap OS_VER=15 FAULT_INJECTION=1 TEST_BUILD=debug",
-                 "OS=opensuse-leap OS_VER=15 FAULT_INJECTION=1 TEST_BUILD=nondebug PUSH_IMAGE=1",
+                 "OS=opensuse-leap OS_VER=15 FAULT_INJECTION=1 TEST_BUILD=nondebug",
+                 "OS=opensuse-leap OS_VER=15 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 PUSH_IMAGE=1",
                  "OS=rockylinux OS_VER=8 TEST_BUILD=debug",
                  "OS=rockylinux OS_VER=8 TEST_BUILD=nondebug",
                  "OS=rockylinux OS_VER=8 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 PUSH_IMAGE=1",
                  "OS=rockylinux OS_VER=9 TEST_BUILD=debug",
                  "OS=rockylinux OS_VER=9 TEST_BUILD=nondebug",
-                 "OS=rockylinux OS_VER=9 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 PUSH_IMAGE=1"
+                 "OS=rockylinux OS_VER=9 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 PUSH_IMAGE=1",
+                 "OS=fedora OS_VER=37 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=debug PUSH_IMAGE=1",
+                 "OS=fedora OS_VER=37 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=nondebug",
                 ]
     steps:
       - name: Clone the git repo
         uses: actions/checkout@v3
         with:
           fetch-depth: 50
 
+      - name: Get system information
+        run: ./$WORKDIR/get-system-info.sh
+
       - name: Pull or rebuild the image
         run: cd $WORKDIR && ${{ matrix.CONFIG }} ./pull-or-rebuild-image.sh rebuild
 

.github/workflows/ras_linux.yml

@@ -24,7 +24,7 @@
 # utils/gha-runners/run-ras-linux.yml
 name: RAS_Linux
 
-on: [push, pull_request]
+on: [push, pull_request, workflow_dispatch]
 
 jobs:
   linux:

.github/workflows/unittest.yml

@@ -3,7 +3,7 @@
 # This workflow is run on 'self-hosted' runners.
 name: PMDK_runners
 
-on: [push, pull_request]
+on: [push, pull_request, workflow_dispatch]
 
 jobs:
   linux:
@@ -14,6 +14,14 @@ jobs:
       fail-fast: false
       matrix:
         os: [[self-hosted, rhel],[self-hosted, opensuse]]
+        config: ['check TEST_BUILD=debug',
+                 'check TEST_BUILD=nondebug',
+                 'check TEST_BUILD=static-debug',
+                 'check TEST_BUILD=static-nondebug',
+                 'pycheck TEST_BUILD=debug',
+                 'pycheck TEST_BUILD=release',
+                 'pycheck TEST_BUILD=static_debug',
+                 'pycheck TEST_BUILD=static_release']
     env:
       WORKDIR: utils/gha-runners
 
@@ -31,4 +39,4 @@ jobs:
         run: ./$WORKDIR/create-testconfig.sh
 
       - name: Run tests
-        run: make check
+        run: make ${{ matrix.config }}

.github/workflows/unittest_long.yml

@@ -18,8 +18,15 @@ jobs:
       fail-fast: false
       matrix:
         os: [[self-hosted, rhel],[self-hosted, opensuse]]
-        config: ['RUNTESTS -t long',
-                 'RUNTESTS.py -t long']
+        config: ['RUNTESTS -t long -b debug',
+                 'RUNTESTS -t long -b nondebug',
+                 'RUNTESTS -t long -b static-debug',
+                 'RUNTESTS -t long -b static-nondebug',
+                 'RUNTESTS.py -t long -b debug',
+                 'RUNTESTS.py -t long -b release',
+                 'RUNTESTS.py -t long -b static_debug',
+                 'RUNTESTS.py -t long -b static_release']
+
     env:
       WORKDIR: utils/gha-runners
 

.github/workflows/unittest_valgrind_py.yml

@@ -17,11 +17,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        config: ['drd', 'pmemcheck', 'memcheck', 'helgrind']
+        build: ['debug', 'release', 'static_debug', 'static_release']
         os: [[self-hosted, rhel],[self-hosted, opensuse]]
-        config: ['RUNTESTS.py --force-enable memcheck',
-                 'RUNTESTS.py --force-enable pmemcheck',
-                 'RUNTESTS.py --force-enable drd',
-                 'RUNTESTS.py --force-enable helgrind']
+
     env:
       WORKDIR: utils/gha-runners
 
@@ -39,4 +38,4 @@ jobs:
         run: ./$WORKDIR/create-testconfig.sh
 
       - name: Run tests
-        run: cd src/test/ && ./${{ matrix.config }}
+        run: cd src/test/ && ./RUNTESTS.py --force-enable ${{ matrix.config }} -b ${{ matrix.build }}

.github/workflows/unittest_valgrind_sh.yml

@@ -17,11 +17,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        config: ['-d', '-p', '-m', '-e']
+        build: ['debug', 'nondebug', 'static-debug', 'static-nondebug']
         os: [[self-hosted, rhel],[self-hosted, opensuse]]
-        config: ['RUNTESTS -m force-enable',
-                 'RUNTESTS -p force-enable',
-                 'RUNTESTS -d force-enable',
-                 'RUNTESTS -e force-enable']
+
     env:
       WORKDIR: utils/gha-runners
 
@@ -39,4 +38,4 @@ jobs:
         run: ./$WORKDIR/create-testconfig.sh
 
       - name: Run tests
-        run: cd src/test/ && ./${{ matrix.config }}
+        run: cd src/test/ && ./RUNTESTS -b ${{ matrix.build }} ${{ matrix.config }} force-enable

ChangeLog

@@ -1,3 +1,31 @@
+Fri May 26 2023 Oksana Sałyk <oksana.salyk@intel.com>
+
+	* Version 1.13.1-rc2
+
+	This release :
+	- Identifies and temporarily disables tests permanently or sporadically
+	failing in some configurations:
+		- ex_libpmem2 (#5593, #5635, #5638, #5641)
+		- obj_defrag_advanced (#5690)
+		- obj_sync (#5643, #5646)
+		- pmem2_badblock (#5652, #5636)
+		- pmem2_future (#5596)
+		- pmem2_map (#5600)
+		- pmem2_map_prot (#5599)
+		- pmem2_mem_ext (#5640)
+		- pmem2_mover (partially fixed) (#5595, #5686, #5694)
+		- pmem2_vm_reservation (#5592)
+
+	- Fixes failing tests:
+		- obj_tx_alloc_mt (#5680)
+		- pmem2_integration (#5637)
+		- pmem2_map_from_existing (#5594)
+
+	Other changes and notable bug fixes:
+	- test: update permissions of DAX devices after clearing bad blocks (#5611)
+	- pmem2: fix valgrind instrumentation in pmem2_map_from_exisiting (#5605)
+	- test: disable avx512f tests (pmem2_mem_ext) under Valgrind (#5658)
+
 Wed Apr 26 2023 Łukasz Stolarczuk <lukasz.stolarczuk@intel.com>
 
 	* Version 1.13.0

src/libpmem2/map_posix.c

@@ -621,10 +621,11 @@ pmem2_map_delete(struct pmem2_map **map_ptr)
 			if (ret)
 				goto err_register_map;
 		}
-
-		if (!map->custom_vdm)
-			mover_delete(map->vdm);
 	}
+
+	if (!map->custom_vdm)
+		mover_delete(map->vdm);
+
 	Free(map);
 	*map_ptr = NULL;
 

src/libpmem2/mover.c

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: BSD-3-Clause
-/* Copyright 2021-2022, Intel Corporation */
+/* Copyright 2021-2023, Intel Corporation */
 
 /*
  * mover.c -- default pmem2 data mover
@@ -12,6 +12,7 @@
 #include "out.h"
 #include "pmem2_utils.h"
 #include "util.h"
+#include "alloc.h"
 #include <inttypes.h>
 #include <stdlib.h>
 #include <unistd.h>
@@ -194,6 +195,7 @@ mover_new(struct pmem2_map *map, struct vdm **vdm)
 		goto membuf_failed;
 	}
 
+	LOG(3, "dms %p", dms);
 	return 0;
 
 	membuf_failed:
@@ -207,8 +209,9 @@ mover_new(struct pmem2_map *map, struct vdm **vdm)
 void
 mover_delete(struct vdm *dms)
 {
+	LOG(3, "dms %p", dms);
 	membuf_delete(((struct data_mover *)dms)->membuf);
-	free((struct data_mover *)dms);
+	Free((struct data_mover *)dms);
 }
 
 /*

src/test/RUNTESTS.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright 2019-2022, Intel Corporation
+# Copyright 2019-2023, Intel Corporation
 
 """Main script for unit tests execution.
 
@@ -132,9 +132,14 @@ def run_tests(self):
         return ret
 
     def _test_failed(self, tc, ctx, fail):
-        self.msg.print('{}: {}FAILED{}\t({}/{})'
+        """Print message specific for failed test"""
+        if self.config.tm:
+            tm = '\t[{:06.3F} s]'.format(tc.elapsed)
+        else:
+            tm = ''
+        self.msg.print('{}: {}FAILED{}\t({}/{}) {}'
                        .format(tc, futils.Color.RED,
-                               futils.Color.END, tc.test_type, ctx))
+                               futils.Color.END, tc.test_type, ctx, tm))
         self.msg.print(fail)
 
         if not self.config.keep_going:
@@ -143,7 +148,7 @@ def _test_failed(self, tc, ctx, fail):
     def _test_passed(self, tc):
         """Print message specific for passed test"""
         if self.config.tm:
-            tm = '\t\t\t[{:06.3F} s]'.format(tc.elapsed)
+            tm = '\t[{:06.3F} s]'.format(tc.elapsed)
         else:
             tm = ''