From 506a5345c061b6a859cc73e60694f3ebcfbd4ba9 Mon Sep 17 00:00:00 2001 From: Nic Watson Date: Wed, 28 Apr 2021 15:44:08 -0400 Subject: [PATCH] Make map resize test more robust for weird-page-size OSs (#296) * Make map resize test more robust for weird-page-size OSs Ubuntu on PPC64LE has a 64K page size. This impacts the minimum map size for an environment, as well as how fast a environment fills up. * Remove Windows/Python 2.7 builds in CI. Microsoft removed the link to download Visual Studio C++ 9.0. Python 2.7 Windows can literally not be built anymore without a cached copy. --- .github/workflows/python-package.yml | 14 ++++++------ tests/crash_test.py | 32 +++++++++++++++++++++------- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 7b7d80d..3c1cb2b 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -32,6 +32,9 @@ jobs: # versions) - python-version: 3.4 os: windows-latest + # Microsoft removed VC 9.0 installer so Python 2.7 modules can no longer be built on Windows + - python-version: 2.7 + os: windows-latest include: - python-version: 3.9 @@ -77,11 +80,6 @@ jobs: sudo apt-get install liblmdb-dev; fi - - name: Install Visual C++ for Python 2.7 - if: runner.os == 'Windows' && matrix.python-version == '2.7' - run: | - choco install vcpython27 -f -y - - name: Install dependencies run: | echo "Linux: Envs are cpython=$LMDB_FORCE_CPYTHON @@ -200,14 +198,16 @@ jobs: impl: cffi - python-version: '3.9' impl: cffi - - # Github actions doesn't have a 3.4/macos-latest distro + # Github Actions doesn't have a 3.4/macos-latest distro - python-version: 3.4 os: macos-latest # patch-ng -> typing has trouble on python 3.4 (TODO: might be resolvable with explicit # versions) - python-version: 3.4 os: windows-latest + # Microsoft removed VC 9.0 installer so Python 2.7 modules can no longer be built on Windows + - python-version: '2.7' + os: windows-latest include: # Ubuntu artifacts apply to all python versions diff --git a/tests/crash_test.py b/tests/crash_test.py index 8c488bb..a4d92a1 100644 --- a/tests/crash_test.py +++ b/tests/crash_test.py @@ -280,8 +280,11 @@ def test_cursor_open_failure(self): txn2 = env.begin(write=False) self.assertRaises(lmdb.InvalidParameterError, txn2.cursor, db=db) +MINDBSIZE = 64 * 1024 * 2 # certain ppcle Linux distros have a 64K page size + if sys.version_info[:2] >= (3, 4): class MapResizeTest(unittest.TestCase): + def tearDown(self): testlib.cleanup() @@ -291,13 +294,26 @@ def do_resize(path): Increase map size and fill up database, making sure that the root page is no longer accessible in the main process. ''' - data = [i.to_bytes(4, 'little') for i in range(400)] - with lmdb.open(path, max_dbs=10, create=False, map_size=32000) as env: + with lmdb.open(path, max_dbs=10, create=False, map_size=MINDBSIZE) as env: env.open_db(b'foo') - env.set_mapsize(64000) - with env.begin(write=True) as txn: - for datum in data: - txn.put(datum, b'0') + env.set_mapsize(MINDBSIZE * 2) + count = 0 + try: + # Figure out how many keyvals we can enter before we run out of space + with env.begin(write=True) as txn: + while True: + datum = count.to_bytes(4, 'little') + txn.put(datum, b'0') + count += 1 + + except lmdb.MapFullError: + # Now put (and commit) just short of that + with env.begin(write=True) as txn: + for i in range(count - 100): + datum = i.to_bytes(4, 'little') + txn.put(datum, b'0') + else: + assert 0 def test_opendb_resize(self): ''' @@ -306,9 +322,9 @@ def test_opendb_resize(self): Would seg fault in cffi implementation ''' mpctx = multiprocessing.get_context('spawn') - path, env = testlib.temp_env(max_dbs=10, map_size=32000) + path, env = testlib.temp_env(max_dbs=10, map_size=MINDBSIZE) env.close() - env = lmdb.open(path, max_dbs=10, map_size=32000, readonly=True) + env = lmdb.open(path, max_dbs=10, map_size=MINDBSIZE, readonly=True) proc = mpctx.Process(target=self.do_resize, args=(path,)) proc.start() proc.join(5)