From c370923f2896d6569ccb0327acf5a273ffa5afa1 Mon Sep 17 00:00:00 2001 From: Noel Kuntze Date: Fri, 20 Aug 2021 10:40:21 +0200 Subject: [PATCH] python: Use libmdbx instead of bsddb --- ECtools/backup/kopano_backup/__init__.py | 30 ++++++++--------------- ECtools/backup/requirements.txt | 2 +- ECtools/search/kopano_search/__init__.py | 10 ++++---- ECtools/search/requirements.txt | 2 +- ECtools/spamd/kopano_spamd/__init__.py | 6 ++--- ECtools/spamd/requirements.txt | 2 +- ECtools/utils/kopano_utils/autorespond.py | 6 ++--- 7 files changed, 24 insertions(+), 34 deletions(-) diff --git a/ECtools/backup/kopano_backup/__init__.py b/ECtools/backup/kopano_backup/__init__.py index ad9d3485f..dd4387c42 100644 --- a/ECtools/backup/kopano_backup/__init__.py +++ b/ECtools/backup/kopano_backup/__init__.py @@ -18,7 +18,7 @@ try: import cPickle as pickle except ImportError: import _pickle as pickle -import bsddb3 as bsddb +import libmdbx from MAPI import ( PT_UNICODE, PT_ERROR, KEEP_OPEN_READWRITE, @@ -107,7 +107,7 @@ def fatal(s): sys.exit(1) def dbopen(path): - return bsddb.hashopen(path, 'c') + return libmdbx.Env(path) def _copy_folder_meta(from_dir, to_dir, keep_db=False): if not os.path.exists(to_dir): @@ -594,7 +594,6 @@ class Service(kopano.Service): stats['folders'] += 1 else: # check all items for deletion - items_deleted = False with closing(dbopen(fpath+'/items')) as db_items: with closing(dbopen(fpath+'/index')) as db_index: delete_items = [] @@ -604,23 +603,12 @@ class Service(kopano.Service): if backup_deleted and (self.timestamp - backup_deleted).days >= self.options.purge: delete_items.append(item) - if delete_items: - items_deleted = True - for item in delete_items: stats['items'] += 1 self.log.debug('purging item: %s', item) del db_index[item] if item in db_items: del db_items[item] - if items_deleted: - self.log.debug('compacting index database') - freed_pages = db_index.db.compact(flags=bsddb.db.DB_FREE_SPACE) - self.log.debug('returned %d pages to the underlying filesystem', freed_pages) - if items_deleted: - self.log.debug('compacting items database') - freed_pages = db_items.db.compact(flags=bsddb.db.DB_FREE_SPACE) - self.log.debug('returned %d pages to the underlying filesystem', freed_pages) self.log.info('purged %d folders and %d items', stats['folders'], stats['items']) @@ -891,11 +879,13 @@ def folder_struct(data_path, options, mapper=None): # XXX deprecate? def folder_deleted(data_path): if os.path.exists(data_path+'/index'): - with closing(bsddb.hashopen(data_path+'/index')) as db: - idx = db.get(b'folder') - if idx and pickle_loads(idx).get(b'backup_deleted'): - return pickle_loads(idx).get(b'backup_deleted') - return None + # implement usage as "with" and use it. + env=libmdbx.Env(data_path+'/index') + idx=env[b'folder'] + if idx: + value=pickle_loads(idx).get(b'backup_deleted') + if value: + return value def show_contents(data_path, options): """ summary of contents of backup directory, at the item or folder level, in CSV format """ @@ -920,7 +910,7 @@ def show_contents(data_path, options): # filter items on date using 'index' database if os.path.exists(data_path+'/index'): - with closing(bsddb.hashopen(data_path+'/index')) as db: + with closing(libmdbx.Env(data_path+'/index')) as db: for key, value in db.items(): d = pickle_loads(value) if ((key == b'folder') or diff --git a/ECtools/backup/requirements.txt b/ECtools/backup/requirements.txt index 6ee1b7b47..2e968c7f0 100644 --- a/ECtools/backup/requirements.txt +++ b/ECtools/backup/requirements.txt @@ -1,3 +1,3 @@ MAPI -bsddb3 +libmdbx kopano diff --git a/ECtools/search/kopano_search/__init__.py b/ECtools/search/kopano_search/__init__.py index 631b40d68..496392a63 100644 --- a/ECtools/search/kopano_search/__init__.py +++ b/ECtools/search/kopano_search/__init__.py @@ -11,7 +11,7 @@ from multiprocessing import Queue, Value import time import sys -import bsddb3 as bsddb +import libmdbx from queue import Empty from kopano_search import plaintext @@ -78,7 +78,7 @@ def db_get(db_path, key): """ get value from db file """ if not isinstance(key, bytes): # python3 key = key.encode('ascii') - with closing(bsddb.hashopen(db_path, 'c')) as db: + with closing(libmdbx.Env(db_path)) as db: value = db.get(key) if value is not None: return db.get(key).decode('ascii') @@ -89,7 +89,7 @@ def db_put(db_path, key, value): key = key.encode('ascii') with open(db_path+'.lock', 'w') as lockfile: fcntl.flock(lockfile.fileno(), fcntl.LOCK_EX) - with closing(bsddb.hashopen(db_path, 'c')) as db: + with closing(libmdbx.Env(db_path)) as db: db[key] = value class SearchWorker(kopano.Worker): @@ -340,8 +340,8 @@ class Service(kopano.Service): worker.start() try: self.state = db_get(self.state_db, 'SERVER') - except bsddb.db.DBAccessError: - self.log.error("Cannot access '%s': permission denied", self.state_db) + except libmdbx.MDBXErrorExc as exc: + self.log.error("Cannot access '%s': %s", self.state_db, exc.message) sys.exit(1) if self.state: diff --git a/ECtools/search/requirements.txt b/ECtools/search/requirements.txt index ba0834df3..04b7c433f 100644 --- a/ECtools/search/requirements.txt +++ b/ECtools/search/requirements.txt @@ -1,3 +1,3 @@ -bsddb3 +libmdbx kopano xapian diff --git a/ECtools/spamd/kopano_spamd/__init__.py b/ECtools/spamd/kopano_spamd/__init__.py index 7fb8f422d..c31afb277 100644 --- a/ECtools/spamd/kopano_spamd/__init__.py +++ b/ECtools/spamd/kopano_spamd/__init__.py @@ -8,7 +8,7 @@ import os import sys import time -import bsddb3 as bsddb +import libmdbx import kopano from kopano import Config, log_exc @@ -53,13 +53,13 @@ class Importer: def mark_spam(self, searchkey): if not isinstance(searchkey, bytes): # python3 searchkey = searchkey.encode('ascii') - with closing(bsddb.btopen(self.spamdb, 'c')) as db: + with closing(libmdbx.Env(self.spamdb)) as db: db[searchkey] = '' def was_spam(self, searchkey): if not isinstance(searchkey, bytes): # python3 searchkey = searchkey.encode('ascii') - with closing(bsddb.btopen(self.spamdb, 'c')) as db: + with closing(libmdbx.Env(self.spamdb)) as db: return searchkey in db def update(self, item, flags): diff --git a/ECtools/spamd/requirements.txt b/ECtools/spamd/requirements.txt index 82a553bb6..f0b35cb8a 100644 --- a/ECtools/spamd/requirements.txt +++ b/ECtools/spamd/requirements.txt @@ -1,2 +1,2 @@ kopano -bsddb3 +libmdbx diff --git a/ECtools/utils/kopano_utils/autorespond.py b/ECtools/utils/kopano_utils/autorespond.py index 1f83a41d2..0664e3390 100644 --- a/ECtools/utils/kopano_utils/autorespond.py +++ b/ECtools/utils/kopano_utils/autorespond.py @@ -9,7 +9,7 @@ import time import kopano from kopano.log import logger -import bsddb3 as bsddb +import libmdbx CONFIG = { "autorespond_cc": kopano.Config.boolean(default=False), @@ -41,7 +41,7 @@ def send_ooo(server, username, msg, copy_to_sentmail): def check_time(senddb, timelimit, username, to): - with closing(bsddb.btopen(senddb, 'c')) as db: + with closing(libmdbx.Env(senddb)) as db: key = username + ":" + to key = key.encode('utf-8') if key in db: @@ -52,7 +52,7 @@ def check_time(senddb, timelimit, username, to): def add_time(senddb, username, to): - with closing(bsddb.btopen(senddb, 'c')) as db: + with closing(libmdbx.Env(senddb)) as db: key = username + ":" + to key = key.encode('utf-8') db[key] = str(int(time.time())) -- 2.33.0