From 0d5a447eb3b5822f659bbed6c7cf2e14147b6589 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Tue, 20 Aug 2013 22:45:11 +0200 Subject: [PATCH] add persistent option for db --- tiramisu/config.py | 31 +++++--- tiramisu/setting.py | 8 +- tiramisu/{plugins => storage}/__init__.py | 0 .../dictionary/__init__.py | 0 .../dictionary/setting.py | 4 +- .../dictionary/storage.py} | 10 +++ .../{plugins => storage}/dictionary/value.py | 5 +- .../{plugins => storage}/sqlite3/__init__.py | 0 .../{plugins => storage}/sqlite3/setting.py | 38 ++++------ .../cache.py => storage/sqlite3/storage.py} | 75 ++++++++++++++----- .../{plugins => storage}/sqlite3/value.py | 31 +++----- tiramisu/value.py | 8 +- 12 files changed, 124 insertions(+), 86 deletions(-) rename tiramisu/{plugins => storage}/__init__.py (100%) rename tiramisu/{plugins => storage}/dictionary/__init__.py (100%) rename tiramisu/{plugins => storage}/dictionary/setting.py (95%) rename tiramisu/{plugins/dictionary/cache.py => storage/dictionary/storage.py} (89%) rename tiramisu/{plugins => storage}/dictionary/value.py (96%) rename tiramisu/{plugins => storage}/sqlite3/__init__.py (100%) rename tiramisu/{plugins => storage}/sqlite3/setting.py (65%) rename tiramisu/{plugins/sqlite3/cache.py => storage/sqlite3/storage.py} (50%) rename tiramisu/{plugins => storage}/sqlite3/value.py (75%) diff --git a/tiramisu/config.py b/tiramisu/config.py index f60497f..e7372d8 100644 --- a/tiramisu/config.py +++ b/tiramisu/config.py @@ -24,11 +24,15 @@ from time import time from tiramisu.error import PropertiesOptionError, ConfigError from tiramisu.option import OptionDescription, Option, SymLinkOption, \ BaseInformation -from tiramisu.setting import groups, Settings, default_encoding, default_storage +from tiramisu.setting import groups, Settings, default_encoding, storage_type from tiramisu.value import Values from tiramisu.i18n import _ +def gen_id(config): + return str(id(config)) + str(time()) + + class SubConfig(BaseInformation): "sub configuration management entry" __slots__ = ('_impl_context', '_impl_descr') @@ -473,6 +477,13 @@ class CommonConfig(SubConfig): "abstract base class for the Config and the MetaConfig" __slots__ = ('_impl_values', '_impl_settings', '_impl_meta') + def _init_storage(self, config_id, is_persistent): + if config_id is None: + config_id = gen_id(self) + import_lib = 'tiramisu.storage.{0}.storage'.format(storage_type) + return __import__(import_lib, globals(), locals(), ['Storage'], + -1).Storage(config_id, is_persistent) + def _impl_build_all_paths(self): self.cfgimpl_get_description().impl_build_cache() @@ -517,7 +528,7 @@ class Config(CommonConfig): "main configuration management entry" __slots__ = tuple() - def __init__(self, descr): + def __init__(self, descr, config_id=None, is_persistent=False): """ Configuration option management master class :param descr: describes the configuration schema @@ -525,9 +536,9 @@ class Config(CommonConfig): :param context: the current root config :type context: `Config` """ - config_id = str(id(self)) + str(time()) - self._impl_settings = Settings(self, config_id, default_storage) - self._impl_values = Values(self, config_id, default_storage) + storage = self._init_storage(config_id, is_persistent) + self._impl_settings = Settings(self, storage) + self._impl_values = Values(self, storage) super(Config, self).__init__(descr, self) self._impl_build_all_paths() self._impl_meta = None @@ -545,7 +556,7 @@ class Config(CommonConfig): class MetaConfig(CommonConfig): __slots__ = ('_impl_children',) - def __init__(self, children, meta=True): + def __init__(self, children, meta=True, config_id=None, is_persistent=False): if not isinstance(children, list): raise ValueError(_("metaconfig's children must be a list")) self._impl_descr = None @@ -564,10 +575,12 @@ class MetaConfig(CommonConfig): raise ValueError(_("child has already a metaconfig's")) child._impl_meta = self - config_id = str(id(self)) + if config_id is None: + config_id = gen_id(self) self._impl_children = children - self._impl_settings = Settings(self, config_id, default_storage) - self._impl_values = Values(self, config_id, default_storage) + storage = self._init_storage(config_id, is_persistent) + self._impl_settings = Settings(self, storage) + self._impl_values = Values(self, storage) self._impl_meta = None self._impl_informations = {} diff --git a/tiramisu/setting.py b/tiramisu/setting.py index 95149c8..b4cdc7a 100644 --- a/tiramisu/setting.py +++ b/tiramisu/setting.py @@ -34,7 +34,7 @@ ro_append = ('frozen', 'disabled', 'validator', 'everything_frozen', rw_remove = ('permissive', 'everything_frozen', 'mandatory') rw_append = ('frozen', 'disabled', 'validator', 'hidden') default_properties = ('expire', 'validator') -default_storage = 'dictionary' +storage_type = 'dictionary' class _const: @@ -183,13 +183,13 @@ class Settings(object): "``Config()``'s configuration options" __slots__ = ('context', '_owner', '_p_') - def __init__(self, context, config_id, plugin_name): + def __init__(self, context, storage): # generic owner self._owner = owners.user self.context = context - import_lib = 'tiramisu.plugins.{0}.setting'.format(plugin_name) + import_lib = 'tiramisu.storage.{0}.setting'.format(storage_type) self._p_ = __import__(import_lib, globals(), locals(), ['Settings'], - -1).Settings(config_id) + -1).Settings(storage) def _getkey(self, opt): if self._p_.key_is_path: diff --git a/tiramisu/plugins/__init__.py b/tiramisu/storage/__init__.py similarity index 100% rename from tiramisu/plugins/__init__.py rename to tiramisu/storage/__init__.py diff --git a/tiramisu/plugins/dictionary/__init__.py b/tiramisu/storage/dictionary/__init__.py similarity index 100% rename from tiramisu/plugins/dictionary/__init__.py rename to tiramisu/storage/dictionary/__init__.py diff --git a/tiramisu/plugins/dictionary/setting.py b/tiramisu/storage/dictionary/setting.py similarity index 95% rename from tiramisu/plugins/dictionary/setting.py rename to tiramisu/storage/dictionary/setting.py index 5b7b67d..519c844 100644 --- a/tiramisu/plugins/dictionary/setting.py +++ b/tiramisu/storage/dictionary/setting.py @@ -17,13 +17,13 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # # ____________________________________________________________ -from tiramisu.plugins.dictionary.cache import Cache +from tiramisu.storage.dictionary.storage import Cache class Settings(Cache): __slots__ = ('_properties', '_permissives') - def __init__(self, config_id): + def __init__(self, storage): # properties attribute: the name of a property enables this property # key is None for global properties self._properties = {} diff --git a/tiramisu/plugins/dictionary/cache.py b/tiramisu/storage/dictionary/storage.py similarity index 89% rename from tiramisu/plugins/dictionary/cache.py rename to tiramisu/storage/dictionary/storage.py index d84964d..2180adf 100644 --- a/tiramisu/plugins/dictionary/cache.py +++ b/tiramisu/storage/dictionary/storage.py @@ -19,6 +19,16 @@ # ____________________________________________________________ +from tiramisu.i18n import _ + + +class Storage(object): + __slots__ = tuple() + def __init__(self, config_id, is_persistent): + if is_persistent: + raise ValueError(_('a dictionary cannot be persistent')) + + class Cache(object): __slots__ = ('_cache',) key_is_path = False diff --git a/tiramisu/plugins/dictionary/value.py b/tiramisu/storage/dictionary/value.py similarity index 96% rename from tiramisu/plugins/dictionary/value.py rename to tiramisu/storage/dictionary/value.py index 501c318..0f80f73 100644 --- a/tiramisu/plugins/dictionary/value.py +++ b/tiramisu/storage/dictionary/value.py @@ -18,14 +18,13 @@ # # ____________________________________________________________ -#FIXME -from tiramisu.plugins.dictionary.cache import Cache +from tiramisu.storage.dictionary.storage import Cache class Values(Cache): __slots__ = ('_values',) - def __init__(self, config_id): + def __init__(self, storage): """init plugin means create values storage """ self._values = {} diff --git a/tiramisu/plugins/sqlite3/__init__.py b/tiramisu/storage/sqlite3/__init__.py similarity index 100% rename from tiramisu/plugins/sqlite3/__init__.py rename to tiramisu/storage/sqlite3/__init__.py diff --git a/tiramisu/plugins/sqlite3/setting.py b/tiramisu/storage/sqlite3/setting.py similarity index 65% rename from tiramisu/plugins/sqlite3/setting.py rename to tiramisu/storage/sqlite3/setting.py index bee0550..5e722d3 100644 --- a/tiramisu/plugins/sqlite3/setting.py +++ b/tiramisu/storage/sqlite3/setting.py @@ -17,53 +17,47 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # # ____________________________________________________________ -from tiramisu.plugins.sqlite3.cache import Cache +from tiramisu.storage.sqlite3.storage import Cache class Settings(Cache): __slots__ = tuple() - def __init__(self, config_id): + def __init__(self, storage): settings_table = 'CREATE TABLE IF NOT EXISTS property(path text primary key, properties text)' permissives_table = 'CREATE TABLE IF NOT EXISTS permissive(path text primary key, permissives text)' # should init cache too - super(Settings, self).__init__(config_id, 'property') - self._cursor.execute(settings_table) - self._cursor.execute(permissives_table) - self._conn.commit() + super(Settings, self).__init__('property', storage) + self.storage.execute(settings_table, commit=False) + self.storage.execute(permissives_table) # propertives def setproperties(self, path, properties): - self._cursor.execute("DELETE FROM property WHERE path = ?", (path,)) - self._cursor.execute("INSERT INTO property(path, properties) VALUES (?, ?)", + self.storage.execute("DELETE FROM property WHERE path = ?", (path,), False) + self.storage.execute("INSERT INTO property(path, properties) VALUES (?, ?)", (path, self._sqlite_encode(properties))) - self._conn.commit() def getproperties(self, path, default_properties): - self._cursor.execute("SELECT properties FROM property WHERE path = ?", (path,)) - value = self._cursor.fetchone() + value = self.storage.select("SELECT properties FROM property WHERE path = ?", (path,)) if value is None: return set(default_properties) else: return set(self._sqlite_decode(value[0])) def hasproperties(self, path): - return self._cursor.execute("SELECT properties FROM property WHERE path = ?", (path,)) is not None + return self.storage.select("SELECT properties FROM property WHERE path = ?", (path,)) is not None def reset_all_propertives(self): - self._cursor.execute("DELETE FROM property") - self._conn.commit() + self.storage.execute("DELETE FROM property") def reset_properties(self, path): - self._cursor.execute("DELETE FROM property WHERE path = ?", (path,)) - self._conn.commit() + self.storage.execute("DELETE FROM property WHERE path = ?", (path,)) def get_properties(self, context): """return all properties in a dictionary """ - self._cursor.execute("SELECT * FROM property") ret = {} - for path, properties in self._cursor.fetchall(): + for path, properties in self.storage.select("SELECT * FROM property", only_one=False): if path == '_none': opt = None else: @@ -74,14 +68,12 @@ class Settings(Cache): # permissive def setpermissive(self, path, permissive): - self._cursor.execute("DELETE FROM permissive WHERE path = ?", (path,)) - self._cursor.execute("INSERT INTO permissive(path, permissives) VALUES (?, ?)", + self.storage.execute("DELETE FROM permissive WHERE path = ?", (path,), False) + self.storage.execute("INSERT INTO permissive(path, permissives) VALUES (?, ?)", (path, self._sqlite_encode(permissive))) - self._conn.commit() def getpermissive(self, path='_none'): - self._cursor.execute("SELECT permissives FROM permissive WHERE path = ?", (path,)) - permissives = self._cursor.fetchone() + permissives = self.storage.select("SELECT permissives FROM permissive WHERE path = ?", (path,)) if permissives is None: return frozenset() else: diff --git a/tiramisu/plugins/sqlite3/cache.py b/tiramisu/storage/sqlite3/storage.py similarity index 50% rename from tiramisu/plugins/sqlite3/cache.py rename to tiramisu/storage/sqlite3/storage.py index 8d47e35..6ce03c4 100644 --- a/tiramisu/plugins/sqlite3/cache.py +++ b/tiramisu/storage/sqlite3/storage.py @@ -19,19 +19,51 @@ # ____________________________________________________________ from pickle import dumps, loads +from os import unlink import sqlite3 -class Cache(object): - __slots__ = ('_conn', '_cursor') - key_is_path = True +class Storage(object): + __slots__ = ('_conn', '_cursor', 'is_persistent', 'db_file') - def __init__(self, config_id, cache_type): - cache_table = 'CREATE TABLE IF NOT EXISTS cache_{0}(path text primary key, value text, time real)'.format(cache_type) - self._conn = sqlite3.connect(config_id + '.db') + def __init__(self, config_id, is_persistent): + self.is_persistent = is_persistent + self.db_file = config_id + '.db' + self._conn = sqlite3.connect(self.db_file) self._conn.text_factory = str self._cursor = self._conn.cursor() - self._cursor.execute(cache_table) + + def execute(self, sql, params=None, commit=True): + if params is None: + params = tuple() + self._cursor.execute(sql, params) + if commit: + self._conn.commit() + + def select(self, sql, params=None, only_one=True): + self.execute(sql, params=params, commit=False) + if only_one: + return self._cursor.fetchone() + else: + return self._cursor.fetchall() + + def __del__(self): + self._cursor.close() + self._conn.close() + if not self.is_persistent: + unlink(self.db_file) + + +class Cache(object): + __slots__ = ('storage',) + key_is_path = True + + def __init__(self, cache_type, storage): + self.storage = storage + cache_table = 'CREATE TABLE IF NOT EXISTS cache_{0}(path '.format( + cache_type) + cache_table += 'text primary key, value text, time real)' + self.storage.execute(cache_table) # value def _sqlite_decode(self, value): @@ -44,38 +76,41 @@ class Cache(object): def setcache(self, cache_type, path, val, time): convert_value = self._sqlite_encode(val) - self._cursor.execute("DELETE FROM cache_{0} WHERE path = ?".format(cache_type), (path,)) - self._cursor.execute("INSERT INTO cache_{0}(path, value, time) VALUES (?, ?, ?)".format(cache_type), + self.storage.execute("DELETE FROM cache_{0} WHERE path = ?".format( + cache_type), (path,), False) + self.storage.execute("INSERT INTO cache_{0}(path, value, time) " + "VALUES (?, ?, ?)".format(cache_type), (path, convert_value, time)) - self._conn.commit() def getcache(self, cache_type, path, exp): - self._cursor.execute("SELECT value FROM cache_{0} WHERE path = ? AND time >= ?".format(cache_type), (path, exp)) - cached = self._cursor.fetchone() + cached = self.storage.select("SELECT value FROM cache_{0} WHERE " + "path = ? AND time >= ?".format( + cache_type), (path, exp)) if cached is None: return False, None else: return True, self._sqlite_decode(cached[0]) def hascache(self, cache_type, path): - self._cursor.execute("SELECT value FROM cache_{0} WHERE path = ?".format(cache_type), (path,)) - return self._cursor.fetchone() is not None + return self.storage.select("SELECT value FROM cache_{0} WHERE " + "path = ?".format(cache_type), + (path,)) is not None def reset_expired_cache(self, cache_type, exp): - self._cursor.execute("DELETE FROM cache_{0} WHERE time < ?".format(cache_type), (exp,)) - self._conn.commit() + self.storage.execute("DELETE FROM cache_{0} WHERE time < ?".format( + cache_type), (exp,)) def reset_all_cache(self, cache_type): - self._cursor.execute("DELETE FROM cache_{0}".format(cache_type)) - self._conn.commit() + self.storage.execute("DELETE FROM cache_{0}".format(cache_type)) def get_cached(self, cache_type, context): """return all values in a dictionary example: {option1: ('value1', 'time1'), option2: ('value2', 'time2')} """ - self._cursor.execute("SELECT * FROM cache_{0}".format(cache_type)) ret = {} - for path, value, time in self._cursor.fetchall(): + for path, value, time in self.storage.select("SELECT * FROM cache_{0}" + "".format(cache_type), + only_one=False): opt = context.cfgimpl_get_description().impl_get_opt_by_path(path) value = self._sqlite_decode(value) ret[opt] = (value, time) diff --git a/tiramisu/plugins/sqlite3/value.py b/tiramisu/storage/sqlite3/value.py similarity index 75% rename from tiramisu/plugins/sqlite3/value.py rename to tiramisu/storage/sqlite3/value.py index 85fb640..525c899 100644 --- a/tiramisu/plugins/sqlite3/value.py +++ b/tiramisu/storage/sqlite3/value.py @@ -18,26 +18,24 @@ # # ____________________________________________________________ -from tiramisu.plugins.sqlite3.cache import Cache +from tiramisu.storage.sqlite3.storage import Cache from tiramisu.setting import owners class Values(Cache): __slots__ = tuple() - def __init__(self, config_id): + def __init__(self, storage): """init plugin means create values storage """ values_table = 'CREATE TABLE IF NOT EXISTS value(path text primary key, value text, owner text)' # should init cache too - super(Values, self).__init__(config_id, 'value') - self._cursor.execute(values_table) - self._conn.commit() + super(Values, self).__init__('value', storage) + self.storage.execute(values_table) # sqlite def _sqlite_select(self, path): - self._cursor.execute("SELECT value FROM value WHERE path = ?", (path,)) - return self._cursor.fetchone() + return self.storage.select("SELECT value FROM value WHERE path = ?", (path,)) # value def setvalue(self, path, value, owner): @@ -45,9 +43,8 @@ class Values(Cache): a specified value must be associated to an owner """ self.resetvalue(path) - self._cursor.execute("INSERT INTO value(path, value, owner) VALUES (?, ?, ?)", + self.storage.execute("INSERT INTO value(path, value, owner) VALUES (?, ?, ?)", (path, self._sqlite_encode(value), str(owner))) - self._conn.commit() def getvalue(self, path): """get value for an option @@ -64,16 +61,14 @@ class Values(Cache): def resetvalue(self, path): """remove value means delete value in storage """ - self._cursor.execute("DELETE FROM value WHERE path = ?", (path,)) - self._conn.commit() + self.storage.execute("DELETE FROM value WHERE path = ?", (path,)) def get_modified_values(self, context): """return all values in a dictionary example: {option1: (owner, 'value1'), option2: (owner, 'value2')} """ - self._cursor.execute("SELECT value") ret = {} - for path, value, owner in self._cursor.fetchall(): + for path, value, owner in self.storage.select("SELECT value", only_one=False): opt = context.cfgimpl_get_description().impl_get_opt_by_path(path) owner = getattr(owners, owner) @@ -85,20 +80,14 @@ class Values(Cache): def setowner(self, path, owner): """change owner for an option """ - self._cursor.execute("UPDATE value SET owner = ? WHERE path = ?", (str(owner), path)) - self._conn.commit() + self.storage.execute("UPDATE value SET owner = ? WHERE path = ?", (str(owner), path)) def getowner(self, path, default): """get owner for an option return: owner object """ - self._cursor.execute("SELECT owner FROM value WHERE path = ?", (path,)) - owner = self._cursor.fetchone() + owner = self.storage.select("SELECT owner FROM value WHERE path = ?", (path,)) if owner is None: return default else: return getattr(owners, owner[0]) - - def __del__(self): - self._cursor.close() - self._conn.close() diff --git a/tiramisu/value.py b/tiramisu/value.py index 60b9255..988fd0b 100644 --- a/tiramisu/value.py +++ b/tiramisu/value.py @@ -20,7 +20,7 @@ from time import time from copy import copy from tiramisu.error import ConfigError, SlaveError -from tiramisu.setting import owners, multitypes, expires_time +from tiramisu.setting import owners, multitypes, expires_time, storage_type from tiramisu.autolib import carry_out_calculation from tiramisu.i18n import _ from tiramisu.option import SymLinkOption @@ -33,7 +33,7 @@ class Values(object): """ __slots__ = ('context', '_p_') - def __init__(self, context, config_id, plugin_name): + def __init__(self, context, storage): """ Initializes the values's dict. @@ -42,9 +42,9 @@ class Values(object): """ self.context = context - import_lib = 'tiramisu.plugins.{0}.value'.format(plugin_name) + import_lib = 'tiramisu.storage.{0}.value'.format(storage_type) self._p_ = __import__(import_lib, globals(), locals(), ['Values'], - -1).Values(config_id) + -1).Values(storage) def _getkey(self, opt): if self._p_.key_is_path: