From b10f02a8e999f98c81f00c4b1c318f91829a2798 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Mon, 25 Jun 2018 21:40:16 +0200 Subject: [PATCH] cache in dictionary storage --- test/test_cache.py | 270 ++++++++++++++------------- tiramisu/option/optiondescription.py | 7 +- tiramisu/setting.py | 61 +++--- tiramisu/storage/dictionary/cache.py | 41 ++++ tiramisu/storage/util.py | 112 +++++++---- tiramisu/value.py | 46 ++--- 6 files changed, 306 insertions(+), 231 deletions(-) create mode 100644 tiramisu/storage/dictionary/cache.py diff --git a/test/test_cache.py b/test/test_cache.py index 286aba8..4ad8fa4 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -376,26 +376,26 @@ def test_cache_not_cache(): # api.property.pop('disabled') # # c.cfgimpl_get_values().force_cache() -# assert c.cfgimpl_get_values()._p_.get_cached() == {'u1': {None: ([], None)}, -# 'u2': {None: (None, None)}, -# 'u3': {None: ([], None)}, -# 'u4': {None: (None, None)}} -# assert c.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'frozen', 'hidden', 'validator', 'warnings']), None)}, -# 'u1': {None: (set(['empty']), None)}, -# 'u2': {None: (set([]), None)}, -# 'u3': {None: (set(['empty']), None)}, -# 'u4': {None: (set(['disabled']), None)}} +# compare(c.cfgimpl_get_values()._p_.get_cached(), {'u1': {None: ([], None)}, +# 'u2': {None: (None, None)}, +# 'u3': {None: ([], None)}, +# 'u4': {None: (None, None)}}) +# compare(c.cfgimpl_get_settings()._p_.get_cached(), {None: {None: (set(['cache', 'frozen', 'hidden', 'validator', 'warnings']), None)}, +# 'u1': {None: (set(['empty']), None)}, +# 'u2': {None: (set([]), None)}, +# 'u3': {None: (set(['empty']), None)}, +# 'u4': {None: (set(['disabled']), None)}}) # api.property.read_only() # # c.cfgimpl_get_values().force_cache() -# assert c.cfgimpl_get_values()._p_.get_cached() == {'u1': {None: ([], None)}, -# 'u2': {None: (None, None)}, -# 'u3': {None: ([], None)}} -# assert c.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'disabled', 'empty', 'everything_frozen', 'frozen', 'mandatory', 'validator', 'warnings']), None)}, -# 'u1': {None: (set(['empty']), None)}, -# 'u2': {None: (set([]), None)}, -# 'u3': {None: (set(['empty']), None)}, -# 'u4': {None: (set(['disabled']), None)}} +# compare(c.cfgimpl_get_values()._p_.get_cached(), {'u1': {None: ([], None)}, +# 'u2': {None: (None, None)}, +# 'u3': {None: ([], None)}}) +# compare(c.cfgimpl_get_settings()._p_.get_cached(), {None: {None: (set(['cache', 'disabled', 'empty', 'everything_frozen', 'frozen', 'mandatory', 'validator', 'warnings']), None)}, +# 'u1': {None: (set(['empty']), None)}, +# 'u2': {None: (set([]), None)}, +# 'u3': {None: (set(['empty']), None)}, +# 'u4': {None: (set(['disabled']), None)}}) # # c.cfgimpl_get_settings().remove('cache') # raises(ConfigError, "c.cfgimpl_get_values().force_cache()") @@ -465,6 +465,15 @@ def return_value(value=None): return value +def compare(calculated, expected): + assert set(calculated.keys()) == set(expected.keys()) + for calculated_key in calculated: + assert set(calculated[calculated_key].keys()) == set(expected[calculated_key].keys()) + for calculated_subkey in calculated[calculated_key]: + # do not check timestamp + assert calculated[calculated_key][calculated_subkey][0] == expected[calculated_key][calculated_subkey][0] + + def test_cache_callback(): val1 = StrOption('val1', "", 'val') val2 = StrOption('val2', "", callback=return_value, callback_params=Params((ParamOption(val1),)), properties=('mandatory',)) @@ -480,81 +489,81 @@ def test_cache_callback(): api.option.make_dict() #assert cfg.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, # 'val1': {None: (set([]), None)}} - assert cfg.cfgimpl_get_values()._p_.get_cached() == {'val1': {None: ('val', None)}, - 'val2': {None: ('val', None)}, - 'val3': {None: ('yes', None)}, - 'val4': {None: ('val', None)}, - 'val5': {None: (['yes'], None)}} + compare(cfg.cfgimpl_get_values()._p_.get_cached(), {'val1': {None: ('val', None)}, + 'val2': {None: ('val', None)}, + 'val3': {None: ('yes', None)}, + 'val4': {None: ('val', None)}, + 'val5': {None: (['yes'], None)}}) api.option('val1').value.set('new') #assert cfg.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, # 'val1': {None: (set([]), None)}} - assert cfg.cfgimpl_get_values()._p_.get_cached() == {'val3': {None: ('yes', None)}, - 'val5': {None: (['yes'], None)}} + compare(cfg.cfgimpl_get_values()._p_.get_cached(), {'val3': {None: ('yes', None)}, + 'val5': {None: (['yes'], None)}}) api.option.make_dict() #assert cfg.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, # 'val1': {None: (set([]), None)}} - assert cfg.cfgimpl_get_values()._p_.get_cached() == {'val1': {None: ('new', None)}, - 'val2': {None: ('new', None)}, - 'val3': {None: ('yes', None)}, - 'val4': {None: ('new', None)}, - 'val5': {None: (['yes'], None)}} + compare(cfg.cfgimpl_get_values()._p_.get_cached(), {'val1': {None: ('new', None)}, + 'val2': {None: ('new', None)}, + 'val3': {None: ('yes', None)}, + 'val4': {None: ('new', None)}, + 'val5': {None: (['yes'], None)}}) api.option('val3').value.set('new2') #assert cfg.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, # 'val1': {None: (set([]), None)}, # 'val3': {None: (set([]), None)}} - assert cfg.cfgimpl_get_values()._p_.get_cached() == {'val1': {None: ('new', None)}, - 'val2': {None: ('new', None)}, - 'val4': {None: ('new', None)}, - 'val5': {None: (['yes'], None)}} + compare(cfg.cfgimpl_get_values()._p_.get_cached(), {'val1': {None: ('new', None)}, + 'val2': {None: ('new', None)}, + 'val4': {None: ('new', None)}, + 'val5': {None: (['yes'], None)}}) api.option.make_dict() #assert cfg.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, # 'val1': {None: (set([]), None)}, # 'val3': {None: (set([]), None)}} - assert cfg.cfgimpl_get_values()._p_.get_cached() == {'val1': {None: ('new', None)}, - 'val2': {None: ('new', None)}, - 'val3': {None: ('new2', None)}, - 'val4': {None: ('new', None)}, - 'val5': {None: (['yes'], None)}} + compare(cfg.cfgimpl_get_values()._p_.get_cached(), {'val1': {None: ('new', None)}, + 'val2': {None: ('new', None)}, + 'val3': {None: ('new2', None)}, + 'val4': {None: ('new', None)}, + 'val5': {None: (['yes'], None)}}) api.option('val4').value.set('new3') #assert cfg.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, # 'val1': {None: (set([]), None)}, # 'val3': {None: (set([]), None)}, # 'val4': {None: (set([]), None)}} - assert cfg.cfgimpl_get_values()._p_.get_cached() == {'val1': {None: ('new', None)}, - 'val2': {None: ('new', None)}, - 'val3': {None: ('new2', None)}, - 'val5': {None: (['yes'], None)}} + compare(cfg.cfgimpl_get_values()._p_.get_cached(), {'val1': {None: ('new', None)}, + 'val2': {None: ('new', None)}, + 'val3': {None: ('new2', None)}, + 'val5': {None: (['yes'], None)}}) api.option.make_dict() #assert cfg.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, # 'val1': {None: (set([]), None)}, # 'val3': {None: (set([]), None)}, # 'val4': {None: (set([]), None)}} - assert cfg.cfgimpl_get_values()._p_.get_cached() == {'val1': {None: ('new', None)}, - 'val2': {None: ('new', None)}, - 'val3': {None: ('new2', None)}, - 'val4': {None: ('new3', None)}, - 'val5': {None: (['yes'], None)}} + compare(cfg.cfgimpl_get_values()._p_.get_cached(), {'val1': {None: ('new', None)}, + 'val2': {None: ('new', None)}, + 'val3': {None: ('new2', None)}, + 'val4': {None: ('new3', None)}, + 'val5': {None: (['yes'], None)}}) api.option('val5').value.set([undefined, 'new4']) #assert cfg.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, # 'val1': {None: (set([]), None)}, # 'val3': {None: (set([]), None)}, # 'val4': {None: (set([]), None)}, # 'val5': {None: (set(['empty']), None)}} - assert cfg.cfgimpl_get_values()._p_.get_cached() == {'val1': {None: ('new', None)}, - 'val2': {None: ('new', None)}, - 'val3': {None: ('new2', None)}, - 'val4': {None: ('new3', None)}} + compare(cfg.cfgimpl_get_values()._p_.get_cached(), {'val1': {None: ('new', None)}, + 'val2': {None: ('new', None)}, + 'val3': {None: ('new2', None)}, + 'val4': {None: ('new3', None)}}) api.option.make_dict() #assert cfg.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, # 'val1': {None: (set([]), None)}, # 'val3': {None: (set([]), None)}, # 'val4': {None: (set([]), None)}, # 'val5': {None: (set(['empty']), None)}} - assert cfg.cfgimpl_get_values()._p_.get_cached() == {'val1': {None: ('new', None)}, - 'val2': {None: ('new', None)}, - 'val3': {None: ('new2', None)}, - 'val4': {None: ('new3', None)}, - 'val5': {None: (['yes', 'new4'], None)}} + compare(cfg.cfgimpl_get_values()._p_.get_cached(), {'val1': {None: ('new', None)}, + 'val2': {None: ('new', None)}, + 'val3': {None: ('new2', None)}, + 'val4': {None: ('new3', None)}, + 'val5': {None: (['yes', 'new4'], None)}}) def test_cache_master_and_slaves_master(): @@ -588,16 +597,16 @@ def test_cache_master_and_slaves_master(): assert cfg.cfgimpl_get_settings()._p_.get_cached() == {} assert cfg.cfgimpl_get_values()._p_.get_cached() == {} else: - assert cfg.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (global_props, None)}, - 'val1': {None: (val1_props, None)}, - 'val1.val1': {None: (val1_val1_props, None)}, - 'val1.val2': {idx_val2: (val1_val2_props, None)}} + compare(cfg.cfgimpl_get_settings()._p_.get_cached(), {None: {None: (global_props, None)}, + 'val1': {None: (val1_props, None)}, + 'val1.val1': {None: (val1_val1_props, None)}, + 'val1.val2': {idx_val2: (val1_val2_props, None)}}) # len is 0 so don't get any value - assert cfg.cfgimpl_get_values()._p_.get_cached() == {'val1.val1': {None: ([], None)}} + compare(cfg.cfgimpl_get_values()._p_.get_cached(), {'val1.val1': {None: ([], None)}}) # api.option('val1.val1').value.set([undefined]) - assert cfg.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, - 'val1': {None: (set([]), None)}} + compare(cfg.cfgimpl_get_settings()._p_.get_cached(), {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, + 'val1': {None: (set([]), None)}}) assert cfg.cfgimpl_get_values()._p_.get_cached() == {} api.option.make_dict() if TIRAMISU_VERSION == 2: @@ -608,17 +617,17 @@ def test_cache_master_and_slaves_master(): idx_val2 = 0 val_val2 = None val_val2_props = {idx_val2: (val1_val2_props, None)} - assert cfg.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (global_props, None)}, - 'val1': {None: (val1_props, None)}, - 'val1.val1': {None: (val1_val1_props, None)}, - 'val1.val2': val_val2_props} - assert cfg.cfgimpl_get_values()._p_.get_cached() == {'val1.val1': {None: ([None], None)}, - 'val1.val2': {idx_val2: (val_val2, None)}} + compare(cfg.cfgimpl_get_settings()._p_.get_cached(), {None: {None: (global_props, None)}, + 'val1': {None: (val1_props, None)}, + 'val1.val1': {None: (val1_val1_props, None)}, + 'val1.val2': val_val2_props}) + compare(cfg.cfgimpl_get_values()._p_.get_cached(), {'val1.val1': {None: ([None], None)}, + 'val1.val2': {idx_val2: (val_val2, None)}}) api.option('val1.val1').value.set([undefined, undefined]) api.option.make_dict() api.option('val1.val2', 1).value.set('oui') - assert cfg.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, - 'val1': {None: (set([]), None)}} + compare(cfg.cfgimpl_get_settings()._p_.get_cached(), {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, + 'val1': {None: (set([]), None)}}) assert cfg.cfgimpl_get_values()._p_.get_cached() == {} if TIRAMISU_VERSION == 2: val1_val2_props = {None: (set([]), None), 0: (set([]), None), 1: (set([]), None)} @@ -665,14 +674,14 @@ def test_cache_master_callback(): assert cfg.cfgimpl_get_settings()._p_.get_cached() == {} assert cfg.cfgimpl_get_values()._p_.get_cached() == {} else: - assert cfg.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (global_props, None)}, - 'val1': {None: (val1_props, None)}, - 'val1.val1': {None: (val1_val1_props, None)}, - 'val1.val2': {None: (val1_val2_props, None)}} - assert cfg.cfgimpl_get_values()._p_.get_cached() == {'val1.val1': {None: ([], None)}} + compare(cfg.cfgimpl_get_settings()._p_.get_cached(), {None: {None: (global_props, None)}, + 'val1': {None: (val1_props, None)}, + 'val1.val1': {None: (val1_val1_props, None)}, + 'val1.val2': {None: (val1_val2_props, None)}}) + compare(cfg.cfgimpl_get_values()._p_.get_cached(), {'val1.val1': {None: ([], None)}}) api.option('val1.val1').value.set([undefined]) - assert cfg.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, - 'val1': {None: (set([]), None)}} + compare(cfg.cfgimpl_get_settings()._p_.get_cached(), {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, + 'val1': {None: (set([]), None)}}) assert cfg.cfgimpl_get_values()._p_.get_cached() == {} api.option.make_dict() @@ -694,35 +703,35 @@ def test_cache_master_callback(): # api.property.read_write() # api.property.pop('expire') # api.option.make_dict() -# assert cfg.cfgimpl_get_values()._p_.get_cached() == {'int': {None: ([0], None)}, -# 'str': {None: ([None], None)}, -# 'str1': {None: ([None], None)}} -# assert cfg.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, -# 'int': {None: (set(['empty']), None)}, -# 'str': {None: (set([]), None), 0: (set([]), None)}, -# 'str1': {None: (set([]), None), 0: (set([]), None)}} +# compare(cfg.cfgimpl_get_values()._p_.get_cached(), {'int': {None: ([0], None)}, +# 'str': {None: ([None], None)}, +# 'str1': {None: ([None], None)}}) +# conver(cfg.cfgimpl_get_settings()._p_.get_cached(), {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, +# 'int': {None: (set(['empty']), None)}, +# 'str': {None: (set([]), None), 0: (set([]), None)}, +# 'str1': {None: (set([]), None), 0: (set([]), None)}}) # api.option('int').value.set([0, 1]) # api.option.make_dict() -# assert cfg.cfgimpl_get_values()._p_.get_cached() == {'int': {None: ([0, 1], None)}, +# compare(cfg.cfgimpl_get_values()._p_.get_cached(), {'int': {None: ([0, 1], None)}, # 'str': {None: ([None, None], None)}, -# 'str1': {None: ([None, None], None)}} -# assert cfg.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, +# 'str1': {None: ([None, None], None)}}) +# compare(cfg.cfgimpl_get_settings()._p_.get_cached(), {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, # 'int': {None: (set(['empty']), None)}, # 'str': {None: (set([]), None), 0: (set([]), None), 1: (set([]), None)}, -# 'str1': {None: (set([]), None), 0: (set([]), None), 1: (set([]), None)}} +# 'str1': {None: (set([]), None), 0: (set([]), None), 1: (set([]), None)}}) # # api.option('str', 1).value.set('1') # api.option.make_dict() # assert set(cfg.cfgimpl_get_values()._p_.get_cached().keys()) == set(['int', 'str', 'str1']) -# assert cfg.cfgimpl_get_values()._p_.get_cached()['int'] == {None: ([0, 1], None)} -# assert cfg.cfgimpl_get_values()._p_.get_cached()['str'] == {None: ([None, '1'], None)} +# compare(cfg.cfgimpl_get_values()._p_.get_cached()['int'], {None: ([0, 1], None)}) +# compare(cfg.cfgimpl_get_values()._p_.get_cached()['str'], {None: ([None, '1'], None)}) # assert cfg.cfgimpl_get_values()._p_.get_cached()['str1'][None][0][0] == None # raises(PropertiesOptionError, "cfg.cfgimpl_get_values()._p_.get_cached()['str1'][None][0][1]") # assert cfg.cfgimpl_get_values()._p_.get_cached()['str1'][None][1] == None -# assert cfg.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, -# 'int': {None: (set(['empty']), None)}, -# 'str': {None: (set([]), None), 0: (set([]), None), 1: (set([]), None)}, -# 'str1': {None: (set([]), None), 0: (set([]), None), 1: (set(['hidden']), None)}} +# compare(cfg.cfgimpl_get_settings()._p_.get_cached(), {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, +# 'int': {None: (set(['empty']), None)}, +# 'str': {None: (set([]), None), 0: (set([]), None), 1: (set([]), None)}, +# 'str1': {None: (set([]), None), 0: (set([]), None), 1: (set(['hidden']), None)}}) # api.property.read_only() # assert cfg.cfgimpl_get_values()._p_.get_cached() == {} # assert cfg.cfgimpl_get_settings()._p_.get_cached() == {} @@ -736,7 +745,6 @@ def test_cache_master_callback(): # 'str1': {None: (set([]), None), 0: (set([]), None), 1: (set(['hidden']), None)}} - def test_cache_requires(): a = BoolOption('activate_service', '', True) b = IPOption('ip_address_service', '', @@ -747,47 +755,46 @@ def test_cache_requires(): api.property.read_write() if TIRAMISU_VERSION == 2: api.property.pop('expire') - #assert c.cfgimpl_get_settings()._p_.get_cached() == {} assert c.cfgimpl_get_values()._p_.get_cached() == {} assert api.option('ip_address_service').value.get() == None - assert c.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, - 'activate_service': {None: (set([]), None)}, - 'ip_address_service': {None: (set([]), None)}} + compare(c.cfgimpl_get_settings()._p_.get_cached(), {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, + 'activate_service': {None: (set([]), None)}, + 'ip_address_service': {None: (set([]), None)}}) if TIRAMISU_VERSION == 2: assert c.cfgimpl_get_values()._p_.get_cached() == {'ip_address_service': {None: (None, None)}} else: - assert c.cfgimpl_get_values()._p_.get_cached() == {'ip_address_service': {None: (None, None)}, - 'activate_service': {None: (True, None)}} + compare(c.cfgimpl_get_values()._p_.get_cached(), {'ip_address_service': {None: (None, None)}, + 'activate_service': {None: (True, None)}}) api.option.make_dict() - assert c.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, - 'activate_service': {None: (set([]), None)}, - 'ip_address_service': {None: (set([]), None)}} + compare(c.cfgimpl_get_settings()._p_.get_cached(), {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, + 'activate_service': {None: (set([]), None)}, + 'ip_address_service': {None: (set([]), None)}}) - assert c.cfgimpl_get_values()._p_.get_cached() == {'ip_address_service': {None: (None, None)}, - 'activate_service': {None: (True, None)}} + compare(c.cfgimpl_get_values()._p_.get_cached(), {'ip_address_service': {None: (None, None)}, + 'activate_service': {None: (True, None)}}) api.option('ip_address_service').value.set('1.1.1.1') - assert c.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, - 'activate_service': {None: (set([]), None)}} + compare(c.cfgimpl_get_settings()._p_.get_cached(), {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, + 'activate_service': {None: (set([]), None)}}) - assert c.cfgimpl_get_values()._p_.get_cached() == {'activate_service': {None: (True, None)}} + compare(c.cfgimpl_get_values()._p_.get_cached(), {'activate_service': {None: (True, None)}}) api.option.make_dict() - assert c.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, - 'activate_service': {None: (set([]), None)}, - 'ip_address_service': {None: (set([]), None)}} + compare(c.cfgimpl_get_settings()._p_.get_cached(), {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, + 'activate_service': {None: (set([]), None)}, + 'ip_address_service': {None: (set([]), None)}}) - assert c.cfgimpl_get_values()._p_.get_cached() == {'ip_address_service': {None: ('1.1.1.1', None)}, - 'activate_service': {None: (True, None)}} + compare(c.cfgimpl_get_values()._p_.get_cached(), {'ip_address_service': {None: ('1.1.1.1', None)}, + 'activate_service': {None: (True, None)}}) api.option('activate_service').value.set(False) - assert c.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}} + compare(c.cfgimpl_get_settings()._p_.get_cached(), {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}}) assert c.cfgimpl_get_values()._p_.get_cached() == {} api.option.make_dict() - assert c.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, - 'activate_service': {None: (set([]), None)}, - 'ip_address_service': {None: (set(['disabled']), None)}} + compare(c.cfgimpl_get_settings()._p_.get_cached(), {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, + 'activate_service': {None: (set([]), None)}, + 'ip_address_service': {None: (set(['disabled']), None)}}) - assert c.cfgimpl_get_values()._p_.get_cached() == {'activate_service': {None: (False, None)}} + compare(c.cfgimpl_get_values()._p_.get_cached(), {'activate_service': {None: (False, None)}}) def test_cache_global_properties(): @@ -798,30 +805,27 @@ def test_cache_global_properties(): c = Config(od) api = getapi(c) api.property.read_write() - if TIRAMISU_VERSION == 2: - api.property.pop('expire') - #assert c.cfgimpl_get_settings()._p_.get_cached() == {} assert c.cfgimpl_get_values()._p_.get_cached() == {} assert api.option('ip_address_service').value.get() == None - assert c.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, + compare(c.cfgimpl_get_settings()._p_.get_cached(), {None: {None: (set(['cache', 'disabled', 'frozen', 'hidden', 'validator', 'warnings']), None)}, 'activate_service': {None: (set([]), None)}, - 'ip_address_service': {None: (set([]), None)}} + 'ip_address_service': {None: (set([]), None)}}) if TIRAMISU_VERSION == 2: assert c.cfgimpl_get_values()._p_.get_cached() == {'ip_address_service': {None: (None, None)}} else: - assert c.cfgimpl_get_values()._p_.get_cached() == {'ip_address_service': {None: (None, None)}, - 'activate_service': {None: (True, None)}} + compare(c.cfgimpl_get_values()._p_.get_cached(), {'ip_address_service': {None: (None, None)}, + 'activate_service': {None: (True, None)}}) api.property.pop('disabled') assert api.option('ip_address_service').value.get() == None - assert c.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'frozen', 'hidden', 'validator', 'warnings']), None)}, - 'activate_service': {None: (set([]), None)}, - 'ip_address_service': {None: (set([]), None)}} + compare(c.cfgimpl_get_settings()._p_.get_cached(), {None: {None: (set(['cache', 'frozen', 'hidden', 'validator', 'warnings']), None)}, + 'activate_service': {None: (set([]), None)}, + 'ip_address_service': {None: (set([]), None)}}) api.property.add('test') assert api.option('ip_address_service').value.get() == None - assert c.cfgimpl_get_settings()._p_.get_cached() == {None: {None: (set(['cache', 'frozen', 'hidden', 'validator', 'warnings', 'test']), None)}, - 'activate_service': {None: (set([]), None)}, - 'ip_address_service': {None: (set([]), None)}} + compare(c.cfgimpl_get_settings()._p_.get_cached(), {None: {None: (set(['cache', 'frozen', 'hidden', 'validator', 'warnings', 'test']), None)}, + 'activate_service': {None: (set([]), None)}, + 'ip_address_service': {None: (set([]), None)}}) def test_callback_value_incr(): diff --git a/tiramisu/option/optiondescription.py b/tiramisu/option/optiondescription.py index c8377e2..758af28 100644 --- a/tiramisu/option/optiondescription.py +++ b/tiramisu/option/optiondescription.py @@ -166,12 +166,15 @@ class CacheOptionDescription(BaseOption): if option.impl_is_master_slaves('slave'): # problem with index raise ConfigError(_('the slave "{0}" cannot have ' - '"force_store_value" property').format(option.impl_get_display_name())) + '"force_store_value" property').format( + option.impl_get_display_name())) if option.issubdyn(): raise ConfigError(_('the dynoption "{0}" cannot have ' - '"force_store_value" property').format(option.impl_get_display_name())) + '"force_store_value" property').format( + option.impl_get_display_name())) if not values._p_.hasvalue(subpath): config_bag = ConfigBag(config=context, option=option) + config_bag.properties = frozenset() value = values.getvalue(subpath, None, config_bag) diff --git a/tiramisu/setting.py b/tiramisu/setting.py index a92869b..1d58224 100644 --- a/tiramisu/setting.py +++ b/tiramisu/setting.py @@ -15,7 +15,6 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . # ____________________________________________________________ -from time import time from copy import copy from logging import getLogger import weakref @@ -337,27 +336,24 @@ class Settings(object): # get properties and permissive methods def get_context_properties(self): - ntime = int(time()) - if self._p_.hascache(None, - None): - is_cached, props = self._p_.getcache(None, - ntime, - None) - else: - is_cached = False - if not is_cached or 'cache' not in props: + is_cached, props = self._p_.getcache(None, + None, + None, + None, + None, + 'context_props') + if not is_cached: meta = self._getcontext().cfgimpl_get_meta() if meta is None: props = self._p_.getproperties(None, default_properties) else: props = meta.cfgimpl_get_settings().get_context_properties() - if 'cache' in props: - if 'expire' in props: - ntime = ntime + expires_time - else: - ntime = None - self._p_.setcache(None, props, ntime, None) + self._p_.setcache(None, + None, + props, + props, + None) return props def getproperties(self, @@ -371,19 +367,17 @@ class Settings(object): if opt.impl_is_symlinkoption(): opt = opt.impl_getopt() path = opt.impl_getpath(self._getcontext()) - is_cached = False - if apply_requires and config_bag.setting_properties is not None: - if 'cache' in config_bag.setting_properties and \ - 'expire' in config_bag.setting_properties: - ntime = int(time()) - else: - ntime = None - if 'cache' in config_bag.setting_properties and self._p_.hascache(path, - index): - is_cached, props = self._p_.getcache(path, - ntime, - index) + if apply_requires: + props = config_bag.setting_properties + is_cached, props = self._p_.getcache(path, + expires_time, + index, + props, + None, + 'self_props') + else: + is_cached = False if not is_cached: meta = self._getcontext().cfgimpl_get_meta() if meta is None: @@ -403,14 +397,12 @@ class Settings(object): opt.impl_get_display_name()) props -= self.getpermissive(opt, path) - if apply_requires and config_bag.setting_properties is not None and \ - 'cache' in config_bag.setting_properties: - if 'expire' in config_bag.setting_properties: - ntime = ntime + expires_time + if apply_requires: self._p_.setcache(path, + index, props, - ntime, - index) + config_bag.setting_properties, + config_bag.setting_properties) return props def get_context_permissive(self): @@ -599,6 +591,7 @@ class Settings(object): """save properties for specified path (never save properties if same has option properties) """ + # should have index !!! if self._getcontext().cfgimpl_get_meta() is not None: raise ConfigError(_('cannot change property with metaconfig')) if path is not None and config_bag.option.impl_getrequires() is not None: diff --git a/tiramisu/storage/dictionary/cache.py b/tiramisu/storage/dictionary/cache.py new file mode 100644 index 0000000..ce75f55 --- /dev/null +++ b/tiramisu/storage/dictionary/cache.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2018 Team tiramisu (see AUTHORS for all contributors) +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . +# ____________________________________________________________ + + +class Cache(object): + __slots__ = ('_cache',) + + def __init__(self): + self._cache = {} + + def _setcache(self, path, index, val, time): + self._cache.setdefault(path, {})[index] = (val, int(time)) + + def _getcache(self, path, index): + values = self._cache.get(path) + if values is None: + return + return values.get(index) + + def _delcache(self, path): + del self._cache[path] + + def _get_cached(self): + return self._cache + + def _reset_all_cache(self): + self._cache.clear() diff --git a/tiramisu/storage/util.py b/tiramisu/storage/util.py index b89e28d..63811d7 100644 --- a/tiramisu/storage/util.py +++ b/tiramisu/storage/util.py @@ -15,6 +15,10 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . # ____________________________________________________________ +from time import time +from .dictionary.cache import Cache as DictCache + + def _display_classname(obj): return(obj.__class__.__name__.lower()) @@ -22,31 +26,78 @@ DEBUG = False #DEBUG = True -class Cache(object): - __slots__ = ('_cache', '_storage') - key_is_path = False +class Cache(DictCache): + __slots__ = ('_storage',) def __init__(self, storage): - self._cache = {} self._storage = storage + super().__init__() - def setcache(self, path, val, time, index): + def setcache(self, path, index, val, props, self_props): """add val in cache for a specified path if slave, add index """ - if DEBUG: - print('setcache', path, val, _display_classname(self), id(self)) - self._cache.setdefault(path, {})[index] = (val, time) - - def getcache(self, path, exp, index): - value, created = self._cache[path][index] - if created is None or exp is None or exp <= created: + if props is None or 'cache' in props or \ + (self_props is not None and 'cache' in self_props): if DEBUG: - print('getcache in cache', path, value, _display_classname(self), id(self), index, exp) - return True, value + print('setcache {} with index {} and value {} in {} ({})'.format(path, index, val, + _display_classname(self), + id(self))) + self._setcache(path, index, val, time()) if DEBUG: - print('getcache not in cache') - return False, None # pragma: no cover + print('not setcache {} with index {} and value {} in {} ({})'.format(path, + index, + val, + _display_classname(self), + id(self))) + return + + def getcache(self, + path, + expires_time, + index, + props, + self_props, + type_): + no_cache = False, None + if props is None or 'cache' in props: + indexed = self._getcache(path, index) + if indexed is None: + return no_cache + value, timestamp = indexed + if type_ == 'context_props': + # cached value is settings properties so value is props + props = value + elif type_ == 'self_props': + # if self_props is None, so cached value is self properties + # so value is self_props + self_props = value + # recheck "cache" value + if props is None or 'cache' in props or (self_props is not None and 'cache' in props): + if expires_time and timestamp and \ + (props is not None and 'expire' in props or \ + self_props is not None and 'expire' in self_props): + ntime = int(time()) + if timestamp + expires_time >= ntime: + if DEBUG: + print('getcache in cache (1)', path, value, _display_classname(self), + id(self), index) + return True, value + else: + if DEBUG: + print('getcache expired value for path {} < {}'.format( + timestamp + expires_time, ntime)) + # if expired, remove from cache + self.delcache(path) + else: + if DEBUG: + print('getcache in cache (2)', path, value, _display_classname(self), + id(self), index) + return True, value + if DEBUG: + print('getcache {} with index {} not in {} cache'.format(path, index, + _display_classname(self))) + return no_cache def delcache(self, path): """remove cache for a specified path @@ -54,36 +105,19 @@ class Cache(object): if DEBUG: print('delcache', path, _display_classname(self), id(self)) if path in self._cache: - del self._cache[path] - - def hascache(self, path, index): - """ path is in the cache - - :param path: the path's option - """ - if DEBUG: - print('hascache', path, _display_classname(self), id(self)) - return path in self._cache and index in self._cache[path] - - def reset_expired_cache(self, exp): - cache_keys = list(self._cache.keys()) - for key in cache_keys: - key_cache_keys = list(self._cache[key].keys()) - for index in key_cache_keys: - val, created = self._cache[key][index] - if created is not None and exp > created: - del(self._cache[key][index]) - if self._cache[key] == {}: - del(self._cache[key]) + self._delcache(path) def reset_all_cache(self): "empty the cache" if DEBUG: print('reset_all_cache', _display_classname(self), id(self)) - self._cache.clear() + self._reset_all_cache() def get_cached(self): """return all values in a dictionary + please only use it in test purpose example: {'path1': {'index1': ('value1', 'time1')}, 'path2': {'index2': ('value2', 'time2', )}} """ - return self._cache + if DEBUG: + print('get_chached', self._cache) + return self._get_cached() diff --git a/tiramisu/value.py b/tiramisu/value.py index 508edd3..3d1659b 100644 --- a/tiramisu/value.py +++ b/tiramisu/value.py @@ -15,7 +15,6 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . # ____________________________________________________________ -from time import time import weakref from .error import ConfigError, PropertiesOptionError from .setting import owners, expires_time, undefined, forbidden_owners @@ -81,19 +80,14 @@ class Values(object): :returns: value """ - ntime = None # try to retrive value in cache setting_properties = config_bag.setting_properties - is_cached = False - if setting_properties and 'cache' in setting_properties and \ - self._p_.hascache(path, - index): - if 'expire' in setting_properties or \ - (config_bag.properties and 'expire' in config_bag.properties): - ntime = int(time()) - is_cached, value = self._p_.getcache(path, - ntime, - index) + is_cached, value = self._p_.getcache(path, + expires_time, + index, + setting_properties, + config_bag.properties, + 'value') if not is_cached: # no cached value so get value @@ -115,13 +109,12 @@ class Values(object): check_error=False, config_bag=config_bag) # store value in cache - if not is_cached and \ - setting_properties and 'cache' in setting_properties: - if 'expire' in setting_properties or 'expire' in config_bag.properties: - if ntime is None: - ntime = int(time()) - ntime = ntime + expires_time - self._p_.setcache(path, value, ntime, index) + if not is_cached: + self._p_.setcache(path, + index, + value, + setting_properties, + config_bag.properties) # and return it return value @@ -198,10 +191,16 @@ class Values(object): context = self._getcontext() opt = config_bag.option def _reset_cache(_value): - if self._p_.hascache(path, index): - is_cache, cache_value = self._p_.getcache(path, None, index) - if is_cache and cache_value == _value: - return + is_cache, cache_value = self._p_.getcache(path, + expires_time, + index, + config_bag.setting_properties, + config_bag.properties, + 'value') + if is_cache and cache_value == _value: + # calculation return same value as previous value, + # so do not invalidate cache + return # calculated value is a new value, so reset cache context.cfgimpl_reset_cache(opt, path, @@ -389,6 +388,7 @@ class Values(object): config_bag, commit=True): + self._getcontext().cfgimpl_reset_cache(config_bag.option, path, config_bag)