diff --git a/doc/source/operations.rst b/doc/source/operations.rst index f744148a..45072364 100644 --- a/doc/source/operations.rst +++ b/doc/source/operations.rst @@ -149,3 +149,64 @@ value of ``dict_reference`` will actually be a dictionary. You should now be ready to :doc:`use reclass `! .. include:: substs.inc + +Function interpolation +---------------------- + +Certain functions can be used to dynamically generate values. They are specified +like this:: + + parameters: + key: $ + +The following functions are supported: + +print +***** + +This function simply concatenates all its parameters and returns a string. For +example, take this:: + + test: $ + +which results in this:: + + test: "first second third" + +Yeah, it's quite useless. Now to something a bit more useful: + +aggregate +********* + +This can be used to extract values from hosts that satisfy certain conditions. +The syntax looks like this:: + + aggregate(filter, extractor) + +The ``filter`` parameter specifies the condition the host has to fulfil, and +the ``extractor`` determines what will be taken from that host. Inside the +function, ``node`` refers to all parameters of a node, and regular python +dictionary functions can be used to access it. + +The return value is always a dictionary that maps the nodename to the extracted +values. + +Here is an example that extracts the IP address of every host in our domain +``example.com``:: + + hosts: $ + +which might result in something like this:: + + hosts: + first.example.com: 192.168.1.1 + second.example.com: 192.168.1.2 + +Note that parameter interpolation can be used inside functions, but one has to +pay attention to proper quoting:: + + hosts: $ + +Note that chained references are not supported: If the result of a function +contains a function itself, this function will not be interpolated, but taken +verbatim. This also prevents circular references. diff --git a/reclass/core.py b/reclass/core.py index 76bd0a8e..e32013e7 100644 --- a/reclass/core.py +++ b/reclass/core.py @@ -133,13 +133,22 @@ def _nodeinfo_as_dict(self, nodename, entity): return ret def nodeinfo(self, nodename): - return self._nodeinfo_as_dict(nodename, self._nodeinfo(nodename)) + return self.inventory()['nodes'][nodename] def inventory(self): entities = {} + + # first run, reference parameters are expanded for n in self._storage.enumerate_nodes(): entities[n] = self._nodeinfo(n) + # second run, function are executed + #all_parameters = {} + #for nodename, info in entities.items(): + # all_parameters.update({nodename: info.parameters}) + for nodename, node in entities.items(): + node.expand_functions(inventory=entities) + nodes = {} applications = {} classes = {} diff --git a/reclass/datatypes/entity.py b/reclass/datatypes/entity.py index 573a28c9..48a1e68a 100644 --- a/reclass/datatypes/entity.py +++ b/reclass/datatypes/entity.py @@ -53,6 +53,9 @@ def _set_parameters(self, parameters): 'instance of type %s' % type(parameters)) self._parameters = parameters + def expand_functions(self, inventory): + self.parameters.interpolate_functions(inventory) + def merge(self, other): self._classes.merge_unique(other._classes) self._applications.merge_unique(other._applications) diff --git a/reclass/datatypes/parameters.py b/reclass/datatypes/parameters.py index 37419fc6..4aee5453 100644 --- a/reclass/datatypes/parameters.py +++ b/reclass/datatypes/parameters.py @@ -10,8 +10,10 @@ from reclass.defaults import PARAMETER_INTERPOLATION_DELIMITER from reclass.utils.dictpath import DictPath -from reclass.utils.refvalue import RefValue -from reclass.errors import InfiniteRecursionError, UndefinedVariableError +from reclass.utils.refvalue import (ReferenceStringParameter, ReferenceParameter, + ReferenceFunction, ReferenceStringFunction) +from reclass.errors import (InfiniteRecursionError, UndefinedVariableError, + UndefinedFunctionError) class Parameters(object): ''' @@ -70,37 +72,37 @@ def as_dict(self): return self._base.copy() def _update_scalar(self, cur, new, path): - if isinstance(cur, RefValue) and path in self._occurrences: - # If the current value already holds a RefValue, we better forget + if isinstance(cur, ReferenceStringParameter) and path in self._occurrences: + # If the current value already holds a ReferenceStringParameter, we better forget # the occurrence, or else interpolate() will later overwrite - # unconditionally. If the new value is a RefValue, the occurrence + # unconditionally. If the new value is a ReferenceStringParameter, the occurrence # will be added again further on del self._occurrences[path] if self.delimiter is None or not isinstance(new, (types.StringTypes, - RefValue)): + ReferenceStringParameter)): # either there is no delimiter defined (and hence no references # are being used), or the new value is not a string (and hence - # cannot be turned into a RefValue), and not a RefValue. We can + # cannot be turned into a ReferenceStringParameter), and not a ReferenceStringParameter. We can # shortcut and just return the new scalar return new - elif isinstance(new, RefValue): - # the new value is (already) a RefValue, so we need not touch it + elif isinstance(new, ReferenceStringParameter): + # the new value is (already) a ReferenceStringParameter, so we need not touch it # at all ret = new else: # the new value is a string, let's see if it contains references, - # by way of wrapping it in a RefValue and querying the result - ret = RefValue(new, self.delimiter) + # by way of wrapping it in a ReferenceStringParameter and querying the result + ret = ReferenceStringParameter(new, self.delimiter) if not ret.has_references(): - # do not replace with RefValue instance if there are no - # references, i.e. discard the RefValue in ret, just return + # do not replace with ReferenceStringParameter instance if there are no + # references, i.e. discard the ReferenceStringParameter in ret, just return # the new value return new - # So we now have a RefValue. Let's, keep a reference to the instance + # So we now have a ReferenceStringParameter. Let's, keep a reference to the instance # we just created, in a dict indexed by the dictionary path, instead # of just a list. The keys are required to resolve dependencies during # interpolation @@ -171,6 +173,21 @@ def merge(self, other): def has_unresolved_refs(self): return len(self._occurrences) > 0 + def interpolate_functions(self, inventory): + self._interpolate_functions_inner(self._base, inventory) + + def _interpolate_functions_inner(self, node, inventory): + for k, v in node.items(): + if isinstance(v, dict): + self._interpolate_functions_inner(node[k], inventory) + elif isinstance(v, types.StringTypes): + refval = ReferenceStringFunction(v) + if refval.has_references(): + try: + node[k] = refval.render(inventory) + except UndefinedFunctionError as e: + raise UndefinedFunctionError(e.var) + def interpolate(self): while self.has_unresolved_refs(): # we could use a view here, but this is simple enough: @@ -182,7 +199,7 @@ def interpolate(self): def _interpolate_inner(self, path, refvalue): self._occurrences[path] = True # mark as seen for ref in refvalue.get_references(): - path_from_ref = DictPath(self.delimiter, ref) + path_from_ref = DictPath(self.delimiter, ref.string) try: refvalue_inner = self._occurrences[path_from_ref] @@ -201,14 +218,13 @@ def _interpolate_inner(self, path, refvalue): # Therefore, if we encounter True instead of a refvalue, # it means that we have already processed it and are now # faced with a cyclical reference. - raise InfiniteRecursionError(path, ref) + raise InfiniteRecursionError(path, ref.string) self._interpolate_inner(path_from_ref, refvalue_inner) except KeyError as e: # not actually an error, but we are done resolving all # dependencies of the current ref, so move on continue - try: new = refvalue.render(self._base) path.set_value(self._base, new) diff --git a/reclass/defaults.py b/reclass/defaults.py index d0662908..24be49f2 100644 --- a/reclass/defaults.py +++ b/reclass/defaults.py @@ -26,3 +26,5 @@ PARAMETER_INTERPOLATION_SENTINELS = ('${', '}') PARAMETER_INTERPOLATION_DELIMITER = ':' + +FUNCTION_INTERPOLATION_SENTINELS = ('$<', '>') diff --git a/reclass/errors.py b/reclass/errors.py index ddb95fdb..479bda27 100644 --- a/reclass/errors.py +++ b/reclass/errors.py @@ -10,7 +10,8 @@ import posix, sys import traceback -from reclass.defaults import PARAMETER_INTERPOLATION_SENTINELS +from reclass.defaults import (PARAMETER_INTERPOLATION_SENTINELS, + FUNCTION_INTERPOLATION_SENTINELS) class ReclassException(Exception): @@ -144,6 +145,16 @@ def set_context(self, context): self._context = context +class UndefinedFunctionError(InterpolationError): + def __init__(self, var): + super(UndefinedFunctionError, self).__init__(msg=None) + self._var = var + var = property(lambda self: self._var) + + def _get_message(self): + return "Unknown function in " + self._var.join(FUNCTION_INTERPOLATION_SENTINELS) + + class IncompleteInterpolationError(InterpolationError): def __init__(self, string, end_sentinel): diff --git a/reclass/utils/function.py b/reclass/utils/function.py new file mode 100644 index 00000000..3f032980 --- /dev/null +++ b/reclass/utils/function.py @@ -0,0 +1,56 @@ +# +# -*- coding: utf-8 -*- +# +# This file is part of reclass (http://github.com/madduck/reclass) +# +# Copyright © 2007–14 martin f. krafft +# Released under the terms of the Artistic Licence 2.0 +# + +from reclass.errors import UndefinedFunctionError + + +def get_function(name): + if name == 'print': + return FunctionPrint() + if name == 'aggregate': + return FunctionAggregate() + else: + raise UndefinedFunctionError(name) + + +class Function(object): + def __init__(self): + pass + + def execute(self, *args, **kwargs): + pass + + +class FunctionPrint(Function): + def __init__(self): + super(FunctionPrint, self).__init__() + + def execute(self, inventory, *args): + return " ".join(args) + + +class FunctionAggregate(Function): + def __init__(self): + super(FunctionAggregate, self).__init__() + + def execute(self, inventory, *args): + func_filter, func_extract = args[0:2] + result = {} + matching_hosts = {} + for hostname, hostinfo in inventory.items(): + node = hostinfo.parameters.as_dict() + try: + if eval(func_filter): + matching_hosts.update({hostname: node}) + for hostname, hostinfo in matching_hosts.items(): + expr = func_extract.replace("node", "hostinfo") + result[hostname] = eval(expr) + except KeyError: + raise + return result diff --git a/reclass/utils/refvalue.py b/reclass/utils/refvalue.py index b8e730be..9f1e9155 100644 --- a/reclass/utils/refvalue.py +++ b/reclass/utils/refvalue.py @@ -11,105 +11,192 @@ from reclass.utils.dictpath import DictPath from reclass.defaults import PARAMETER_INTERPOLATION_SENTINELS, \ - PARAMETER_INTERPOLATION_DELIMITER + PARAMETER_INTERPOLATION_DELIMITER, \ + FUNCTION_INTERPOLATION_SENTINELS from reclass.errors import IncompleteInterpolationError, \ - UndefinedVariableError + UndefinedVariableError, \ + UndefinedFunctionError +from reclass.utils.function import get_function -_SENTINELS = [re.escape(s) for s in PARAMETER_INTERPOLATION_SENTINELS] -_RE = '{0}\s*(.+?)\s*{1}'.format(*_SENTINELS) +_SENTINELS_PARAMETER = [re.escape(s) for s in PARAMETER_INTERPOLATION_SENTINELS] +_SENTINELS_FUNCTIONS = [re.escape(s) for s in FUNCTION_INTERPOLATION_SENTINELS] -class RefValue(object): +_RE_PARAMETER = '{0}\s*(.+?)\s*{1}'.format(*_SENTINELS_PARAMETER) +_RE_FUNCTIONS = '{0}\s*(.+?)\s*{1}'.format(*_SENTINELS_FUNCTIONS) + +# matches a string like 'function, args)' +_RE_FUNC = '([^(]+)\((.*)\)' +_RE_FUNC = re.compile(_RE_FUNC) + + +class Reference(object): + def __init__(self, string): + self.string = string + + +class ReferenceFunction(Reference): + def __init__(self, string): + super(ReferenceFunction, self).__init__(string) + + def resolve(self, inventory, *args, **kwargs): + return self._execute(inventory) + + def _execute(self, inventory): + match = _RE_FUNC.match(self.string) + func_name = match.group(1) + func_args = match.groups()[1].split(',') + + func_args = [f.strip(' ') for f in func_args] + + try: + func = get_function(func_name) + ret = func.execute(inventory, *func_args) + return ret + except UndefinedFunctionError: + raise UndefinedFunctionError(self.string) + + +class ReferenceParameter(Reference): + def __init__(self, string): + super(ReferenceParameter, self).__init__(string) + + def resolve(self, context, *args, **kwargs): + path = DictPath(kwargs['delim'], self.string) + try: + return path.get_value(context) + except KeyError as e: + raise UndefinedVariableError(self.string) + + +class ReferenceString(object): + def __init__(self, string): + self._strings = [] + self._refs = [] + self._parse(string) + + def has_references(self): + return len(self._refs) > 0 + + def get_references(self): + return self._refs + + def render(self, inventory): + pass + + def _check_strings(self, orig, strings, sentinel): + for s in strings: + pos = s.find(sentinel[0]) + if pos >= 0: + raise IncompleteInterpolationError(orig, sentinel[1]) + + def _assemble(self, resolver): + if not self.has_references(): + return self._strings[0] + + if self._strings == ['', '']: + # preserve the type of the referenced variable + ret = resolver(self._refs[0]) + else: + + # reassemble the string by taking a string and str(ref) pairwise + ret = '' + for i in range(0, len(self._refs)): + ret += self._strings[i] + str(resolver(self._refs[i])) + if len(self._strings) > len(self._refs): + # and finally append a trailing string, if any + ret += self._strings[-1] + return ret + + +class ReferenceStringFunction(ReferenceString): + + INTERPOLATION_RE_FUNCTIONS = re.compile(_RE_FUNCTIONS) + + def __init__(self, string): + super(ReferenceStringFunction,self).__init__(string) + + def _parse(self, string): + strings, refs = self._parse_functions(string) + self._strings = strings + self._refs = [ReferenceFunction(ref) for ref in refs] + + def _parse_functions(self, string): + parts = self.INTERPOLATION_RE_FUNCTIONS.split(string) + strings = parts[0:][::2] + functions = parts[1:][::2] + self._check_strings(string, strings, FUNCTION_INTERPOLATION_SENTINELS) + return (strings, functions) + + + def _resolve(self, ref, inventory): + return ref.resolve(inventory) + + def render(self, inventory): + resolver = lambda s: self._resolve(s, inventory) + ret = self._assemble(resolver) + return ret + + +class ReferenceStringParameter(ReferenceString): ''' Isolates references in string values - RefValue can be used to isolate and eventually expand references to other + ReferenceStringParameter can be used to isolate and eventually expand references to other parameters in strings. Those references can then be iterated and rendered in the context of a dictionary to resolve those references. - RefValue always gets constructed from a string, because templating + ReferenceStringParameter always gets constructed from a string, because templating — essentially this is what's going on — is necessarily always about - strings. Therefore, generally, the rendered value of a RefValue instance + strings. Therefore, generally, the rendered value of a ReferenceStringParameter instance will also be a string. - Nevertheless, as this might not be desirable, RefValue will return the + Nevertheless, as this might not be desirable, ReferenceStringParameter will return the referenced variable without casting it to a string, if the templated string contains nothing but the reference itself. For instance: mydict = {'favcolour': 'yellow', 'answer': 42, 'list': [1,2,3]} - RefValue('My favourite colour is ${favolour}').render(mydict) + ReferenceStringParameter('My favourite colour is ${favolour}').render(mydict) → 'My favourite colour is yellow' # a string - RefValue('The answer is ${answer}').render(mydict) + ReferenceStringParameter('The answer is ${answer}').render(mydict) → 'The answer is 42' # a string - RefValue('${answer}').render(mydict) + ReferenceStringParameter('${answer}').render(mydict) → 42 # an int - RefValue('${list}').render(mydict) + ReferenceStringParameter('${list}').render(mydict) → [1,2,3] # an list The markers used to identify references are set in reclass.defaults, as is the default delimiter. ''' - INTERPOLATION_RE = re.compile(_RE) + INTERPOLATION_RE_PARAMETER = re.compile(_RE_PARAMETER) def __init__(self, string, delim=PARAMETER_INTERPOLATION_DELIMITER): - self._strings = [] - self._refs = [] self._delim = delim - self._parse(string) + super(ReferenceStringParameter,self).__init__(string) + def _parse(self, string): - parts = RefValue.INTERPOLATION_RE.split(string) + parts = ReferenceStringParameter.INTERPOLATION_RE_PARAMETER.split(string) self._refs = parts[1:][::2] + self._refs = [ReferenceParameter(ref) for ref in self._refs] self._strings = parts[0:][::2] - self._check_strings(string) - - def _check_strings(self, orig): - for s in self._strings: - pos = s.find(PARAMETER_INTERPOLATION_SENTINELS[0]) - if pos >= 0: - raise IncompleteInterpolationError(orig, - PARAMETER_INTERPOLATION_SENTINELS[1]) - - def _resolve(self, ref, context): - path = DictPath(self._delim, ref) - try: - return path.get_value(context) - except KeyError as e: - raise UndefinedVariableError(ref) - - def has_references(self): - return len(self._refs) > 0 + self._check_strings(string, self._strings, PARAMETER_INTERPOLATION_SENTINELS) - def get_references(self): - return self._refs - def _assemble(self, resolver): - if not self.has_references(): - return self._strings[0] + def _resolve(self, ref, context, additional_info): + return ref.resolve(context, additional_info, delim=self._delim) - if self._strings == ['', '']: - # preserve the type of the referenced variable - return resolver(self._refs[0]) - - # reassemble the string by taking a string and str(ref) pairwise - ret = '' - for i in range(0, len(self._refs)): - ret += self._strings[i] + str(resolver(self._refs[i])) - if len(self._strings) > len(self._refs): - # and finally append a trailing string, if any - ret += self._strings[-1] + def render(self, context, additional_info=None): + resolver = lambda s: self._resolve(s, context, additional_info) + ret = self._assemble(resolver) return ret - def render(self, context): - resolver = lambda s: self._resolve(s, context) - return self._assemble(resolver) - def __repr__(self): - do_not_resolve = lambda s: s.join(PARAMETER_INTERPOLATION_SENTINELS) - return 'RefValue(%r, %r)' % (self._assemble(do_not_resolve), + do_not_resolve = lambda s: s.string.join(PARAMETER_INTERPOLATION_SENTINELS) + return 'ReferenceStringParameter(%r, %r)' % (self._assemble(do_not_resolve), self._delim) diff --git a/reclass/utils/tests/test_refvalue.py b/reclass/utils/tests/test_refvalue.py index 23d7e7b0..a99784d8 100644 --- a/reclass/utils/tests/test_refvalue.py +++ b/reclass/utils/tests/test_refvalue.py @@ -7,7 +7,7 @@ # Released under the terms of the Artistic Licence 2.0 # -from reclass.utils.refvalue import RefValue +from reclass.utils.refvalue import ReferenceStringParameter from reclass.defaults import PARAMETER_INTERPOLATION_SENTINELS, \ PARAMETER_INTERPOLATION_DELIMITER from reclass.errors import UndefinedVariableError, \ @@ -31,17 +31,17 @@ def _var(s): def _poor_mans_template(s, var, value): return s.replace(_var(var), value) -class TestRefValue(unittest.TestCase): +class TestReferenceStringParameter(unittest.TestCase): def test_simple_string(self): s = 'my cat likes to hide in boxes' - tv = RefValue(s) + tv = ReferenceStringParameter(s) self.assertFalse(tv.has_references()) self.assertEquals(tv.render(CONTEXT), s) def _test_solo_ref(self, key): s = _var(key) - tv = RefValue(s) + tv = ReferenceStringParameter(s) res = tv.render(CONTEXT) self.assertTrue(tv.has_references()) self.assertEqual(res, CONTEXT[key]) @@ -63,7 +63,7 @@ def test_solo_ref_bool(self): def test_single_subst_bothends(self): s = 'I like ' + _var('favcolour') + ' and I like it' - tv = RefValue(s) + tv = ReferenceStringParameter(s) self.assertTrue(tv.has_references()) self.assertEqual(tv.render(CONTEXT), _poor_mans_template(s, 'favcolour', @@ -71,7 +71,7 @@ def test_single_subst_bothends(self): def test_single_subst_start(self): s = _var('favcolour') + ' is my favourite colour' - tv = RefValue(s) + tv = ReferenceStringParameter(s) self.assertTrue(tv.has_references()) self.assertEqual(tv.render(CONTEXT), _poor_mans_template(s, 'favcolour', @@ -79,7 +79,7 @@ def test_single_subst_start(self): def test_single_subst_end(self): s = 'I like ' + _var('favcolour') - tv = RefValue(s) + tv = ReferenceStringParameter(s) self.assertTrue(tv.has_references()) self.assertEqual(tv.render(CONTEXT), _poor_mans_template(s, 'favcolour', @@ -88,7 +88,7 @@ def test_single_subst_end(self): def test_deep_subst_solo(self): var = PARAMETER_INTERPOLATION_DELIMITER.join(('motd', 'greeting')) s = _var(var) - tv = RefValue(s) + tv = ReferenceStringParameter(s) self.assertTrue(tv.has_references()) self.assertEqual(tv.render(CONTEXT), _poor_mans_template(s, var, @@ -97,7 +97,7 @@ def test_deep_subst_solo(self): def test_multiple_subst(self): greet = PARAMETER_INTERPOLATION_DELIMITER.join(('motd', 'greeting')) s = _var(greet) + ' I like ' + _var('favcolour') + '!' - tv = RefValue(s) + tv = ReferenceStringParameter(s) self.assertTrue(tv.has_references()) want = _poor_mans_template(s, greet, CONTEXT['motd']['greeting']) want = _poor_mans_template(want, 'favcolour', CONTEXT['favcolour']) @@ -106,7 +106,7 @@ def test_multiple_subst(self): def test_multiple_subst_flush(self): greet = PARAMETER_INTERPOLATION_DELIMITER.join(('motd', 'greeting')) s = _var(greet) + ' I like ' + _var('favcolour') - tv = RefValue(s) + tv = ReferenceStringParameter(s) self.assertTrue(tv.has_references()) want = _poor_mans_template(s, greet, CONTEXT['motd']['greeting']) want = _poor_mans_template(want, 'favcolour', CONTEXT['favcolour']) @@ -114,14 +114,14 @@ def test_multiple_subst_flush(self): def test_undefined_variable(self): s = _var('no_such_variable') - tv = RefValue(s) + tv = ReferenceStringParameter(s) with self.assertRaises(UndefinedVariableError): tv.render(CONTEXT) def test_incomplete_variable(self): s = PARAMETER_INTERPOLATION_SENTINELS[0] + 'incomplete' with self.assertRaises(IncompleteInterpolationError): - tv = RefValue(s) + tv = ReferenceStringParameter(s) if __name__ == '__main__': unittest.main()