From 0313eceac203e538e1abcfc5e6e3ca93853b1375 Mon Sep 17 00:00:00 2001 From: Pedro-Juan Ferrer Matoses Date: Wed, 8 May 2013 11:59:09 +0200 Subject: [PATCH 1/4] Changed the first split condition in order to accept also descriptions that include ";" in the text. --- pypcaxis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypcaxis.py b/pypcaxis.py index 957e547..34655e9 100644 --- a/pypcaxis.py +++ b/pypcaxis.py @@ -59,7 +59,7 @@ def parse(path): def read_data(path): return [t.strip() for t in - open(path).read().decode('ISO-8859-1').split(';')] + open(path).read().decode('ISO-8859-1').split(';\n')] def create_dimension(title, values): From 73787922117cb7d4187622035a867cb087d61038 Mon Sep 17 00:00:00 2001 From: vehrka Date: Wed, 8 May 2013 13:12:13 +0200 Subject: [PATCH 2/4] The Spanish government also includes an aditional "\n" to some lines --- pypcaxis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypcaxis.py b/pypcaxis.py index 34655e9..8f60574 100644 --- a/pypcaxis.py +++ b/pypcaxis.py @@ -64,7 +64,7 @@ def read_data(path): def create_dimension(title, values): # values are defined like: "foo","bar","zap" - values = values.replace('\r\n', '')[1:-1].split('","') + values = values.replace('\r\n', '').replace('\n', '')[1:-1].split('","') return Dimension(title, values) From d5aa23de746b295122ea3a4eaec3af1c366bc6b2 Mon Sep 17 00:00:00 2001 From: omniumie Date: Mon, 10 Jun 2013 09:26:03 +0200 Subject: [PATCH 3/4] Changed dimensions end line recognition for more Spanish oddities --- pypcaxis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypcaxis.py b/pypcaxis.py index 8f60574..7460b69 100644 --- a/pypcaxis.py +++ b/pypcaxis.py @@ -59,7 +59,7 @@ def parse(path): def read_data(path): return [t.strip() for t in - open(path).read().decode('ISO-8859-1').split(';\n')] + open(path).read().decode('IBM850').replace(';\r\n', ';\n').split(';\n')] def create_dimension(title, values): From 3cb1fe1faf341de61d06f5ed2e3772a2d5adf85f Mon Sep 17 00:00:00 2001 From: Pedro-Juan Ferrer Date: Thu, 15 May 2014 13:23:08 +0200 Subject: [PATCH 4/4] Added python Pandas transformation --- pypcaxis.py | 196 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 117 insertions(+), 79 deletions(-) diff --git a/pypcaxis.py b/pypcaxis.py index 7460b69..74a43c4 100644 --- a/pypcaxis.py +++ b/pypcaxis.py @@ -1,79 +1,117 @@ -import re -from itertools import product -from operator import mul - - -class Dimension(object): - - def __init__(self, title, values): - self.title = title - self.values = values - - def __len__(self): - return len(self.values) - - -class Table(object): - - def __init__(self): - self.dimensions = [] - self.data = None - - def add_dimension(self, dimension): - self.dimensions.append(dimension) - - def get_by(self, title, value): - #FIXME does not work!!! - title_index = [dim.title for dim in self.dimensions].index(title) - dims = [dim.values for dim in self.dimensions] - dims[title_index] = [value] - table = Table() - table.dimension = [d for d in self.dimensions if d.title != title] - table.data = [self.get(*criteria) for criteria in reversed(list(product(*dims)))] - return table - - def get(self, *criteria): - dim_lenghts = [len(dim) for dim in self.dimensions] - dim_indices = [dim.values.index(c) for (dim, c) - in zip(self.dimensions, criteria)] - return self.data[sum(reduce(mul, dim_lenghts[i+1:], 1) * index - for i, index in enumerate(dim_indices))] - - -def parse(path): - data = read_data(path) - value_regex = re.compile(r'VALUES\(\"(.*)\"\)') - table = Table() - for item in data: - if not item: - continue - name, values = [t.strip() for t in item.split('=', 1)] - value_match = value_regex.match(name) - if value_match: - title = value_match.group(1) - table.add_dimension(create_dimension(title, values)) - if name == 'DATA': - table.data = [i.strip() for i in values.split(' ')] - return table - - -def read_data(path): - return [t.strip() for t in - open(path).read().decode('IBM850').replace(';\r\n', ';\n').split(';\n')] - - -def create_dimension(title, values): - # values are defined like: "foo","bar","zap" - values = values.replace('\r\n', '').replace('\n', '')[1:-1].split('","') - return Dimension(title, values) - - -if __name__ == '__main__': - table = parse('examples/tulot.px') - print table.get('2008', 'Tuusula - Tusby', 'Veronalaiset tulot, mediaani') - print table.get('2009', 'Tuusula - Tusby', 'Veronalaiset tulot, mediaani') - print table.get('2007', u'Hyvink\xe4\xe4 - Hyvinge', 'Tulonsaajia') - print table.get_by('Vuosi', '2007').get(u'Hyvink\xe4\xe4 - Hyvinge', 'Tulonsaajia') - print table.get('2008', 'Tuusula - Tusby', 'Veronalaiset tulot, mediaani') - table = parse('examples/vaalit.px') - print table.get('Uudenmaan vaalipiiri', 'VIHR', u'Yhteens\xe4', u'78 vuotta') +import re +from itertools import product +from operator import mul +from pandas import DataFrame + +class Dimension(object): + + def __init__(self, title, values): + self.title = title + self.values = values + + def __len__(self): + return len(self.values) + + +class Table(object): + + def __init__(self): + self.dimensions = [] + self.data = None + + def add_dimension(self, dimension): + self.dimensions.append(dimension) + + def get_by(self, title, value): + #FIXME does not work!!! + title_index = [dim.title for dim in self.dimensions].index(title) + dims = [dim.values for dim in self.dimensions] + dims[title_index] = [value] + table = Table() + table.dimension = [d for d in self.dimensions if d.title != title] + table.data = [self.get(*criteria) for criteria in reversed(list(product(*dims)))] + return table + + def get(self, *criteria): + dim_lenghts = [len(dim) for dim in self.dimensions] + dim_indices = [dim.values.index(c) for (dim, c) + in zip(self.dimensions, criteria)] + return self.data[sum(reduce(mul, dim_lenghts[i+1:], 1) * index + for i, index in enumerate(dim_indices))] + + +def parse(path): + data = read_data(path) + value_regex = re.compile(r'VALUES\(\"(.*)\"\)') + table = Table() + for item in data: + if not item: + continue + name, values = [t.strip() for t in item.split('=', 1)] + value_match = value_regex.match(name) + if value_match: + title = value_match.group(1) + table.add_dimension(create_dimension(title, values)) + if name == 'DATA': + table.data = [i.strip() for i in values.split(' ')] + return table + + +def read_data(path): + return [t.strip() for t in + open(path).read().decode('IBM850').replace(';\r\n', ';\n').split(';\n')] + + +def create_dimension(title, values): + # values are defined like: "foo","bar","zap" + values = values.replace('\r\n', '').replace('\n', '')[1:-1].split('","') + return Dimension(title, values) + +def to_pandas_df(table): + """Creates a Pandas DataFrame with the data in the table""" + largs = table.dimensions + total = 1 + ldims = [] + + for dim in largs: + ldims.append(len(dim)) + total *= len(dim) + + vresult = total * [0] + + for i, dim in enumerate(largs): + ldim = ldims[i] + tblo = total / ldim + if i < (len(ldims) - 1): + rdim = 1 + for x in range(i, len(ldims) - 1): + rdim *= ldims[x+1] + else: + rdim = 1 + for udi in range(ldim): + if i == 0: + for j in range(rdim): + vresult[j+(udi * rdim)]=[dim.values[udi]] + else: + for j in range(0, total, ldim * rdim): + for k in range(rdim): + vresult[j + k + (udi * rdim)].extend([dim.values[udi]]) + for i, fila in enumerate(vresult): + fila.extend([table.data[i]]) + + colnames = [dim.title for dim in table.dimensions] + colnames.extend(['values']) + + return DataFrame(vresult, columns=colnames) + + + +if __name__ == '__main__': + table = parse('examples/tulot.px') + print table.get('2008', 'Tuusula - Tusby', 'Veronalaiset tulot, mediaani') + print table.get('2009', 'Tuusula - Tusby', 'Veronalaiset tulot, mediaani') + print table.get('2007', u'Hyvink\xe4\xe4 - Hyvinge', 'Tulonsaajia') + print table.get_by('Vuosi', '2007').get(u'Hyvink\xe4\xe4 - Hyvinge', 'Tulonsaajia') + print table.get('2008', 'Tuusula - Tusby', 'Veronalaiset tulot, mediaani') + table = parse('examples/vaalit.px') + print table.get('Uudenmaan vaalipiiri', 'VIHR', u'Yhteens\xe4', u'78 vuotta')