diff --git a/changes/155.changes b/changes/155.changes new file mode 100644 index 00000000..c4a003a9 --- /dev/null +++ b/changes/155.changes @@ -0,0 +1 @@ +Create test sample recombinant definition with example of custom primary key generation diff --git a/ckanext/recombinant/cli.py b/ckanext/recombinant/cli.py index 187ea961..697f1b55 100644 --- a/ckanext/recombinant/cli.py +++ b/ckanext/recombinant/cli.py @@ -536,7 +536,11 @@ def _create_triggers(dataset_types: Optional[List[str]], """ lc = LocalCKAN() for dtype in _expand_dataset_types(dataset_types, all_types): - for chromo in get_geno(dtype)['resources']: + geno = get_geno(dtype) + if seq := geno.get('datastore_create_sequence'): + lc = LocalCKAN() + lc.action.datastore_sequence_create(name=seq, if_not_exists=True) + for chromo in geno['resources']: _update_triggers(lc, chromo) @@ -618,10 +622,9 @@ def _load_one_csv_file(name: str) -> int: dataset_type = chromo['dataset_type'] method = 'upsert' if chromo.get('datastore_primary_key') else 'insert' - lc = LocalCKAN() + lc = LocalCKAN(context={'datastore_import': True}) # required for importing _id errors = 0 - # dynamic fields dynamic_fields = [ 'owner_org', 'owner_org_title', @@ -770,9 +773,10 @@ def _write_one_csv(lc: LocalCKAN, chromo: Dict[str, Any], outfile: TextIO): out = csv.writer(outfile) - column_ids = [ - f['datastore_id'] for f in chromo['fields'] if - not f.get('published_resource_computed_field')] + \ + column_ids = \ + (['_id'] if chromo.get('edit_using__id') else []) + \ + [f['datastore_id'] for f in chromo['fields'] if + not f.get('published_resource_computed_field')] + \ chromo.get('csv_org_extras', []) + \ ['owner_org', 'owner_org_title'] out.writerow(column_ids) diff --git a/ckanext/recombinant/read_csv.py b/ckanext/recombinant/read_csv.py index 3bc36123..0b4b0de0 100644 --- a/ckanext/recombinant/read_csv.py +++ b/ckanext/recombinant/read_csv.py @@ -45,11 +45,12 @@ def csv_data_batch(csv_path: str, if f not in ignore_fields] if strict: - expected = [f['datastore_id'] for f in chromo['fields'] if not f.get( - 'published_resource_computed_field')] + expected = \ + (['_id'] if chromo.get('edit_using__id') else []) + \ + [f['datastore_id'] for f in chromo['fields'] if not f.get( + 'published_resource_computed_field')] if ignore_fields: - expected = [f['datastore_id'] for f in chromo['fields'] if - f['datastore_id'] not in ignore_fields] + expected = [f for f in expected if f not in ignore_fields] assert cols == expected, 'column mismatch:\n{0}\n{1}'.format( cols, expected) diff --git a/ckanext/recombinant/read_excel.py b/ckanext/recombinant/read_excel.py index 69177359..69908f8b 100644 --- a/ckanext/recombinant/read_excel.py +++ b/ckanext/recombinant/read_excel.py @@ -66,27 +66,51 @@ def read_excel(f: Union[str, FlaskFileStorage, FieldStorage], names_row = next(rowiter) org_name = organization_row[0].value - if org_name and names_row[0].value != 'v3': + sig = names_row[0].value + if org_name and not sig.startswith('v3'): # v2 template yield ( _sheetname, org_name, [c.value for c in names_row], # type_ignore_reason: incomplete typing - _filter_bumf(rowiter, HEADER_ROWS_V2)) + _filter_bumf(rowiter, HEADER_ROWS_V2), + 'upsert') continue # type_ignore_reason: incomplete typing next(rowiter) example_row = next(rowiter) - if example_row[0].value != 'e.g.' and example_row[0].value != 'ex.': + if ( + sig != 'v3-update' # we remove the example from update-only template + and example_row[0].value not in ('e.g.', 'ex.') + ): raise BadExcelData('Example record on row 5 is missing') - yield ( - _sheetname, - names_row[1].value, - [c.value for c in names_row[2:]], - _filter_bumf((row[2:] for row in rowiter), HEADER_ROWS_V3)) + if sig == 'v3-update': + yield ( + _sheetname, + names_row[1].value, + ['_id'] + [c.value for c in names_row[2:]], + _filter_bumf((row[1:] for row in rowiter), HEADER_ROWS_V3), + 'update') + elif sig == 'v3-insert': + yield ( + _sheetname, + names_row[1].value, + [c.value for c in names_row[2:]], + _filter_bumf((row[2:] for row in rowiter), HEADER_ROWS_V3), + 'insert') + elif sig == 'v3': + yield ( + _sheetname, + names_row[1].value, + [c.value for c in names_row[2:]], + _filter_bumf((row[2:] for row in rowiter), HEADER_ROWS_V3), + 'upsert') + else: + raise BadExcelData( + f'Unknown signature {sig!r}, must be one: v3, v3-insert, v3-update') def _filter_bumf(rowiter: Iterator[Any], diff --git a/ckanext/recombinant/tests/samples/sample_pk_deluxe.yaml b/ckanext/recombinant/tests/samples/sample_pk_deluxe.yaml new file mode 100644 index 00000000..5d16c3fd --- /dev/null +++ b/ckanext/recombinant/tests/samples/sample_pk_deluxe.yaml @@ -0,0 +1,117 @@ +dataset_type: sampledeluxe +target_dataset: sampledeluxe + + +title: Sample PK-Deluxe Dataset Type +shortname: Sample PK-Deluxe Dataset Type +notes: Sample PK-Deluxe Dataset Type + +datastore_create_sequence: sample_pk_deluxe_seq + +template_version: 3 + +portal_type: dataset +collection: sample + +resources: +- title: Sample Resource + resource_name: sampledeluxe + + published_resource_id: 1495491e-338c-43ec-9995-ecb48c67beef + edit_using__id: true + + create_form: true + edit_form: true + fields: + - datastore_id: year + datastore_type: year + label: Year + description: + en: Sample Year Field + fr: SampleYear Field FR + obligation: Mandatory + excel_required: true + form_required: true + validation: This field must not be empty + excel_column_width: 10 + extract_date_year: true + form_attrs: + size: 20 + + - datastore_id: value + datastore_type: int + label: Value + obligation: Mandatory + excel_required: true + form_required: true + validation: This field must not be empty + excel_column_width: 10 + + datastore_primary_key: year + datastore_indexes: "" + + default_preview_sort: year + + excel_example_height: 100 + excel_data_num_rows: 500 + + triggers: + - test_trigger_1: | + DECLARE + errors text[][] := '{{}}'; + crval RECORD; + importing boolean NOT NULL := (SELECT importing + FROM datastore_user LIMIT 1); + BEGIN + errors := errors || required_error(NEW.year, 'year'); + errors := errors || required_error(NEW.value, 'value'); + + IF TG_OP = 'UPDATE' THEN + errors := errors || required_error(NEW._id, '_id'); + END IF; + + IF errors = '{{}}' THEN + IF NOT importing AND TG_OP = 'INSERT' THEN + -- open_canada_id generation rule -- + + -- Assign unique _id across *all* records and so that (_id % 97) == 5 + -- These _id values are exported as open_canada_id in the combined csv dataset + + -- The last two _id digits are check digits. Mod 97 prevents almost all numeric + -- typos (Mod 97 is also used for international bank account check digits) + + -- 5 chosen only because this is a Sample and 5 looks like an S. + -- open_canada_id values for other record types use values + -- different than 5 so that open_canada_ids are unique across all record types + + -- Use check digits from 03-99 because numbers ending in 00 look approximate, + -- i.e. for all open_canada_id values (_id % 100) > 2 + + NEW._id := nextval('sample_pk_deluxe_seq') * 100 + 99; + NEW._id := NEW._id - (NEW._id - 5) % 97; + END IF; + RETURN NEW; + END IF; + RAISE EXCEPTION E'TAB-DELIMITED\t%', array_to_string(errors, E'\t'); + END; + + examples: + record: + year: 2026 + value: 42 + filter_one: + year: 2026 + sort: year desc + +excel_edge_style: + PatternFill: + fgColor: FF336B87 + patternType: solid +excel_header_style: + PatternFill: + patternType: solid + fgColor: FF6832e3 +excel_column_heading_style: + PatternFill: + patternType: solid + fgColor: FFEFEFEF diff --git a/ckanext/recombinant/views.py b/ckanext/recombinant/views.py index 925f7ced..804e23cb 100644 --- a/ckanext/recombinant/views.py +++ b/ckanext/recombinant/views.py @@ -300,12 +300,13 @@ def template(dataset_type: str, lang: str, owner_org: str) -> Union[Response, st except NotFound: return abort(404, _('Not found')) - try: - book = excel_template(dataset_type, org) - except RecombinantException as e: - return abort(400, _('Unable to download template.\n%s') % e) + if request.method == 'GET': + try: + book = excel_template(dataset_type, org) + except RecombinantException as e: + return abort(400, _('Unable to download template.\n%s') % e) - if request.method == 'POST': + elif request.method == 'POST': filters = {} resource_name = request.form.get('resource_name', '') for r in dataset['resources']: @@ -320,6 +321,10 @@ def template(dataset_type: str, lang: str, owner_org: str) -> Union[Response, st chromo = get_chromo(resource['name']) record_data = [] + edit_using__id = bool(chromo.get('edit_using__id')) + if edit_using__id: + pk_fields = [{'datastore_id': '_id'}] + for keys in primary_keys: temp = keys.split(",") for f, pkf in zip(temp, pk_fields): @@ -333,7 +338,13 @@ def template(dataset_type: str, lang: str, owner_org: str) -> Union[Response, st record_data += result['records'] try: - append_data(book, record_data, chromo) + book = excel_template( + dataset_type, org, edit_using__id, len(record_data)) + except RecombinantException as e: + return abort(400, _('Unable to download template.\n%s') % e) + + try: + append_data(book, record_data, chromo, edit_using__id) except RecombinantFieldError as e: h.flash_error(render('recombinant/snippets/outdated_error.html', extra_vars={'key_errors': str(e).replace("'", ''), @@ -792,7 +803,7 @@ def _process_upload_file(lc: LocalCKAN, try: while True: try: - sheet_name, org_name, column_names, rows = next(upload_data) + sheet_name, org_name, column_names, rows, method = next(upload_data) except StopIteration: break except BadExcelData as e: @@ -841,6 +852,12 @@ def _process_upload_file(lc: LocalCKAN, expected_columns = [f['datastore_id'] for f in chromo['fields'] if f.get('import_template_include', True) and not f.get('published_resource_computed_field')] + if method == 'update': + expected_columns = ['_id'] + expected_columns + pk = ['_id'] + else: + pk = chromo.get('datastore_primary_key', []) + if column_names != expected_columns: raise BadExcelData( _("This template is out of date. " @@ -850,7 +867,6 @@ def _process_upload_file(lc: LocalCKAN, "{support} so we may investigate.").format( support=h.support_email_address())) - pk = chromo.get('datastore_primary_key', []) choice_fields = { f['datastore_id']: 'full' if f.get('excel_full_text_choices') else True @@ -859,12 +875,13 @@ def _process_upload_file(lc: LocalCKAN, records = get_records( rows, - [f for f in chromo['fields'] if f.get( + ([{'datastore_id': '_id', 'datastore_type': 'int'}] + if method == 'update' else []) + + [f for f in chromo['fields'] if f.get( 'import_template_include', True) and not f.get( 'published_resource_computed_field')], pk, choice_fields) - method = 'upsert' if pk else 'insert' total_records += len(records) if not records: continue @@ -896,9 +913,12 @@ def _process_upload_file(lc: LocalCKAN, else: key = _('unknown') raise RecombinantFieldError(key) - else: + elif 'records' in e.error_dict: # type_ignore_reason: incomplete typing pgerror = e.error_dict['records'][0] # type: ignore + elif 'key' in e.error_dict: + # type_ignore_reason: incomplete typing + pgerror = e.error_dict['key'][0] # type: ignore if isinstance(pgerror, dict): pgerror = '; '.join( (h.recombinant_language_text( diff --git a/ckanext/recombinant/write_excel.py b/ckanext/recombinant/write_excel.py index dc01d13c..bd975998 100644 --- a/ckanext/recombinant/write_excel.py +++ b/ckanext/recombinant/write_excel.py @@ -115,10 +115,15 @@ def excel_template(dataset_type: str, - org: Dict[str, Any]) -> Workbook: + org: Dict[str, Any], + edit_using__id: bool = False, + edit_rows: Optional[int] = None) -> Workbook: """ return an openpyxl.Workbook object containing the sheet and header fields for passed dataset_type and org. Supports version 3 templates. + + if `edit_using__id` is True the template will include a read-only `_id` + column and can only be used for editing existing records. """ geno = get_geno(dataset_type) version = geno.get('template_version', DEFAULT_TEMPLATE_VERSION) @@ -138,7 +143,8 @@ def excel_template(dataset_type: str, if version == 3: _append_resource_ref_header(geno, refs, rnum) choice_ranges.append(_populate_excel_sheet( - book, sheet, geno, chromo, org, refs, rnum)) + book, sheet, geno, chromo, org, refs, rnum, + edit_using__id, edit_rows)) sheet.protection.enabled = SHEET_PROTECTION sheet.protection.formatRows = False sheet.protection.formatColumns = False @@ -172,15 +178,23 @@ def excel_template(dataset_type: str, def append_data(book: Workbook, record_data: List[Dict[str, Any]], - chromo: Dict[str, Any]): + chromo: Dict[str, Any], + edit_using__id: bool = False): """ fills rows of an openpyxl.Workbook with selected data from a datastore resource """ # type_ignore_reason: incomplete typing sheet: Worksheet = book[chromo['resource_name']] + + if edit_using__id: + # make RPAD column wider to use for record _id values + sheet.column_dimensions[RPAD_COL].width = REF_KEY_WIDTH + current_row = DATA_FIRST_ROW for record in record_data: + if edit_using__id: + sheet.cell(row=current_row, column=RPAD_COL_NUM).value = record['_id'] for col_num, field in template_cols_fields(chromo): if field['datastore_id'] not in record: raise RecombinantFieldError(field['datastore_id']) @@ -333,7 +347,10 @@ def _populate_excel_sheet(book: Workbook, chromo: Dict[str, Any], org: Dict[str, Any], refs: List[Tuple[Optional[str], List[Any]]], - resource_num: int) -> Dict[str, Any]: + resource_num: int, + edit_using__id: bool, + edit_rows: Optional[int], + ) -> Dict[str, Any]: """ Format openpyxl sheet for the resource definition chromo and org. (Version 3) @@ -347,7 +364,10 @@ def _populate_excel_sheet(book: Workbook, sheet.title = chromo['resource_name'] cranges = {} - data_num_rows = chromo.get('excel_data_num_rows', DEFAULT_DATA_NUM_ROWS) + if edit_using__id and edit_rows is not None: + data_num_rows = edit_rows + else: + data_num_rows = chromo.get('excel_data_num_rows', DEFAULT_DATA_NUM_ROWS) required_style = dict( dict(DEFAULT_EDGE_STYLE, **geno.get('excel_edge_style', {})), @@ -366,28 +386,48 @@ def _populate_excel_sheet(book: Workbook, for i in range(1, DATA_FIRST_ROW + data_num_rows): sheet.cell(row=i, column=1).value = None - sheet.merge_cells(EXAMPLE_MERGE) - fill_cell(sheet, EXAMPLE_ROW, 1, _('e.g.'), 'reco_example') + if edit_using__id: + sheet.row_dimensions[EXAMPLE_ROW].hidden = True + fill_cell( + sheet, + CHEADINGS_ROW, + RPAD_COL_NUM, + _('Record ID'), + 'reco_cheading') + else: + sheet.merge_cells(EXAMPLE_MERGE) + fill_cell(sheet, EXAMPLE_ROW, 1, _('e.g.'), 'reco_example') - fill_cell( - sheet, - DATA_FIRST_ROW, - RPAD_COL_NUM, - '=IF(r{rnum}!{col}{row},"","▶")'.format( - rnum=resource_num, - col=RPAD_COL, - row=DATA_FIRST_ROW), - TYPE_HERE_STYLE) + fill_cell( + sheet, + DATA_FIRST_ROW, + RPAD_COL_NUM, + '=IF(r{rnum}!{col}{row},"","▶")'.format( + rnum=resource_num, + col=RPAD_COL, + row=DATA_FIRST_ROW), + TYPE_HERE_STYLE) + + restriction = '' + sig = 'v3' + if chromo.get('edit_using__id'): + # template is update-only or insert-only + restriction = ' \N{em dash} ' + ( + _('Edit existing records') if edit_using__id + else _('Create new records')) + sig = 'v3-update' if edit_using__id else 'v3-insert' fill_cell( sheet, HEADER_ROW, DATA_FIRST_COL_NUM, recombinant_language_text(chromo['title']) - + ' \N{em dash} ' + org_title_lang_hack(org['title']), + + ' \N{em dash} ' + org_title_lang_hack(org['title']) + + restriction, 'reco_header') - sheet.cell(row=CODE_ROW, column=1).value = 'v3' # template version + # template version + sheet.cell(row=CODE_ROW, column=1).value = sig # allow only upload to this org sheet.cell(row=CODE_ROW, column=2).value = org['name']