From 13122bf62f1a023da5fddce3dd9a6c15f6814904 Mon Sep 17 00:00:00 2001 From: jm Date: Sat, 16 Feb 2013 20:59:02 +0100 Subject: [PATCH] UFAL implementation of OAI-PMH for both importing (with ORE if supported) and exporting (proof of concept) --- .gitignore | 4 + metashare/media/css/screen.css | 19 + metashare/oai_pmh/__init__.py | 480 +++++++++++++++++ metashare/oai_pmh/_metadatareader.py | 25 + metashare/oai_pmh/_utils.py | 79 +++ metashare/oai_pmh/oaipmh/__init__.py | 1 + metashare/oai_pmh/oaipmh/client.py | 371 ++++++++++++++ metashare/oai_pmh/oaipmh/common.py | 192 +++++++ metashare/oai_pmh/oaipmh/datestamp.py | 81 +++ metashare/oai_pmh/oaipmh/error.py | 61 +++ metashare/oai_pmh/oaipmh/interfaces.py | 170 ++++++ metashare/oai_pmh/oaipmh/metadata.py | 109 ++++ metashare/oai_pmh/oaipmh/server.py | 485 ++++++++++++++++++ metashare/oai_pmh/oaipmh/validation.py | 88 ++++ metashare/repository/forms.py | 48 ++ metashare/repository/urls.py | 6 + metashare/repository/views.py | 174 +++++++ metashare/templates/header.html | 4 + .../templates/repository/oai_pmh_export.html | 32 ++ .../templates/repository/oai_pmh_import.html | 105 ++++ 20 files changed, 2534 insertions(+) create mode 100644 metashare/oai_pmh/__init__.py create mode 100644 metashare/oai_pmh/_metadatareader.py create mode 100644 metashare/oai_pmh/_utils.py create mode 100644 metashare/oai_pmh/oaipmh/__init__.py create mode 100644 metashare/oai_pmh/oaipmh/client.py create mode 100644 metashare/oai_pmh/oaipmh/common.py create mode 100644 metashare/oai_pmh/oaipmh/datestamp.py create mode 100644 metashare/oai_pmh/oaipmh/error.py create mode 100644 metashare/oai_pmh/oaipmh/interfaces.py create mode 100644 metashare/oai_pmh/oaipmh/metadata.py create mode 100644 metashare/oai_pmh/oaipmh/server.py create mode 100644 metashare/oai_pmh/oaipmh/validation.py create mode 100644 metashare/templates/repository/oai_pmh_export.html create mode 100644 metashare/templates/repository/oai_pmh_import.html diff --git a/.gitignore b/.gitignore index 42b7eeb1d..64e1e1c7b 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,7 @@ lighttpd.pid lighttpd/ opt/ ._* +metashare/store +*.bat +*.idea +metashare-git-vidiecan \ No newline at end of file diff --git a/metashare/media/css/screen.css b/metashare/media/css/screen.css index ddbe58cf5..047800b72 100644 --- a/metashare/media/css/screen.css +++ b/metashare/media/css/screen.css @@ -791,3 +791,22 @@ table.result_table tr.odd{ } +table.oai_table tbody tr td { + overflow: auto; + max-width: 500px !important; +} + +table.oai_table tbody tr:nth-child(even) { + background: #efefef; +} + +input.clickable : hover { + cursor: pointer; + cursor: hand; +} + +.simplebutton { + white-space:nowrap; + width:125px !important; +} + diff --git a/metashare/oai_pmh/__init__.py b/metashare/oai_pmh/__init__.py new file mode 100644 index 000000000..b8e0a4869 --- /dev/null +++ b/metashare/oai_pmh/__init__.py @@ -0,0 +1,480 @@ +# coding=utf-8 +# pylint: + +""" + OAI-PMH module. + + @author: jm (UFAL, Charles University in Prague) + @www: http://ufal-point.mff.cuni.cz + @version: 0.2 +""" + +import sys +import os +import logging +import re +import codecs +from zipfile import ZipFile +from _utils import * +from collections import OrderedDict +from datetime import datetime +import oaipmh.error +from oai_pmh.oaipmh import common + + +# ensure local libs get loaded +sys.path.append(os.path.dirname(os.path.abspath(__file__))) +# setup logging support +#noinspection PyBroadException +try: + _logger = logging.getLogger(__name__) + from metashare.settings import LOG_HANDLER + _logger.addHandler(LOG_HANDLER) +except: + pass + + +#noinspection PyUnresolvedReferences +def check_system(): + """ + Check the system for support of oai-pmh. + """ + try: + from lxml import etree + except ImportError: + _logger.warning(u"Install lxml") + return False, u"Please, install lxml" + + try: + from oaipmh.client import Client + except ImportError: + _logger.warning(u"Install pyoai") + return False, u"Please, install pyoai" + + return True, None + + +#========================== +# Interface helpers to pyoai +#========================== + +def _client( url_str, registry=None ): + """ + """ + from oaipmh.client import Client + client = Client(url_str, registry) + return client + + +def _reader( metadata_str ): + """ + + """ + from oaipmh.metadata import MetadataRegistry + registry = MetadataRegistry() + + if "dc" in metadata_str: + from oaipmh.metadata import oai_dc_reader + registry.registerReader(metadata_str, oai_dc_reader) + + elif "metashare" in metadata_str: + from _metadatareader import metashare_schema_reader + registry.registerReader(metadata_str, metashare_schema_reader()) + + elif "ore" in metadata_str: + from _metadatareader import metashare_schema_reader + registry.registerReader(metadata_str, metashare_schema_reader()) + + else: + raise NotImplementedError("Metadata reading not implemented for [%s]" % metadata_str) + + return registry + + +#========================== +# The actual work +#========================== + +def list_records( kwargs ): + """ + List all records. + @required: url, metadata_str + @return: dict of displayable items + """ + #noinspection PyUnresolvedReferences + from lxml import etree + import cgi + + url_str, metadata_str = params(kwargs, ("url", "metadata_str")) + client = _client(url_str, _reader(metadata_str)) + d = {} + for header, metadata, _1 in client.listRecords(metadataPrefix=metadata_str): + data_str = prehtmlify(metadata.getMap() if not metadata is None else {}, + add_pre=False) + data_str = data_str.replace(u"\\n", u"\n").splitlines() + data_str = [cgi.escape(x) for x in data_str if not 0 == len(x.strip())] + d[header.identifier()] = u"
".join(data_str) + return d + + +def list_sets( kwargs ): + """ + List all sets. + @required: url + @return: dict of displayable items + """ + url_str = params(kwargs, ("url",)) + client = _client(url_str) + d = {} + for spec, name, description in client.listSets(): + d[spec] = prehtmlify(( + u"name: %s" % name, + u"spec: %s" % spec, + u"description: %s" % description, + )) + return d + + +def list_formats( kwargs ): + """ + List all metadata formats. + @required: url + @return: dict of displayable items + """ + url_str = params(kwargs, ("url",)) + client = _client(url_str) + d = {} + for prefix, schema, ns in client.listMetadataFormats(): + d[prefix] = prehtmlify(( + u"prefix: %s" % prefix, + u"schema: %s" % schema, + u"ns: %s" % ns, + )) + return d + + +def list_identifiers( kwargs ): + """ + List all metadata identifiers. + @required: url, metadata_str + @return: dict of displayable items + """ + url_str, metadata_str = params(kwargs, ("url", "metadata_str")) + client = _client(url_str) + d = {} + for id_ in client.listIdentifiers(metadataPrefix=metadata_str): + id_str = id_.setSpec() + if isinstance(id_str, (tuple, list)): + id_str = id_str[0] + d[id_.identifier()] = id_str + return d + + +def identify( kwargs ): + """ + List server identification. + @required: url, metadata_str + @return: dict of displayable items + """ + url_str = params(kwargs, ("url",)) + client = _client(url_str) + d = {} + id_ = client.identify() + d["Name"] = id_.repositoryName() + d["Url"] = id_.baseURL() + d["Protocol ver."] = id_.protocolVersion() + d["Admin emails"] = id_.adminEmails() + return d + + +def import_id( kwargs ): + """ + Import selected set id. + + What it does? + 1) get the settings of what we want + 2) get xml string of metashare metadata + 3) do the metadata import + 4) get ORE files + 5) copy the file to the proper place and update metashare node + + + @required: url, metadata_str, itemid + @return: dict of displayable items + + """ + from metashare.xml_utils import import_from_string + from metashare.storage.models import PUBLISHED, MASTER + import metashare.storage_admin + + ## + # 1. settings + # + save_to_dir = None + # noinspection PyBroadException + try: + # noinspection PyUnresolvedReferences + from metashare.settings import OAI_PMH_IMPORT_SAVE_DIR + save_to_dir = OAI_PMH_IMPORT_SAVE_DIR + if not os.path.exists(save_to_dir): + os.makedirs(save_to_dir) + except: + pass + + ## + # 2. get metashare metadata + # + OK_MSG = u"OK" + url_str, metadata_str, itemid, import_from_dir = \ + params(kwargs, ("url", "metadata_str", "itemid", "import_from_dir")) + if 0 == len(itemid.strip()): + itemid = None + else: + itemid = u"/".join(itemid.split(u"/")[-2:]) + + # what else than specific metashare? + if not "metashare" in metadata_str: + return {"Error": "how shall i import? (you need specific metashare metadata profile)"} + + ret_d = OrderedDict() + # make metashare friendly + re_subs = ( re.compile(u"^]+>", re.M | re.U), + re.compile(u"$", re.M | re.U) ) + + # + # + re_id = re.compile(u".*([^<]+).*", + re.MULTILINE | re.DOTALL | re.UNICODE) + re_sub_id = re.compile(u"().*()", re.UNICODE) + + # Metashare glitch + def _id_from_meta( raw_xml_record ): + m = re_id.search(raw_xml_record) + return m.group(1) if m else None + + from metashare.storage.models import StorageObject + + storage_objects = StorageObject.objects.all() + so_map = {} + for so in storage_objects: + so_map[_id_from_meta(so.metadata)] = so + + def _find_id_in_current_db( raw_xml_record ): + id_str = _id_from_meta(raw_xml_record) + return so_map[id_str] if id_str in so_map else None + + pos = 0 + client = _client(url_str, _reader(metadata_str)) + for header, metadata, _2 in client.listRecords(metadataPrefix=metadata_str): + pos += 1 + try: + if header.isDeleted(): + ret_d[header.identifier()] = html_mark_warning(u"This record is deleted - delete it manually!!!"), \ + header.identifier(), \ + None + continue + + # try to publish it on this node + raw_xml_record = metadata.getField("raw_xml") + for re_sub in re_subs: + raw_xml_record = re_sub.sub(u"", raw_xml_record) + + if not itemid is None and not itemid in header.identifier(): + _logger.info(u"Skipping [%s]", header.identifier()) + continue + + # e.g., for debugging + if not save_to_dir is None: + with codecs.open(os.path.join(save_to_dir, "UFAL-metashare-%s.xml" % pos), + mode="w+", encoding="utf-8") as fout: + fout.write(raw_xml_record) + + # 3. do the import + # + # 2.5 find the id and check if we already have it + obj = _find_id_in_current_db(raw_xml_record) + if obj: + raw_xml_record = re_sub_id.sub(u"%s" % obj.identifier, + raw_xml_record, count=1) + _logger.info(u"Item [%s]->[%s] found in database", + _id_from_meta(raw_xml_record), obj.identifier) + + resource = import_from_string(raw_xml_record, PUBLISHED, MASTER) + ret_d[header.identifier()] = OK_MSG, \ + repr(resource), \ + resource.storage_object._storage_folder() + except Exception, e: + ret_d[header.identifier()] = html_mark_error(u"Failed!"), \ + html_mark_error(repr(e)), \ + None + + # 4. get ORE + # + only_in_ore = u"Found item in ORE but not in metadata format" + ret_d[only_in_ore] = [] + + metadata_str = "ore" + client = _client(url_str, _reader(metadata_str)) + if import_from_dir: + for header, metadata, _2 in client.listRecords(metadataPrefix=metadata_str): + + if not header.identifier() in ret_d or metadata is None: + ret_d[only_in_ore].append(header.identifier()) + continue + + # noinspection PyBroadException,PyUnusedLocal + try: + raw_xml_record = metadata.getField("raw_xml") + info, resource, import_to_path = ret_d[header.identifier()] + + if OK_MSG != info: + continue + + # find the path + imported_storage_files = re.compile(u"([^<]+)") \ + .findall(raw_xml_record) + imported_storage_files = [os.path.join(import_from_dir, x) for x in imported_storage_files] + for imported_storage_file in imported_storage_files: + if not os.path.exists(imported_storage_file): + warn_msg = u" | Could not find imported file [%s]" % imported_storage_file + ret_d[header.identifier()] = info + warn_msg, \ + resource + raw_xml_record, \ + import_to_path + imported_storage_files = [x for x in imported_storage_files if os.path.exists(x)] + + # hardcoded in storage/models.py + new_zip_path = os.path.join(import_to_path, 'archive.zip') + # we know where, we know from where + if os.path.exists(new_zip_path): + warn_msg = u" | Removed existing zip file" + ret_d[header.identifier()] = info + warn_msg, \ + resource + raw_xml_record, \ + import_to_path + safe_remove(new_zip_path) + + # 5. copy and update + # + with ZipFile(new_zip_path, 'w') as new_zip: + for f in imported_storage_files: + new_zip.write(f, arcname=os.path.basename(f)) + + # all is where it should be and now update + object_id = os.path.basename(os.path.dirname(new_zip_path)) + # Metashare glitch + #noinspection PyUnresolvedReferences + metashare.storage_admin.StorageObject = StorageObject + #noinspection PyUnresolvedReferences + metashare.storage_admin.checksum(object_id) + except Exception, e: + _logger.exception( u"Got exception while importing" ) + + # make html friendly + for k, v in ret_d.iteritems(): + ret_d[k] = prehtmlify(v) + + return ret_d + + +class MetashareOaiServer(object): + """ + The metashare implementation of the OAI verbs. + + This is the first shot on a real implementation which is left for + the original authors; however, the basics and the api usage + is shown below. + """ + + def __init__( self, env ): + """ + Settings, required arguments are `name`, `url`, `adminEmails`. + """ + self.env = env + + def identify(self): + """ + OAI-PMH ?verb=identify. + See http://www.openarchives.org/OAI/openarchivesprotocol.html#Identify + """ + return common.Identify( + repositoryName=self.env["name"], + baseURL=self.env["url"], + protocolVersion="2.0", + adminEmails=self.env["adminEmails"], + earliestDatestamp=datetime(2004, 1, 1), + deletedRecord='no', + granularity='YYYY-MM-DDThh:mm:ssZ', + compression=['identity']) + + def getRecord(self, metadataPrefix, identifier): + """ + TODO! one item is composed of + (common.Header(str(i), datestamp, '', False), + common.Metadata({'title': ['Title %s' % i]}), + None) # about + + """ + try: + return [] + except IndexError: + raise oaipmh.error.IdDoesNotExistError, "Id does not exist: %s" % identifier + + def listIdentifiers(self, metadataPrefix=None, from_=None, until=None, set=None): + """ + TODO! check metadataPrefix + """ + from metashare.storage.models import StorageObject + + # get all objects according to id + re_id = re.compile(u".*([^<]+).*", + re.MULTILINE | re.DOTALL | re.UNICODE) + + def _id_from_meta( raw_xml_record ): + m = re_id.search(raw_xml_record) + return m.group(1) if m else None + + storage_objects = StorageObject.objects.all() + so_map = {} + result = [] + for so in storage_objects: + i = _id_from_meta(so.metadata) + so_map[i] = so + result.append( + common.Header(str(i), so.created, '', False) ) + return result + + def listRecords(self, metadataPrefix=None, from_=None, until=None, set=None): + """ + TODO! + resource = get_object_or_404(resourceInfoType_model, + storage_object__identifier=object_id, + storage_object__publication_status=PUBLISHED) + """ + result = [] + return result + + +#========================== +# Supported commands +#========================== + +key_list_ids_for_import = u"list identifiers (use for import)" +key_import = u"Import the resource(s)" +key_list_metadata_format = u"list formats" + +supported_commands = OrderedDict() +supported_commands[key_import] = import_id +supported_commands[key_list_ids_for_import] = list_identifiers + +# available as separate buttons +supported_commands.update({ + u"list records": list_records, + u"identify server": identify, + u"list sets": list_sets, + key_list_metadata_format: list_formats, +}) + + +# +# +if __name__ == "__main__": + pass diff --git a/metashare/oai_pmh/_metadatareader.py b/metashare/oai_pmh/_metadatareader.py new file mode 100644 index 000000000..9fd8028dd --- /dev/null +++ b/metashare/oai_pmh/_metadatareader.py @@ -0,0 +1,25 @@ +# coding=utf-8 +# pylint: + +""" + OAI-PMH metashare scheme (specific) reader. + + @author: jm (UFAL, Charles University in Prague) + @www: http://ufal-point.mff.cuni.cz + @version: 0.1 +""" + +from oaipmh import common +from lxml import etree + + +class metashare_schema_reader( object ): + """ + An implementation of a reader. + """ + def __init__(self): + pass + + def __call__( self, lxml_elem ): + xml = etree.tostring( lxml_elem, pretty_print=True ) + return common.Metadata( { "raw_xml": xml } ) diff --git a/metashare/oai_pmh/_utils.py b/metashare/oai_pmh/_utils.py new file mode 100644 index 000000000..d25097904 --- /dev/null +++ b/metashare/oai_pmh/_utils.py @@ -0,0 +1,79 @@ +# coding=utf-8 +# pylint: + +""" + Helpers. + + @author: jm (UFAL, Charles University in Prague) + @www: http://ufal-point.mff.cuni.cz + @version: 0.1 +""" + +import os +import json +import time + + +def params( d, param_list ): + """ + Check parameters and return the values or raise + AttributeError if missing. + """ + for param in d: + if not param in d: + raise AttributeError("Missing [%s] parameter", param) + if 1 == len(param_list): + return d[param_list[0]] + else: + return [d[param] for param in param_list] + + +def prehtmlify( obj, add_pre=True ): + """ + Make object nicely displayable in 
.
+    """
+    if not obj is None:
+        str_ = u"%s" % json.dumps(obj, indent=2)
+        return u"
%s
" % str_ \
+            if add_pre else str_
+    return u"Invalid"
+
+
+def smart_extend( dict_inst, *dict_arr ):
+    """
+        Recursively add items at every level to
+        dict_inst rather than simply update.
+    """
+    for a in dict_arr:
+        dict_inst.update(a)
+    return dict_inst
+
+
+def time_it( func, *args, **kwargs ):
+    """
+        Time a function.
+        Alternative implement as a decorator.
+    """
+    t1 = time.time()
+    ret = func(*args, **kwargs)
+    t2 = time.time()
+    return t2 - t1, ret
+
+
+# noinspection PyBroadException
+def safe_remove( file_str ):
+    """ Do not raise if not successful. """
+    try:
+        os.remove(file_str)
+    except:
+        pass
+
+
+def html_mark_error( s ):
+    """ Simple mark error using html and css. """
+    return u"
%s
" % s
+
+
+def html_mark_warning( s ):
+    """ Simple mark warning using html and css. """
+    return u"
%s
" % s
diff --git a/metashare/oai_pmh/oaipmh/__init__.py b/metashare/oai_pmh/oaipmh/__init__.py
new file mode 100644
index 000000000..1e0ca6a5f
--- /dev/null
+++ b/metashare/oai_pmh/oaipmh/__init__.py
@@ -0,0 +1 @@
+# 
diff --git a/metashare/oai_pmh/oaipmh/client.py b/metashare/oai_pmh/oaipmh/client.py
new file mode 100644
index 000000000..72ce5e28b
--- /dev/null
+++ b/metashare/oai_pmh/oaipmh/client.py
@@ -0,0 +1,371 @@
+# Copyright 2003, 2004, 2005 Infrae
+# Released under the BSD license (see LICENSE.txt)
+from __future__ import nested_scopes
+import urllib2
+import base64
+from urllib import urlencode
+from StringIO import StringIO
+from types import SliceType
+from lxml import etree
+import time
+
+from oaipmh import common, metadata, validation, error
+from oaipmh.datestamp import datestamp_to_datetime, datetime_to_datestamp
+
+WAIT_DEFAULT = 120 # two minutes
+WAIT_MAX = 5
+
+class Error(Exception):
+    pass
+
+class BaseClient(common.OAIPMH):
+
+    def __init__(self, metadata_registry=None):
+        self._metadata_registry = (
+            metadata_registry or metadata.global_metadata_registry)
+        self._ignore_bad_character_hack = 0
+        self._day_granularity = False
+
+    def updateGranularity(self):
+        """Update the granularity setting dependent on that the server says.
+        """
+        identify = self.identify()
+        granularity = identify.granularity()
+        if granularity == 'YYYY-MM-DD':
+            self._day_granularity = True
+        elif granularity == 'YYYY-MM-DDThh:mm:ssZ':
+            self._day_granularity= False
+        else:
+            raise Error, "Non-standard granularity on server: %s" % granularity
+            
+    def handleVerb(self, verb, kw):
+        # validate kw first
+        validation.validateArguments(verb, kw)
+        # encode datetimes as datestamps
+        from_ = kw.get('from_')
+        if from_ is not None:
+            # turn it into 'from', not 'from_' before doing actual request
+            kw['from'] = datetime_to_datestamp(from_,
+                                               self._day_granularity)
+        if 'from_' in kw:
+            # always remove it from the kw, no matter whether it be None or not
+            del kw['from_']
+    
+        until = kw.get('until')
+        if until is not None:
+            kw['until'] = datetime_to_datestamp(until,
+                                                self._day_granularity)
+        elif 'until' in kw:
+            # until is None but is explicitly in kw, remove it
+            del kw['until']
+        
+        # now call underlying implementation
+        method_name = verb + '_impl'
+        return getattr(self, method_name)(
+            kw, self.makeRequestErrorHandling(verb=verb, **kw))    
+
+    def getNamespaces(self):
+        """Get OAI namespaces.
+        """
+        return {'oai': 'http://www.openarchives.org/OAI/2.0/'}
+
+    def getMetadataRegistry(self):
+        """Return the metadata registry in use.
+
+        Do we want to allow the returning of the global registry?
+        """
+        return self._metadata_registry
+
+    def ignoreBadCharacters(self, true_or_false): 	 
+        """Set to ignore bad characters in UTF-8 input. 	 
+        This is a hack to get around well-formedness errors of 	 
+        input sources which *should* be in UTF-8 but for some reason 	 
+        aren't completely. 	 
+        """ 	 
+        self._ignore_bad_character_hack = true_or_false 	 
+
+    def parse(self, xml): 	 
+        """Parse the XML to a lxml tree. 	 
+        """
+        # XXX this is only safe for UTF-8 encoded content, 	 
+        # and we're basically hacking around non-wellformedness anyway,
+        # but oh well
+        if self._ignore_bad_character_hack: 	 
+            xml = unicode(xml, 'UTF-8', 'replace') 	 
+            # also get rid of character code 12 	 
+            xml = xml.replace(chr(12), '?')
+            xml = xml.encode('UTF-8')
+        return etree.XML(xml)
+
+    def GetRecord_impl(self, args, tree):
+        records, token = self.buildRecords(
+            args['metadataPrefix'],
+            self.getNamespaces(),
+            self._metadata_registry,
+            tree
+            )
+        assert token is None
+        return records[0]
+
+    # implementation of the various methods, delegated here by
+    # handleVerb method
+    
+    def Identify_impl(self, args, tree):
+        namespaces = self.getNamespaces()
+        evaluator = etree.XPathEvaluator(tree, namespaces=namespaces)
+        identify_node = evaluator.evaluate(
+            '/oai:OAI-PMH/oai:Identify')[0]
+        identify_evaluator = etree.XPathEvaluator(identify_node,
+                                                  namespaces=namespaces)
+        e = identify_evaluator.evaluate
+
+        repositoryName = e('string(oai:repositoryName/text())')
+        baseURL = e('string(oai:baseURL/text())')
+        protocolVersion = e('string(oai:protocolVersion/text())')
+        adminEmails = e('oai:adminEmail/text()')
+        earliestDatestamp = datestamp_to_datetime(
+            e('string(oai:earliestDatestamp/text())'))
+        deletedRecord = e('string(oai:deletedRecord/text())')
+        granularity = e('string(oai:granularity/text())')
+        compression = e('oai:compression/text()')
+        # XXX description
+        identify = common.Identify(
+            repositoryName, baseURL, protocolVersion,
+            adminEmails, earliestDatestamp,
+            deletedRecord, granularity, compression)
+        return identify
+
+    def ListIdentifiers_impl(self, args, tree):
+        namespaces = self.getNamespaces()
+        def firstBatch():
+            return self.buildIdentifiers(namespaces, tree)
+        def nextBatch(token):
+            tree = self.makeRequestErrorHandling(verb='ListIdentifiers',
+                                                 resumptionToken=token)
+            return self.buildIdentifiers(namespaces, tree)
+        return ResumptionListGenerator(firstBatch, nextBatch)
+
+    def ListMetadataFormats_impl(self, args, tree):
+        namespaces = self.getNamespaces()
+        evaluator = etree.XPathEvaluator(tree, 
+                                         namespaces=namespaces)
+
+        metadataFormat_nodes = evaluator.evaluate(
+            '/oai:OAI-PMH/oai:ListMetadataFormats/oai:metadataFormat')
+        metadataFormats = []
+        for metadataFormat_node in metadataFormat_nodes:
+            e = etree.XPathEvaluator(metadataFormat_node, 
+                                     namespaces=namespaces).evaluate
+            metadataPrefix = e('string(oai:metadataPrefix/text())')
+            schema = e('string(oai:schema/text())')
+            metadataNamespace = e('string(oai:metadataNamespace/text())')
+            metadataFormat = (metadataPrefix, schema, metadataNamespace)
+            metadataFormats.append(metadataFormat)
+
+        return metadataFormats
+
+    def ListRecords_impl(self, args, tree):
+        namespaces = self.getNamespaces()
+        metadata_prefix = args['metadataPrefix']
+        metadata_registry = self._metadata_registry
+        def firstBatch():
+            return self.buildRecords(
+                metadata_prefix, namespaces,
+                metadata_registry, tree)
+        def nextBatch(token):
+            tree = self.makeRequestErrorHandling(
+                verb='ListRecords',
+                resumptionToken=token)
+            return self.buildRecords(
+                metadata_prefix, namespaces,
+                metadata_registry, tree)
+        return ResumptionListGenerator(firstBatch, nextBatch)
+
+    def ListSets_impl(self, args, tree):
+        namespaces = self.getNamespaces()
+        def firstBatch():
+            return self.buildSets(namespaces, tree)
+        def nextBatch(token):
+            tree = self.makeRequestErrorHandling(
+                verb='ListSets',
+                resumptionToken=token)
+            return self.buildSets(namespaces, tree)
+        return ResumptionListGenerator(firstBatch, nextBatch)
+
+    # various helper methods
+    
+    def buildRecords(self,
+                     metadata_prefix, namespaces, metadata_registry, tree):
+        # first find resumption token if available
+        evaluator = etree.XPathEvaluator(tree,
+                                         namespaces=namespaces)
+        token = evaluator.evaluate(
+            'string(/oai:OAI-PMH/*/oai:resumptionToken/text())')
+        if token.strip() == '':
+            token = None
+        record_nodes = evaluator.evaluate(
+            '/oai:OAI-PMH/*/oai:record')
+        result = []
+        for record_node in record_nodes:
+            record_evaluator = etree.XPathEvaluator(record_node, 
+                                                    namespaces=namespaces)
+            e = record_evaluator.evaluate
+            # find header node
+            header_node = e('oai:header')[0]
+            # create header
+            header = buildHeader(header_node, namespaces)
+            # find metadata node
+            metadata_list = e('oai:metadata')
+            if metadata_list:
+                metadata_node = metadata_list[0]
+                # create metadata
+                metadata = metadata_registry.readMetadata(metadata_prefix,
+                                                          metadata_node)
+            else:
+                metadata = None
+            # XXX TODO: about, should be third element of tuple
+            result.append((header, metadata, None))
+        return result, token
+
+    def buildIdentifiers(self, namespaces, tree):
+        evaluator = etree.XPathEvaluator(tree, 
+                                         namespaces=namespaces)
+        # first find resumption token is available
+        token = evaluator.evaluate(
+            'string(/oai:OAI-PMH/oai:ListIdentifiers/oai:resumptionToken/text())')
+        if token.strip() == '':
+            token = None    
+        header_nodes = evaluator.evaluate(
+                '/oai:OAI-PMH/oai:ListIdentifiers/oai:header')            
+        result = []
+        for header_node in header_nodes:
+            header = buildHeader(header_node, namespaces)
+            result.append(header)
+        return result, token
+
+    def buildSets(self, namespaces, tree):
+        evaluator = etree.XPathEvaluator(tree, 
+                                         namespaces=namespaces)
+        # first find resumption token if available
+        token = evaluator.evaluate(
+            'string(/oai:OAI-PMH/oai:ListSets/oai:resumptionToken/text())')
+        if token.strip() == '':
+            token = None  
+        set_nodes = evaluator.evaluate(
+            '/oai:OAI-PMH/oai:ListSets/oai:set')
+        sets = []
+        for set_node in set_nodes:
+            e = etree.XPathEvaluator(set_node, 
+                                     namespaces=namespaces).evaluate
+            # make sure we get back unicode strings instead
+            # of lxml.etree._ElementUnicodeResult objects.
+            setSpec = unicode(e('string(oai:setSpec/text())'))
+            setName = unicode(e('string(oai:setName/text())'))
+            # XXX setDescription nodes
+            sets.append((setSpec, setName, None))
+        return sets, token
+
+    def makeRequestErrorHandling(self, **kw):
+        xml = self.makeRequest(**kw)
+        try:
+            tree = self.parse(xml)
+        except SyntaxError:
+            raise error.XMLSyntaxError(kw)
+        # check whether there are errors first
+        e_errors = tree.xpath('/oai:OAI-PMH/oai:error',
+                              namespaces=self.getNamespaces())
+        if e_errors:
+            # XXX right now only raise first error found, does not
+            # collect error info
+            for e_error in e_errors:
+                code = e_error.get('code')
+                msg = e_error.text
+                if code not in ['badArgument', 'badResumptionToken',
+                                'badVerb', 'cannotDisseminateFormat',
+                                'idDoesNotExist', 'noRecordsMatch',
+                                'noMetadataFormats', 'noSetHierarchy']:
+                    raise error.UnknownError,\
+                          "Unknown error code from server: %s, message: %s" % (
+                        code, msg)
+                # find exception in error module and raise with msg
+                raise getattr(error, code[0].upper() + code[1:] + 'Error'), msg
+        return tree
+    
+    def makeRequest(self, **kw):
+        raise NotImplementedError
+    
+class Client(BaseClient):
+    def __init__(
+            self, base_url, metadata_registry=None, credentials=None):
+        BaseClient.__init__(self, metadata_registry)
+        self._base_url = base_url
+        if credentials is not None:
+            self._credentials = base64.encodestring('%s:%s' % credentials)
+        else:
+            self._credentials = None
+            
+    def makeRequest(self, **kw):
+        """Actually retrieve XML from the server.
+        """
+        # XXX include From header?
+        headers = {'User-Agent': 'pyoai'}
+        if self._credentials is not None:
+            headers['Authorization'] = 'Basic ' + self._credentials.strip()
+        request = urllib2.Request(
+            self._base_url, data=urlencode(kw), headers=headers)
+        return retrieveFromUrlWaiting(request)
+
+def buildHeader(header_node, namespaces):
+    e = etree.XPathEvaluator(header_node, 
+                            namespaces=namespaces).evaluate
+    identifier = e('string(oai:identifier/text())')
+    datestamp = datestamp_to_datetime(
+        str(e('string(oai:datestamp/text())')))
+    setspec = [str(s) for s in e('oai:setSpec/text()')]
+    deleted = e("@status = 'deleted'") 
+    return common.Header(identifier, datestamp, setspec, deleted)
+
+def ResumptionListGenerator(firstBatch, nextBatch):
+    result, token = firstBatch()
+    while 1:
+        for item in result:
+            yield item
+        if token is None:
+            break
+        result, token = nextBatch(token)
+
+def retrieveFromUrlWaiting(request,
+                           wait_max=WAIT_MAX, wait_default=WAIT_DEFAULT):
+    """Get text from URL, handling 503 Retry-After.
+    """
+    for i in range(wait_max):
+        try:
+            f = urllib2.urlopen(request)
+            text = f.read()
+            f.close()
+            # we successfully opened without having to wait
+            break
+        except urllib2.HTTPError, e:
+            if e.code == 503:
+                try:
+                    retryAfter = int(e.hdrs.get('Retry-After'))
+                except TypeError:
+                    retryAfter = None
+                if retryAfter is None:
+                    time.sleep(wait_default)
+                else:
+                    time.sleep(retryAfter)
+            else:
+                # reraise any other HTTP error
+                raise
+    else:
+        raise Error, "Waited too often (more than %s times)" % wait_max
+    return text
+
+class ServerClient(BaseClient):
+    def __init__(self, server, metadata_registry=None):
+        BaseClient.__init__(self, metadata_registry)
+        self._server = server
+        
+    def makeRequest(self, **kw):
+        return self._server.handleRequest(kw)
diff --git a/metashare/oai_pmh/oaipmh/common.py b/metashare/oai_pmh/oaipmh/common.py
new file mode 100644
index 000000000..7335b0b2e
--- /dev/null
+++ b/metashare/oai_pmh/oaipmh/common.py
@@ -0,0 +1,192 @@
+import pkg_resources
+
+
+class Header(object):
+    def __init__(self, identifier, datestamp, setspec, deleted):
+        # force identifier to be a string, it might be 
+        # an lxml.etree._ElementStringResult...
+        self._identifier = str(identifier)
+        self._datestamp = datestamp
+        self._setspec = setspec
+        self._deleted = deleted
+        
+    def identifier(self):
+        return self._identifier
+    
+    def datestamp(self):
+        return self._datestamp
+
+    def setSpec(self):
+        return self._setspec
+
+    def isDeleted(self):
+        return self._deleted
+
+class Metadata(object):
+    def __init__(self, map):
+        self._map = map
+
+    def getMap(self):
+        return self._map
+    
+    def getField(self, name):
+        return self._map[name]
+
+    __getitem__ = getField
+
+class Identify(object):
+    def __init__(self, repositoryName, baseURL, protocolVersion, adminEmails,
+                 earliestDatestamp, deletedRecord, granularity, compression,
+                 toolkit_description=True):
+        self._repositoryName = repositoryName
+        self._baseURL = baseURL
+        self._protocolVersion = protocolVersion
+        self._adminEmails = adminEmails
+        self._earliestDatestamp = earliestDatestamp
+        self._deletedRecord = deletedRecord
+        self._granularity = granularity
+        self._compression = compression
+        self._descriptions = []
+        
+        if toolkit_description:
+            req = pkg_resources.Requirement.parse('pyoai')
+            egg = pkg_resources.working_set.find(req)
+            if egg:
+                version = '%s' % egg.version
+            else:
+                version = ''
+            self.add_description(
+                ''
+                'pyoai'
+                '%s'
+                'http://infrae.com/products/oaipack'
+                '' % version)
+        
+    def repositoryName(self):
+        return self._repositoryName
+
+    def baseURL(self):
+        return self._baseURL
+
+    def protocolVersion(self):
+        return self._protocolVersion
+
+    def adminEmails(self):
+        return self._adminEmails
+
+    def earliestDatestamp(self):
+        return self._earliestDatestamp
+
+    def deletedRecord(self):
+        return self._deletedRecord
+
+    def granularity(self):
+        return self._granularity
+
+    def compression(self):
+        return self._compression
+
+    def add_description(self, xml_string):
+        self._descriptions.append(xml_string)
+
+    def descriptions(self):
+        return self._descriptions
+    
+def ResumptionTokenSpec(dict):
+    dict = dict.copy()
+    dict['resumptionToken'] = 'exclusive'
+    return dict
+
+class OAIMethodImpl(object):
+    def __init__(self, verb):
+        self._verb = verb
+        
+    def __call__(self, bound_self, **kw):
+        return bound_self.handleVerb(self._verb, kw)
+        
+def OAIMethod(verb):
+    obj = OAIMethodImpl(verb)
+    def method(self, **kw):
+        return obj(self, **kw)
+    return method
+
+class OAIPMH(object):
+    """Mixin that implements the Python-level OAI-PMH interface.
+
+    It does not include resumptionToken handling.
+    
+    It passes the calls on to the 'handleVerb' method, which should be
+    overridden in a subclass.
+    """
+    def handleVerb(self, verb, kw):
+        raise NotImplementedError
+    
+    getRecord = OAIMethod(
+        'GetRecord',
+        )
+    
+    identify = OAIMethod(
+        'Identify',
+        )
+
+    listIdentifiers = OAIMethod(
+        'ListIdentifiers',
+        )
+
+    listMetadataFormats = OAIMethod(
+        'ListMetadataFormats',
+        )
+
+    listRecords = OAIMethod(
+        'ListRecords',
+        )
+
+    listSets = OAIMethod(
+        'ListSets',
+        )
+    
+class ResumptionOAIPMH(object):
+    """Mixin that implements the Resumption-capable OAI-PMH interface.
+
+    It passes the arguments on to the 'handleVerb' method, which
+    should be overridden in a subclass.
+
+    The listIdentifiers, listSets and listRecords methods return
+    tuples of a list and resumptionToken. If the resumptionToken
+    returned is None, this indicates the end of the list is reached.
+    """
+
+    def handleVerb(self, verb, kw):
+        raise NotImplementedError
+    
+    getRecord = OAIMethod(
+        'GetRecord',
+        )
+    
+    identify = OAIMethod(
+        'Identify',
+        )
+
+    listIdentifiers = OAIMethod(
+        'ListIdentifiers',
+        )
+
+    listMetadataFormats = OAIMethod(
+        'ListMetadataFormats',
+        )
+
+    listRecords = OAIMethod(
+        'ListRecords',
+        )
+
+    listSets = OAIMethod(
+        'ListSets',
+        )
+
+def getMethodForVerb(server, verb):
+    return getattr(server, verb[0].lower() + verb[1:])
+
diff --git a/metashare/oai_pmh/oaipmh/datestamp.py b/metashare/oai_pmh/oaipmh/datestamp.py
new file mode 100644
index 000000000..f109cccfd
--- /dev/null
+++ b/metashare/oai_pmh/oaipmh/datestamp.py
@@ -0,0 +1,81 @@
+import datetime
+from oaipmh.error import DatestampError
+
+def datetime_to_datestamp(dt, day_granularity=False):
+    assert dt.tzinfo is None # only accept timezone naive datetimes
+    # ignore microseconds
+    dt = dt.replace(microsecond=0)
+    result = dt.isoformat() + 'Z'
+    if day_granularity:
+        result = result[:-10]
+    return result
+
+# handy utility function not used by pyoai itself yet
+def date_to_datestamp(d, day_granularity=False): 	 
+    return datetime_to_datestamp( 	 
+        datetime.datetime.combine(d, datetime.time(0)), day_granularity)
+
+def datestamp_to_datetime(datestamp, inclusive=False):
+    try:
+        return _datestamp_to_datetime(datestamp, inclusive)
+    except ValueError:
+        raise DatestampError(datestamp)
+    
+def _datestamp_to_datetime(datestamp, inclusive=False):
+    splitted = datestamp.split('T')
+    if len(splitted) == 2:
+        d, t = splitted
+        if not t or t[-1] != 'Z':
+            raise DatestampError(datestamp)
+        # strip off 'Z'
+        t = t[:-1]
+    else:
+        d = splitted[0]
+        if inclusive:
+            # used when a date was specified as ?until parameter
+            t = '23:59:59'
+        else:
+            t = '00:00:00'
+    YYYY, MM, DD = d.split('-')
+    hh, mm, ss = t.split(':') # this assumes there's no timezone info
+    return datetime.datetime(
+        int(YYYY), int(MM), int(DD), int(hh), int(mm), int(ss))
+
+def tolerant_datestamp_to_datetime(datestamp):
+    """A datestamp to datetime that's more tolerant of diverse inputs.
+
+    Not used inside pyoai itself right now, but can be used when defining
+    your own metadata schema if that has a broader variety of datetimes
+    in there.
+    """
+    splitted = datestamp.split('T')
+    if len(splitted) == 2:
+        d, t = splitted
+        # if no Z is present, raise error
+        if t[-1] != 'Z':
+            raise DatestampError(datestamp)
+        # split off Z at the end
+        t = t[:-1]
+    else:
+        d = splitted[0]
+        t = '00:00:00'
+    d_splitted = d.split('-')
+    if len(d_splitted) == 3:
+        YYYY, MM, DD = d_splitted
+    elif len(d_splitted) == 2:
+        YYYY, MM = d_splitted
+        DD = '01'
+    elif len(d_splitted) == 1:
+        YYYY = d_splitted[0]
+        MM = '01'
+        DD = '01'   
+    else:
+        raise DatestampError(datestamp)
+    
+    t_splitted = t.split(':')
+    if len(t_splitted) == 3:
+        hh, mm, ss = t_splitted
+    else:
+        raise DatestampError(datestamp)
+    return datetime.datetime(
+        int(YYYY), int(MM), int(DD), int(hh), int(mm), int(ss))
diff --git a/metashare/oai_pmh/oaipmh/error.py b/metashare/oai_pmh/oaipmh/error.py
new file mode 100644
index 000000000..ddf1a0cff
--- /dev/null
+++ b/metashare/oai_pmh/oaipmh/error.py
@@ -0,0 +1,61 @@
+
+class ErrorBase(Exception):
+    def oainame(self):
+        name = self.__class__.__name__
+        # strip off 'Error' part
+        name = name[:-5]
+        # lowercase error name
+        name = name[0].lower() + name[1:]
+        return name
+
+class BadArgumentError(ErrorBase):
+    pass
+
+class BadVerbError(ErrorBase):
+    pass
+
+class BadResumptionTokenError(ErrorBase):
+    pass
+
+class CannotDisseminateFormatError(ErrorBase):
+    pass
+
+class IdDoesNotExistError(ErrorBase):
+    pass
+
+class NoRecordsMatchError(ErrorBase):
+    pass
+
+class NoMetadataFormatsError(ErrorBase):
+    pass
+
+class NoSetHierarchyError(ErrorBase):
+    pass
+
+class UnknownError(ErrorBase):
+    pass
+
+# errors not defined by OAI-PMH but which can occur in a client when
+# the server is somehow misbehaving
+class ClientError(Exception):
+    def details(self):
+        """Error details in human readable text.
+        """
+        raise NotImplementedError
+
+class XMLSyntaxError(ClientError):
+    """The OAI-PMH XML can not be parsed as it is not well-formed.
+    """
+    def details(self):
+        return ("The data delivered by the server could not be parsed, as it "
+                "is not well-formed XML.")
+    
+class DatestampError(ClientError):
+    """The OAI-PMH datestamps were not proper UTC datestamps as by spec.
+    """
+    def __init__(self, datestamp):
+        self.datestamp = datestamp
+
+    def details(self):
+        return ("An illegal datestamp was encountered: %s" % self.datestamp)
+    
diff --git a/metashare/oai_pmh/oaipmh/interfaces.py b/metashare/oai_pmh/oaipmh/interfaces.py
new file mode 100644
index 000000000..aec38bf48
--- /dev/null
+++ b/metashare/oai_pmh/oaipmh/interfaces.py
@@ -0,0 +1,170 @@
+class IOAI:
+    def getRecord(metadataPrefix, identifier):
+        """Get a record for a metadataPrefix and identifier.
+
+        metadataPrefix - identifies metadata set to retrieve
+        identifier - repository-unique identifier of record
+        
+        Should raise error.CannotDisseminateFormatError if
+        metadataPrefix is unknown or not supported by identifier.
+        
+        Should raise error.IdDoesNotExistError if identifier is
+        unknown or illegal.
+
+        Returns a header, metadata, about tuple describing the record.
+        """
+
+    def identify():
+        """Retrieve information about the repository.
+
+        Returns an Identify object describing the repository.
+        """
+
+    def listIdentifiers(metadataPrefix, set=None, from_=None, until=None):
+        """Get a list of header information on records.
+
+        metadataPrefix - identifies metadata set to retrieve
+        set - set identifier; only return headers in set (optional)
+        from_ - only retrieve headers from from_ date forward (optional)
+        until - only retrieve headers with dates up to and including
+                until date (optional)
+
+        Should raise error.CannotDisseminateFormatError if metadataPrefix
+        is not supported by the repository.
+
+        Should raise error.NoSetHierarchyError if the repository does not
+        support sets.
+        
+        Returns an iterable of headers.
+        """
+        
+    def listMetadataFormats(identifier=None):
+        """List metadata formats supported by repository or record.
+
+        identifier - identify record for which we want to know all
+                     supported metadata formats. if absent, list all metadata
+                     formats supported by repository. (optional)
+
+
+        Should raise error.IdDoesNotExistError if record with
+        identifier does not exist.
+        
+        Should raise error.NoMetadataFormatsError if no formats are
+        available for the indicated record.
+
+        Returns an iterable of metadataPrefix, schema, metadataNamespace
+        tuples (each entry in the tuple is a string).
+        """
+        
+    def listRecords(metadataPrefix, set=None, from_=None, until=None):
+        """Get a list of header, metadata and about information on records.
+
+        metadataPrefix - identifies metadata set to retrieve
+        set - set identifier; only return records in set (optional)
+        from_ - only retrieve records from from_ date forward (optional)
+        until - only retrieve records with dates up to and including
+                until date (optional)
+
+        Should raise error.CannotDisseminateFormatError if metadataPrefix
+        is not supported by the repository.
+
+        Should raise error.NoSetHierarchyError if the repository does not
+        support sets.
+
+        Returns an iterable of header, metadata, about tuples.
+        """
+
+    def listSets():
+        """Get a list of sets in the repository.
+
+        Should raise error.NoSetHierarchyError if the repository does not
+        support sets.
+
+        Returns an iterable of setSpec, setName tuples (strings).
+        """
+        
+class IBatchingOAI:
+    """Very similar to IOAI, but the implementation can be batch-aware.
+
+    Methods that support resumption will get two extra arguments,
+    cursor and batch_size, which indicate the batch currently being
+    requested.
+    """
+    
+    def getRecord(metadataPrefix, identifier):
+        pass
+    
+    def identify():
+        pass
+    
+    def listIdentifiers(metadataPrefix, set=None, from_=None, until=None,
+                        cursor=0, batch_size=10):
+        pass
+    
+    def listMetadataFormats(identifier=None):
+        pass
+    
+    def listRecords(metadataPrefix, set=None, from_=None, until=None,
+                    cursor=0, batch_size=10):
+        pass
+    
+    def listSets():
+        pass
+    
+class IIdentify:
+    def repositoryName():
+        """Name of repository.
+        """
+
+    def baseURL():
+        """Base URL for OAI-PMH requests.
+        """
+
+    def protocolVersion():
+        """OAI-PMH protocol version (should always be '2.0')
+        """
+
+    def adminEmails():
+        """List of email addresses of repository administrators.
+        """
+
+    def earliestDateStamp():
+        """The datetime (datestamp) of the earliest record in repository.
+        """
+
+    def deletedRecord():
+        """Way the repository handles deleted records.
+
+        Either 'no', 'transient' or 'persistent'.
+        """
+
+    def granularity():
+        """Datetime granularity of datestamps in repository.
+
+        Either YYYY-MM-DD or YYYY-MM-DDThh:mm:ssZ
+        """
+
+    def compression():
+        """List of types of compression schemes supported by repository.
+
+        'identity' is the 'do-nothing' scheme.
+        """
+        
+class IHeader:
+    def identifier():
+        """Repository-unique identifier of this record.
+        """
+
+    def datestamp():
+        """Datetime of creation, last modification or deletion of the record.
+
+        This can be used for selective harvesting.
+        """
+
+    def setSpec():
+        """A list of sets this record is a member of.
+        """
+
+    def isDeleted():
+        """If true, record has been deleted.
+        """
diff --git a/metashare/oai_pmh/oaipmh/metadata.py b/metashare/oai_pmh/oaipmh/metadata.py
new file mode 100644
index 000000000..3dd2347a7
--- /dev/null
+++ b/metashare/oai_pmh/oaipmh/metadata.py
@@ -0,0 +1,109 @@
+from lxml import etree
+from lxml.etree import SubElement
+from oaipmh import common
+
+class MetadataRegistry(object):
+    """A registry that contains readers and writers of metadata.
+
+    a reader is a function that takes a chunk of (parsed) XML and
+    returns a metadata object.
+
+    a writer is a function that takes a takes a metadata object and
+    produces a chunk of XML in the right format for this metadata.
+    """
+    def __init__(self):
+        self._readers = {}
+        self._writers = {}
+        
+    def registerReader(self, metadata_prefix, reader):
+        self._readers[metadata_prefix] = reader
+
+    def registerWriter(self, metadata_prefix, writer):
+        self._writers[metadata_prefix] = writer
+
+    def hasReader(self, metadata_prefix):
+        return metadata_prefix in self._readers
+    
+    def hasWriter(self, metadata_prefix):
+        return metadata_prefix in self._writers
+    
+    def readMetadata(self, metadata_prefix, element):
+        """Turn XML into metadata object.
+
+        element - element to read in
+
+        returns - metadata object
+        """
+        return self._readers[metadata_prefix](element)
+
+    def writeMetadata(self, metadata_prefix, element, metadata):
+        """Write metadata as XML.
+        
+        element - ElementTree element to write under
+        metadata - metadata object to write
+        """
+        self._writers[metadata_prefix](element, metadata)
+
+global_metadata_registry = MetadataRegistry()
+
+class Error(Exception):
+    pass
+
+class MetadataReader(object):
+    """A default implementation of a reader based on fields.
+    """
+    def __init__(self, fields, namespaces=None):
+        self._fields = fields
+        self._namespaces = namespaces or {}
+
+    def __call__(self, element):
+        map = {}
+        # create XPathEvaluator for this element
+        xpath_evaluator = etree.XPathEvaluator(element, 
+                                               namespaces=self._namespaces)
+        
+        e = xpath_evaluator.evaluate
+        # now extra field info according to xpath expr
+        for field_name, (field_type, expr) in self._fields.items():
+            if field_type == 'bytes':
+                value = str(e(expr))
+            elif field_type == 'bytesList':
+                value = [str(item) for item in e(expr)]
+            elif field_type == 'text':
+                # make sure we get back unicode strings instead
+                # of lxml.etree._ElementUnicodeResult objects.
+                value = unicode(e(expr))
+            elif field_type == 'textList':
+                # make sure we get back unicode strings instead
+                # of lxml.etree._ElementUnicodeResult objects.
+                value = [unicode(v) for v in e(expr)]
+            else:
+                raise Error, "Unknown field type: %s" % field_type
+            map[field_name] = value
+        return common.Metadata(map)
+
+oai_dc_reader = MetadataReader(
+    fields={
+    'title':       ('textList', 'oai_dc:dc/dc:title/text()'),
+    'creator':     ('textList', 'oai_dc:dc/dc:creator/text()'),
+    'subject':     ('textList', 'oai_dc:dc/dc:subject/text()'),
+    'description': ('textList', 'oai_dc:dc/dc:description/text()'),
+    'publisher':   ('textList', 'oai_dc:dc/dc:publisher/text()'),
+    'contributor': ('textList', 'oai_dc:dc/dc:contributor/text()'),
+    'date':        ('textList', 'oai_dc:dc/dc:date/text()'),
+    'type':        ('textList', 'oai_dc:dc/dc:type/text()'),
+    'format':      ('textList', 'oai_dc:dc/dc:format/text()'),
+    'identifier':  ('textList', 'oai_dc:dc/dc:identifier/text()'),
+    'source':      ('textList', 'oai_dc:dc/dc:source/text()'),
+    'language':    ('textList', 'oai_dc:dc/dc:language/text()'),
+    'relation':    ('textList', 'oai_dc:dc/dc:relation/text()'),
+    'coverage':    ('textList', 'oai_dc:dc/dc:coverage/text()'),
+    'rights':      ('textList', 'oai_dc:dc/dc:rights/text()')
+    },
+    namespaces={
+    'oai_dc': 'http://www.openarchives.org/OAI/2.0/oai_dc/',
+    'dc' : 'http://purl.org/dc/elements/1.1/'}
+    )
+
+
+    
diff --git a/metashare/oai_pmh/oaipmh/server.py b/metashare/oai_pmh/oaipmh/server.py
new file mode 100644
index 000000000..8ffe19e2b
--- /dev/null
+++ b/metashare/oai_pmh/oaipmh/server.py
@@ -0,0 +1,485 @@
+from lxml.etree import ElementTree, Element, SubElement
+from lxml import etree
+from datetime import datetime
+from urllib import quote, unquote, urlencode
+import sys, cgi
+
+from oaipmh import common, metadata, validation, error
+from oaipmh.datestamp import datestamp_to_datetime, datetime_to_datestamp, DatestampError
+
+NS_OAIPMH = 'http://www.openarchives.org/OAI/2.0/'
+NS_XSI = 'http://www.w3.org/2001/XMLSchema-instance'
+NS_OAIDC = 'http://www.openarchives.org/OAI/2.0/oai_dc/'
+NS_DC = "http://purl.org/dc/elements/1.1/"
+
+NSMAP = {
+    None: NS_OAIPMH,
+    }
+
+class XMLTreeServer(object):
+    """A server that responds to messages by returning XML trees.
+
+    This is an implementation class that normally would not be exposed
+    to the outside world.
+
+    Takes a server object conforming to the ResumptionOAIPMH interface.
+    """
+    def __init__(self, server, metadata_registry, nsmap=None):
+        if nsmap is None:
+            nsmap = {}
+        self._nsmap = NSMAP.copy()
+        self._nsmap.update(nsmap)
+        self._server = server
+        self._metadata_registry = (
+            metadata_registry or metadata.global_metadata_registry)
+        
+    def getRecord(self, **kw):
+        envelope, e_getRecord = self._outputEnvelope(
+            verb='GetRecord', **kw)
+        header, metadata, about = self._server.getRecord(**kw)
+        e_record = SubElement(e_getRecord, nsoai('record'))
+        self._outputHeader(e_record, header)   
+        if not header.isDeleted():
+            self._outputMetadata(e_record, kw['metadataPrefix'], metadata)
+        return envelope
+    
+    def identify(self):
+        envelope, e_identify = self._outputEnvelope(verb='Identify')
+        identify = self._server.identify()
+        e_repositoryName = SubElement(e_identify, nsoai('repositoryName'))
+        e_repositoryName.text = identify.repositoryName()
+        e_baseURL = SubElement(e_identify, nsoai('baseURL'))
+        e_baseURL.text = identify.baseURL()
+        e_protocolVersion = SubElement(e_identify, nsoai('protocolVersion'))
+        e_protocolVersion.text = identify.protocolVersion()
+        for adminEmail in identify.adminEmails():
+            e = SubElement(e_identify, nsoai('adminEmail'))
+            e.text = adminEmail
+        e_earliestDatestamp = SubElement(e_identify,
+                                         nsoai('earliestDatestamp'))
+        e_earliestDatestamp.text = datetime_to_datestamp(
+            identify.earliestDatestamp())
+        e_deletedRecord = SubElement(e_identify,
+                                     nsoai('deletedRecord'))
+        e_deletedRecord.text = identify.deletedRecord()
+        e_granularity = SubElement(e_identify, nsoai('granularity'))
+        e_granularity.text = identify.granularity()
+        compressions = identify.compression()
+        if compressions != ['identity']:
+            for compression in compressions:
+                e_compression = SubElement(e_identify, nsoai('compression'))
+                e_compression.text = compression
+
+        for description in identify.descriptions():
+            e_description = SubElement(e_identify, nsoai('description'))
+            e_description.append(etree.fromstring(description))
+        return envelope
+
+    def listMetadataFormats(self, **kw):
+        envelope, e_listMetadataFormats = self._outputEnvelope(
+            verb="ListMetadataFormats", **kw)
+        for (metadataPrefix, schema,
+             metadataNamespace) in self._server.listMetadataFormats(**kw):
+            e_metadataFormat = SubElement(e_listMetadataFormats,
+                                          nsoai('metadataFormat'))
+            e_metadataPrefix = SubElement(e_metadataFormat,
+                                          nsoai('metadataPrefix'))
+            e_metadataPrefix.text = metadataPrefix
+            e_schema = SubElement(e_metadataFormat,
+                                  nsoai('schema'))
+            e_schema.text = schema
+            e_metadataNamespace = SubElement(e_metadataFormat,
+                                             nsoai('metadataNamespace'))
+            e_metadataNamespace.text = metadataNamespace
+        return envelope            
+
+    def listIdentifiers(self, **kw):
+        envelope, e_listIdentifiers = self._outputEnvelope(
+            verb='ListIdentifiers', **kw)
+        def outputFunc(element, headers, token_kw):
+            for header in headers:
+                self._outputHeader(element, header)
+        self._outputResuming(
+            e_listIdentifiers,
+            self._server.listIdentifiers,
+            outputFunc,
+            kw)
+        return envelope
+    
+    def listRecords(self, **kw):
+        envelope, e_listRecords = self._outputEnvelope(
+            verb="ListRecords", **kw)
+        def outputFunc(element, records, token_kw):
+            metadataPrefix = token_kw['metadataPrefix']
+            for header, metadata, about in records:
+                e_record = SubElement(e_listRecords, nsoai('record'))
+                self._outputHeader(e_record, header)
+                if not header.isDeleted():
+                    self._outputMetadata(e_record, metadataPrefix, metadata)
+                # XXX about
+        self._outputResuming(
+            e_listRecords,
+            self._server.listRecords,
+            outputFunc,
+            kw)
+        return envelope
+
+    def listSets(self, **kw):
+        envelope, e_listSets = self._outputEnvelope(
+            verb='ListSets', **kw)
+        def outputFunc(element, sets, token_kw):
+            for setSpec, setName, setDescription in sets:
+                e_set = SubElement(e_listSets, nsoai('set'))
+                e_setSpec = SubElement(e_set, nsoai('setSpec'))
+                e_setSpec.text = setSpec
+                e_setName = SubElement(e_set, nsoai('setName'))
+                e_setName.text = setName
+                # XXX ignore setDescription
+        self._outputResuming(
+            e_listSets,
+            self._server.listSets,
+            outputFunc,
+            kw)
+        return envelope
+
+    def handleException(self, exception):
+        if isinstance(exception, error.ErrorBase):
+            envelope = self._outputErrors(
+                [(exception.oainame(), str(exception))])
+            return envelope
+        # unhandled exception, so raise again
+        raise
+    
+    def _outputBasicEnvelope(self, **kw):
+        e_oaipmh = Element(nsoai('OAI-PMH'), nsmap=self._nsmap)
+        e_oaipmh.set('{%s}schemaLocation' % NS_XSI,
+                     ('http://www.openarchives.org/OAI/2.0/ '
+                      'http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd'))
+        e_tree = ElementTree(element=e_oaipmh)
+        e_responseDate = SubElement(e_oaipmh, nsoai('responseDate'))
+        # date should be first possible moment
+        e_responseDate.text = datetime_to_datestamp(
+            datetime.utcnow().replace(microsecond=0))
+        e_request = SubElement(e_oaipmh, nsoai('request'))
+        for key, value in kw.items():
+            if key == 'from_':
+                key = 'from'
+            if key == 'from' or key == 'until':
+                value = datetime_to_datestamp(value)
+            e_request.set(key, value)
+        # XXX this is potentially slow..
+        e_request.text = self._server.identify().baseURL()
+        return e_tree, e_oaipmh
+    
+    def _outputEnvelope(self, **kw):
+        e_tree, e_oaipmh = self._outputBasicEnvelope(**kw)
+        e_element = SubElement(e_oaipmh, nsoai(kw['verb']))
+        return e_tree, e_element
+
+    def _outputErrors(self, errors, **kw):
+        # only pass functional arguments
+        e_tree, e_oaipmh = self._outputBasicEnvelope(**kw)
+        for error_code, error_msg in errors:
+            e_error = SubElement(e_oaipmh, nsoai('error'))
+            e_error.set('code', error_code)
+            e_error.text = error_msg
+        return e_tree
+    
+    def _outputResuming(self, element, input_func, output_func, kw):
+        if 'resumptionToken' in kw:
+            resumptionToken = kw['resumptionToken']
+            result, token = input_func(resumptionToken=resumptionToken)
+            # unpack keywords from resumption token
+            token_kw, dummy = decodeResumptionToken(resumptionToken)
+        else:
+            result, token = input_func(**kw)
+            # if we don't get results for the first request,
+            # then no records match
+            # XXX this will also be triggered if there are no sets,
+            # but input_func (listSets) should have already raised
+            # NoSetHierarchyError in that case
+            if not result:
+                raise error.NoRecordsMatchError,\
+                      "No records match for request."
+            # without resumption token keys are fine
+            token_kw = kw
+        output_func(element, result, token_kw)
+        if token is not None:
+            e_resumptionToken = SubElement(element, nsoai('resumptionToken'))
+            e_resumptionToken.text = token
+            
+    def _outputHeader(self, element, header):
+        e_header = SubElement(element, nsoai('header'))
+        if header.isDeleted():
+            e_header.set('status', 'deleted')
+        e_identifier = SubElement(e_header, nsoai('identifier'))
+        e_identifier.text = header.identifier()
+        e_datestamp = SubElement(e_header, nsoai('datestamp'))
+        e_datestamp.text = datetime_to_datestamp(header.datestamp())
+        for set in header.setSpec():
+            e = SubElement(e_header, nsoai('setSpec'))
+            e.text = set
+    
+    def _outputMetadata(self, element, metadata_prefix, metadata):
+        e_metadata = SubElement(element, nsoai('metadata'))
+        if not self._metadata_registry.hasWriter(metadata_prefix):
+            raise error.CannotDisseminateFormatError,\
+                  "Unknown metadata format: %s" % metadata_prefix
+        self._metadata_registry.writeMetadata(
+            metadata_prefix, e_metadata, metadata)
+
+class ServerBase(common.ResumptionOAIPMH):
+    """A server that responds to messages by returning OAI-PMH compliant XML.
+
+    Takes a server object complying with the ResumptionOAIPMH interface.
+    """
+    def __init__(self, server, metadata_registry=None, nsmap=None):
+        self._tree_server = XMLTreeServer(server, metadata_registry, nsmap)
+
+    def handleRequest(self, request_kw):
+        """Handles incoming OAI-PMH request.
+
+        request_kw is a dictionary containing request parameters, including
+        verb.
+        """
+        # try to get verb, if not, we have an argument handling error
+        try:
+            new_kw = {}
+            try:
+                for key, value in request_kw.items():
+                    new_kw[str(key)] = value
+            except UnicodeError:
+                raise error.BadVerbError,\
+                      "Non-ascii keys in request."
+            request_kw = new_kw
+            try:
+                verb = request_kw.pop('verb')
+            except KeyError:
+                verb = 'unknown'
+                raise error.BadVerbError,\
+                      "Required verb argument not found."
+            if verb not in ['GetRecord', 'Identify', 'ListIdentifiers',
+                            'ListMetadataFormats', 'ListRecords', 'ListSets']:
+                raise error.BadVerbError, "Illegal verb: %s" % verb
+            # replace from and until arguments if necessary
+            from_ = request_kw.get('from')
+            if from_ is not None:
+                # rename to from_ for internal use
+                try:
+                    request_kw['from_'] = datestamp_to_datetime(from_)
+                except DatestampError, err:
+                    raise error.BadArgumentError(
+                        "The value '%s' of the argument "
+                        "'%s' is not valid." %(from_, 'from'))
+                del request_kw['from']
+            until = request_kw.get('until')
+            if until is not None:
+                try:
+                    request_kw['until'] = datestamp_to_datetime(until,
+                                                                inclusive=True)
+                except DatestampError, err:
+                    raise error.BadArgumentError(
+                        "The value '%s' of the argument "
+                        "'%s' is not valid." %(until, 'until'))
+
+            if from_ is not None and until is not None:
+                if (('T' in from_ and not 'T' in until) or
+                    ('T' in until and not 'T' in from_)):
+                    raise error.BadArgumentError(
+                        "The request has different granularities for"
+                        " the from and until parameters")
+                
+            # now validate parameters
+            try:
+                validation.validateResumptionArguments(verb, request_kw)
+            except validation.BadArgumentError, e:
+                # have to raise this as a error.BadArgumentError
+                raise error.BadArgumentError, str(e)
+            # now handle verb
+            return self.handleVerb(verb, request_kw)            
+        except:
+            # in case of exception, call exception handler
+            return self.handleException(request_kw, sys.exc_info())
+        
+    def handleVerb(self, verb, kw):
+        method = common.getMethodForVerb(self._tree_server, verb)
+        return etree.tostring(method(**kw).getroot(), 
+                              encoding='UTF-8',
+                              xml_declaration=True,
+                              pretty_print=True)
+  
+    def handleException(self, kw, exc_info):
+        type, value, traceback = exc_info
+        return etree.tostring(
+            self._tree_server.handleException(value).getroot(),
+            encoding='UTF-8',
+            xml_declaration=True,
+            pretty_print=True)
+
+class Server(ServerBase):
+    """Expects to be initialized with a IOAI server implementation.
+    """
+    def __init__(self, server, metadata_registry=None, nsmap=None,
+                 resumption_batch_size=10):
+        super(Server, self).__init__(
+            Resumption(server, resumption_batch_size),
+            metadata_registry,
+            nsmap)
+
+class BatchingServer(ServerBase):
+    """Expects to be initialized with a IBatchingOAI server implementation.
+    """
+    def __init__(self, server, metadata_registry=None, nsmap=None,
+                 resumption_batch_size=10):
+        super(BatchingServer, self).__init__(
+            BatchingResumption(server, resumption_batch_size),
+            metadata_registry,
+            nsmap)
+
+class Resumption(common.ResumptionOAIPMH):
+    """
+    The Resumption class can turn a plain IOAIPMH interface into
+    a ResumptionOAIPMH interface
+
+    This implementation is not particularly efficient for large
+    result sets, as the complete result set needs to be reconstructed each
+    time.
+    """
+    def __init__(self, server, batch_size=10):
+        self._server = server
+        self._batch_size = batch_size
+    
+    def handleVerb(self, verb, kw):
+        # do original query
+        method = common.getMethodForVerb(self._server, verb)
+        # if we're handling a resumption token
+        if 'resumptionToken' in kw:
+            kw, cursor = decodeResumptionToken(
+                kw['resumptionToken'])
+            end_batch = cursor + self._batch_size
+            # do query again with original parameters
+            result = method(**kw)
+            # XXX defeat laziness of any generators..
+            result = list(result)
+            if end_batch < len(result):
+                resumptionToken = encodeResumptionToken(
+                    kw, end_batch)
+            else:
+                resumptionToken = None
+            return result[cursor:end_batch], resumptionToken
+
+        # we're not handling resumption token, so do request
+        result = method(**kw)
+
+        # now handle resumption system
+        if verb in ['ListSets', 'ListIdentifiers', 'ListRecords']:
+            # XXX defeat the laziness effect of any generators..
+            result = list(result)
+            end_batch = self._batch_size
+            if end_batch < len(result):
+                resumptionToken = encodeResumptionToken(
+                    kw, end_batch)
+            else:
+                resumptionToken = None
+            return result[0:end_batch], resumptionToken
+        return result
+
+class BatchingResumption(common.ResumptionOAIPMH):
+    """
+    The BatchingResumption class can turn a IBatchingOAIPMH interface into
+    a ResumptionOAIPMH interface.
+    """
+    
+    def __init__(self, server, batch_size=10):
+        self._server = server
+        self._batch_size = batch_size
+        
+    def handleVerb(self, verb, kw):
+        if 'resumptionToken' in kw:
+            kw, cursor = decodeResumptionToken(
+                kw['resumptionToken'])
+            kw['cursor'] = cursor
+            
+        method = common.getMethodForVerb(self._server, verb)
+
+        # now handle resumption system
+        if verb in ['ListSets', 'ListIdentifiers', 'ListRecords']:
+            kw = kw.copy()
+            cursor = kw.get('cursor', None)
+            if cursor is None:
+                kw['cursor'] = cursor = 0
+            # we request 1 beyond the batch size, so that
+            # if we retrieve <= batch_size items, we know we
+            # don't need to output another resumption token
+            kw['batch_size'] = self._batch_size + 1  
+            result = method(**kw)
+            result = list(result)
+            if len(result) > self._batch_size:
+                # more results are expected, so encode resumption token
+                resumptionToken = encodeResumptionToken(
+                    kw, cursor + self._batch_size)
+                # we also want to result only the batch_size, so pop the
+                # last one
+                result.pop()
+            else:
+                # no more results are expected
+                resumptionToken = None
+            return result, resumptionToken
+        return method(**kw)
+    
+def encodeResumptionToken(kw, cursor):
+    kw = kw.copy()
+    kw['cursor'] = str(cursor)
+    from_ = kw.get('from_')
+    if from_ is not None:
+        kw['from_'] = datetime_to_datestamp(from_)
+    until = kw.get('until')
+    if until is not None:
+        kw['until'] = datetime_to_datestamp(until)
+    return quote(urlencode(kw))
+
+def decodeResumptionToken(token):
+    token = str(unquote(token))
+    
+    try:
+        kw = cgi.parse_qs(token, True, True)
+    except ValueError:
+        raise error.BadResumptionTokenError,\
+              "Unable to decode resumption token: %s" % token
+    result = {}
+    for key, value in kw.items():
+        value = value[0]
+        if key == 'from_' or key == 'until':
+            value = datestamp_to_datetime(value)
+        result[key] = value
+    try:
+        cursor = int(result.pop('cursor'))
+    except (KeyError, ValueError):
+        raise error.BadResumptionTokenError,\
+              "Unable to decode resumption token (bad cursor): %s" % token
+    # XXX should also validate result contents. Need verb information
+    # for this, and somewhat more flexible verb validation support
+    return result, cursor
+    
+def oai_dc_writer(element, metadata):
+    e_dc = SubElement(element, nsoaidc('dc'),
+                      nsmap={'oai_dc': NS_OAIDC, 'dc': NS_DC, 'xsi': NS_XSI})
+    e_dc.set('{%s}schemaLocation' % NS_XSI,
+             '%s http://www.openarchives.org/OAI/2.0/oai_dc.xsd' % NS_DC)
+    map = metadata.getMap()
+    for name in [
+        'title', 'creator', 'subject', 'description', 'publisher',
+        'contributor', 'date', 'type', 'format', 'identifier',
+        'source', 'language', 'relation', 'coverage', 'rights']:
+        for value in map.get(name, []):
+            e = SubElement(e_dc, nsdc(name))
+            e.text = value
+               
+def nsoai(name):
+    return '{%s}%s' % (NS_OAIPMH, name)
+
+def nsoaidc(name):
+    return '{%s}%s' % (NS_OAIDC, name)
+
+def nsdc(name):
+    return '{%s}%s' % (NS_DC, name)
diff --git a/metashare/oai_pmh/oaipmh/validation.py b/metashare/oai_pmh/oaipmh/validation.py
new file mode 100644
index 000000000..5b395d1b0
--- /dev/null
+++ b/metashare/oai_pmh/oaipmh/validation.py
@@ -0,0 +1,88 @@
+
+#
+class BadArgumentError(Exception):
+    pass
+
+def validate(argspec, dict):
+    exclusive = None
+    for arg_name, arg_type in argspec.items():
+        if arg_type == 'exclusive':
+            exclusive = arg_name
+    # check if we have unknown arguments
+    for key, value in dict.items():
+        if not argspec.has_key(key):
+            msg = "Unknown argument: %s" % key
+            raise BadArgumentError, msg
+    # first investigate if we have exclusive argument
+    if dict.has_key(exclusive):
+        if len(dict) > 1:
+            msg = ("Exclusive argument %s is used but other "
+                   "arguments found." % exclusive)
+            raise BadArgumentError, msg
+        return
+    # if not exclusive, check for required
+    for arg_name, arg_type in argspec.items(): 
+        if arg_type == 'required':
+            msg = "Argument required but not found: %s" % arg_name
+            if not dict.has_key(arg_name):
+                raise BadArgumentError, msg 
+    return
+        
+class ValidationSpec(object):
+    GetRecord = {
+        'identifier':'required',
+        'metadataPrefix':'required'
+        }
+    
+    Identify = {
+        }
+
+    ListIdentifiers = {
+        'from_':'optional',
+        'until':'optional',
+        'metadataPrefix':'required',
+        'set':'optional',
+        }
+
+    ListMetadataFormats = {
+        'identifier':'optional'
+        }
+
+    ListRecords = {
+        'from_':'optional',
+        'until':'optional',
+        'set':'optional',
+        'metadataPrefix':'required',
+        }
+
+    ListSets = {
+        }
+
+class ResumptionValidationSpec(ValidationSpec):
+
+    ListIdentifiers = {
+        'from_':'optional',
+        'until':'optional',
+        'metadataPrefix':'required',
+        'set':'optional',
+        'resumptionToken':'exclusive',
+        }
+    
+    ListRecords = {
+        'from_':'optional',
+        'until':'optional',
+        'set':'optional',
+        'metadataPrefix':'required',
+        'resumptionToken':'exclusive',
+        }
+
+    ListSets = {
+        'resumptionToken':'exclusive',
+        }
+
+def validateArguments(verb, kw):
+    validate(getattr(ValidationSpec, verb), kw)
+
+def validateResumptionArguments(verb, kw):
+    validate(getattr(ResumptionValidationSpec, verb), kw)
+    
diff --git a/metashare/repository/forms.py b/metashare/repository/forms.py
index 4452f46a3..2d145f039 100644
--- a/metashare/repository/forms.py
+++ b/metashare/repository/forms.py
@@ -16,6 +16,14 @@
 from haystack.forms import FacetedSearchForm
 from haystack.query import SQ
 
+#noinspection PyBroadException
+try:
+    OAI_PMH_IMPORT_FROM_DIR = None
+    from metashare.settings import OAI_PMH_IMPORT_FROM_DIR
+except:
+    pass
+
+
 
 # Setup logging support.
 LOGGER = logging.getLogger(__name__)
@@ -221,3 +229,43 @@ class DownloadContactForm(forms.Form):
     """
     userEmail = forms.EmailField(label=_("Your e-mail"))
     message = forms.CharField(label=_("Your message"), widget=forms.Textarea())
+
+
+from oai_pmh import supported_commands
+
+class OaiPmhForm(forms.Form):
+    """
+    	Settings form for OAI-PMH import.
+    """
+    url = forms.URLField(
+        label=u"Harvested URL supporting OAI-PMH",
+        initial=u"http://ufal-point.mff.cuni.cz/oai/request",
+        help_text=u"URL with OAI-PMH support (for importing file contents you need a little updated ORE support on the remote server)",
+    )
+
+    verb = forms.ChoiceField(
+        label=u"What to do",
+        choices=[ (name, name) for name in supported_commands.keys() ],
+        help_text=u"Select OAI-PMH action (fill out item id if applicable)",
+    )
+
+    metadata_str = forms.CharField(
+        label=u"OAI metadata format",
+        help_text=u"Use list metadata for choices",
+        initial=u"oai_metasharev3",
+        required=False,
+    )
+
+    itemid = forms.CharField(
+        label=u"OAI item ID",
+        help_text=u"Leave empty for the whole collection or if not applicable",
+        required=False,
+    )
+
+    import_from_dir = forms.CharField(
+        label=u"Imported directory",
+        help_text=u"Because of licensing, we use ORE only for information about directories and use local " + \
+                  u"(or remote) directories for importing. Should be provided in local_settings.py",
+        initial=OAI_PMH_IMPORT_FROM_DIR,
+        required=False,
+    )
diff --git a/metashare/repository/urls.py b/metashare/repository/urls.py
index edaa98dd5..6af22bcfc 100644
--- a/metashare/repository/urls.py
+++ b/metashare/repository/urls.py
@@ -71,6 +71,12 @@
     'download'),
   (r'^download_contact/(?P\w+)/$',
     'download_contact'),
+  (r'^oai-pmh-import/$',
+   'oai_pmh_import'),
+  (r'^oai-pmh-export/$',
+   'oai_pmh_export'),
+  (r'^oai.*$',
+   'oai_server'),
   url(r'^search/$',
     search_view_factory(view_class=MetashareFacetedSearchView,
                         form_class=FacetedBrowseForm,
diff --git a/metashare/repository/views.py b/metashare/repository/views.py
index 198d41273..d268c96ae 100644
--- a/metashare/repository/views.py
+++ b/metashare/repository/views.py
@@ -14,6 +14,7 @@
 from django.template.loader import render_to_string
 from django.core.mail import send_mail
 from django.utils.translation import ugettext as _
+from django.utils.safestring import mark_safe
 
 from haystack.views import FacetedSearchView
 
@@ -34,6 +35,10 @@
     get_download_recommendations, get_view_recommendations, \
     get_more_from_same_creators_qs, get_more_from_same_projects_qs
 
+from metashare.repository.forms import OaiPmhForm
+from metashare import oai_pmh
+from oai_pmh import time_it, smart_extend
+
 
 MAXIMUM_READ_BLOCK_SIZE = 4096
 
@@ -959,3 +964,172 @@ def show_subfilter(self, facet, sel_facets, facet_fields, results):
                                'addable': addable})
 
         return results
+
+
+
+# PAI-PMH import
+#
+@login_required
+def oai_pmh_import(request):
+    """
+        Handle the OAI-PMH importing.
+
+        There are 2 features *not* out of the box ready when you use an arbitrary
+        digital library supporting OAI-PMH:
+        1) the MetaShare metadata format (according to the v3.0 schema)
+        2) ORE must give a hint to local paths (because of our specific and strict
+            licensing on the other point)
+
+        Please, consult UFAL, Charles University for more detail. We use it
+        for DSpace <-> Metashare synchronisation.
+    """
+
+    def _mark_obj_as_html_ok( data_dict ):
+        """ We return html strings, ouch """
+        if isinstance(data_dict, dict):
+            for k, v in data_dict.iteritems():
+                if isinstance(v, basestring):
+                    data_dict[k] = mark_safe(v)
+
+    def _get_what_to_do_from_fast_buttons( form ):
+        """
+            Simplification for the user if another button was clicked than
+            the Process button.
+        """
+        verb = form.cleaned_data['verb']
+        for key in form.data.keys():
+            if key.startswith("verb_"):
+                verb = key.replace("verb_", "")
+                break
+        return verb
+
+    def _remove_all_items():
+        from metashare.storage.models import StorageObject
+        from metashare.sync.sync_utils import remove_resource
+
+        d = {}
+        all_sos = StorageObject.objects.all()
+        # somehow it is not always working (could be known windows delete problem)
+        for so in all_sos:
+            d[so.id] = "Removed"
+            try:
+                remove_resource(so)
+            except:
+                pass
+            try:
+                so.delete()
+            except:
+                pass
+        return d
+
+
+    form = OaiPmhForm()
+    # the user posted something so do what he wanted
+    if "POST" == request.method:
+
+        data_info = {}
+        form = OaiPmhForm(request.POST)
+        if form.is_valid():
+            # find out what to do and then do it
+            what_to_do = _get_what_to_do_from_fast_buttons(form)
+
+            try:
+
+                if what_to_do in oai_pmh.supported_commands:
+
+                    # call the function associated with the task
+                    #
+                    ftor = oai_pmh.supported_commands[what_to_do]
+                    data_dict_title = what_to_do
+                    tm, data_dict = time_it(ftor, smart_extend(locals(), form.cleaned_data))
+                    data_info = {
+                        "Item size": len(data_dict),
+                        "Time taken": tm,
+                    }
+                    _mark_obj_as_html_ok(data_dict)
+
+                    # ok so use javascript if needed
+                    #
+                    if oai_pmh.key_list_ids_for_import == what_to_do:
+                        javascript_for_id = 1
+                        import_verb = oai_pmh.key_import
+                    elif oai_pmh.key_list_metadata_format == what_to_do:
+                        javascript_for_id = 2
+
+                # special case
+                elif "removeall" == what_to_do:
+                    tm, data_dict = time_it(_remove_all_items)
+                    data_info = {
+                        "Item size": len(data_dict),
+                        "Time taken": tm,
+                    }
+
+                else:
+                    error_str = u"Invalid command [%s]" % what_to_do
+
+            except Exception, e:
+                error_str = repr(e)
+
+
+    # give it all params together with form params
+    params = smart_extend(
+        {
+            'showform': "1",
+            'form_input': form,
+        },
+        locals(),
+        getattr(form, 'cleaned_data', {}),
+    )
+
+    # and show it
+    return render_to_response(
+        'repository/oai_pmh_import.html',
+        params,
+        context_instance=RequestContext(request))
+
+
+# PAI-PMH export
+#
+oai_url = DJANGO_URL.rstrip("/") + "/repository/oai"
+
+from oai_pmh import MetashareOaiServer
+from oai_pmh.oaipmh.server import Server
+from metashare.settings import ADMINS, METASHARE_VERSION
+
+__env_dict = {
+    "name": u"Metashare " + METASHARE_VERSION,
+    "url": oai_url,
+    "adminEmails": [x[1] for x in ADMINS],
+}
+oai_server_inst = Server(server=MetashareOaiServer(__env_dict))
+
+
+@login_required
+def oai_pmh_export(request):
+    """
+        Simple OAI-PMH page with testing and settings information.
+    """
+    params = {
+        "oai_url": oai_url,
+    }
+    return render_to_response(
+        'repository/oai_pmh_export.html',
+        params,
+        context_instance=RequestContext(request))
+
+
+def oai_server(request):
+    """
+        Handle the OAI-PMH exporting.
+    """
+    if not "verb" in request.GET:
+        invalid_html = oai_server_inst.handleRequest({})
+        return HttpResponse(content=invalid_html,
+                            status=400,
+                            content_type="text/xml")
+
+    verb = request.GET.get("verb")
+    content = oai_server_inst.handleVerb(verb, {})
+    return HttpResponse(content=content,
+                        status=200,
+                        content_type="text/xml")
diff --git a/metashare/templates/header.html b/metashare/templates/header.html
index 70fd2040d..b7ffd2e5e 100644
--- a/metashare/templates/header.html
+++ b/metashare/templates/header.html
@@ -24,6 +24,10 @@
   {# drop-down menu created with http://javascript-array.com/scripts/jquery_simple_drop_down_menu/ #}
   
    
     
  • Browse Resources
    • 
+    {% if user.is_staff and user.get_profile.has_editor_permission %}
+        
  • Import OAI-PMH
    • 
+        
  • Export OAI-PMH
    • 
+    {% endif %}
     {% if user.get_profile.has_editor_permission %}
     
  • Manage Resources
       
      
diff --git a/metashare/templates/repository/oai_pmh_export.html b/metashare/templates/repository/oai_pmh_export.html
new file mode 100644
index 000000000..1791f8b85
--- /dev/null
+++ b/metashare/templates/repository/oai_pmh_export.html
@@ -0,0 +1,32 @@
+{% extends "base.html" %}
+
+{% block title %}
+ – Export OAI-PMH –
+{% endblock %}
+
+{% block content %}
+
      OAI-PMH export
      
+
+{% if error_str %}
+    
      {{ error_str }}
      
+{% endif %}
+
+
      
+    
      OAI-PMH Settings
      
+      
      
+          
+              
+                  
+              
+              
+                  
+              
+              
+                  
+              
+          
      OAI-PMH Identifytest
      OAI-PMH ListIdentifierstest
      OAI-PMH errortest
      
+      
      
+
      
+
+
+{% endblock %}
diff --git a/metashare/templates/repository/oai_pmh_import.html b/metashare/templates/repository/oai_pmh_import.html
new file mode 100644
index 000000000..e63bcbce0
--- /dev/null
+++ b/metashare/templates/repository/oai_pmh_import.html
@@ -0,0 +1,105 @@
+{% extends "base.html" %}
+
+{% block title %}
+ – Import OAI-PMH –
+{% endblock %}
+
+{% block content %}
+
      OAI-PMH (UFAL, Charles University specific implementation)
      
+
+{% if error_str %}
+    
      {{ error_str }}
      
+{% endif %}
+
+
      
+  {% if showform == "1" %}
+    
      OAI-PMH Settings
      
+      
      
+          
      
+              
+                {% csrf_token %}
+                {{ form_input.as_table }}
+                
+                
+              
      
+                        
+                        
+                            
+                            
+                            
+                            
+                            
+                        
+                    
      
+          
      
+      
      
+  {% endif %}
+
+
+    
      
+    {% if data_dict %}
+        
      
+            
+                 
      Result: {{ data_dict_title }}
      
+                  
+            
+                Info detail:{{ data_info }}
+            
+        
      
+        
      
+        
+            {% for k, v in data_dict.iteritems %}
+                
+                    
+                    
+                
+            {% endfor %}
+        
      {{ k }}{{ v }}
      
+        
      
+    {% endif %}
+
+    {% if data_info %}
+        
      
+            
      Info detail
      {{ data_info }}
+        
      
+    {% endif %}
+
+    {% if javascript_for_id %}
+
+        
+
+    {% endif %}
+
+
      
+
+
+{% endblock %}