Transformers for OAI-PMH harvested records.
A transformer converts what the harvester's reader yields into the data the writer
stores - {"record": <data for the model's service or None>, "oai_record": <OAI record>}.
This package ships oai-import, which
converts the metadata with one of the target model's imports, and
skip-if-owned, which skips records
written by another harvester.
How to write your own readers and transformers, what they receive and produce and how
they report errors is described in
Readers, transformers and writers.
Modules:
| Name |
Description |
oai_import |
oai-import transformer - converts harvested XML with the target model's import.
|
skip_if_owned |
skip-if-owned transformer - lets a fallback harvester give way to other harvesters.
|
Bases: BaseTransformer
Transformer for importing OAI-PMH records.
This transformers needs a model name to find the appropriate importer
for the metadata format being imported.
Source code in oarepo_oaipmh_harvester/transformers/oai_import.py
| class OAIImportTransformer(BaseTransformer):
"""Transformer for importing OAI-PMH records.
This transformers needs a model name to find the appropriate importer
for the metadata format being imported.
"""
def __init__(self, model: str, *_args: Any, **_kwargs: Any) -> None:
"""Initialize the OAI import transformer."""
super().__init__()
self.model = model
def apply(self, stream_entry: StreamEntry, *_args: Any, **_kwargs: Any) -> StreamEntry:
"""Apply the transformation to the entry.
Returns:
StreamEntry: The transformed entry.
Raises:
TransformerError: In case of errors.
"""
if not isinstance(stream_entry.entry, dict) or not isinstance(
stream_entry.entry["record"],
OAIRecord,
):
raise TransformerError(
"OAIImport transformer requires entries to be oaipmh_scythe Record instances, "
f"it is {type(stream_entry)}."
)
if stream_entry.entry["record"].header.deleted:
# deleted record, nothing to import
return StreamEntry(
entry={"record": None, "oai_record": stream_entry.entry["record"]},
op_type=stream_entry.op_type,
)
# parse the metadata via the appropriate importer
metadata_elem = self._get_metadata_element(stream_entry.entry["record"].xml)
q = etree.QName(metadata_elem)
namespace_uri = q.namespace
local_name = q.localname
imp = self._find_importer(namespace_uri, local_name)
if xml_deserializer := getattr(imp.deserializer, "deserialize_xml", None):
# if the deserializer can natively handle XML elements, use that
data = xml_deserializer(metadata_elem)
else:
# otherwise convert to string and use the default deserialize method
data = imp.deserializer.deserialize(etree.tostring(metadata_elem, encoding=str))
return StreamEntry(
entry={"record": data, "oai_record": stream_entry.entry["record"]},
op_type=stream_entry.op_type,
)
def _find_importer(self, namespace_uri: str, local_name: str) -> Import:
"""Find the appropriate importer for the given OAI element."""
for imp in current_runtime.models[self.model].imports:
if not imp.oai_name:
continue
if imp.oai_name == (namespace_uri, local_name):
return imp
raise TransformerError(
f"No import found for OAI element {{{namespace_uri}}}{local_name} in model {self.model}."
)
def _get_metadata_element(self, record_xml: etree._Element) -> etree._Element:
"""Extract the metadata element from an oaipmh_scythe Record XML.
look for {http://www.openarchives.org/OAI/2.0/}record /
{http://www.openarchives.org/OAI/2.0/}metadata and get the first child
"""
oai_metadata_elem = record_xml.find("{http://www.openarchives.org/OAI/2.0/}metadata")
if oai_metadata_elem is None:
raise TransformerError("No metadata element found in OAI record.")
children = [x for x in oai_metadata_elem if x.tag]
if not children:
raise TransformerError("No metadata child elements found in OAI record.")
if len(children) > 1:
raise TransformerError("Multiple metadata child elements found in OAI record.")
return children[0]
|
Initialize the OAI import transformer.
Source code in oarepo_oaipmh_harvester/transformers/oai_import.py
| def __init__(self, model: str, *_args: Any, **_kwargs: Any) -> None:
"""Initialize the OAI import transformer."""
super().__init__()
self.model = model
|
Apply the transformation to the entry.
Returns:
| Name | Type |
Description |
StreamEntry |
StreamEntry
|
|
Raises:
| Type |
Description |
TransformerError
|
|
Source code in oarepo_oaipmh_harvester/transformers/oai_import.py
| def apply(self, stream_entry: StreamEntry, *_args: Any, **_kwargs: Any) -> StreamEntry:
"""Apply the transformation to the entry.
Returns:
StreamEntry: The transformed entry.
Raises:
TransformerError: In case of errors.
"""
if not isinstance(stream_entry.entry, dict) or not isinstance(
stream_entry.entry["record"],
OAIRecord,
):
raise TransformerError(
"OAIImport transformer requires entries to be oaipmh_scythe Record instances, "
f"it is {type(stream_entry)}."
)
if stream_entry.entry["record"].header.deleted:
# deleted record, nothing to import
return StreamEntry(
entry={"record": None, "oai_record": stream_entry.entry["record"]},
op_type=stream_entry.op_type,
)
# parse the metadata via the appropriate importer
metadata_elem = self._get_metadata_element(stream_entry.entry["record"].xml)
q = etree.QName(metadata_elem)
namespace_uri = q.namespace
local_name = q.localname
imp = self._find_importer(namespace_uri, local_name)
if xml_deserializer := getattr(imp.deserializer, "deserialize_xml", None):
# if the deserializer can natively handle XML elements, use that
data = xml_deserializer(metadata_elem)
else:
# otherwise convert to string and use the default deserialize method
data = imp.deserializer.deserialize(etree.tostring(metadata_elem, encoding=str))
return StreamEntry(
entry={"record": data, "oai_record": stream_entry.entry["record"]},
op_type=stream_entry.op_type,
)
|
Bases: BaseTransformer
Mark the entry as filtered if another harvester owns its local record.
Source code in oarepo_oaipmh_harvester/transformers/skip_if_owned.py
| class SkipIfOwnedTransformer(BaseTransformer):
"""Mark the entry as filtered if another harvester owns its local record."""
def __init__(
self,
model: str,
*_args: Any,
harvester_id: str,
**_kwargs: Any,
) -> None:
"""Initialize the transformer.
Args:
model: Name of the target model (`current_runtime.models`).
harvester_id: Harvester the entries belong to (set by the harvest task).
"""
super().__init__()
self._model = model
self._harvester_id = harvester_id
@override
def apply(self, stream_entry: StreamEntry, *_args: Any, **_kwargs: Any) -> StreamEntry:
"""Mark the entry as filtered, with the reason in `entry["skip_reason"]`, if the pid is owned."""
oai_identifier = stream_entry.entry["oai_record"].header.identifier
pid_value = self._pid_value(oai_identifier, stream_entry.entry.get("record"))
if not pid_value:
return stream_entry
owner = (
db.session.query(OAIHarvestedRecord.harvester_id, OAIHarvestedRecord.oai_identifier)
.filter(
OAIHarvestedRecord.harvester_id != self._harvester_id,
OAIHarvestedRecord.record_type == self._model,
OAIHarvestedRecord.record_pid == pid_value,
OAIHarvestedRecord.deleted.is_(False),
# a skipped harvest record does not own the pid - two fallback harvesters
# would otherwise skip each other's records forever
OAIHarvestedRecord.has_warnings.is_(False),
)
.first()
)
if owner is not None:
stream_entry.filtered = True
stream_entry.entry[SKIP_REASON] = (
f"Record {pid_value} is harvested by harvester {owner.harvester_id} ({owner.oai_identifier})"
)
return stream_entry
def _pid_value(self, oai_identifier: str, record_data: dict | None) -> str | None:
"""Local pid of the entry - from its harvest record, or from the transformed data."""
record_pid = db.session.query(OAIHarvestedRecord.record_pid).filter_by(oai_identifier=oai_identifier).scalar()
if record_pid:
return record_pid
if not record_data:
# deleted at the source and never written - nothing to protect
return None
return get_pid_value_from_record(current_runtime.models[self._model].service, record_data)
|
Initialize the transformer.
Parameters:
| Name |
Type |
Description |
Default |
model
|
str
|
Name of the target model (current_runtime.models).
|
required
|
harvester_id
|
str
|
Harvester the entries belong to (set by the harvest task).
|
required
|
Source code in oarepo_oaipmh_harvester/transformers/skip_if_owned.py
| def __init__(
self,
model: str,
*_args: Any,
harvester_id: str,
**_kwargs: Any,
) -> None:
"""Initialize the transformer.
Args:
model: Name of the target model (`current_runtime.models`).
harvester_id: Harvester the entries belong to (set by the harvest task).
"""
super().__init__()
self._model = model
self._harvester_id = harvester_id
|
Mark the entry as filtered, with the reason in entry["skip_reason"], if the pid is owned.
Source code in oarepo_oaipmh_harvester/transformers/skip_if_owned.py
| @override
def apply(self, stream_entry: StreamEntry, *_args: Any, **_kwargs: Any) -> StreamEntry:
"""Mark the entry as filtered, with the reason in `entry["skip_reason"]`, if the pid is owned."""
oai_identifier = stream_entry.entry["oai_record"].header.identifier
pid_value = self._pid_value(oai_identifier, stream_entry.entry.get("record"))
if not pid_value:
return stream_entry
owner = (
db.session.query(OAIHarvestedRecord.harvester_id, OAIHarvestedRecord.oai_identifier)
.filter(
OAIHarvestedRecord.harvester_id != self._harvester_id,
OAIHarvestedRecord.record_type == self._model,
OAIHarvestedRecord.record_pid == pid_value,
OAIHarvestedRecord.deleted.is_(False),
# a skipped harvest record does not own the pid - two fallback harvesters
# would otherwise skip each other's records forever
OAIHarvestedRecord.has_warnings.is_(False),
)
.first()
)
if owner is not None:
stream_entry.filtered = True
stream_entry.entry[SKIP_REASON] = (
f"Record {pid_value} is harvested by harvester {owner.harvester_id} ({owner.oai_identifier})"
)
return stream_entry
|