Skip to content

oarepo_oaipmh_harvester.transformers

oarepo_oaipmh_harvester.transformers

Transformers for OAI-PMH harvested records.

A transformer converts what the harvester's reader yields into the data the writer stores - {"record": <data for the model's service or None>, "oai_record": <OAI record>}. This package ships oai-import, which converts the metadata with one of the target model's imports, and skip-if-owned, which skips records written by another harvester.

How to write your own readers and transformers, what they receive and produce and how they report errors is described in Readers, transformers and writers.

Modules:

Name Description
oai_import

oai-import transformer - converts harvested XML with the target model's import.

skip_if_owned

skip-if-owned transformer - lets a fallback harvester give way to other harvesters.

OAIImportTransformer

Bases: BaseTransformer

Transformer for importing OAI-PMH records.

This transformers needs a model name to find the appropriate importer for the metadata format being imported.

Source code in oarepo_oaipmh_harvester/transformers/oai_import.py
class OAIImportTransformer(BaseTransformer):
    """Transformer for importing OAI-PMH records.

    This transformers needs a model name to find the appropriate importer
    for the metadata format being imported.
    """

    def __init__(self, model: str, *_args: Any, **_kwargs: Any) -> None:
        """Initialize the OAI import transformer."""
        super().__init__()
        self.model = model

    def apply(self, stream_entry: StreamEntry, *_args: Any, **_kwargs: Any) -> StreamEntry:
        """Apply the transformation to the entry.

        Returns:
            StreamEntry: The transformed entry.

        Raises:
            TransformerError: In case of errors.

        """
        if not isinstance(stream_entry.entry, dict) or not isinstance(
            stream_entry.entry["record"],
            OAIRecord,
        ):
            raise TransformerError(
                "OAIImport transformer requires entries to be oaipmh_scythe Record instances, "
                f"it is {type(stream_entry)}."
            )
        if stream_entry.entry["record"].header.deleted:
            # deleted record, nothing to import
            return StreamEntry(
                entry={"record": None, "oai_record": stream_entry.entry["record"]},
                op_type=stream_entry.op_type,
            )
        # parse the metadata via the appropriate importer
        metadata_elem = self._get_metadata_element(stream_entry.entry["record"].xml)
        q = etree.QName(metadata_elem)
        namespace_uri = q.namespace
        local_name = q.localname
        imp = self._find_importer(namespace_uri, local_name)
        if xml_deserializer := getattr(imp.deserializer, "deserialize_xml", None):
            # if the deserializer can natively handle XML elements, use that
            data = xml_deserializer(metadata_elem)
        else:
            # otherwise convert to string and use the default deserialize method
            data = imp.deserializer.deserialize(etree.tostring(metadata_elem, encoding=str))
        return StreamEntry(
            entry={"record": data, "oai_record": stream_entry.entry["record"]},
            op_type=stream_entry.op_type,
        )

    def _find_importer(self, namespace_uri: str, local_name: str) -> Import:
        """Find the appropriate importer for the given OAI element."""
        for imp in current_runtime.models[self.model].imports:
            if not imp.oai_name:
                continue
            if imp.oai_name == (namespace_uri, local_name):
                return imp
        raise TransformerError(
            f"No import found for OAI element {{{namespace_uri}}}{local_name} in model {self.model}."
        )

    def _get_metadata_element(self, record_xml: etree._Element) -> etree._Element:
        """Extract the metadata element from an oaipmh_scythe Record XML.

        look for {http://www.openarchives.org/OAI/2.0/}record /
        {http://www.openarchives.org/OAI/2.0/}metadata and get the first child
        """
        oai_metadata_elem = record_xml.find("{http://www.openarchives.org/OAI/2.0/}metadata")
        if oai_metadata_elem is None:
            raise TransformerError("No metadata element found in OAI record.")
        children = [x for x in oai_metadata_elem if x.tag]
        if not children:
            raise TransformerError("No metadata child elements found in OAI record.")
        if len(children) > 1:
            raise TransformerError("Multiple metadata child elements found in OAI record.")
        return children[0]

__init__(model: str, *_args: Any, **_kwargs: Any) -> None

Initialize the OAI import transformer.

Source code in oarepo_oaipmh_harvester/transformers/oai_import.py
def __init__(self, model: str, *_args: Any, **_kwargs: Any) -> None:
    """Initialize the OAI import transformer."""
    super().__init__()
    self.model = model

apply(stream_entry: StreamEntry, *_args: Any, **_kwargs: Any) -> StreamEntry

Apply the transformation to the entry.

Returns:

Name Type Description
StreamEntry StreamEntry

The transformed entry.

Raises:

Type Description
TransformerError

In case of errors.

Source code in oarepo_oaipmh_harvester/transformers/oai_import.py
def apply(self, stream_entry: StreamEntry, *_args: Any, **_kwargs: Any) -> StreamEntry:
    """Apply the transformation to the entry.

    Returns:
        StreamEntry: The transformed entry.

    Raises:
        TransformerError: In case of errors.

    """
    if not isinstance(stream_entry.entry, dict) or not isinstance(
        stream_entry.entry["record"],
        OAIRecord,
    ):
        raise TransformerError(
            "OAIImport transformer requires entries to be oaipmh_scythe Record instances, "
            f"it is {type(stream_entry)}."
        )
    if stream_entry.entry["record"].header.deleted:
        # deleted record, nothing to import
        return StreamEntry(
            entry={"record": None, "oai_record": stream_entry.entry["record"]},
            op_type=stream_entry.op_type,
        )
    # parse the metadata via the appropriate importer
    metadata_elem = self._get_metadata_element(stream_entry.entry["record"].xml)
    q = etree.QName(metadata_elem)
    namespace_uri = q.namespace
    local_name = q.localname
    imp = self._find_importer(namespace_uri, local_name)
    if xml_deserializer := getattr(imp.deserializer, "deserialize_xml", None):
        # if the deserializer can natively handle XML elements, use that
        data = xml_deserializer(metadata_elem)
    else:
        # otherwise convert to string and use the default deserialize method
        data = imp.deserializer.deserialize(etree.tostring(metadata_elem, encoding=str))
    return StreamEntry(
        entry={"record": data, "oai_record": stream_entry.entry["record"]},
        op_type=stream_entry.op_type,
    )

SkipIfOwnedTransformer

Bases: BaseTransformer

Mark the entry as filtered if another harvester owns its local record.

Source code in oarepo_oaipmh_harvester/transformers/skip_if_owned.py
class SkipIfOwnedTransformer(BaseTransformer):
    """Mark the entry as filtered if another harvester owns its local record."""

    def __init__(
        self,
        model: str,
        *_args: Any,
        harvester_id: str,
        **_kwargs: Any,
    ) -> None:
        """Initialize the transformer.

        Args:
            model: Name of the target model (`current_runtime.models`).
            harvester_id: Harvester the entries belong to (set by the harvest task).

        """
        super().__init__()
        self._model = model
        self._harvester_id = harvester_id

    @override
    def apply(self, stream_entry: StreamEntry, *_args: Any, **_kwargs: Any) -> StreamEntry:
        """Mark the entry as filtered, with the reason in `entry["skip_reason"]`, if the pid is owned."""
        oai_identifier = stream_entry.entry["oai_record"].header.identifier
        pid_value = self._pid_value(oai_identifier, stream_entry.entry.get("record"))
        if not pid_value:
            return stream_entry

        owner = (
            db.session.query(OAIHarvestedRecord.harvester_id, OAIHarvestedRecord.oai_identifier)
            .filter(
                OAIHarvestedRecord.harvester_id != self._harvester_id,
                OAIHarvestedRecord.record_type == self._model,
                OAIHarvestedRecord.record_pid == pid_value,
                OAIHarvestedRecord.deleted.is_(False),
                # a skipped harvest record does not own the pid - two fallback harvesters
                # would otherwise skip each other's records forever
                OAIHarvestedRecord.has_warnings.is_(False),
            )
            .first()
        )
        if owner is not None:
            stream_entry.filtered = True
            stream_entry.entry[SKIP_REASON] = (
                f"Record {pid_value} is harvested by harvester {owner.harvester_id} ({owner.oai_identifier})"
            )
        return stream_entry

    def _pid_value(self, oai_identifier: str, record_data: dict | None) -> str | None:
        """Local pid of the entry - from its harvest record, or from the transformed data."""
        record_pid = db.session.query(OAIHarvestedRecord.record_pid).filter_by(oai_identifier=oai_identifier).scalar()
        if record_pid:
            return record_pid
        if not record_data:
            # deleted at the source and never written - nothing to protect
            return None
        return get_pid_value_from_record(current_runtime.models[self._model].service, record_data)

__init__(model: str, *_args: Any, harvester_id: str, **_kwargs: Any) -> None

Initialize the transformer.

Parameters:

Name Type Description Default
model str

Name of the target model (current_runtime.models).

required
harvester_id str

Harvester the entries belong to (set by the harvest task).

required
Source code in oarepo_oaipmh_harvester/transformers/skip_if_owned.py
def __init__(
    self,
    model: str,
    *_args: Any,
    harvester_id: str,
    **_kwargs: Any,
) -> None:
    """Initialize the transformer.

    Args:
        model: Name of the target model (`current_runtime.models`).
        harvester_id: Harvester the entries belong to (set by the harvest task).

    """
    super().__init__()
    self._model = model
    self._harvester_id = harvester_id

apply(stream_entry: StreamEntry, *_args: Any, **_kwargs: Any) -> StreamEntry

Mark the entry as filtered, with the reason in entry["skip_reason"], if the pid is owned.

Source code in oarepo_oaipmh_harvester/transformers/skip_if_owned.py
@override
def apply(self, stream_entry: StreamEntry, *_args: Any, **_kwargs: Any) -> StreamEntry:
    """Mark the entry as filtered, with the reason in `entry["skip_reason"]`, if the pid is owned."""
    oai_identifier = stream_entry.entry["oai_record"].header.identifier
    pid_value = self._pid_value(oai_identifier, stream_entry.entry.get("record"))
    if not pid_value:
        return stream_entry

    owner = (
        db.session.query(OAIHarvestedRecord.harvester_id, OAIHarvestedRecord.oai_identifier)
        .filter(
            OAIHarvestedRecord.harvester_id != self._harvester_id,
            OAIHarvestedRecord.record_type == self._model,
            OAIHarvestedRecord.record_pid == pid_value,
            OAIHarvestedRecord.deleted.is_(False),
            # a skipped harvest record does not own the pid - two fallback harvesters
            # would otherwise skip each other's records forever
            OAIHarvestedRecord.has_warnings.is_(False),
        )
        .first()
    )
    if owner is not None:
        stream_entry.filtered = True
        stream_entry.entry[SKIP_REASON] = (
            f"Record {pid_value} is harvested by harvester {owner.harvester_id} ({owner.oai_identifier})"
        )
    return stream_entry