Skip to content

oarepo_oaipmh_harvester.oai_record.dumpers

oarepo_oaipmh_harvester.oai_record.dumpers

Dumpers for OAI harvested records.

MAX_KEYWORD_LENGTH = 512 module-attribute

ignore_above of the errors.* keyword fields - longer values are not indexed.

AddHarvesterDumperExt

Bases: SearchDumperExt

Dumper extension adding the harvester's name and managers to the search document.

Source code in oarepo_oaipmh_harvester/oai_record/dumpers.py
class AddHarvesterDumperExt(SearchDumperExt):
    """Dumper extension adding the harvester's name and managers to the search document."""

    @override
    def dump(self, record: Any, data: dict[str, Any]) -> None:
        """Add the harvester name and managers to the dumped data."""
        harvester = record.harvester
        data["harvester_name"] = harvester.name
        data["harvest_managers"] = harvester.harvest_managers or []

    @override
    def load(self, data: dict[str, Any], record_cls: type) -> None:
        """Load the data.

        Reverse the changes made by the dump method.
        """
        data.pop("harvester_name", None)
        data.pop("harvest_managers", None)

dump(record: Any, data: dict[str, Any]) -> None

Add the harvester name and managers to the dumped data.

Source code in oarepo_oaipmh_harvester/oai_record/dumpers.py
@override
def dump(self, record: Any, data: dict[str, Any]) -> None:
    """Add the harvester name and managers to the dumped data."""
    harvester = record.harvester
    data["harvester_name"] = harvester.name
    data["harvest_managers"] = harvester.harvest_managers or []

load(data: dict[str, Any], record_cls: type) -> None

Load the data.

Reverse the changes made by the dump method.

Source code in oarepo_oaipmh_harvester/oai_record/dumpers.py
@override
def load(self, data: dict[str, Any], record_cls: type) -> None:
    """Load the data.

    Reverse the changes made by the dump method.
    """
    data.pop("harvester_name", None)
    data.pop("harvest_managers", None)

ErrorsFacetDumperExt

Bases: SearchDumperExt

Make the errors of the search document usable in facets.

The errors.* fields are keywords with ignore_above: 512 - a longer message or a traceback in location would not be indexed at all, so the record would be missing in the error facets. The search document gets single-line messages truncated to fit, the innermost traceback frame as the location and a placeholder for an empty message; the database keeps the full errors (shown in the harvest record detail).

Source code in oarepo_oaipmh_harvester/oai_record/dumpers.py
class ErrorsFacetDumperExt(SearchDumperExt):
    """Make the errors of the search document usable in facets.

    The `errors.*` fields are keywords with `ignore_above: 512` - a longer message or a
    traceback in `location` would not be indexed at all, so the record would be missing in
    the error facets. The search document gets single-line messages truncated to fit, the
    innermost traceback frame as the location and a placeholder for an empty message;
    the database keeps the full errors (shown in the harvest record detail).
    """

    @override
    def dump(self, record: Any, data: dict[str, Any]) -> None:
        """Shorten the errors of the search document."""
        data["errors"] = [
            {
                **error,
                "type": _keyword(error.get("type")),
                "message": _keyword(error.get("message") or f"{error.get('type') or 'Error'} (no message)"),
                "location": _keyword(_innermost_frame(error.get("location"))),
            }
            for error in data.get("errors") or []
        ]

    @override
    def load(self, data: dict[str, Any], record_cls: type) -> None:
        """Keep the shortened errors - the full ones are only in the database."""

dump(record: Any, data: dict[str, Any]) -> None

Shorten the errors of the search document.

Source code in oarepo_oaipmh_harvester/oai_record/dumpers.py
@override
def dump(self, record: Any, data: dict[str, Any]) -> None:
    """Shorten the errors of the search document."""
    data["errors"] = [
        {
            **error,
            "type": _keyword(error.get("type")),
            "message": _keyword(error.get("message") or f"{error.get('type') or 'Error'} (no message)"),
            "location": _keyword(_innermost_frame(error.get("location"))),
        }
        for error in data.get("errors") or []
    ]

load(data: dict[str, Any], record_cls: type) -> None

Keep the shortened errors - the full ones are only in the database.

Source code in oarepo_oaipmh_harvester/oai_record/dumpers.py
@override
def load(self, data: dict[str, Any], record_cls: type) -> None:
    """Keep the shortened errors - the full ones are only in the database."""