Skip to content

oarepo_oaipmh_harvester.oai_record.models

oarepo_oaipmh_harvester.oai_record.models

Models for OAI-PMH harvested records.

OAIHarvestedRecord

Bases: Model

Metadata about a harvested record.

Always contains the latest metadata of the record, no previous versions are stored.

Source code in oarepo_oaipmh_harvester/oai_record/models.py
class OAIHarvestedRecord(db.Model):
    """Metadata about a harvested record.

    Always contains the latest metadata of the record, no previous versions are stored.
    """

    __tablename__ = "oai_harvest_records"

    oai_identifier = db.Column(db.String(255), primary_key=True)
    """OAI identifier of the record."""

    record_type = db.Column(db.String(32), nullable=True)
    """Type of the record."""

    record_pid = db.Column(db.String(255), nullable=True)
    """Persistent identifier (Record.pid) of the record (local identifier).

    Note that this is not the database ID but the PID value.
    """

    datestamp = db.Column(db.UTCDateTime, nullable=False)
    """Datestamp of the record."""

    harvested_at = db.Column(db.UTCDateTime, nullable=False)
    """Time when the record was harvested."""

    deleted = db.Column(db.Boolean, default=False, nullable=False)
    """True if the record was deleted during the harvest."""

    has_errors = db.Column(db.Boolean, default=False, nullable=False)
    """True if the record has errors during the harvest."""

    has_warnings = db.Column(db.Boolean, default=False, nullable=False)
    """True if the record has warnings during the harvest."""

    errors = db.Column(
        db.JSON()
        .with_variant(
            postgresql.JSONB(none_as_null=True),
            "postgresql",
        )
        .with_variant(
            JSONType(),
            "sqlite",
        )
        .with_variant(
            JSONType(),
            "mysql",
        ),
        default=dict,
        nullable=False,
    )
    """Errors."""

    original_data = db.Column(
        db.JSON()
        .with_variant(
            postgresql.JSONB(none_as_null=True),
            "postgresql",
        )
        .with_variant(
            JSONType(),
            "sqlite",
        )
        .with_variant(
            JSONType(),
            "mysql",
        ),
        default=dict,
        nullable=False,
    )
    """Original data."""

    transformed_data = db.Column(
        db.JSON()
        .with_variant(
            postgresql.JSONB(none_as_null=True),
            "postgresql",
        )
        .with_variant(
            JSONType(),
            "sqlite",
        )
        .with_variant(
            JSONType(),
            "mysql",
        ),
        default=dict,
        nullable=False,
    )
    """Transformed data before they were stored into the target record."""

    harvester_id = db.Column(
        db.String(30),
        db.ForeignKey("oai_harvesters.id"),
        nullable=False,
    )
    """ID of the harvester that harvested this record."""

    harvester = db.relationship(OAIHarvester)

    job_run_id = db.Column(UUIDType(binary=False), nullable=True)
    """Run ID of the harvest run (from invenio-jobs)."""

oai_identifier = db.Column(db.String(255), primary_key=True) class-attribute instance-attribute

OAI identifier of the record.

record_type = db.Column(db.String(32), nullable=True) class-attribute instance-attribute

Type of the record.

record_pid = db.Column(db.String(255), nullable=True) class-attribute instance-attribute

Persistent identifier (Record.pid) of the record (local identifier).

Note that this is not the database ID but the PID value.

datestamp = db.Column(db.UTCDateTime, nullable=False) class-attribute instance-attribute

Datestamp of the record.

harvested_at = db.Column(db.UTCDateTime, nullable=False) class-attribute instance-attribute

Time when the record was harvested.

deleted = db.Column(db.Boolean, default=False, nullable=False) class-attribute instance-attribute

True if the record was deleted during the harvest.

has_errors = db.Column(db.Boolean, default=False, nullable=False) class-attribute instance-attribute

True if the record has errors during the harvest.

has_warnings = db.Column(db.Boolean, default=False, nullable=False) class-attribute instance-attribute

True if the record has warnings during the harvest.

errors = db.Column(db.JSON().with_variant(postgresql.JSONB(none_as_null=True), 'postgresql').with_variant(JSONType(), 'sqlite').with_variant(JSONType(), 'mysql'), default=dict, nullable=False) class-attribute instance-attribute

Errors.

original_data = db.Column(db.JSON().with_variant(postgresql.JSONB(none_as_null=True), 'postgresql').with_variant(JSONType(), 'sqlite').with_variant(JSONType(), 'mysql'), default=dict, nullable=False) class-attribute instance-attribute

Original data.

transformed_data = db.Column(db.JSON().with_variant(postgresql.JSONB(none_as_null=True), 'postgresql').with_variant(JSONType(), 'sqlite').with_variant(JSONType(), 'mysql'), default=dict, nullable=False) class-attribute instance-attribute

Transformed data before they were stored into the target record.

harvester_id = db.Column(db.String(30), db.ForeignKey('oai_harvesters.id'), nullable=False) class-attribute instance-attribute

ID of the harvester that harvested this record.

job_run_id = db.Column(UUIDType(binary=False), nullable=True) class-attribute instance-attribute

Run ID of the harvest run (from invenio-jobs).

OAIRecordAggregateModel

Bases: AggregateMetadata

Aggregate data model for harvested OAI-PMH records.

Source code in oarepo_oaipmh_harvester/oai_record/models.py
class OAIRecordAggregateModel(AggregateMetadata):
    """Aggregate data model for harvested OAI-PMH records."""

    _model_obj: OAIHarvestedRecord | None

    # If you add properties here you likely also want to add a ModelField on
    # the OAIRecordAggregate API class.
    _properties = (
        "oai_identifier",
        "record_pid",
        "datestamp",
        "harvested_at",
        "deleted",
        "has_errors",
        "has_warnings",
        "errors",
        "original_data",
        "transformed_data",
        "harvester_id",
        "harvester",
    )
    """Properties of this object that can be accessed."""

    _set_properties: tuple[str, ...] = ()
    """Properties of this object that can be set."""

    @property
    @override
    def model_obj(self) -> OAIHarvestedRecord:
        """The actual model object behind this record aggregate."""
        if self._model_obj is None:
            id_ = (self._data or {}).get("id")
            with db.session.no_autoflush:
                self._model_obj = OAIHarvestedRecord.query.get(id_)
        if self._model_obj is None:
            raise ValueError("OAIHarvestedRecord not found")
        return self._model_obj

    @property
    def version_id(self) -> int:
        """Return the version ID of the record."""
        return 1

    @property
    def created(self) -> dt.datetime:
        """Return the creation date of the record."""
        return self.datestamp

    @property
    def updated(self) -> dt.datetime:
        """Return the last update date of the record."""
        return self.datestamp

    @property
    def id(self) -> str:
        """Return the ID of the record."""
        return self.oai_identifier

model_obj: OAIHarvestedRecord property

The actual model object behind this record aggregate.

version_id: int property

Return the version ID of the record.

created: dt.datetime property

Return the creation date of the record.

updated: dt.datetime property

Return the last update date of the record.

id: str property

Return the ID of the record.