Harvester merge unmerge

conftest.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2023 CERN.
+#
+# CDS-RDM is free software; you can redistribute it and/or modify it under
+# the terms of the MIT License; see LICENSE file for more details.
+
+"""Pytest configuration."""
+
+pytest_plugins = ("celery.contrib.pytest",)

site/cds_rdm/components.py

@@ -99,7 +99,7 @@ class SubjectsValidationComponent(ServiceComponent):
     def _validate_subject_changes(self, identity, updated_data, original_data):
         """Validate that the subject changes are allowed."""
         user = getattr(identity, "user", None)
-        if user and user.has_role("administration"):
+        if identity.id == "system" or user and user.has_role("administration"):
             return
         updated_collection_subjects = {
             s["subject"]
@@ -111,6 +111,8 @@ def _validate_subject_changes(self, identity, updated_data, original_data):
             for s in original_data
             if s.get("subject", "").startswith("collection:")
         }
+        print(updated_collection_subjects, "=============")
+        print(original_collection_subjects, "++++++++++++++++")
         if updated_collection_subjects != original_collection_subjects:
             raise ValidationError(
                 "Collection subjects cannot be updated.",

site/cds_rdm/inspire_harvester/__init__.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2025 CERN.
+# Copyright (C) 2026 CERN.
 #
 # CDS-RDM is free software; you can redistribute it and/or modify it under
-# the terms of the GPL-2.0 License; see LICENSE file for more details.
+# the terms of the MIT License; see LICENSE file for more details.
 
-"""INSPIRE to CDS harvester module."""
+"""INSPIRE to CDS harvester context module."""

site/cds_rdm/inspire_harvester/jobs.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2025 CERN.
+# Copyright (C) 2026 CERN.
 #
 # CDS-RDM is free software; you can redistribute it and/or modify it under
-# the terms of the GPL-2.0 License; see LICENSE file for more details.
+# the terms of the MIT License; see LICENSE file for more details.
 
 """Jobs module."""
 

site/cds_rdm/inspire_harvester/reader.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2025 CERN.
+# Copyright (C) 2026 CERN.
 #
 # CDS-RDM is free software; you can redistribute it and/or modify it under
-# the terms of the GPL-2.0 License; see LICENSE file for more details.
+# the terms of the MIT License; see LICENSE file for more details.
 
 """Reader component."""
 from urllib.parse import urlencode
@@ -38,7 +38,8 @@ def __init__(
 
     def _iter(self, url, *args, **kwargs):
         """Yields HTTP response."""
-        headers = {"Accept": "application/json"}
+        # header set to include additional data (external file URLs and more detailed metadata
+        headers = {"Accept": "application/vnd+inspire.record.expanded+json"}
 
         while url:  # Continue until there is no "next" link
             current_app.logger.info(f"Querying URL: {url}.")

site/cds_rdm/inspire_harvester/transform/__init__.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 CERN.
+#
+# CDS-RDM is free software; you can redistribute it and/or modify it under
+# the terms of the MIT License; see LICENSE file for more details.
+
+"""INSPIRE to CDS harvester context module."""

site/cds_rdm/inspire_harvester/transform/config.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 CERN.
+#
+# CDS-RDM is free software; you can redistribute it and/or modify it under
+# the terms of the MIT License; see LICENSE file for more details.
+
+"""INSPIRE to CDS harvester config module."""
+
+from cds_rdm.inspire_harvester.transform.mappers.basic_metadata import (
+    AdditionalDescriptionsMapper,
+    AdditionalTitlesMapper,
+    CopyrightMapper,
+    DescriptionMapper,
+    LanguagesMapper,
+    PublicationDateMapper,
+    PublisherMapper,
+    ResourceTypeMapper,
+    SubjectsMapper,
+    TitleMapper,
+)
+from cds_rdm.inspire_harvester.transform.mappers.contributors import (
+    AuthorsMapper,
+    ContributorsMapper,
+)
+from cds_rdm.inspire_harvester.transform.mappers.custom_fields import (
+    CERNFieldsMapper,
+    ImprintMapper,
+)
+from cds_rdm.inspire_harvester.transform.mappers.files import FilesMapper
+from cds_rdm.inspire_harvester.transform.mappers.identifiers import (
+    DOIMapper,
+    IdentifiersMapper,
+    RelatedIdentifiersMapper,
+)
+from cds_rdm.inspire_harvester.transform.mappers.thesis import (
+    ThesisContributorsMapper,
+    ThesisDefenceDateMapper,
+    ThesisPublicationDateMapper,
+    ThesisTypeMappers,
+    ThesisUniversityMappers,
+)
+from cds_rdm.inspire_harvester.transform.policies import MapperPolicy
+from cds_rdm.inspire_harvester.transform.resource_types import ResourceType
+
+BASE_MAPPERS = (
+    FilesMapper(),
+    ResourceTypeMapper(),
+    DOIMapper(),
+    TitleMapper(),
+    AdditionalTitlesMapper(),
+    AuthorsMapper(),
+    ContributorsMapper(),
+    PublisherMapper(),
+    PublicationDateMapper(),
+    CopyrightMapper(),
+    DescriptionMapper(),
+    AdditionalDescriptionsMapper(),
+    SubjectsMapper(),
+    LanguagesMapper(),
+    ImprintMapper(),
+    CERNFieldsMapper(),
+    IdentifiersMapper(),
+    RelatedIdentifiersMapper(),
+)
+
+THESIS_MAPPERS = (ThesisDefenceDateMapper(), ThesisUniversityMappers(),
+                  ThesisTypeMappers(),
+                  )
+
+inspire_mapper_policy = MapperPolicy(base=BASE_MAPPERS)
+
+mapper_policy = MapperPolicy(
+    base=BASE_MAPPERS,
+    add={
+        ResourceType.THESIS: THESIS_MAPPERS,
+    },
+    # if you had a generic mapper in base, you'd replace it here
+    replace={
+        (
+            ResourceType.THESIS,
+            "metadata.publication_date",
+        ): ThesisPublicationDateMapper(),
+        (ResourceType.THESIS, "metadata.contributors"): ThesisContributorsMapper(),
+    },
+
+)

site/cds_rdm/inspire_harvester/transform/context.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 CERN.
+#
+# CDS-RDM is free software; you can redistribute it and/or modify it under
+# the terms of the MIT License; see LICENSE file for more details.
+
+"""INSPIRE to CDS harvester context module."""
+
+from dataclasses import dataclass, field
+from typing import List
+
+from cds_rdm.inspire_harvester.transform.resource_types import ResourceType
+
+
+@dataclass(frozen=True)
+class MetadataSerializationContext:
+    """Metadata serializing context."""
+
+    resource_type: ResourceType
+    inspire_id: str
+    errors: List[str] = field(default_factory=list)

site/cds_rdm/inspire_harvester/transform/mappers/__init__.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 CERN.
+#
+# CDS-RDM is free software; you can redistribute it and/or modify it under
+# the terms of the MIT License; see LICENSE file for more details.
+
+"""INSPIRE to CDS harvester context module."""