Support csw dcat harvest

setup.py

@@ -44,6 +44,7 @@ def pip(filename):
         ],
         'udata.harvesters': [
             'dcat = udata.harvest.backends.dcat:DcatBackend',
+            'csw-dcat = udata.harvest.backends.dcat:CswDcatBackend',
         ],
         'udata.avatars': [
             'internal = udata.features.identicon.backends:internal',

udata/harvest/backends/base.py

@@ -268,8 +268,6 @@ def process(self, item):
         raise NotImplementedError
 
     def add_item(self, identifier, *args, **kwargs):
-        if identifier is None:
-            raise ValueError('DCT.identifier is required for all DCAT.Dataset records')
         item = HarvestItem(remote_id=str(identifier), args=args, kwargs=kwargs)
         self.job.items.append(item)
         return item

udata/harvest/backends/dcat.py

@@ -4,6 +4,7 @@
 
 from rdflib import Graph, URIRef, BNode
 from rdflib.namespace import RDF
+import xml.etree.ElementTree as ET
 from typing import List
 
 from udata.rdf import (
@@ -110,6 +111,8 @@ def get_node_from_item(self, item):
             return URIRef(nid) if item.kwargs['type'] == 'uriref' else BNode(nid)
 
     def process(self, item):
+        if item.remote_id == 'None':
+            raise ValueError('The DCT.identifier is missing on this DCAT.Dataset record')
         graph = Graph(namespace_manager=namespace_manager)
         data = self.job.data['graphs'][item.kwargs['page']]
         format = self.job.data['format']
@@ -120,3 +123,62 @@ def process(self, item):
         dataset = self.get_dataset(item.remote_id)
         dataset = dataset_from_rdf(graph, dataset, node=node)
         return dataset
+
+
+class CswDcatBackend(DcatBackend):
+    display_name = 'CSW-DCAT'
+
+    def parse_graph(self, url, fmt):
+        body = '''<csw:GetRecords xmlns:csw="http://www.opengis.net/cat/csw/2.0.2"
+                                  xmlns:gmd="http://www.isotc211.org/2005/gmd"
+                                  service="CSW" version="2.0.2" resultType="results"
+                                  startPosition="{start}" maxPosition="15"
+                                  outputSchema="http://www.w3.org/ns/dcat#">
+                    <csw:Query typeNames="gmd:MD_Metadata">
+                        <csw:ElementSetName>full</csw:ElementSetName>
+                        <csw:Constraint version="1.1.0">
+                            <Filter xmlns="http://www.opengis.net/ogc"><PropertyIsEqualTo>
+                                <PropertyName>documentStandard</PropertyName>
+                                <Literal>iso19139</Literal>
+                            </PropertyIsEqualTo></Filter>
+                        </csw:Constraint>
+                    </csw:Query>
+                </csw:GetRecords>'''
+        headers = {"Content-Type": "application/xml"}
+
+        graphs = []
+        page = 0
+
+        content = requests.post(url, data=body.format(start=1), headers=headers).text
+        tree = ET.fromstring(content)
+        while tree:
+            graph = Graph(namespace_manager=namespace_manager)
+            # TODO: could we find a better way to deal with namespaces?
+            namespace = tree.tag.split('}')[0].strip('{}')
+            search_results = tree.find('csw:SearchResults', {'csw': namespace})
+            if not search_results:
+                # TODO: may be worth an investigation if it happens
+                log.error(f'No search results found for {url} on page {page}')
+                break
+            for child in search_results:
+                subgraph = Graph(namespace_manager=namespace_manager)
+                subgraph.parse(data=ET.tostring(child), format=fmt)
+                graph += subgraph
+
+                for node in subgraph.subjects(RDF.type, DCAT.Dataset):
+                    id = subgraph.value(node, DCT.identifier)
+                    kwargs = {'nid': str(node), 'page': page}
+                    kwargs['type'] = 'uriref' if isinstance(node, URIRef) else 'blank'
+                    self.add_item(id, **kwargs)
+            graphs.append(graph)
+            page += 1
+
+            if int(search_results.attrib['nextRecord']) == 0 or \
+                    self.max_items and len(self.job.items) >= self.max_items:
+                break
+
+            tree = ET.fromstring(
+                requests.post(url, data=body.format(start=search_results.attrib['nextRecord']),
+                              headers=headers).text)
+
+        return graphs