[PATCH 13 of 14 eac] import dates, items, and place_entries as child

Guillaume Vandevelde guillaume.vandevelde at logilab.fr
Tue Jul 2 14:54:02 CEST 2019


# HG changeset patch
# User Guillaume Vandevelde <gvandevelde at logilab.fr>
# Date 1561971826 -7200
#      Mon Jul 01 11:03:46 2019 +0200
# Node ID 5115998f280adfa7916aa435931197186006702e
# Parent  907de60ffc8a32d6d3d0dea1207c456edce52ae9
# Available At http://hg.logilab.org/review/cubes/eac
#              hg pull http://hg.logilab.org/review/cubes/eac -r 5115998f280a
import dates, items, and place_entries as child

Change the import of the dates, items and place_entries.
They are now imported as child of their upper element.
The child concatenation is based on the previous decorator that handled the appending of `Citation`. It have been modified to handle any child appending that is based on the same model.
The child concatenation is quite ugly. Some functions have more decorators lines for child appending than they have for their own code.
Nonetheless this mechanic allows to transforms quickly the data tree and to adapt tags for beeing imported as list.

This change is needed because these elements can appear as a list into a lot of tags.

Because this modification change the data structure of some previously exported entities, it break the export tests. I'll handle that problem later, and focus on the integration of this changes in francearchives.

Differential Revision: https://phab.logilab.fr/D3674

diff -r 907de60ffc8a -r 5115998f280a cubicweb_eac/dataimport.py
--- a/cubicweb_eac/dataimport.py	Thu Jun 27 17:46:09 2019 +0200
+++ b/cubicweb_eac/dataimport.py	Mon Jul 01 11:03:46 2019 +0200
@@ -185,8 +185,9 @@
             builders = {'has_citation': self.build_citation,
                         'has_event': self.build_event,
                         'simple_name_relation': self.build_parallel_name_entry,
-                        'date_relation': self.build_parallel_date_entry,
-                        'has_item': self.build_item}
+                        'date_relation': self.build_date_entry,
+                        'has_item': self.build_item,
+                        'place_entry_relation': self.build_place_entry}
             build_child = builders[relation]
             for extentity in func(self, elem):
                 if extentity.etype == etype:
@@ -463,8 +464,9 @@
 
     @filter_empty
     @filter_none
-    def build_parallel_date_entry(self, elem):
-        """Build DateEntity linked to a ParallelNames entity"""
+    @elem_maybe_none
+    def build_date_entry(self, elem):
+        """Build DateEntitys linked to a parent entity"""
         for extentity in self.extract_dates_from(elem, 'eac:useDates'):
             yield extentity
         for date_set in self._elem_findall(elem, 'eac:dateSet'):
@@ -537,6 +539,7 @@
             for extentity in self.build_generalcontext(context):
                 yield extentity
 
+    @elem_maybe_none
     def build_item(self, elem):
         for item in self.find_nested(elem, 'eac:item', 'eac:list'):
             values = {}
@@ -630,7 +633,10 @@
     @elem_maybe_none
     @relate_to_record_through('LegalStatus', 'legal_status_agent')
     @filter_empty
+    @add_child_for('LegalStatus', 'place_entry_relation')
+    @add_child_for('LegalStatus', 'has_item')
     @add_child_for('LegalStatus', 'has_citation')
+    @add_child_for('LegalStatus', 'date_relation')
     @equivalent_concept('eac:term', 'LegalStatus')
     def build_legal_status(self, elem, **kwargs):
         """Build a `LegalStatus` external entity.
@@ -641,19 +647,15 @@
         term = self._elem_find(elem, 'eac:term')
         if term is not None and term.text:
             values['term'] = set([text_type(term.text)])
-        dateelem = self._elem_find(elem, 'eac:dateRange')
-        dates = self.parse_daterange(dateelem)
-        if dates:
-            values.update(dates)
-        legalstatus = ExtEntity('LegalStatus', self._gen_extid(), values)
-        if dateelem is not None:
-            self.record_visited(dateelem, legalstatus)
-        yield legalstatus
+        yield ExtEntity('LegalStatus', self._gen_extid(), values)
 
     @elem_maybe_none
     @relate_to_record_through('Mandate', 'mandate_agent')
     @filter_empty
+    @add_child_for('Mandate', 'place_entry_relation')
+    @add_child_for('Mandate', 'has_item')
     @add_child_for('Mandate', 'has_citation')
+    @add_child_for('Mandate', 'date_relation')
     @equivalent_concept('eac:term', 'Mandate')
     def build_mandate(self, elem, **kwargs):
         """Build a `Mandate` external entity.
@@ -664,10 +666,6 @@
         term = self._elem_find(elem, 'eac:term')
         if term is not None and term.text:
             values['term'] = set([text_type(term.text)])
-        dateelem = self._elem_find(elem, 'eac:dateRange')
-        dates = self.parse_daterange(dateelem)
-        if dates:
-            values.update(dates)
         yield ExtEntity('Mandate', self._gen_extid(), values)
 
     @elem_maybe_none
@@ -706,6 +704,8 @@
                 values['abstract'] = set([text_type(abstract.text)])
             yield ExtEntity('History', self._gen_extid(), values)
 
+    @add_child_for('Event', 'date_relation')
+    @add_child_for('Event', 'place_entry_relation')
     @filter_none
     @filter_empty
     @elem_maybe_none
@@ -715,18 +715,10 @@
         if citems is not None:
             for citem in citems:
                 values = {}
-                date = self._elem_find(citem, 'eac:date')
-                date_range = self._elem_find(citem, 'eac:dateRange')
                 event = self._elem_find(citem, 'eac:event')
                 if event is not None and event.text:
                     values['event'] = set([text_type(event.text)])
-                    if date is None and date_range is not None:
-                        date_range = self.parse_daterange(date_range)
-                        values.update(date_range)
-                    elif date is not None:
-                        values.update({'start_date': set([self.parse_date(date)]),
-                                       'end_date': set([self.parse_date(date)])})
-                    yield ExtEntity('Event', self._gen_extid(), values)
+                yield ExtEntity('Event', self._gen_extid(), values)
 
     @elem_maybe_none
     @relate_to_record_through('Structure', 'structure_agent')
@@ -740,6 +732,9 @@
 
     @relate_to_record_through('AgentPlace', 'place_agent')
     @filter_empty
+    @add_child_for('AgentPlace', 'place_entry_relation')
+    @add_child_for('AgentPlace', 'date_relation')
+    @add_child_for('AgentPlace', 'has_item')
     @add_child_for('AgentPlace', 'has_citation')
     @equivalent_concept('eac:placeEntry', 'AgentPlace')
     def build_place(self, elem):
@@ -758,6 +753,14 @@
                     yield extentity
         yield ExtEntity('AgentPlace', self._gen_extid(), values)
 
+    @filter_none
+    @filter_empty
+    @elem_maybe_none
+    def build_place_entry(self, elem):
+        for entry in self._elem_findall(elem, 'eac:placeEntry'):
+            values = {'place_entry': set([text_type(etree.tostring(entry))])}
+            yield ExtEntity('PlaceEntry', self._gen_extid(), values)
+
     def build_address(self, elem):
         """Build `PostalAddress`s external entity"""
         address_entity = {}
@@ -771,6 +774,9 @@
 
     @relate_to_record_through('AgentFunction', 'function_agent')
     @filter_empty
+    @add_child_for('AgentFunction', 'place_entry_relation')
+    @add_child_for('AgentFunction', 'date_relation')
+    @add_child_for('AgentFunction', 'has_item')
     @add_child_for('AgentFunction', 'has_citation')
     @equivalent_concept('eac:term', 'AgentFunction')
     def build_function(self, elem):
@@ -783,7 +789,10 @@
 
     @relate_to_record_through('Occupation', 'occupation_agent')
     @filter_empty
+    @add_child_for('Occupation', 'place_entry_relation')
+    @add_child_for('Occupation', 'has_item')
     @add_child_for('Occupation', 'has_citation')
+    @add_child_for('Occupation', 'date_relation')
     @equivalent_concept('eac:term', 'Occupation')
     def build_occupation(self, elem):
         """Build a `Occupation`s external entities"""
@@ -791,13 +800,10 @@
         term = self._elem_find(elem, 'eac:term')
         if term is not None:
             values['term'] = set([text_type(term.text)])
-        dateelem = self._elem_find(elem, 'eac:dateRange')
-        dates = self.parse_daterange(dateelem)
-        if dates:
-            values.update(dates)
         yield ExtEntity('Occupation', self._gen_extid(), values)
 
     @relate_to_record_through('GeneralContext', 'general_context_of')
+    @add_child_for('GeneralContext', 'has_item')
     @add_child_for('GeneralContext', 'has_citation')
     def build_generalcontext(self, elem):
         """Build a `GeneralContext` external entity"""
@@ -878,6 +884,12 @@
 
     @add_xml_wrap_for('AssociationRelation', 'ChronologicalRelation',
                       'HierarchicalRelation')
+    @add_child_for('AssociationRelation', 'date_relation')
+    @add_child_for('ChronologicalRelation', 'date_relation')
+    @add_child_for('HierarchicalRelation', 'date_relation')
+    @add_child_for('AssociationRelation', 'place_entry_relation')
+    @add_child_for('ChronologicalRelation', 'place_entry_relation')
+    @add_child_for('HierarchicalRelation', 'place_entry_relation')
     def build_relation(self, elem):
         """Build a relation between records external entity (with proper type)."""
         relationship = elem.attrib.get('cpfRelationType')
@@ -926,13 +938,11 @@
         rentry = self._elem_find(elem, 'eac:relationEntry')
         if rentry is not None and rentry.text.strip():
             values['entry'] = set([text_type(rentry.text)])
-        dates = self.parse_daterange(
-            self._elem_find(elem, 'eac:dateRange'))
-        if dates:
-            values.update(dates)
         values.update(self.parse_tag_description(elem))
         yield ExtEntity(etype, self._gen_extid(), values)
 
+    @add_child_for('EACFunctionRelation', 'place_entry_relation')
+    @add_child_for('EACFunctionRelation', 'date_relation')
     @add_xml_wrap_for('EACFunctionRelation')
     def build_function_relation(self, elem):
         """Build a relation between function entities
@@ -956,10 +966,6 @@
         # Yield the ExternalUri object
         yield external_uri(obj_uri)
         values = self.parse_tag_description(elem)
-        dates = self.parse_daterange(
-            self._elem_find(elem, 'eac:dateRange'))
-        if dates:
-            values.update(dates)
         values.update({
             'r_type': set([text_type(relationship)]),
             'function_relation_function': set([text_type(obj_uri)]),
@@ -977,6 +983,8 @@
             values.update({'attributes': set([text_type(attrib)])})
         yield ExtEntity('EACFunctionRelation', self._gen_extid(), values)
 
+    @add_child_for('EACResourceRelation', 'place_entry_relation')
+    @add_child_for('EACResourceRelation', 'date_relation')
     @add_xml_wrap_for('EACResourceRelation')
     def build_resource_relation(self, elem):
         """Build a `EACResourceRelation` external entity (along with
@@ -999,9 +1007,6 @@
         agent_role = elem.attrib.get('resourceRelationType')
         if agent_role:
             values['agent_role'] = set([text_type(agent_role)])
-        dates = self.parse_daterange(self._elem_find(elem, 'eac:dateRange'))
-        if dates:
-            values.update(dates)
         values.update(self.parse_tag_description(elem))
         attrib = {k: v for (k, v) in dict(elem.attrib).items() if k not in {
             'relationRelationType',
@@ -1036,7 +1041,10 @@
                 self.record_visited(other_record_id, extentity)
                 yield extentity
         publication_status = self._elem_find(control, 'eac:publicationStatus')
-        p_status = publication_status.text.strip()
+        if publication_status is not None and publication_status.text.strip():
+            p_status = publication_status.text.strip()
+        else:
+            p_status = None
         self.record_visited(publication_status, self.record)
         maintenance_status = self._elem_find(control, 'eac:maintenanceStatus')
         if maintenance_status is None:
diff -r 907de60ffc8a -r 5115998f280a cubicweb_eac/migration/0.9.0_Any.py
--- a/cubicweb_eac/migration/0.9.0_Any.py	Thu Jun 27 17:46:09 2019 +0200
+++ b/cubicweb_eac/migration/0.9.0_Any.py	Mon Jul 01 11:03:46 2019 +0200
@@ -1,21 +1,37 @@
-add_attribute('History', 'abstract')
+# Entities to add
 add_entity_type('Event')
 add_entity_type('Convention')
 add_entity_type('Language')
 add_entity_type('LanguageDec')
 add_entity_type('ParallelNames')
 add_entity_type('MaintenanceAg')
+add_entity_type('EACFunctionRelation')
+add_entity_type('NameEntryS')
+add_entity_type('DateEntity')
+add_entity_type('Item')
+add_entity_type('PlaceEntry')
+
+# Attributes to update
+add_attribute('EACResourceRelation', 'attributes')
+
+add_attribute('History', 'abstract')
+
 add_attribute('MaintenanceAg', 'maintenance_status')
 add_attribute('MaintenanceAg', 'publication_status')
-add_entity_type('EACFunctionRelation')
-add_attribute('EACResourceRelation', 'attributes')
-add_attribute('EACFunctionRelation', 'attributes')
-add_entity_type('NameEntryS')
-add_attribute('NameEntry', 'language')
-add_attribute('NameEntry', 'preferred_form')
-add_attribute('NameEntry', 'alternative_form')
-add_attribute('NameEntry', 'authorized_form')
-add_attribute('NameEntry', 'script_code')
-add_entity_type('DateEntity')
-add_entity_type('Item')
+
+for attrib in ('language', 'preferred_form',
+               'alternative_form', 'authorized_form',
+               'script_code'):
+    add_attribute('NameEntry', attrib)
+
 add_attribute('Activity', 'agent_type')
+
+for etype in ('AssociationRelation', 'ChronologicalRelation',
+              'HierarchicalRelation', 'Mandate', 'LegalStatus',
+              'Event', 'Occupation', 'EACFunctionRelation',
+              'EACResourceRelation'):
+    drop_attribute(etype, 'start_date')
+    drop_attribute(etype, 'end_date')
+
+# Apply relations modifications
+sync_schema_props_perms()
diff -r 907de60ffc8a -r 5115998f280a cubicweb_eac/schema.py
--- a/cubicweb_eac/schema.py	Thu Jun 27 17:46:09 2019 +0200
+++ b/cubicweb_eac/schema.py	Mon Jul 01 11:03:46 2019 +0200
@@ -97,7 +97,8 @@
 
 
 class has_item(RelationDefinition):
-    subject = 'History'
+    subject = ('GeneralContext', 'Mandate', 'Occupation', 'History',
+               'AgentFunction', 'LegalStatus', 'AgentPlace', 'Convention')
     object = 'Item'
     cardinality = '*1'
     composite = 'subject'
@@ -110,13 +111,31 @@
 
 
 class date_relation(RelationDefinition):
-    subject = 'ParallelNames'
+    subject = ('ParallelNames', 'HierarchicalRelation', 'Event',
+               'ChronologicalRelation', 'AssociationRelation',
+               'AgentFunction', 'EACFunctionRelation', 'LegalStatus',
+               'Mandate', 'Occupation', 'AgentPlace', 'EACResourceRelation')
     object = 'DateEntity'
     cardinality = '*1'
     composite = 'subject'
     fulltext_container = 'subject'
 
 
+class PlaceEntry(EntityType):
+    place_entry = String(fulltextindexed=True)
+
+
+class place_entry_relation(RelationDefinition):
+    subject = ('HierarchicalRelation', 'Event',
+               'ChronologicalRelation', 'AssociationRelation',
+               'AgentFunction', 'EACFunctionRelation', 'LegalStatus',
+               'Mandate', 'Occupation', 'AgentPlace', 'EACResourceRelation')
+    object = 'PlaceEntry'
+    cardinality = '*1'
+    composite = 'subject'
+    fulltext_container = 'subject'
+
+
 class simple_name_relation(RelationDefinition):
     subject = 'ParallelNames'
     object = 'NameEntryS'
@@ -272,7 +291,6 @@
 
 
 @xml_wrap
- at dated_entity_type
 class AssociationRelation(EntityType):
     """Association relation between authority records"""
     entry = String()
@@ -288,7 +306,6 @@
 
 
 @xml_wrap
- at dated_entity_type
 class ChronologicalRelation(EntityType):
     """Chronological relation between authority records"""
     entry = String()
@@ -304,7 +321,6 @@
 
 
 @xml_wrap
- at dated_entity_type
 class HierarchicalRelation(EntityType):
     """Hierarchical relation between authority records"""
     entry = String()
@@ -329,14 +345,12 @@
     object = 'AuthorityRecord'
 
 
- at dated_entity_type
 class Mandate(EntityType):
     """Reference text coming from an authority"""
     term = String(fulltextindexed=True)
     description = RichString(fulltextindexed=True)
 
 
- at dated_entity_type
 class LegalStatus(EntityType):
     """Information relative to the legal status of an authority"""
     term = String(fulltextindexed=True)
@@ -374,11 +388,9 @@
     text = RichString(fulltextindexed=True)
 
 
- at dated_entity_type
 class Event(EntityType):
     """Number of dates and event description linked to an History object"""
     event = RichString(fulltextindexed=True)
-    place_entry = String(fulltextindexed=True)
 
 
 class has_event(RelationDefinition):
@@ -395,7 +407,6 @@
     description = RichString(fulltextindexed=True)
 
 
- at dated_entity_type
 class Occupation(EntityType):
     term = String(fulltextindexed=True)
     description = RichString(fulltextindexed=True)
@@ -467,7 +478,6 @@
 
 
 @xml_wrap
- at dated_entity_type
 class EACFunctionRelation(EntityType):
     """Represent a relation between an AuthorityRecord and a function"""
     r_type = String(internationalizable=True,
@@ -496,7 +506,6 @@
 
 
 @xml_wrap
- at dated_entity_type
 class EACResourceRelation(EntityType):
     """Represent a relation between an AuthorityRecord and a remote resource in the
     EAC-CPF model.
diff -r 907de60ffc8a -r 5115998f280a test/test_dataimport.py
--- a/test/test_dataimport.py	Thu Jun 27 17:46:09 2019 +0200
+++ b/test/test_dataimport.py	Mon Jul 01 11:03:46 2019 +0200
@@ -73,7 +73,7 @@
         return importer.external_entities()
 
     def test_parse_FRAD033_EAC_00001(self):
-        _gen_extid = map(str, (x for x in count() if x != 3)).next
+        _gen_extid = map(str, (x for x in count() if x not in (3, 40))).next
         expected = [
             ('EACOtherRecordId', _gen_extid(),
              {'eac_other_record_id_of': set(['FRAD033_EAC_00001']),
@@ -220,31 +220,50 @@
              {'name': set([u'Bordeaux (Gironde, France)']),
               'role': set([u'siege']),
               'place_agent': set(['FRAD033_EAC_00001']),
+              'place_entry_relation': set(['25']),
               'place_address': set(['23']),
               'equivalent_concept': set(['http://catalogue.bnf.fr/ark:/12148/cb152418385']),
              },
             ),
+            ('PlaceEntry', _gen_extid(),
+             {'place_entry': set([u'<placeEntry xmlns="urn:isbn:1-931666-33-4" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xlink="http://www.w3.org/1999/xlink" vocabularySource="http://catalogue.bnf.fr/ark:/12148/cb152418385">Bordeaux (Gironde, France)</placeEntry>\n          ']) # noqa
+             },
+            ),
             ('AgentPlace', _gen_extid(),
              {'name': set([u'Toulouse (France)']),
               'place_agent': set(['FRAD033_EAC_00001']),
+              'place_entry_relation': set(['27']),
               'role': set([u'domicile']),
              },
             ),
+            ('PlaceEntry', _gen_extid(),
+             {'place_entry': set([u'<placeEntry xmlns="urn:isbn:1-931666-33-4" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xlink="http://www.w3.org/1999/xlink">Toulouse (France)</placeEntry>\n      ']), # noqa
+             },
+            ),
             ('AgentPlace', _gen_extid(),
              {'name': set([u'Lit']),
               'place_agent': set(['FRAD033_EAC_00001']),
               'role': set([u'dodo']),
+              'place_entry_relation': set(['29']),
+             },
+            ),
+            ('PlaceEntry', _gen_extid(),
+             {'place_entry': set([u'<placeEntry xmlns="urn:isbn:1-931666-33-4" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xlink="http://www.w3.org/1999/xlink">Lit</placeEntry>\n          ']), # noqa
              },
             ),
             ('LegalStatus', _gen_extid(),
              {'term': set([u'Collectivité territoriale']),
-              'start_date': set([datetime.date(1234, 1, 1)]),
-              'end_date': set([datetime.date(3000, 1, 1)]),
+              'date_relation': set(['31']),
               'description': set([u'Description du statut']),
               'description_format': set([u'text/plain']),
               'legal_status_agent': set(['FRAD033_EAC_00001']),
              },
             ),
+            ('DateEntity', _gen_extid(),
+             {'start_date': set([datetime.date(1234, 1, 1)]),
+              'end_date': set([datetime.date(3000, 1, 1)])
+             },
+            ),
             ('Mandate', _gen_extid(),
              {'term': set([u'1. Constitutions françaises']),
               'description': set([u'Description du mandat']),
@@ -275,9 +294,9 @@
               ]),
               'text_format': set([u'text/html']),
               'history_agent': set(['FRAD033_EAC_00001']),
-              'has_citation': set(['32', '33']),
-              'has_event': set(['35', '34']),
-              'has_item': set(['38', '37', '36'])
+              'has_citation': set(['37', '36']),
+              'has_event': set(['38', '39']),
+              'has_item': set(['41', '42', '43'])
              },
             ),
             ('Citation', _gen_extid(),
@@ -287,14 +306,12 @@
              {'uri': set(['http://pifgadget']), 'note': set(['Voir aussi pifgadget'])},
             ),
             ('Event', _gen_extid(),
-             {'event': [u'Left Mer and moved to the mainland.\n\t      Worked at various jobs including canecutter\n\t      and railway labourer.\n\t      '], # noqa
-              'end_date': set([datetime.date(1957, 1, 1)]),
-              'start_date': set([datetime.date(1957, 1, 1)])}
+             {'event': set([u'Left Mer and moved to the mainland.\n\t      Worked at various jobs including canecutter\n\t      and railway labourer.\n\t      ']) # noqa
+             },
             ),
             ('Event', _gen_extid(),
-             {'event': set([u'Union representative, Townsville-\n\t      Mount Isa rail construction project.\n\t      ']), # noqa
-              'end_date': set([datetime.date(1961, 1, 1)]),
-              'start_date': set([datetime.date(1960, 1, 1)])}
+             {'event': set([u'Union representative, Townsville-\n\t      Mount Isa rail construction project.\n\t      ']) # noqa
+             },
             ),
             ('Item', _gen_extid(),
              {'span': set([u'<span xmlns="urn:isbn:1-931666-33-4" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xlink="http://www.w3.org/1999/xlink" style="font-       style:italic">1450-1950\n\t      </span>\n\t      (1929)']) # noqa
@@ -340,15 +357,19 @@
             ),
             ('Occupation', _gen_extid(),
              {'term': set([u'Réunioniste']),
-              'start_date': set([datetime.date(1987, 1, 1)]),
-              'end_date': set([datetime.date(2099, 1, 1)]),
+              'date_relation': set(['49']),
               'description': set([u'Organisation des réunions ...']),
               'description_format': set([u'text/plain']),
               'occupation_agent': set(['FRAD033_EAC_00001']),
-              'has_citation': set(['44']),
+              'has_citation': set(['50']),
               'equivalent_concept': set(['http://pifgadget.com']),
              },
             ),
+            ('DateEntity', _gen_extid(),
+             {'start_date': set([datetime.date(1987, 1, 1)]),
+              'end_date': set([datetime.date(2099, 1, 1)])
+             },
+            ),
             ('Citation', _gen_extid(),
              {'note': set([u'la bible']),
              },
@@ -358,7 +379,7 @@
                               u'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" '
                               u'xmlns:xlink="http://www.w3.org/1999/xlink">very famous</p>']),
               'content_format': set([u'text/html']),
-              'has_citation': set(['46']),
+              'has_citation': set(['52']),
               'general_context_of': set(['FRAD033_EAC_00001']),
               }
             ),
@@ -372,16 +393,20 @@
              },
             ),
             ('HierarchicalRelation', _gen_extid(),
-             {'start_date': set([datetime.date(2008, 1, 1)]),
-              'end_date': set([datetime.date(2099, 1, 1)]),
-              'entry': set([u"Gironde. Conseil général. Direction de l'administration et de "
+             {'entry': set([u"Gironde. Conseil général. Direction de l'administration et de "
                             u"la sécurité juridique"]),
+              'date_relation': set(['54']),
               'description': set([u'Coucou']),
               'description_format': set([u'text/plain']),
               'hierarchical_parent': set(['CG33-DIRADSJ']),
               'hierarchical_child': set(['FRAD033_EAC_00001']),
              },
             ),
+            ('DateEntity', _gen_extid(),
+             {'start_date': set([datetime.date(2008, 1, 1)]),
+              'end_date': set([datetime.date(2099, 1, 1)])
+             },
+            ),
             ('ExternalUri', 'whatever',
              {'uri': set([u'whatever']),
               'cwuri': set([u'whatever']),
@@ -395,19 +420,27 @@
             ('ChronologicalRelation', _gen_extid(),
              {'chronological_predecessor': set(['whatever']),
               'chronological_successor': set(['FRAD033_EAC_00001']),
-              'start_date': set([datetime.date(1917, 1, 1)]),
-              'end_date': set([datetime.date(2009, 1, 1)]),
+              'date_relation': set(['56']),
               'entry': set([u'CG32']),
              },
             ),
+            ('DateEntity', _gen_extid(),
+             {'start_date': set([datetime.date(1917, 1, 1)]),
+              'end_date': set([datetime.date(2009, 1, 1)])
+             },
+            ),
             ('ChronologicalRelation', _gen_extid(),
              {'chronological_predecessor': set(['FRAD033_EAC_00001']),
               'chronological_successor': set(['/dev/null']),
-              'start_date': set([datetime.date(2042, 1, 1)]),
+              'date_relation': set(['58']),
               'xml_wrap': set(['<gloups xmlns="urn:isbn:1-931666-33-4" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xlink="http://www.w3.org/1999/xlink">hips</gloups>']),  # noqa
               'entry': set([u'Trash']),
              },
             ),
+            ('DateEntity', _gen_extid(),
+             {'start_date': set([datetime.date(2042, 1, 1)])
+             },
+            ),
             ('AssociationRelation', _gen_extid(),
              {'association_from': set(['FRAD033_EAC_00001']),
               'association_to': set(['agent-x']),
@@ -415,16 +448,20 @@
             ),
             ('EACResourceRelation', _gen_extid(),
              {'agent_role': set([u'creatorOf']),
+              'date_relation': set(['61']),
               'attributes': set([u"{'{http://www.w3.org/1999/xlink}show': 'new', '{http://www.w3.org/1999/xlink}actuate': 'onRequest', '{http://www.w3.org/1999/xlink}type': 'simple', 'resourceRelationType': 'creatorOf'}"]), # noqa
               'resource_role': set([u'Fonds d\'archives']),
               'resource_relation_resource': set([
                   'http://gael.gironde.fr/ead.html?id=FRAD033_IR_N']),
               'resource_relation_agent': set(['FRAD033_EAC_00001']),
-              'start_date': set([datetime.date(1673, 1, 1)]),
-              'end_date': set([datetime.date(1963, 1, 1)]),
               'xml_wrap': set(['<he xmlns="urn:isbn:1-931666-33-4" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">joe</he>']),  # noqa
              },
             ),
+            ('DateEntity', _gen_extid(),
+             {'start_date': set([datetime.date(1673, 1, 1)]),
+              'end_date': set([datetime.date(1963, 1, 1)])
+             },
+            ),
             ('EACFunctionRelation', _gen_extid(),
              {'description': set([u'<p xmlns="urn:isbn:1-931666-33-4" xmlns:xsi="http:/'
                                   '/www.w3.org/2001/XMLSchema-instance" xmlns:xlink="http:'
@@ -450,11 +487,15 @@
                                   'The second responsibility of the\n\t  '
                                   'Department is to control the establishment\n\t  '
                                   'and abolishment of schools.\n\t  </p>']),
-              'end_date': set([datetime.date(2001, 1, 1)]),
               'r_type': set([u'controls']),
               'description_format': set([u'text/html']),
+              'date_relation': set(['64']),
               'relation_entry': set([u'Establishment and abolishment\n\tof schools\n\t']),
-              'start_date': set([datetime.date(1922, 1, 1)])
+             },
+            ),
+            ('DateEntity', _gen_extid(),
+             {'start_date': set([datetime.date(1922, 1, 1)]),
+              'end_date': set([datetime.date(2001, 1, 1)])
              },
             ),
             ('ExternalUri', 'FRAD033_IR_N',
@@ -479,10 +520,10 @@
             ('ExternalUri', 'http://catalogue.bnf.fr/ark:/12148/cb152418385',
              {'uri': set([u'http://catalogue.bnf.fr/ark:/12148/cb152418385']),
               'cwuri': set([u'http://catalogue.bnf.fr/ark:/12148/cb152418385'])},
-           ),
-             ('ExternalUri', 'http://pifgadget.com',
+            ),
+            ('ExternalUri', 'http://pifgadget.com',
              {'uri': set([u'http://pifgadget.com']),
-              'cwuri': set([u'http://pifgadget.com'])},
+            'cwuri': set([u'http://pifgadget.com'])},
             ),
             ('AuthorityRecord', 'FRAD033_EAC_00001',
              {'isni': set([u'22330001300016']),
@@ -501,6 +542,8 @@
         importer = dataimport.EACCPFImporter(fpath, import_log, mock_,
                                              extid_generator=extid_generator)
         entities = list(importer.external_entities())
+        # Used for an easier handling of the order error while generating the 2 lists
+        self.check_order_entities(entities, expected)
         self.check_external_entities(entities, expected)
         visited = set([])
         for x in importer._visited.values():
@@ -517,6 +560,16 @@
                           'biogHist': set([307, 310]),  # empty.
                           })
 
+    def check_order_entities(self, entities, expected):
+
+        def get_sorted(elems):
+            sorted([(e.etype, e.extid) for e in elems
+                    if e.etype != 'ExternalUri'], key=lambda e: e[1])
+
+        a_lst = get_sorted(entities)
+        e_lst = get_sorted(expected)
+        self.assertEqual(a_lst, e_lst)
+
     def test_mandate_under_mandates(self):
         """In FRAD033_EAC_00003.xml, <mandate> element are within <mandates>."""
         entities = list(self.file_extentities('FRAD033_EAC_00003.xml'))
@@ -612,7 +665,7 @@
                                cwuri=u'http://data.culture.fr/thesaurus/page/ark:/67717/T1-1074')
             cnx.commit()
             created, updated = testutils.eac_import(cnx, fpath)
-            self.assertEqual(len(created), 63)
+            self.assertEqual(len(created), 73)
             self.assertEqual(updated, set())
             rset = cnx.find('AuthorityRecord', isni=u'22330001300016')
             self.assertEqual(len(rset), 1)
@@ -759,10 +812,6 @@
         self.assertEqual(func_relation.function_relation_agent[0], record)
         self.assertEqual(func_relation.function_relation_function[0].uri,
                          u'FRAD033_IR_N')
-        self.assertEqual(func_relation.start_date,
-                         datetime.date(1922, 1, 1))
-        self.assertEqual(func_relation.end_date,
-                         datetime.date(2001, 1, 1))
 
     def _check_parallel_relations(self, cnx, record):
         rset = cnx.find('ParallelNames', parallel_names_of=record).sorted_rset(lambda x: x.eid)
diff -r 907de60ffc8a -r 5115998f280a test/test_schema.py
--- a/test/test_schema.py	Thu Jun 27 17:46:09 2019 +0200
+++ b/test/test_schema.py	Mon Jul 01 11:03:46 2019 +0200
@@ -103,13 +103,24 @@
                 'GeneralContext', 'Mandate', 'Occupation', 'AgentFunction',
                 'AgentPlace', 'History', 'LegalStatus', 'Convention',
             ])},
+            'Item': {('has_item', 'object'): set([
+                'GeneralContext', 'Mandate', 'Occupation', 'AgentFunction',
+                'AgentPlace', 'History', 'LegalStatus', 'Convention',
+            ])},
+            'DateEntity': {('date_relation', 'object'): set([
+                'ParallelNames', 'Event', 'AgentFunction', 'EACFunctionRelation',
+                'LegalStatus', 'Mandate', 'Occupation', 'AgentPlace', 'EACResourceRelation'
+            ])},
+            'PlaceEntry': {('place_entry_relation', 'object'): set([
+                'Event', 'AgentFunction', 'EACFunctionRelation',
+                'LegalStatus', 'Mandate', 'Occupation', 'AgentPlace', 'EACResourceRelation'
+            ])},
             'EACFunctionRelation': {('function_relation_agent', 'subject'):
                                     set(['AuthorityRecord'])},
             'EACOtherRecordId': {('eac_other_record_id_of', 'subject'):
                                  set(['AuthorityRecord'])},
             'EACResourceRelation': {('resource_relation_agent', 'subject'):
                                     set(['AuthorityRecord'])},
-            'DateEntity': {('date_relation', 'object'): set(['ParallelNames'])},
             'Convention': {('convention_of', 'subject'): set(['AuthorityRecord'])},
             'LanguageDec': {('language_declaration_of', 'subject'): set(['AuthorityRecord'])},
             'Language': {('language_used_in', 'subject'): set(['AuthorityRecord'])},
@@ -126,7 +137,6 @@
             'Structure': {('structure_agent', 'subject'): set(['AuthorityRecord'])},
             'ParallelNames': {('parallel_names_of', 'subject'): set(['AuthorityRecord'])},
             'NameEntryS': {('simple_name_relation', 'object'): set(['ParallelNames'])},
-            'Item': {('has_item', 'object'): set(['History'])}
         }
         struct = dict(
             (k, dict((rel, set(targets)) for rel, targets in v.items()))


More information about the saem-devel mailing list