[PATCH 1 of 2 saem_ref V2] [rdf] Prefetch ORM cache for activities when exporting ConceptScheme

Sylvain Thenault sylvain.thenault at logilab.fr
Fri Mar 17 09:11:07 CET 2017


# HG changeset patch
# User Sylvain Thénault <sylvain.thenault at logilab.fr>
# Date 1489576155 -3600
#      Wed Mar 15 12:09:15 2017 +0100
# Node ID 98711dbe50d49f9349ec17f9a99ea325fdaedf3a
# Parent  06537e435c3d0c4281597b0311430ed17b5aadf0
[rdf] Prefetch ORM cache for activities when exporting ConceptScheme

as this may take age on medium sized scheme.

To properly test this, patch the warm_cache method to ensure no more query
may be done.

Related to #12175187

diff --git a/cubicweb_saem_ref/entities/rdf.py b/cubicweb_saem_ref/entities/rdf.py
--- a/cubicweb_saem_ref/entities/rdf.py
+++ b/cubicweb_saem_ref/entities/rdf.py
@@ -279,10 +279,40 @@ class ConceptSchemeRDFPrimaryAdapter(sko
 
     def accept(self, entity):
         """Return True if the entity should be recursivly added to the graph."""
         return entity.cw_etype == 'Activity'
 
+    def warm_caches(self):
+        super(ConceptSchemeRDFPrimaryAdapter, self).warm_caches()
+        scheme = self.entity
+        cnx = self._cw
+
+        activities, rset = _warm_activity_cache(cnx, scheme, 'CS identity X, A generated X')
+        scheme._cw_related_cache['generated_object'] = (rset, activities)
+        scheme._cw_related_cache['used_object'] = (rset, activities)
+
+        activities, rset = _warm_activity_cache(cnx, scheme, 'X in_scheme CS, A generated X')
+        concepts = tuple(rset.entities(1))
+        skos.cache_entities_relations(concepts, rset, 'generated', 'object',
+                                      entity_col=0, target_col=1)
+        skos.cache_entities_relations(concepts, rset, 'used', 'object',
+                                      entity_col=0, target_col=1)
+
+
+def _warm_activity_cache(cnx, scheme, rql_expr):
+    activities_rql = skos._select_attributes(
+        'Any X,A WHERE CS eid %(cs)s,' + rql_expr,
+        'A', _ACTIVITY_ATTRIBUTES)
+    rset = cnx.execute(activities_rql, {'cs': scheme.eid})
+    activities = tuple(rset.entities(1))
+    skos.cache_entities_relations(activities, rset, 'generated', 'subject')
+    skos.cache_entities_relations(activities, rset, 'used', 'subject')
+    user_rset = cnx.execute('Any U,A WHERE A associated_with U, CS eid %(cs)s, ' + rql_expr,
+                            {'cs': scheme.eid})
+    skos.cache_entities_relations(activities, user_rset, 'associated_with', 'subject')
+    return activities, rset
+
 
 class ConceptRDFPrimaryAdapter(skos.ConceptRDFPrimaryAdapter):
 
     def register_rdf_mapping(self, reg):
         super(ConceptRDFPrimaryAdapter, self).register_rdf_mapping(reg)
diff --git a/dev-requirements.txt b/dev-requirements.txt
--- a/dev-requirements.txt
+++ b/dev-requirements.txt
@@ -1,5 +1,6 @@
+mock
 pytest
 webtest
 http://hg.logilab.org/review/cubes/eac/archive/tip.tar.bz2#egg=cubicweb-eac
 http://hg.logilab.org/review/cubes/compound/archive/tip.tar.bz2#egg=cubicweb-compound
 http://hg.logilab.org/master/cubes/relationwidget/archive/tip.tar.bz2#egg=cubicweb-relationwidget
diff --git a/test/test_entities_rdf.py b/test/test_entities_rdf.py
--- a/test/test_entities_rdf.py
+++ b/test/test_entities_rdf.py
@@ -15,15 +15,18 @@
 # with this program. If not, see <http://www.gnu.org/licenses/>.
 """Tests for agent entities"""
 
 import datetime
 
+from mock import patch
+
 from cubicweb.devtools.testlib import CubicWebTC
 
 from cubes.skos.rdfio import RDFLibRDFGraph, RDFRegistry
 
 from cubicweb_saem_ref import permanent_url
+from cubicweb_saem_ref.entities import rdf
 
 import testutils
 
 
 class RDFExportTC(testutils.XmlTestMixin, CubicWebTC):
@@ -227,13 +230,29 @@ class RDFExportTC(testutils.XmlTestMixin
         with self.admin_access.client_cnx() as cnx:
             scheme = testutils.setup_scheme(cnx, u'some vocab')
             cnx.commit()
             concept = scheme.add_concept(u'some concept')
             cnx.commit()
-            scheme.cw_clear_all_caches()
 
-            triples = self._rdf_triples(scheme)
+        # get scheme and concept from a fresh connection to avoid unexpected cache side effects
+        # interference (where cw_clear_all_caches is not enough)
+        with self.admin_access.client_cnx() as cnx:
+            scheme = cnx.entity_from_eid(scheme.eid)
+            concept = cnx.entity_from_eid(concept.eid)
+
+            orig_warm_caches = rdf.ConceptSchemeRDFPrimaryAdapter.warm_caches
+
+            def warm_caches(self):
+                orig_warm_caches(self)
+                # ensure no further RQL query is done past this point by hacking .execute
+                self._cw.execute = lambda rql, args=None: 1 / 0
+
+            with patch('cubicweb_saem_ref.entities.rdf.ConceptSchemeRDFPrimaryAdapter'
+                       '.warm_caches', autospec=True) as meth:
+                meth.side_effect = warm_caches
+                triples = self._rdf_triples(scheme)
+
             # scheme has two activities: its creation and addition of a concept
             self.assertHasActivity(scheme, scheme.reverse_generated[0], triples)
             self.assertHasActivity(scheme, scheme.reverse_generated[1], triples)
             self.assertHasActivity(concept, concept.reverse_generated[0], triples)
 


More information about the saem-devel mailing list