Merge branch 'master' into 5.0.0-dev

* master: (44 commits)
  quote cleanup OCD
  serializer/parser alias for 'ntriples'
  serializer/parser alias for 'ttl'
  cleanup
  remove outdated always skipped test
  a bit of changelog
  add a NTSerializer sub-class for nt11 (#700)
  Restrict normalization to unicode-compatible values (#674)
  fixes for turtle/trig namespace handling
  skip serialising empty default graph
  skip round-trip test, unfixable until 5.0
  prefix test for #428
  Added additional trig unit tests to highlight some currently occurring issues.
  remove ancient and broken 2.3 support code. (#681)
  updating deprecated testing syntax (#697)
  docs: clarify the use of an identifier when persisting a triplestore (#654)
  removing pyparsing version requirement (#696)
  made min/max aggregate functions support all literals (#694)
  actually fix projection from sub-queries
  added dawg tests for #607
  ...

CHANGELOG.md

@@ -1,3 +1,44 @@
+2017/01/2X RELEASE 4.2.2
+========================
+
+This is a bug-fix release, and the last release in the 4.X series.
+
+Bug fixes:
+----------
+* Many SPARQL Engine fixes for aggregates, subqueries, variable-scope
+  [#404](https://github.com/RDFLib/rdflib/issues/404)
+  [#580](https://github.com/RDFLib/rdflib/issues/580)
+  [#607](https://github.com/RDFLib/rdflib/issues/607)
+  [#615](https://github.com/RDFLib/rdflib/issues/615)
+  [#619](https://github.com/RDFLib/rdflib/issues/619)
+  [#628](https://github.com/RDFLib/rdflib/issues/628)
+  [#631](https://github.com/RDFLib/rdflib/issues/631)
+
+
+* initBindings in SPARQL queries finally work everywhere
+  [#294](https://github.com/RDFLib/rdflib/issues/294)
+
+* A handful of Trig serialisation issues
+
+  [#433](https://github.com/RDFLib/rdflib/issues/433)
+  [#679](https://github.com/RDFLib/rdflib/issues/679)
+  [#428](https://github.com/RDFLib/rdflib/issues/428)
+
+
+* Improved support for base64 encoded literals
+  [#646](https://github.com/RDFLib/rdflib/issues/646)
+
+* A handful of bugs around orderability on py3
+  [#676](https://github.com/RDFLib/rdflib/issues/676)
+  [#613](https://github.com/RDFLib/rdflib/issues/613)
+  [#648](https://github.com/RDFLib/rdflib/issues/648)
+  [#653](https://github.com/RDFLib/rdflib/issues/653)
+
+* Improved performance of managing large collections
+  [#609](https://github.com/RDFLib/rdflib/issues/609)
+
+* And many other minor things!
+
 2015/08/12 RELEASE 4.2.1
 ========================
 

docs/persistence.rst

@@ -36,11 +36,13 @@ Most cases passing the name of the store to the Graph constructor is enough:
     graph = Graph(store='Sleepycat')
 
 
-Most store offering on-disk persistence will need to be opened before reading or writing :
+Most store offering on-disk persistence will need to be opened before reading or writing.
+When peristing a triplestore (instead of a ConjuntiveGraph quadstore), you need to specify
+an identifier with which you can open the graph:
 
 .. code-block:: python
 
-   graph = Graph('Sleepycat')
+   graph = Graph('Sleepycat', identifier='mygraph')
 
    # first time create the store:
    graph.open('/home/user/data/myRDFLibStore', create = True) 
@@ -49,7 +51,7 @@ Most store offering on-disk persistence will need to be opened before reading or
    graph.add( mytriples ) 
 
    # when done!
-   graph.close() 
+   graph.close()
 
 
 

rdflib/collection.py

@@ -1,7 +1,6 @@
 from rdflib.namespace import RDF
 from rdflib.term import BNode
 from rdflib.term import Literal
-from rdflib.graph import Graph
 from rdflib.py3compat import format_doctest_out
 
 __all__ = ['Collection']
@@ -43,8 +42,7 @@ class Collection(object):
     def __init__(self, graph, uri, seq=[]):
         self.graph = graph
         self.uri = uri or BNode()
-        for item in seq:
-            self.append(item)
+        self += seq
 
     def n3(self):
         """
@@ -82,15 +80,7 @@ def _get_container(self, index):
 
     def __len__(self):
         """length of items in collection."""
-        count = 0
-        links = set()
-        for item in self.graph.items(self.uri):
-            assert item not in links, \
-                "There is a loop in the RDF list! " + \
-                "(%s has been processed before)" % item
-            links.add(item)
-            count += 1
-        return count
+        return len(list(self.graph.items(self.uri)))
 
     def index(self, item):
         """
@@ -192,6 +182,16 @@ def __iter__(self):
         """Iterator over items in Collections"""
         return self.graph.items(self.uri)
 
+    def _end(self):
+        # find end of list
+        container = self.uri
+        while True:
+            rest = self.graph.value(container, RDF.rest)
+            if rest == None or rest == RDF.nil:
+                return container
+            else:
+                container = rest
+
     def append(self, item):
         """
         >>> from rdflib.graph import Graph
@@ -200,29 +200,36 @@ def append(self, item):
         >>> c = Collection(g,listName,[Literal(1),Literal(2)])
         >>> links = [
         ...     list(g.subjects(object=i, predicate=RDF.first))[0] for i in c]
-        >>> len([i for i in links if (i,RDF.rest, RDF.nil) in g])
+        >>> len([i for i in links if (i, RDF.rest, RDF.nil) in g])
         1
 
         """
-        container = self.uri
-        graph = self.graph
-        # iterate to the end of the linked list
-        rest = graph.value(container, RDF.rest)
-        while rest:
-            if rest == RDF.nil:
-                # the end, append to the end of the linked list
-                node = BNode()
-                graph.set((container, RDF.rest, node))
-                container = node
-                break
-            else:
-                # move down one link
-                if container != self.uri:
-                    rest = graph.value(rest, RDF.rest)
-                if not rest == RDF.nil:
-                    container = rest
-        graph.add((container, RDF.first, item))
-        graph.add((container, RDF.rest, RDF.nil))
+
+        end = self._end()
+        if (end, RDF.first, None) in self.graph:
+            # append new node to the end of the linked list
+            node = BNode()
+            self.graph.set((end, RDF.rest, node))
+            end = node
+
+        self.graph.add((end, RDF.first, item))
+        self.graph.add((end, RDF.rest, RDF.nil))
+
+    def __iadd__(self, other):
+
+        end = self._end()
+        self.graph.remove((end, RDF.rest, None))
+
+        for item in other:
+            if (end, RDF.first, None) in self.graph:
+                nxt = BNode()
+                self.graph.add((end, RDF.rest, nxt))
+                end = nxt
+
+            self.graph.add((end, RDF.first, item))
+
+
+        self.graph.add((end, RDF.rest, RDF.nil))
 
     def clear(self):
         container = self.uri
@@ -241,6 +248,7 @@ def test():
 if __name__ == "__main__":
     test()
 
+    from rdflib import Graph
     g = Graph()
 
     c = Collection(g, BNode())

rdflib/compat.py

@@ -23,3 +23,60 @@ def numeric_greater(a, b):
 
     def numeric_greater(a, b):
         return a > b
+
+
+try:
+  from lxml import etree
+except ImportError:
+  try:
+    # Python 2.5
+    import xml.etree.cElementTree as etree
+  except ImportError:
+    try:
+      # Python 2.5
+      import xml.etree.ElementTree as etree
+    except ImportError:
+      try:
+        # normal cElementTree install
+        import cElementTree as etree
+      except ImportError:
+        try:
+          # normal ElementTree install
+          import elementtree.ElementTree as etree
+        except ImportError:
+          raise Exception("Failed to import ElementTree from any known place")
+
+try:
+    etree_register_namespace = etree.register_namespace
+except AttributeError:
+
+    import xml.etree.ElementTree as etreenative
+
+    def etree_register_namespace(prefix, uri):
+        etreenative._namespace_map[uri] = prefix
+
+try:
+    from functools import cmp_to_key
+except ImportError:
+    # Backport from Py2.7 for Py2.6:
+    def cmp_to_key(mycmp):
+        """Convert a cmp= function into a key= function"""
+        class K(object):
+            __slots__ = ['obj']
+            def __init__(self, obj, *args):
+                self.obj = obj
+            def __lt__(self, other):
+                return mycmp(self.obj, other.obj) < 0
+            def __gt__(self, other):
+                return mycmp(self.obj, other.obj) > 0
+            def __eq__(self, other):
+                return mycmp(self.obj, other.obj) == 0
+            def __le__(self, other):
+                return mycmp(self.obj, other.obj) <= 0
+            def __ge__(self, other):
+                return mycmp(self.obj, other.obj) >= 0
+            def __ne__(self, other):
+                return mycmp(self.obj, other.obj) != 0
+            def __hash__(self):
+                raise TypeError('hash not implemented')
+        return K

rdflib/graph.py

@@ -266,6 +266,7 @@
 from rdflib.parser import create_input_source
 from rdflib.namespace import NamespaceManager
 from rdflib.resource import Resource
+from rdflib.collection import Collection
 from rdflib import py3compat
 b = py3compat.b
 
@@ -1172,6 +1173,28 @@ def all_nodes(self):
         res.update(self.subjects())
         return res
 
+    def collection(self, identifier):
+        """Create a new ``Collection`` instance.
+
+        Parameters:
+
+        - ``identifier``: a URIRef or BNode instance.
+
+        Example::
+
+            >>> graph = Graph()
+            >>> uri = URIRef("http://example.org/resource")
+            >>> collection = graph.collection(uri)
+            >>> assert isinstance(collection, Collection)
+            >>> assert collection.uri is uri
+            >>> assert collection.graph is graph
+            >>> collection += [ Literal(1), Literal(2) ]
+        """
+
+        return Collection(self, identifier)
+
+
+
     def resource(self, identifier):
         """Create a new ``Resource`` instance.
 

rdflib/plugin.py

@@ -166,15 +166,25 @@ def plugins(name=None, kind=None):
 register(
     'turtle', Serializer,
     'rdflib.plugins.serializers.turtle', 'TurtleSerializer')
+register(
+    'ttl', Serializer,
+    'rdflib.plugins.serializers.turtle', 'TurtleSerializer')
 register(
     'trig', Serializer,
     'rdflib.plugins.serializers.trig', 'TrigSerializer')
 register(
     'application/n-triples', Serializer,
     'rdflib.plugins.serializers.nt', 'NTSerializer')
+register(
+    'ntriples', Serializer,
+    'rdflib.plugins.serializers.nt', 'NTSerializer')
 register(
     'nt', Serializer,
     'rdflib.plugins.serializers.nt', 'NTSerializer')
+register(
+    'nt11', Serializer,
+    'rdflib.plugins.serializers.nt', 'NT11Serializer')
+
 register(
     'pretty-xml', Serializer,
     'rdflib.plugins.serializers.rdfxml', 'PrettyXMLSerializer')
@@ -185,10 +195,10 @@ def plugins(name=None, kind=None):
     'application/trix', Serializer,
     'rdflib.plugins.serializers.trix', 'TriXSerializer')
 register(
-    "application/n-quads", Serializer,
+    'application/n-quads', Serializer,
     'rdflib.plugins.serializers.nquads', 'NQuadsSerializer')
 register(
-    "nquads", Serializer,
+    'nquads', Serializer,
     'rdflib.plugins.serializers.nquads', 'NQuadsSerializer')
 
 register(
@@ -209,12 +219,21 @@ def plugins(name=None, kind=None):
 register(
     'turtle', Parser,
     'rdflib.plugins.parsers.notation3', 'TurtleParser')
+register(
+    'ttl', Parser,
+    'rdflib.plugins.parsers.notation3', 'TurtleParser')
 register(
     'application/n-triples', Parser,
     'rdflib.plugins.parsers.nt', 'NTParser')
+register(
+    'ntriples', Parser,
+    'rdflib.plugins.parsers.nt', 'NTParser')
 register(
     'nt', Parser,
     'rdflib.plugins.parsers.nt', 'NTParser')
+register(
+    'nt11', Parser,
+    'rdflib.plugins.parsers.nt', 'NTParser')
 register(
     'application/n-quads', Parser,
     'rdflib.plugins.parsers.nquads', 'NQuadsParser')

rdflib/plugins/serializers/nquads.py

@@ -4,7 +4,7 @@
 from rdflib.serializer import Serializer
 from rdflib.py3compat import b
 
-from rdflib.plugins.serializers.nt import _xmlcharref_encode, _quoteLiteral
+from rdflib.plugins.serializers.nt import _quoteLiteral
 
 __all__ = ['NQuadsSerializer']
 
@@ -36,11 +36,10 @@ def _nq_row(triple, context):
     if isinstance(triple[2], Literal):
         return u"%s %s %s %s .\n" % (triple[0].n3(),
                                      triple[1].n3(),
-                                     _xmlcharref_encode(
-                                         _quoteLiteral(triple[2])),
+                                     _quoteLiteral(triple[2]),
                                      context.n3())
     else:
         return u"%s %s %s %s .\n" % (triple[0].n3(),
                                      triple[1].n3(),
-                                     _xmlcharref_encode(triple[2].n3()),
+                                     triple[2].n3(),
                                      context.n3())