Store JSON objects instead of pickle objects

.github/workflows/continuous-integration.yml

@@ -42,6 +42,7 @@ jobs:
         python-version: [3.7, 3.8]
         database: ["DUMMY", "postgres", "cockroachdb"]
         db_schema: ["custom", "public"]
+        db_serializer: ["pickle", "json"]
         exclude:
           - database: "DUMMY"
             db_schema: "custom"
@@ -52,6 +53,7 @@ jobs:
     env:
       DATABASE: ${{ matrix.database }}
       DB_SCHEMA: ${{ matrix.db_schema }}
+      DB_SERIALIZER: ${{ matrix.db_serializer }}
 
     steps:
       - name: Checkout the repository

contrib-requirements.txt

@@ -5,8 +5,7 @@ aiosmtplib==1.0.6
 pre-commit==1.18.2
 flake8==3.7.7
 codecov==2.0.15
-mypy==0.720
-mypy-zope==0.2.0
+mypy-zope==0.2.8
 black==19.10b0
 isort==4.3.21
 jinja2==2.11.1

guillotina/_settings.py

@@ -91,7 +91,7 @@
     "check_writable_request": "guillotina.writable.check_writable_request",
     "indexer": "guillotina.catalog.index.Indexer",
     "search_parser": "default",
-    "object_reader": "guillotina.db.reader.reader",
+    "db_serializer": "pickle",
     "thread_pool_workers": 32,
     "server_settings": {"uvicorn": {"timeout_keep_alive": 5, "http": "h11"}},
     "valid_id_characters": string.digits + string.ascii_lowercase + ".-_@$^()+ =",

guillotina/contrib/catalog/pg.py

@@ -9,6 +9,7 @@
 from guillotina.catalog.utils import get_index_definition
 from guillotina.catalog.utils import iter_indexes
 from guillotina.catalog.utils import parse_query
+from guillotina.component import get_adapter
 from guillotina.component import get_utility
 from guillotina.const import TRASHED_ID
 from guillotina.db.interfaces import IPostgresStorage
@@ -706,7 +707,7 @@ async def _process_object(self, obj, data):
 
         uuid = obj.__uuid__
 
-        writer = IWriter(obj)
+        writer = get_adapter(obj, IWriter, name=app_settings["db_serializer"])
         await self._index(uuid, writer, data["transaction"], data["table_name"])
 
         data["count"] += 1

guillotina/db/interfaces.py

@@ -17,6 +17,10 @@ class IWriter(Interface):
     """Serializes the object for DB storage"""
 
 
+class IReader(Interface):
+    """Deserializes the object from DB storage"""
+
+
 class ITransaction(Interface):
     _db_conn = Attribute("")
     _query_count_end = Attribute("")

guillotina/db/serializers/__init__.py

@@ -0,0 +1,2 @@
+from . import json  # noqa
+from . import pickle  # noqa

guillotina/db/serializers/json/__init__.py

@@ -0,0 +1,4 @@
+from .reader import *  # noqa
+from .value_deserializers import *  # noqa
+from .value_serializers import *  # noqa
+from .writer import *  # noqa

guillotina/db/serializers/json/interfaces.py

@@ -0,0 +1,9 @@
+from zope.interface import Interface
+
+
+class IStorageDeserializer(Interface):
+    pass
+
+
+class IStorageSerializer(Interface):
+    pass

guillotina/db/serializers/json/reader.py

@@ -0,0 +1,47 @@
+from .interfaces import IStorageDeserializer
+from guillotina import configure
+from guillotina.component import get_adapter
+from guillotina.component import query_adapter
+from guillotina.db.interfaces import IReader
+from guillotina.db.orm.interfaces import IBaseObject
+from guillotina.utils import resolve_dotted_name
+from zope.interface.interface import Interface
+
+import asyncpg
+import orjson
+
+
+def recursive_load(d):
+    if isinstance(d, dict):
+        if "__class__" in d:
+            adapter = query_adapter(d, IStorageDeserializer, name=d["__class__"])
+            if adapter is None:
+                adapter = get_adapter(d, IStorageDeserializer, name="$.AnyObjectDict")
+            return adapter()
+        else:
+            for k, v in d.items():
+                d[k] = recursive_load(v)
+            return d
+    elif isinstance(d, list):
+        return [recursive_load(v) for v in d]
+    else:
+        return d
+
+
+@configure.adapter(for_=(Interface), provides=IReader, name="json")
+def reader(result_: asyncpg.Record) -> IBaseObject:
+    result = dict(result_)
+    state = orjson.loads(result["state"])
+    dotted_class = state.pop("__class__")
+    type_class = resolve_dotted_name(dotted_class)
+    obj = type_class.__new__(type_class)
+
+    state = recursive_load(state)
+
+    obj.__dict__.update(state)
+    obj.__uuid__ = result["zoid"]
+    obj.__serial__ = result["tid"]
+    obj.__name__ = result["id"]
+    obj.__provides__ = obj.__providedBy__
+
+    return obj

guillotina/db/serializers/json/value_deserializers.py

@@ -0,0 +1,128 @@
+from .interfaces import IStorageDeserializer
+from .reader import recursive_load
+from dateutil.parser import parse
+from guillotina import configure
+from guillotina.interfaces.security import PermissionSetting
+from guillotina.security.securitymap import SecurityMap
+from guillotina.utils import resolve_dotted_name
+from zope.interface import Interface
+from zope.interface.declarations import Implements
+
+import base64
+import pickle
+
+
+class PickleDeserializer:
+    def __init__(self, data):
+        self.data = data
+
+    def __call__(self):
+        return pickle.loads(base64.b64decode(self.data["__pickle__"]))
+
+
+@configure.adapter(
+    for_=Interface, provides=IStorageDeserializer, name="$.AnyObjectDict",
+)
+class AnyObjectDeserializer:
+    def __init__(self, data):
+        self.data = data
+
+    def __call__(self):
+        klass = self.data.pop("__class__")
+        type_class = resolve_dotted_name(klass)
+        obj = type_class.__new__(type_class)
+        obj.__dict__ = recursive_load(self.data)
+        return obj
+
+
+@configure.adapter(
+    for_=Interface, provides=IStorageDeserializer, name="guillotina.fields.annotation.BucketListValue",
+)
+class BucketListValueDeserializer(PickleDeserializer):
+    pass
+
+
+@configure.adapter(
+    for_=Interface, provides=IStorageDeserializer, name="guillotina.blob.Blob",
+)
+class BlobDeserializer(PickleDeserializer):
+    pass
+
+
+@configure.adapter(for_=Interface, provides=IStorageDeserializer, name="builtins.set")
+@configure.adapter(for_=Interface, provides=IStorageDeserializer, name="builtins.frozenset")
+class SetDeseializer:
+    def __init__(self, data):
+        self.data = data
+
+    def __call__(self):
+        data = self.data
+        if data["__class__"] == "builtins.set":
+            return set(data["__value__"])
+        else:
+            return frozenset(data["__value__"])
+
+
+@configure.adapter(for_=Interface, provides=IStorageDeserializer, name="datetime.datetime")
+class DatetimeDeserializer:
+    def __init__(self, data):
+        self.data = data
+
+    def __call__(self):
+        return parse(self.data["__value__"])
+
+
+@configure.adapter(
+    for_=Interface, provides=IStorageDeserializer, name="zope.interface.declarations.Implements",
+)
+class ImplementsDeseializer:
+    def __init__(self, data):
+        self.data = data
+
+    def __call__(self):
+        data = self.data
+        obj = Implements()
+        obj.__bases__ = [resolve_dotted_name(iface) for iface in data["__ifaces__"]]
+        obj.__name__ = data["__name__"]
+        return obj
+
+
+@configure.adapter(
+    for_=Interface, provides=IStorageDeserializer, name="zope.interface.Provides",
+)
+class ProvidesDeserializer:
+    def __init__(self, data):
+        self.data = data
+
+    def __call__(self):
+        # from zope.interface import Provides  # type: ignore
+        # obj = Provides(*[resolve_dotted_name(iface) for iface in self.data["__ifaces__"]])
+        # return obj
+        return pickle.loads(base64.b64decode(self.data["__pickle__"]))
+
+
+@configure.adapter(
+    for_=Interface, provides=IStorageDeserializer, name="guillotina.security.securitymap.SecurityMap",
+)
+class SecurityMapDeserializer:
+    def __init__(self, data):
+        self.data = data
+
+    def __call__(self):
+        sec_map = SecurityMap.__new__(SecurityMap)
+        sec_map.__dict__.update(
+            {
+                # byrow
+                k: {
+                    # Role
+                    k2: {
+                        # Principal
+                        k3: PermissionSetting(v3)
+                        for k3, v3 in v2.items()
+                    }
+                    for k2, v2 in v.items()
+                }
+                for k, v in self.data["__dict__"].items()
+            }
+        )
+        return sec_map

guillotina/db/serializers/json/value_serializers.py

@@ -0,0 +1,127 @@
+from .interfaces import IStorageSerializer
+from guillotina import configure
+from guillotina.utils import get_dotted_name
+from zope.interface import Interface
+
+import base64
+import pickle
+
+
+class PickleSerializer:
+    dotted_name: str
+
+    def __init__(self, obj):
+        self.obj = obj
+
+    def __call__(self):
+        return {
+            "__class__": self.dotted_name,
+            "__pickle__": base64.b64encode(pickle.dumps(self.obj)).decode(),
+        }
+
+
+@configure.adapter(
+    for_=Interface, provides=IStorageSerializer, name="$.AnyObjectDict",
+)
+class AnyObjectSerializer:
+    def __init__(self, obj):
+        self.obj = obj
+
+    def __call__(self):
+        return {
+            **{"__class__": get_dotted_name(self.obj)},
+            **self.obj.__dict__,
+        }
+
+
+@configure.adapter(
+    for_=Interface, provides=IStorageSerializer, name="guillotina.fields.annotation.BucketListValue",
+)
+class BucketListValueSerializer(PickleSerializer):
+    """
+    The internal structure of the BucketListValue contains a dictionary
+    whose keys are of type other than string
+    """
+
+    dotted_name = "guillotina.fields.annotation.BucketListValue"
+
+
+@configure.adapter(
+    for_=Interface, provides=IStorageSerializer, name="guillotina.blob.Blob",
+)
+class BlobSerializer(PickleSerializer):
+    """
+    The internal structure of the Blob contains a dictionary
+    whose keys are of type other than string
+    """
+
+    dotted_name = "guillotina.blob.Blob"
+
+
+@configure.adapter(for_=Interface, provides=IStorageSerializer, name="builtins.set")
+@configure.adapter(for_=Interface, provides=IStorageSerializer, name="builtins.frozenset")
+class SetSerializer:
+    def __init__(self, obj):
+        self.obj = obj
+
+    def __call__(self):
+        return {
+            "__class__": get_dotted_name(type(self.obj)),
+            "__value__": list(self.obj),
+        }
+
+
+@configure.adapter(for_=Interface, provides=IStorageSerializer, name="datetime.datetime")
+class DatetimeSerializer:
+    def __init__(self, obj):
+        self.obj = obj
+
+    def __call__(self):
+        return {
+            "__class__": get_dotted_name(type(self.obj)),
+            "__value__": self.obj.isoformat(),
+        }
+
+
+@configure.adapter(
+    for_=Interface, provides=IStorageSerializer, name="zope.interface.declarations.Implements",
+)
+class ImplementsSerializer:
+    def __init__(self, obj):
+        self.obj = obj
+
+    def __call__(self):
+        return {
+            "__class__": "zope.interface.declarations.Implements",
+            "__name__": self.obj.__name__,
+            "__ifaces__": [i.__identifier__ for i in self.obj.flattened()],
+        }
+
+
+@configure.adapter(
+    for_=Interface, provides=IStorageSerializer, name="zope.interface.Provides",
+)
+class ProvidesSerializer:
+    def __init__(self, obj):
+        self.obj = obj
+
+    def __call__(self):
+        return {
+            "__class__": "zope.interface.Provides",
+            # "__ifaces__": [i.__identifier__ for i in self.obj.flattened()],
+            "__pickle__": base64.b64encode(pickle.dumps(self.obj)).decode(),
+        }
+
+
+@configure.adapter(
+    for_=Interface, provides=IStorageSerializer, name="guillotina.security.securitymap.SecurityMap",
+)
+class SecurityMapSerializer:
+    def __init__(self, obj):
+        self.obj = obj
+
+    def __call__(self):
+        return {
+            "__class__": "guillotina.security.securitymap.SecurityMap",
+            "__dict__": self.obj.__dict__,
+        }