feat: serialize/deserialize logic

feat: basic replication graph for dependency hanfling
refacor: replicateddatablock

replication.py

@@ -1,5 +1,6 @@
 import logging
 from uuid import uuid4
+import json
 try:
     from .libs import umsgpack
 
@@ -78,17 +79,16 @@ class ReplicatedDatablock(object):
     owner = None    # Data owner            (string)
     state = None    # Data state            (RepState)
 
-    def __init__(self, owner=None, data=None, uuid=None, buffer=None):
+    def __init__(self, owner=None, pointer=None, uuid=None, buffer=None):
         self.uuid = uuid if uuid else str(uuid4())
         assert(owner)
         self.owner = owner
 
-        if data:
+        if pointer:
-            self.pointer = data
+            self.pointer = pointer
+            self.buffer = self.dump()
         elif buffer:
-            self.buffer = self.deserialize(buffer)
+            self.buffer = buffer
-        else:
-            raise ValueError("Not enought parameter in constructor")
 
         self.str_type = type(self).__name__
 
@@ -96,9 +96,11 @@ class ReplicatedDatablock(object):
         """
         Here send data over the wire:
             - serialize the data
-            - send them as a multipart frame
+            - send them as a multipart frame thought the given socket
         """
-        data = self.serialize(self.pointer)
+        assert(self.buffer)
+
+        data = self.serialize(self.buffer)
         assert(isinstance(data, bytes))
         owner = self.owner.encode()
         key = self.uuid.encode()
@@ -119,10 +121,12 @@ class ReplicatedDatablock(object):
         owner = owner.decode()
         uuid = uuid.decode()
 
-        instance = factory.construct_from_net(str_type)(owner=owner, uuid=uuid, buffer=data)
+        instance = factory.construct_from_net(str_type)(owner=owner, uuid=uuid)
-        # instance.data = instance.deserialize(data)
+        instance.buffer = instance.deserialize(data) 
+
         return instance
 
+
     def store(self, dict, persistent=False):
         """
         I want to store my replicated data. Persistent means into the disk
@@ -137,6 +141,7 @@ class ReplicatedDatablock(object):
 
             return self.uuid
 
+
     def deserialize(self, data):
         """
         BUFFER -> JSON
@@ -146,16 +151,21 @@ class ReplicatedDatablock(object):
 
     def serialize(self, data):
         """
-        I want to load data from DCC
+        JSON -> BUFFER
-
-        DCC -> JSON
-
-        MUST RETURN A BYTE ARRAY
         """
         raise NotImplementedError
     
 
-    def apply(self,target=None):
+    def dump(self):
+        """
+        DCC -> JSON
+        """
+        assert(self.pointer)
+
+        return json.dumps(self.pointer)
+
+
+    def load(self,target=None):
         """
         JSON -> DCC
         """
@@ -171,18 +181,15 @@ class ReplicatedDatablock(object):
         raise NotImplementedError
 
 
-    def dump(self):
-        return self.deserialize(self.buffer)
 
 class RepCommand(ReplicatedDatablock):
-    
     def serialize(self,data):
         return pickle.dumps(data)
 
     def deserialize(self,data):
         return pickle.loads(data)
     
-    def apply(self,target):
+    def load(self,target):
         target = self.pointer
 
 # class RepObject(ReplicatedDatablock):

replication_client.py

@@ -3,6 +3,7 @@ import logging
 import zmq
 import time
 from replication import ReplicatedDatablock, RepCommand
+from replication_graph import ReplicationGraph
 
 logging.basicConfig(level=logging.DEBUG)
 logger = logging.getLogger(__name__)
@@ -16,7 +17,7 @@ class Client(object):
     def __init__(self,factory=None, id='default'):
         assert(factory)
 
-        self._rep_store = {}
+        self._rep_store = ReplicationGraph()
         self._net_client = ClientNetService(
             store_reference=self._rep_store,
             factory=factory,
@@ -36,10 +37,13 @@ class Client(object):
     def register(self, object):
         """
         Register a new item for replication
+        TODO: Dig in the replication comportement,
+        find a better way to handle replication behavior
         """
         assert(object)
         
-        new_item = self._factory.construct_from_dcc(object)(owner="client", data=object)
+        # Construct the coresponding replication type
+        new_item = self._factory.construct_from_dcc(object)(owner="client", pointer=object)
 
         if new_item:
             logger.info("Registering {} on {}".format(object,new_item.uuid))
@@ -55,7 +59,6 @@ class Client(object):
     def pull(self,object=None):
         pass
     
-
     def unregister(self,object):
         pass
 
@@ -87,8 +90,10 @@ class ClientNetService(threading.Thread):
             
             self.subscriber = self.context.socket(zmq.SUB)
             self.subscriber.setsockopt_string(zmq.SUBSCRIBE, '')
+            # self.subscriber.setsockopt(zmq.IDENTITY, self._id.encode())
             self.subscriber.connect("tcp://{}:{}".format(address, port+1))
             self.subscriber.linger = 0
+            time.sleep(.5)
 
             self.publish = self.context.socket(zmq.PUSH)
             self.publish.connect("tcp://{}:{}".format(address, port+2))
@@ -105,8 +110,7 @@ class ClientNetService(threading.Thread):
         while not self._exit_event.is_set():
             """NET OUT 
                 Given the net state we do something:
-                SYNCING : Ask for snapshots
+                INITIAL : Ask for snapshots 
-                ACTIVE : Do nothing 
             """
             if self.state == STATE_INITIAL:
                 logger.debug('{} : request snapshot'.format(self._id))
@@ -116,8 +120,8 @@ class ClientNetService(threading.Thread):
 
             """NET IN 
                 Given the net state we do something:
-                SYNCING : Ask for snapshots
+                SYNCING : load snapshots
-                ACTIVE : Do nothing 
+                ACTIVE : listen for updates 
             """
             items = dict(poller.poll(1))
 
@@ -125,7 +129,7 @@ class ClientNetService(threading.Thread):
                 if self.state == STATE_SYNCING:
                     datablock = ReplicatedDatablock.pull(self.snapshot, self._factory)
 
-                    if datablock.buffer == 'SNAPSHOT_END':
+                    if 'SNAPSHOT_END' in datablock.buffer:
                         self.state = STATE_ACTIVE
                         logger.debug('{} : snapshot done'.format(self._id))
 
@@ -148,9 +152,6 @@ class ClientNetService(threading.Thread):
 
         self._exit_event.clear()
     
-    def setup(self,id="Client"):
-        pass
-
     def stop(self):
         self._exit_event.set()
 
@@ -194,7 +195,7 @@ class ServerNetService(threading.Thread):
         self.pull = None
         self.state = 0
         self.factory = factory
-        self.clients = []
+        self.clients = {}
 
     
     def listen(self, port=5560):
@@ -207,10 +208,11 @@ class ServerNetService(threading.Thread):
 
             # Update all clients
             self.publisher = self.context.socket(zmq.PUB)
-            # self.publisher.setsockopt(zmq.IDENTITY,b'SERVER')
+            # self.publisher.setsockopt(zmq.IDENTITY,b'SERVER_DATA')
             self.publisher.setsockopt(zmq.SNDHWM, 60)
             self.publisher.bind("tcp://*:{}".format(port+1))
-            time.sleep(0.2)
+            self.publisher.setsockopt(zmq.SNDHWM, 60)
+            self.publisher.linger = 0
 
             # Update collector
             self.pull = self.context.socket(zmq.PULL)
@@ -221,6 +223,11 @@ class ServerNetService(threading.Thread):
         except zmq.error.ZMQError:
             logger.error("Address already in use, change net config")
 
+    def add_client(self, identity):
+        if identity in self.clients.keys():
+            logger.debug("client already added")
+        else:
+            self.clients[identity.decode()] = identity
 
     def run(self):
         logger.debug("Server is online")
@@ -241,6 +248,8 @@ class ServerNetService(threading.Thread):
                 identity = msg[0]
                 request = msg[1]
 
+                self.add_client(identity)
+
                 if request == b"SNAPSHOT_REQUEST":
                     # Sending snapshots
                     for key, item in self._rep_store.items():
@@ -249,18 +258,22 @@ class ServerNetService(threading.Thread):
                     
                     # Snapshot end
                     self.snapshot.send(identity, zmq.SNDMORE)
-                    RepCommand(owner='server',data='SNAPSHOT_END').push(self.snapshot)
+                    RepCommand(owner='server',pointer='SNAPSHOT_END').push(self.snapshot)              
 
 
-
+            # Regular update routing (Clients / Server / Clients)
-            # Regular update routing (Clients / Client)
             if self.pull in socks:
-                logger.debug("Receiving changes from client")
+                logger.debug("SERVER: Receiving changes from client")
                 datablock = ReplicatedDatablock.pull(self.pull, self.factory)
                 
                 datablock.store(self._rep_store)
                 
                 # Update all clients
+                # for cli_name,cli_id in self.clients.items():
+                #     logger.debug("SERVER: Broadcast changes to {}".format(cli_name))
+                #     self.publisher.send(cli_id, zmq.SNDMORE)
+                #     datablock.push(self.publisher)
+                
                 datablock.push(self.publisher)
                 
         self.snapshot.close()
@@ -269,6 +282,7 @@ class ServerNetService(threading.Thread):
 
         self._exit_event.clear()
 
+
     def stop(self):
         self._exit_event.set()
 

replication_graph.py

@@ -0,0 +1,30 @@
+import collections
+
+class ReplicationGraph(collections.MutableMapping):
+    """
+    Structure to hold replicated data relation graph
+    """
+
+    def __init__(self, *args, **kwargs):
+        self.store = dict()
+        self.update(dict(*args, **kwargs))  # use the free update to set keys
+
+    def __getitem__(self, key):
+        return self.store[self.__keytransform__(key)]
+
+    def __setitem__(self, key, value):
+        self.store[self.__keytransform__(key)] = value
+
+    def __delitem__(self, key):
+        del self.store[self.__keytransform__(key)]
+
+    def __iter__(self):
+        return iter(self.store)
+
+    def __len__(self):
+        return len(self.store)
+
+    def __keytransform__(self, key):
+        return key
+
+   

test_replication.py

@@ -4,6 +4,8 @@ import umsgpack
 import logging
 from replication_client import Client, Server
 import time
+import cProfile
+import re
 
 
 log = logging.getLogger(__name__)
@@ -24,13 +26,23 @@ class RepSampleData(ReplicatedDatablock):
 
         return pickle.loads(data)
     
+    def dump(self):
+        import json
+        output = {}
+        output['map'] = json.dumps(self.pointer.map)
+        return output
+        
+
+    def load(self,target=None):
+        target = self.buffer
+
 
 class TestDataFactory(unittest.TestCase):
     def test_data_factory(self):
         factory = ReplicatedDataFactory()
         factory.register_type(SampleData, RepSampleData)
         data_sample = SampleData()
-        rep_sample = factory.construct_from_dcc(data_sample)(owner="toto", data=data_sample)
+        rep_sample = factory.construct_from_dcc(data_sample)(owner="toto", pointer=data_sample)
         
         self.assertEqual(isinstance(rep_sample,RepSampleData), True)
 
@@ -63,7 +75,7 @@ class TestClient(unittest.TestCase):
         factory.register_type(SampleData, RepSampleData)
 
         server = Server(factory=factory)
-        client = Client(factory=factory, id="client_test_callback")
+        client = Client(factory=factory, id="cli_test_filled_snapshot")
         client2 = Client(factory=factory, id="client_2")
         
         server.serve(port=5575)
@@ -86,24 +98,25 @@ class TestClient(unittest.TestCase):
 
     def test_register_client_data(self):
         # Setup environment
+        
         factory = ReplicatedDataFactory()
         factory.register_type(SampleData, RepSampleData)
 
         server = Server(factory=factory)
         server.serve(port=5560)
 
-        client = Client(factory=factory, id="client_1")
+        client = Client(factory=factory, id="cli_test_register_client_data")
         client.connect(port=5560)
 
-        client2 = Client(factory=factory, id="client_2")
+        client2 = Client(factory=factory, id="cli2_test_register_client_data")
         client2.connect(port=5560)
         
+       
         # Test the key registering
         data_sample_key = client.register(SampleData())
  
+        time.sleep(4)
         #Waiting for server to receive the datas
-        time.sleep(.5)
-
         rep_test_key = client2._rep_store[data_sample_key].uuid
         
         
@@ -122,21 +135,21 @@ class TestClient(unittest.TestCase):
         server = Server(factory=factory)
         server.serve(port=5560)
 
-        client = Client(factory=factory, id="client_1")
+        client = Client(factory=factory, id="cli_test_client_data_intergity")
         client.connect(port=5560)
 
-        client2 = Client(factory=factory, id="client_2")
+        client2 = Client(factory=factory, id="cli2_test_client_data_intergity")
         client2.connect(port=5560)
        
         test_map = {"toto":"test"}
         # Test the key registering
-        client.register(SampleData(map=test_map))
+        data_sample_key = client.register(SampleData(map=test_map))
 
         test_map_result = {}
         #Waiting for server to receive the datas
-        time.sleep(.5)
+        time.sleep(1)
 
-        rep_test_key = client2._rep_store[data_sample_key].uuid
+        client2._rep_store[data_sample_key].load(target=test_map_result)
 
         
         client.disconnect()
@@ -144,7 +157,7 @@ class TestClient(unittest.TestCase):
         server.stop()
 
 
-        self.assertEqual(rep_test_key, data_sample_key)
+        self.assertEqual(test_map_result["toto"], test_map["toto"])
 
 def suite():
     suite = unittest.TestSuite()
@@ -152,9 +165,10 @@ def suite():
     suite.addTest(TestClient('test_empty_snapshot'))
     suite.addTest(TestClient('test_filled_snapshot'))
     suite.addTest(TestClient('test_register_client_data'))
-
+    # suite.addTest(TestClient('test_client_data_intergity'))
     return suite
 
 if __name__ == '__main__':
     runner = unittest.TextTestRunner(failfast=True,verbosity=2)
     runner.run(suite())
+