Backups/multi-user
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: serialize/deserialize logic

Browse Source 
feat: basic replication graph for dependency hanfling
refacor: replicateddatablock
This commit is contained in:
Swann Martinez
2019-07-22 17:42:38 +02:00
parent 589dcd6979
commit 1c705de7d1
4 changed files with 121 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
replication.py
View File

@ -1,5 +1,6 @@
import logging
from uuid import uuid4
import json
try:
from .libs import umsgpack
@ -78,17 +79,16 @@ class ReplicatedDatablock(object):
owner = None # Data owner (string)
state = None # Data state (RepState)
def __init__(self, owner=None, data=None, uuid=None, buffer=None):
def __init__(self, owner=None, pointer=None, uuid=None, buffer=None):
self.uuid = uuid if uuid else str(uuid4())
assert(owner)
self.owner = owner
if data:
self.pointer = data
if pointer:
self.pointer = pointer
self.buffer = self.dump()
elif buffer:
self.buffer = self.deserialize(buffer)
else:
raise ValueError("Not enought parameter in constructor")
self.buffer = buffer
self.str_type = type(self).__name__
@ -96,9 +96,11 @@ class ReplicatedDatablock(object):
"""
Here send data over the wire:
- serialize the data
- send them as a multipart frame
- send them as a multipart frame thought the given socket
"""
data = self.serialize(self.pointer)
assert(self.buffer)
data = self.serialize(self.buffer)
assert(isinstance(data, bytes))
owner = self.owner.encode()
key = self.uuid.encode()
@ -119,10 +121,12 @@ class ReplicatedDatablock(object):
owner = owner.decode()
uuid = uuid.decode()
instance = factory.construct_from_net(str_type)(owner=owner, uuid=uuid, buffer=data)
# instance.data = instance.deserialize(data)
instance = factory.construct_from_net(str_type)(owner=owner, uuid=uuid)
instance.buffer = instance.deserialize(data)
return instance
def store(self, dict, persistent=False):
"""
I want to store my replicated data. Persistent means into the disk
@ -137,6 +141,7 @@ class ReplicatedDatablock(object):
return self.uuid
def deserialize(self, data):
"""
BUFFER -> JSON
@ -146,16 +151,21 @@ class ReplicatedDatablock(object):
def serialize(self, data):
"""
I want to load data from DCC
DCC -> JSON
MUST RETURN A BYTE ARRAY
JSON -> BUFFER
"""
raise NotImplementedError
def apply(self,target=None):
def dump(self):
"""
DCC -> JSON
"""
assert(self.pointer)
return json.dumps(self.pointer)
def load(self,target=None):
"""
JSON -> DCC
"""
@ -171,18 +181,15 @@ class ReplicatedDatablock(object):
raise NotImplementedError
def dump(self):
return self.deserialize(self.buffer)
class RepCommand(ReplicatedDatablock):
def serialize(self,data):
return pickle.dumps(data)
def deserialize(self,data):
return pickle.loads(data)
def apply(self,target):
def load(self,target):
target = self.pointer
# class RepObject(ReplicatedDatablock):

52
replication_client.py
View File

@ -3,6 +3,7 @@ import logging
import zmq
import time
from replication import ReplicatedDatablock, RepCommand
from replication_graph import ReplicationGraph
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
@ -16,7 +17,7 @@ class Client(object):
def __init__(self,factory=None, id='default'):
assert(factory)
self._rep_store = {}
self._rep_store = ReplicationGraph()
self._net_client = ClientNetService(
store_reference=self._rep_store,
factory=factory,
@ -36,10 +37,13 @@ class Client(object):
def register(self, object):
"""
Register a new item for replication
TODO: Dig in the replication comportement,
find a better way to handle replication behavior
"""
assert(object)
new_item = self._factory.construct_from_dcc(object)(owner="client", data=object)
# Construct the coresponding replication type
new_item = self._factory.construct_from_dcc(object)(owner="client", pointer=object)
if new_item:
logger.info("Registering {} on {}".format(object,new_item.uuid))
@ -55,7 +59,6 @@ class Client(object):
def pull(self,object=None):
pass
def unregister(self,object):
pass
@ -87,8 +90,10 @@ class ClientNetService(threading.Thread):
self.subscriber = self.context.socket(zmq.SUB)
self.subscriber.setsockopt_string(zmq.SUBSCRIBE, '')
# self.subscriber.setsockopt(zmq.IDENTITY, self._id.encode())
self.subscriber.connect("tcp://{}:{}".format(address, port+1))
self.subscriber.linger = 0
time.sleep(.5)
self.publish = self.context.socket(zmq.PUSH)
self.publish.connect("tcp://{}:{}".format(address, port+2))
@ -105,8 +110,7 @@ class ClientNetService(threading.Thread):
while not self._exit_event.is_set():
"""NET OUT
Given the net state we do something:
SYNCING : Ask for snapshots
ACTIVE : Do nothing
INITIAL : Ask for snapshots
"""
if self.state == STATE_INITIAL:
logger.debug('{} : request snapshot'.format(self._id))
@ -116,8 +120,8 @@ class ClientNetService(threading.Thread):
"""NET IN
Given the net state we do something:
SYNCING : Ask for snapshots
ACTIVE : Do nothing
SYNCING : load snapshots
ACTIVE : listen for updates
"""
items = dict(poller.poll(1))
@ -125,7 +129,7 @@ class ClientNetService(threading.Thread):
if self.state == STATE_SYNCING:
datablock = ReplicatedDatablock.pull(self.snapshot, self._factory)
if datablock.buffer == 'SNAPSHOT_END':
if 'SNAPSHOT_END' in datablock.buffer:
self.state = STATE_ACTIVE
logger.debug('{} : snapshot done'.format(self._id))
@ -148,9 +152,6 @@ class ClientNetService(threading.Thread):
self._exit_event.clear()
def setup(self,id="Client"):
pass
def stop(self):
self._exit_event.set()
@ -194,7 +195,7 @@ class ServerNetService(threading.Thread):
self.pull = None
self.state = 0
self.factory = factory
self.clients = []
self.clients = {}
def listen(self, port=5560):
@ -207,10 +208,11 @@ class ServerNetService(threading.Thread):
# Update all clients
self.publisher = self.context.socket(zmq.PUB)
# self.publisher.setsockopt(zmq.IDENTITY,b'SERVER')
# self.publisher.setsockopt(zmq.IDENTITY,b'SERVER_DATA')
self.publisher.setsockopt(zmq.SNDHWM, 60)
self.publisher.bind("tcp://*:{}".format(port+1))
time.sleep(0.2)
self.publisher.setsockopt(zmq.SNDHWM, 60)
self.publisher.linger = 0
# Update collector
self.pull = self.context.socket(zmq.PULL)
@ -221,6 +223,11 @@ class ServerNetService(threading.Thread):
except zmq.error.ZMQError:
logger.error("Address already in use, change net config")
def add_client(self, identity):
if identity in self.clients.keys():
logger.debug("client already added")
else:
self.clients[identity.decode()] = identity
def run(self):
logger.debug("Server is online")
@ -241,6 +248,8 @@ class ServerNetService(threading.Thread):
identity = msg[0]
request = msg[1]
self.add_client(identity)
if request == b"SNAPSHOT_REQUEST":
# Sending snapshots
for key, item in self._rep_store.items():
@ -249,26 +258,31 @@ class ServerNetService(threading.Thread):
# Snapshot end
self.snapshot.send(identity, zmq.SNDMORE)
RepCommand(owner='server',data='SNAPSHOT_END').push(self.snapshot)
RepCommand(owner='server',pointer='SNAPSHOT_END').push(self.snapshot)
# Regular update routing (Clients / Client)
# Regular update routing (Clients / Server / Clients)
if self.pull in socks:
logger.debug("Receiving changes from client")
logger.debug("SERVER: Receiving changes from client")
datablock = ReplicatedDatablock.pull(self.pull, self.factory)
datablock.store(self._rep_store)
# Update all clients
# for cli_name,cli_id in self.clients.items():
# logger.debug("SERVER: Broadcast changes to {}".format(cli_name))
# self.publisher.send(cli_id, zmq.SNDMORE)
# datablock.push(self.publisher)
datablock.push(self.publisher)
self.snapshot.close()
self.pull.close()
self.publisher.close()
self._exit_event.clear()
def stop(self):
self._exit_event.set()

30
replication_graph.py Normal file
View File

@ -0,0 +1,30 @@
import collections
class ReplicationGraph(collections.MutableMapping):
"""
Structure to hold replicated data relation graph
"""
def __init__(self, *args, **kwargs):
self.store = dict()
self.update(dict(*args, **kwargs)) # use the free update to set keys
def __getitem__(self, key):
return self.store[self.__keytransform__(key)]
def __setitem__(self, key, value):
self.store[self.__keytransform__(key)] = value
def __delitem__(self, key):
del self.store[self.__keytransform__(key)]
def __iter__(self):
return iter(self.store)
def __len__(self):
return len(self.store)
def __keytransform__(self, key):
return key

48
test_replication.py
View File

@ -4,6 +4,8 @@ import umsgpack
import logging
from replication_client import Client, Server
import time
import cProfile
import re
log = logging.getLogger(__name__)
@ -16,13 +18,23 @@ class SampleData():
class RepSampleData(ReplicatedDatablock):
def serialize(self,data):
import pickle
return pickle.dumps(data)
def deserialize(self,data):
import pickle
return pickle.loads(data)
def dump(self):
import json
output = {}
output['map'] = json.dumps(self.pointer.map)
return output
def load(self,target=None):
target = self.buffer
class TestDataFactory(unittest.TestCase):
@ -30,7 +42,7 @@ class TestDataFactory(unittest.TestCase):
factory = ReplicatedDataFactory()
factory.register_type(SampleData, RepSampleData)
data_sample = SampleData()
rep_sample = factory.construct_from_dcc(data_sample)(owner="toto", data=data_sample)
rep_sample = factory.construct_from_dcc(data_sample)(owner="toto", pointer=data_sample)
self.assertEqual(isinstance(rep_sample,RepSampleData), True)
@ -63,7 +75,7 @@ class TestClient(unittest.TestCase):
factory.register_type(SampleData, RepSampleData)
server = Server(factory=factory)
client = Client(factory=factory, id="client_test_callback")
client = Client(factory=factory, id="cli_test_filled_snapshot")
client2 = Client(factory=factory, id="client_2")
server.serve(port=5575)
@ -86,26 +98,27 @@ class TestClient(unittest.TestCase):
def test_register_client_data(self):
# Setup environment
factory = ReplicatedDataFactory()
factory.register_type(SampleData, RepSampleData)
server = Server(factory=factory)
server.serve(port=5560)
client = Client(factory=factory, id="client_1")
client = Client(factory=factory, id="cli_test_register_client_data")
client.connect(port=5560)
client2 = Client(factory=factory, id="client_2")
client2 = Client(factory=factory, id="cli2_test_register_client_data")
client2.connect(port=5560)
# Test the key registering
data_sample_key = client.register(SampleData())
time.sleep(4)
#Waiting for server to receive the datas
time.sleep(.5)
rep_test_key = client2._rep_store[data_sample_key].uuid
client.disconnect()
client2.disconnect()
@ -122,21 +135,21 @@ class TestClient(unittest.TestCase):
server = Server(factory=factory)
server.serve(port=5560)
client = Client(factory=factory, id="client_1")
client = Client(factory=factory, id="cli_test_client_data_intergity")
client.connect(port=5560)
client2 = Client(factory=factory, id="client_2")
client2 = Client(factory=factory, id="cli2_test_client_data_intergity")
client2.connect(port=5560)
test_map = {"toto":"test"}
# Test the key registering
client.register(SampleData(map=test_map))
data_sample_key = client.register(SampleData(map=test_map))
test_map_result = {}
#Waiting for server to receive the datas
time.sleep(.5)
time.sleep(1)
rep_test_key = client2._rep_store[data_sample_key].uuid
client2._rep_store[data_sample_key].load(target=test_map_result)
client.disconnect()
@ -144,7 +157,7 @@ class TestClient(unittest.TestCase):
server.stop()
self.assertEqual(rep_test_key, data_sample_key)
self.assertEqual(test_map_result["toto"], test_map["toto"])
def suite():
suite = unittest.TestSuite()
@ -152,9 +165,10 @@ def suite():
suite.addTest(TestClient('test_empty_snapshot'))
suite.addTest(TestClient('test_filled_snapshot'))
suite.addTest(TestClient('test_register_client_data'))
# suite.addTest(TestClient('test_client_data_intergity'))
return suite
if __name__ == '__main__':
runner = unittest.TextTestRunner(failfast=True,verbosity=2)
runner.run(suite())
runner.run(suite())