Backups/multi-user
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refacor: code formatting

Browse Source
This commit is contained in:
Swann
2019-07-23 20:18:51 +02:00
parent ea645044c6
commit 7ff01273be
3 changed files with 116 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
replication.py
View File

@ -1,19 +1,14 @@
import logging
from uuid import uuid4
import json import json
try: import logging
from .libs import umsgpack
except:
# Server import
from libs import umsgpack
import zmq
import pickle import pickle
from enum import Enum from enum import Enum
from uuid import uuid4
import zmq
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class RepState(Enum): class RepState(Enum):
ADDED = 0 ADDED = 0
COMMITED = 1 COMMITED = 1
@ -29,10 +24,9 @@ class ReplicatedDataFactory(object):
self.supported_types = [] self.supported_types = []
# Default registered types # Default registered types
self.register_type(str,RepCommand) self.register_type(str, RepCommand)
self.register_type(RepDeleteCommand, RepDeleteCommand) self.register_type(RepDeleteCommand, RepDeleteCommand)
def register_type(self, dtype, implementation): def register_type(self, dtype, implementation):
""" """
Register a new replicated datatype implementation Register a new replicated datatype implementation
@ -127,7 +121,6 @@ class ReplicatedDatablock(object):
return instance return instance
def store(self, dict, persistent=False): def store(self, dict, persistent=False):
""" """
I want to store my replicated data. Persistent means into the disk I want to store my replicated data. Persistent means into the disk
@ -142,21 +135,18 @@ class ReplicatedDatablock(object):
return self.uuid return self.uuid
def deserialize(self, data): def deserialize(self, data):
""" """
BUFFER -> JSON BUFFER -> JSON
""" """
raise NotImplementedError raise NotImplementedError
def serialize(self, data): def serialize(self, data):
""" """
JSON -> BUFFER JSON -> BUFFER
""" """
raise NotImplementedError raise NotImplementedError
def dump(self): def dump(self):
""" """
DCC -> JSON DCC -> JSON
@ -165,14 +155,12 @@ class ReplicatedDatablock(object):
return json.dumps(self.pointer) return json.dumps(self.pointer)
def load(self, target=None):
def load(self,target=None):
""" """
JSON -> DCC JSON -> DCC
""" """
raise NotImplementedError raise NotImplementedError
def resolve(self): def resolve(self):
""" """
I want to resolve my orphan data to an existing one I want to resolve my orphan data to an existing one
@ -181,7 +169,6 @@ class ReplicatedDatablock(object):
""" """
raise NotImplementedError raise NotImplementedError
def __repr__(self): def __repr__(self):
return "{uuid} - owner: {owner} - type: {type}".format( return "{uuid} - owner: {owner} - type: {type}".format(
uuid=self.uuid, uuid=self.uuid,
@ -189,31 +176,32 @@ class ReplicatedDatablock(object):
type=self.str_type type=self.str_type
) )
class RepCommand(ReplicatedDatablock): class RepCommand(ReplicatedDatablock):
def serialize(self,data): def serialize(self, data):
return pickle.dumps(data) return pickle.dumps(data)
def deserialize(self,data): def deserialize(self, data):
return pickle.loads(data) return pickle.loads(data)
def load(self,target): def load(self, target):
target = self.pointer target = self.pointer
class RepDeleteCommand(ReplicatedDatablock): class RepDeleteCommand(ReplicatedDatablock):
def serialize(self,data): def serialize(self, data):
return pickle.dumps(data) return pickle.dumps(data)
def deserialize(self,data): def deserialize(self, data):
return pickle.loads(data) return pickle.loads(data)
def store(self,rep_store): def store(self, rep_store):
assert(self.buffer) assert(self.buffer)
if rep_store and self.buffer in rep_store.keys(): if rep_store and self.buffer in rep_store.keys():
del rep_store[self.buffer] del rep_store[self.buffer]
# class RepObject(ReplicatedDatablock): # class RepObject(ReplicatedDatablock):
# def deserialize(self): # def deserialize(self):
# try: # try:

65
replication_client.py
View File

@ -1,8 +1,10 @@
import threading
import logging import logging
import zmq import threading
import time import time
from replication import ReplicatedDatablock, RepCommand,RepDeleteCommand
import zmq
from replication import RepCommand, RepDeleteCommand, ReplicatedDatablock
from replication_graph import ReplicationGraph from replication_graph import ReplicationGraph
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -13,7 +15,7 @@ STATE_ACTIVE = 2
class Client(object): class Client(object):
def __init__(self,factory=None, id='default'): def __init__(self, factory=None, id='default'):
assert(factory) assert(factory)
self._rep_store = ReplicationGraph() self._rep_store = ReplicationGraph()
@ -23,11 +25,11 @@ class Client(object):
id=id) id=id)
self._factory = factory self._factory = factory
def connect(self,address="127.0.0.1",port=5560): def connect(self, address="127.0.0.1", port=5560):
""" """
Connect to the server Connect to the server
""" """
self._net_client.connect(address=address,port=port) self._net_client.connect(address=address, port=port)
def disconnect(self): def disconnect(self):
""" """
@ -54,10 +56,11 @@ class Client(object):
assert(object) assert(object)
# Construct the coresponding replication type # Construct the coresponding replication type
new_item = self._factory.construct_from_dcc(object)(owner="client", pointer=object) new_item = self._factory.construct_from_dcc(
object)(owner="client", pointer=object)
if new_item: if new_item:
logger.info("Registering {} on {}".format(object,new_item.uuid)) logger.info("Registering {} on {}".format(object, new_item.uuid))
new_item.store(self._rep_store) new_item.store(self._rep_store)
logger.info("Pushing new registered value") logger.info("Pushing new registered value")
@ -67,7 +70,7 @@ class Client(object):
else: else:
raise TypeError("Type not supported") raise TypeError("Type not supported")
def unregister(self,object_uuid,clean=False): def unregister(self, object_uuid, clean=False):
""" """
Unregister for replication the given Unregister for replication the given
object. object.
@ -76,21 +79,24 @@ class Client(object):
""" """
if object_uuid in self._rep_store.keys(): if object_uuid in self._rep_store.keys():
delete_command = RepDeleteCommand(owner='client', buffer=object_uuid) delete_command = RepDeleteCommand(
owner='client', buffer=object_uuid)
# remove the key from our store
delete_command.store(self._rep_store) delete_command.store(self._rep_store)
delete_command.push(self._net_client.publish) delete_command.push(self._net_client.publish)
else: else:
raise KeyError("Cannot unregister key") raise KeyError("Cannot unregister key")
def pull(self,object=None): def pull(self, object=None):
""" """
Asynchonous pull Asynchonous pull
Here we want to pull all waiting changes and apply them Here we want to pull all waiting changes and apply them
""" """
pass pass
class ClientNetService(threading.Thread): class ClientNetService(threading.Thread):
def __init__(self,store_reference=None, factory=None,id="default"): def __init__(self, store_reference=None, factory=None, id="default"):
# Threading # Threading
threading.Thread.__init__(self) threading.Thread.__init__(self)
@ -108,12 +114,12 @@ class ClientNetService(threading.Thread):
self.context = zmq.Context.instance() self.context = zmq.Context.instance()
self.state = STATE_INITIAL self.state = STATE_INITIAL
def connect(self,address='127.0.0.1', port=5560): def connect(self, address='127.0.0.1', port=5560):
""" """
Network socket setup Network socket setup
""" """
if self.state == STATE_INITIAL: if self.state == STATE_INITIAL:
logger.debug("connecting on {}:{}".format(address,port)) logger.debug("connecting on {}:{}".format(address, port))
self.snapshot = self.context.socket(zmq.DEALER) self.snapshot = self.context.socket(zmq.DEALER)
self.snapshot.setsockopt(zmq.IDENTITY, self._id.encode()) self.snapshot.setsockopt(zmq.IDENTITY, self._id.encode())
self.snapshot.connect("tcp://{}:{}".format(address, port)) self.snapshot.connect("tcp://{}:{}".format(address, port))
@ -147,7 +153,6 @@ class ClientNetService(threading.Thread):
self.snapshot.send(b"SNAPSHOT_REQUEST") self.snapshot.send(b"SNAPSHOT_REQUEST")
self.state = STATE_SYNCING self.state = STATE_SYNCING
"""NET IN """NET IN
Given the net state we do something: Given the net state we do something:
SYNCING : load snapshots SYNCING : load snapshots
@ -157,7 +162,8 @@ class ClientNetService(threading.Thread):
if self.snapshot in items: if self.snapshot in items:
if self.state == STATE_SYNCING: if self.state == STATE_SYNCING:
datablock = ReplicatedDatablock.pull(self.snapshot, self._factory) datablock = ReplicatedDatablock.pull(
self.snapshot, self._factory)
if 'SNAPSHOT_END' in datablock.buffer: if 'SNAPSHOT_END' in datablock.buffer:
self.state = STATE_ACTIVE self.state = STATE_ACTIVE
@ -168,14 +174,15 @@ class ClientNetService(threading.Thread):
# We receive updates from the server ! # We receive updates from the server !
if self.subscriber in items: if self.subscriber in items:
if self.state == STATE_ACTIVE: if self.state == STATE_ACTIVE:
logger.debug("{} : Receiving changes from server".format(self._id)) logger.debug(
datablock = ReplicatedDatablock.pull(self.subscriber, self._factory) "{} : Receiving changes from server".format(self._id))
datablock = ReplicatedDatablock.pull(
self.subscriber, self._factory)
datablock.store(self._store_reference) datablock.store(self._store_reference)
if not items: if not items:
logger.error("No request ") logger.error("No request ")
self.snapshot.close() self.snapshot.close()
self.subscriber.close() self.subscriber.close()
self.publish.close() self.publish.close()
@ -185,20 +192,20 @@ class ClientNetService(threading.Thread):
def stop(self): def stop(self):
self._exit_event.set() self._exit_event.set()
#Wait the end of the run # Wait the end of the run
while self._exit_event.is_set(): while self._exit_event.is_set():
time.sleep(.1) time.sleep(.1)
self.state = 0 self.state = 0
class Server(): class Server():
def __init__(self,config=None, factory=None): def __init__(self, config=None, factory=None):
self._rep_store = {} self._rep_store = {}
self._net = ServerNetService(store_reference=self._rep_store, factory=factory) self._net = ServerNetService(
store_reference=self._rep_store, factory=factory)
def serve(self,port=5560): def serve(self, port=5560):
self._net.listen(port=port) self._net.listen(port=port)
def state(self): def state(self):
@ -209,7 +216,7 @@ class Server():
class ServerNetService(threading.Thread): class ServerNetService(threading.Thread):
def __init__(self,store_reference=None, factory=None): def __init__(self, store_reference=None, factory=None):
# Threading # Threading
threading.Thread.__init__(self) threading.Thread.__init__(self)
self.name = "ServerNetLink" self.name = "ServerNetLink"
@ -227,7 +234,6 @@ class ServerNetService(threading.Thread):
self.factory = factory self.factory = factory
self.clients = {} self.clients = {}
def listen(self, port=5560): def listen(self, port=5560):
try: try:
# Update request # Update request
@ -288,8 +294,8 @@ class ServerNetService(threading.Thread):
# Snapshot end # Snapshot end
self.snapshot.send(identity, zmq.SNDMORE) self.snapshot.send(identity, zmq.SNDMORE)
RepCommand(owner='server',pointer='SNAPSHOT_END').push(self.snapshot) RepCommand(owner='server', pointer='SNAPSHOT_END').push(
self.snapshot)
# Regular update routing (Clients / Server / Clients) # Regular update routing (Clients / Server / Clients)
if self.pull in socks: if self.pull in socks:
@ -312,11 +318,10 @@ class ServerNetService(threading.Thread):
self._exit_event.clear() self._exit_event.clear()
def stop(self): def stop(self):
self._exit_event.set() self._exit_event.set()
#Wait the end of the run # Wait the end of the run
while self._exit_event.is_set(): while self._exit_event.is_set():
time.sleep(.1) time.sleep(.1)

55
test_replication.py
View File

@ -1,42 +1,43 @@
import unittest
from replication import ReplicatedDatablock, ReplicatedDataFactory
import umsgpack
import logging
from replication_client import Client, Server
import time
import cProfile import cProfile
import logging
import re import re
import time
import unittest
logging.basicConfig() logging.basicConfig()
logging.getLogger().setLevel(logging.INFO) logging.getLogger().setLevel(logging.INFO)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
from replication import ReplicatedDatablock, ReplicatedDataFactory
from replication_client import Client, Server
class SampleData(): class SampleData():
def __init__(self, map={"sample":"data"}): def __init__(self, map={"sample": "data"}):
self.map = map self.map = map
class RepSampleData(ReplicatedDatablock): class RepSampleData(ReplicatedDatablock):
def serialize(self,data): def serialize(self, data):
import pickle import pickle
return pickle.dumps(data) return pickle.dumps(data)
def deserialize(self,data): def deserialize(self, data):
import pickle import pickle
return pickle.loads(data) return pickle.loads(data)
def dump(self): def dump(self):
import json import json
output = {} output = {}
output['map'] = json.dumps(self.pointer.map) output['map'] = json.dumps(self.pointer.map)
return output return output
def load(self, target=None):
def load(self,target=None):
import json import json
if target is None: if target is None:
target = SampleData() target = SampleData()
@ -49,13 +50,14 @@ class TestDataFactory(unittest.TestCase):
factory = ReplicatedDataFactory() factory = ReplicatedDataFactory()
factory.register_type(SampleData, RepSampleData) factory.register_type(SampleData, RepSampleData)
data_sample = SampleData() data_sample = SampleData()
rep_sample = factory.construct_from_dcc(data_sample)(owner="toto", pointer=data_sample) rep_sample = factory.construct_from_dcc(
data_sample)(owner="toto", pointer=data_sample)
self.assertEqual(isinstance(rep_sample,RepSampleData), True) self.assertEqual(isinstance(rep_sample, RepSampleData), True)
class TestClient(unittest.TestCase): class TestClient(unittest.TestCase):
def __init__(self,methodName='runTest'): def __init__(self, methodName='runTest'):
unittest.TestCase.__init__(self, methodName) unittest.TestCase.__init__(self, methodName)
def test_empty_snapshot(self): def test_empty_snapshot(self):
@ -91,7 +93,6 @@ class TestClient(unittest.TestCase):
# Test the key registering # Test the key registering
data_sample_key = client.register(SampleData()) data_sample_key = client.register(SampleData())
client2.connect(port=5575) client2.connect(port=5575)
time.sleep(0.2) time.sleep(0.2)
rep_test_key = client2._rep_store[data_sample_key].uuid rep_test_key = client2._rep_store[data_sample_key].uuid
@ -117,20 +118,17 @@ class TestClient(unittest.TestCase):
client2 = Client(factory=factory, id="cli2_test_register_client_data") client2 = Client(factory=factory, id="cli2_test_register_client_data")
client2.connect(port=5560) client2.connect(port=5560)
# Test the key registering # Test the key registering
data_sample_key = client.register(SampleData()) data_sample_key = client.register(SampleData())
time.sleep(0.3) time.sleep(0.3)
#Waiting for server to receive the datas # Waiting for server to receive the datas
rep_test_key = client2._rep_store[data_sample_key].uuid rep_test_key = client2._rep_store[data_sample_key].uuid
client.disconnect() client.disconnect()
client2.disconnect() client2.disconnect()
server.stop() server.stop()
self.assertEqual(rep_test_key, data_sample_key) self.assertEqual(rep_test_key, data_sample_key)
def test_client_data_intergity(self): def test_client_data_intergity(self):
@ -147,22 +145,20 @@ class TestClient(unittest.TestCase):
client2 = Client(factory=factory, id="cli2_test_client_data_intergity") client2 = Client(factory=factory, id="cli2_test_client_data_intergity")
client2.connect(port=5560) client2.connect(port=5560)
test_map = {"toto":"test"} test_map = {"toto": "test"}
# Test the key registering # Test the key registering
data_sample_key = client.register(SampleData(map=test_map)) data_sample_key = client.register(SampleData(map=test_map))
test_map_result = SampleData() test_map_result = SampleData()
#Waiting for server to receive the datas # Waiting for server to receive the datas
time.sleep(1) time.sleep(1)
client2._rep_store[data_sample_key].load(target=test_map_result) client2._rep_store[data_sample_key].load(target=test_map_result)
client.disconnect() client.disconnect()
client2.disconnect() client2.disconnect()
server.stop() server.stop()
self.assertEqual(test_map_result.map["toto"], test_map["toto"]) self.assertEqual(test_map_result.map["toto"], test_map["toto"])
def test_client_unregister_key(self): def test_client_unregister_key(self):
@ -179,13 +175,13 @@ class TestClient(unittest.TestCase):
client2 = Client(factory=factory, id="cli2_test_client_data_intergity") client2 = Client(factory=factory, id="cli2_test_client_data_intergity")
client2.connect(port=5560) client2.connect(port=5560)
test_map = {"toto":"test"} test_map = {"toto": "test"}
# Test the key registering # Test the key registering
data_sample_key = client.register(SampleData(map=test_map)) data_sample_key = client.register(SampleData(map=test_map))
test_map_result = SampleData() test_map_result = SampleData()
#Waiting for server to receive the datas # Waiting for server to receive the datas
time.sleep(.1) time.sleep(.1)
client2._rep_store[data_sample_key].load(target=test_map_result) client2._rep_store[data_sample_key].load(target=test_map_result)
@ -234,7 +230,7 @@ class TestStressClient(unittest.TestCase):
while len(client2._rep_store.keys()) < 10000: while len(client2._rep_store.keys()) < 10000:
time.sleep(0.00001) time.sleep(0.00001)
total_time+=0.00001 total_time += 0.00001
# test_num_items = len(client2._rep_store.keys()) # test_num_items = len(client2._rep_store.keys())
server.stop() server.stop()
@ -242,7 +238,8 @@ class TestStressClient(unittest.TestCase):
client2.disconnect() client2.disconnect()
logger.info("{} s for 10000 values".format(total_time)) logger.info("{} s for 10000 values".format(total_time))
self.assertLess(total_time,1) self.assertLess(total_time, 1)
def suite(): def suite():
suite = unittest.TestSuite() suite = unittest.TestSuite()
@ -261,7 +258,7 @@ def suite():
return suite return suite
if __name__ == '__main__': if __name__ == '__main__':
runner = unittest.TextTestRunner(verbosity=2) runner = unittest.TextTestRunner(verbosity=2)
runner.run(suite()) runner.run(suite())