Backups/multi-user
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refacor: code formatting

Browse Source
This commit is contained in:
Swann
2019-07-23 20:18:51 +02:00
parent ea645044c6
commit 7ff01273be
3 changed files with 116 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
replication.py
View File

@ -1,19 +1,14 @@
import logging
from uuid import uuid4
import json
try:
from .libs import umsgpack
except:
# Server import
from libs import umsgpack
import zmq
import logging
import pickle
from enum import Enum
from uuid import uuid4
import zmq
logger = logging.getLogger(__name__)
class RepState(Enum):
ADDED = 0
COMMITED = 1
@ -29,10 +24,9 @@ class ReplicatedDataFactory(object):
self.supported_types = []
# Default registered types
self.register_type(str,RepCommand)
self.register_type(str, RepCommand)
self.register_type(RepDeleteCommand, RepDeleteCommand)
def register_type(self, dtype, implementation):
"""
Register a new replicated datatype implementation
@ -127,7 +121,6 @@ class ReplicatedDatablock(object):
return instance
def store(self, dict, persistent=False):
"""
I want to store my replicated data. Persistent means into the disk
@ -142,21 +135,18 @@ class ReplicatedDatablock(object):
return self.uuid
def deserialize(self, data):
"""
BUFFER -> JSON
"""
raise NotImplementedError
def serialize(self, data):
"""
JSON -> BUFFER
"""
raise NotImplementedError
def dump(self):
"""
DCC -> JSON
@ -165,14 +155,12 @@ class ReplicatedDatablock(object):
return json.dumps(self.pointer)
def load(self,target=None):
def load(self, target=None):
"""
JSON -> DCC
"""
raise NotImplementedError
def resolve(self):
"""
I want to resolve my orphan data to an existing one
@ -181,7 +169,6 @@ class ReplicatedDatablock(object):
"""
raise NotImplementedError
def __repr__(self):
return "{uuid} - owner: {owner} - type: {type}".format(
uuid=self.uuid,
@ -189,31 +176,32 @@ class ReplicatedDatablock(object):
type=self.str_type
)
class RepCommand(ReplicatedDatablock):
def serialize(self,data):
def serialize(self, data):
return pickle.dumps(data)
def deserialize(self,data):
def deserialize(self, data):
return pickle.loads(data)
def load(self,target):
def load(self, target):
target = self.pointer
class RepDeleteCommand(ReplicatedDatablock):
def serialize(self,data):
def serialize(self, data):
return pickle.dumps(data)
def deserialize(self,data):
def deserialize(self, data):
return pickle.loads(data)
def store(self,rep_store):
def store(self, rep_store):
assert(self.buffer)
if rep_store and self.buffer in rep_store.keys():
del rep_store[self.buffer]
# class RepObject(ReplicatedDatablock):
# def deserialize(self):
# try:

65
replication_client.py
View File

@ -1,8 +1,10 @@
import threading
import logging
import zmq
import threading
import time
from replication import ReplicatedDatablock, RepCommand,RepDeleteCommand
import zmq
from replication import RepCommand, RepDeleteCommand, ReplicatedDatablock
from replication_graph import ReplicationGraph
logger = logging.getLogger(__name__)
@ -13,7 +15,7 @@ STATE_ACTIVE = 2
class Client(object):
def __init__(self,factory=None, id='default'):
def __init__(self, factory=None, id='default'):
assert(factory)
self._rep_store = ReplicationGraph()
@ -23,11 +25,11 @@ class Client(object):
id=id)
self._factory = factory
def connect(self,address="127.0.0.1",port=5560):
def connect(self, address="127.0.0.1", port=5560):
"""
Connect to the server
"""
self._net_client.connect(address=address,port=port)
self._net_client.connect(address=address, port=port)
def disconnect(self):
"""
@ -54,10 +56,11 @@ class Client(object):
assert(object)
# Construct the coresponding replication type
new_item = self._factory.construct_from_dcc(object)(owner="client", pointer=object)
new_item = self._factory.construct_from_dcc(
object)(owner="client", pointer=object)
if new_item:
logger.info("Registering {} on {}".format(object,new_item.uuid))
logger.info("Registering {} on {}".format(object, new_item.uuid))
new_item.store(self._rep_store)
logger.info("Pushing new registered value")
@ -67,7 +70,7 @@ class Client(object):
else:
raise TypeError("Type not supported")
def unregister(self,object_uuid,clean=False):
def unregister(self, object_uuid, clean=False):
"""
Unregister for replication the given
object.
@ -76,21 +79,24 @@ class Client(object):
"""
if object_uuid in self._rep_store.keys():
delete_command = RepDeleteCommand(owner='client', buffer=object_uuid)
delete_command = RepDeleteCommand(
owner='client', buffer=object_uuid)
# remove the key from our store
delete_command.store(self._rep_store)
delete_command.push(self._net_client.publish)
else:
raise KeyError("Cannot unregister key")
def pull(self,object=None):
def pull(self, object=None):
"""
Asynchonous pull
Here we want to pull all waiting changes and apply them
"""
pass
class ClientNetService(threading.Thread):
def __init__(self,store_reference=None, factory=None,id="default"):
def __init__(self, store_reference=None, factory=None, id="default"):
# Threading
threading.Thread.__init__(self)
@ -108,12 +114,12 @@ class ClientNetService(threading.Thread):
self.context = zmq.Context.instance()
self.state = STATE_INITIAL
def connect(self,address='127.0.0.1', port=5560):
def connect(self, address='127.0.0.1', port=5560):
"""
Network socket setup
"""
if self.state == STATE_INITIAL:
logger.debug("connecting on {}:{}".format(address,port))
logger.debug("connecting on {}:{}".format(address, port))
self.snapshot = self.context.socket(zmq.DEALER)
self.snapshot.setsockopt(zmq.IDENTITY, self._id.encode())
self.snapshot.connect("tcp://{}:{}".format(address, port))
@ -147,7 +153,6 @@ class ClientNetService(threading.Thread):
self.snapshot.send(b"SNAPSHOT_REQUEST")
self.state = STATE_SYNCING
"""NET IN
Given the net state we do something:
SYNCING : load snapshots
@ -157,7 +162,8 @@ class ClientNetService(threading.Thread):
if self.snapshot in items:
if self.state == STATE_SYNCING:
datablock = ReplicatedDatablock.pull(self.snapshot, self._factory)
datablock = ReplicatedDatablock.pull(
self.snapshot, self._factory)
if 'SNAPSHOT_END' in datablock.buffer:
self.state = STATE_ACTIVE
@ -168,14 +174,15 @@ class ClientNetService(threading.Thread):
# We receive updates from the server !
if self.subscriber in items:
if self.state == STATE_ACTIVE:
logger.debug("{} : Receiving changes from server".format(self._id))
datablock = ReplicatedDatablock.pull(self.subscriber, self._factory)
logger.debug(
"{} : Receiving changes from server".format(self._id))
datablock = ReplicatedDatablock.pull(
self.subscriber, self._factory)
datablock.store(self._store_reference)
if not items:
logger.error("No request ")
self.snapshot.close()
self.subscriber.close()
self.publish.close()
@ -185,20 +192,20 @@ class ClientNetService(threading.Thread):
def stop(self):
self._exit_event.set()
#Wait the end of the run
# Wait the end of the run
while self._exit_event.is_set():
time.sleep(.1)
self.state = 0
class Server():
def __init__(self,config=None, factory=None):
def __init__(self, config=None, factory=None):
self._rep_store = {}
self._net = ServerNetService(store_reference=self._rep_store, factory=factory)
self._net = ServerNetService(
store_reference=self._rep_store, factory=factory)
def serve(self,port=5560):
def serve(self, port=5560):
self._net.listen(port=port)
def state(self):
@ -209,7 +216,7 @@ class Server():
class ServerNetService(threading.Thread):
def __init__(self,store_reference=None, factory=None):
def __init__(self, store_reference=None, factory=None):
# Threading
threading.Thread.__init__(self)
self.name = "ServerNetLink"
@ -227,7 +234,6 @@ class ServerNetService(threading.Thread):
self.factory = factory
self.clients = {}
def listen(self, port=5560):
try:
# Update request
@ -288,8 +294,8 @@ class ServerNetService(threading.Thread):
# Snapshot end
self.snapshot.send(identity, zmq.SNDMORE)
RepCommand(owner='server',pointer='SNAPSHOT_END').push(self.snapshot)
RepCommand(owner='server', pointer='SNAPSHOT_END').push(
self.snapshot)
# Regular update routing (Clients / Server / Clients)
if self.pull in socks:
@ -312,11 +318,10 @@ class ServerNetService(threading.Thread):
self._exit_event.clear()
def stop(self):
self._exit_event.set()
#Wait the end of the run
# Wait the end of the run
while self._exit_event.is_set():
time.sleep(.1)

55
test_replication.py
View File

@ -1,42 +1,43 @@
import unittest
from replication import ReplicatedDatablock, ReplicatedDataFactory
import umsgpack
import logging
from replication_client import Client, Server
import time
import cProfile
import logging
import re
import time
import unittest
logging.basicConfig()
logging.getLogger().setLevel(logging.INFO)
logger = logging.getLogger(__name__)
from replication import ReplicatedDatablock, ReplicatedDataFactory
from replication_client import Client, Server
class SampleData():
def __init__(self, map={"sample":"data"}):
def __init__(self, map={"sample": "data"}):
self.map = map
class RepSampleData(ReplicatedDatablock):
def serialize(self,data):
def serialize(self, data):
import pickle
return pickle.dumps(data)
def deserialize(self,data):
def deserialize(self, data):
import pickle
return pickle.loads(data)
def dump(self):
import json
output = {}
output['map'] = json.dumps(self.pointer.map)
return output
def load(self,target=None):
def load(self, target=None):
import json
if target is None:
target = SampleData()
@ -49,13 +50,14 @@ class TestDataFactory(unittest.TestCase):
factory = ReplicatedDataFactory()
factory.register_type(SampleData, RepSampleData)
data_sample = SampleData()
rep_sample = factory.construct_from_dcc(data_sample)(owner="toto", pointer=data_sample)
rep_sample = factory.construct_from_dcc(
data_sample)(owner="toto", pointer=data_sample)
self.assertEqual(isinstance(rep_sample,RepSampleData), True)
self.assertEqual(isinstance(rep_sample, RepSampleData), True)
class TestClient(unittest.TestCase):
def __init__(self,methodName='runTest'):
def __init__(self, methodName='runTest'):
unittest.TestCase.__init__(self, methodName)
def test_empty_snapshot(self):
@ -91,7 +93,6 @@ class TestClient(unittest.TestCase):
# Test the key registering
data_sample_key = client.register(SampleData())
client2.connect(port=5575)
time.sleep(0.2)
rep_test_key = client2._rep_store[data_sample_key].uuid
@ -117,20 +118,17 @@ class TestClient(unittest.TestCase):
client2 = Client(factory=factory, id="cli2_test_register_client_data")
client2.connect(port=5560)
# Test the key registering
data_sample_key = client.register(SampleData())
time.sleep(0.3)
#Waiting for server to receive the datas
# Waiting for server to receive the datas
rep_test_key = client2._rep_store[data_sample_key].uuid
client.disconnect()
client2.disconnect()
server.stop()
self.assertEqual(rep_test_key, data_sample_key)
def test_client_data_intergity(self):
@ -147,22 +145,20 @@ class TestClient(unittest.TestCase):
client2 = Client(factory=factory, id="cli2_test_client_data_intergity")
client2.connect(port=5560)
test_map = {"toto":"test"}
test_map = {"toto": "test"}
# Test the key registering
data_sample_key = client.register(SampleData(map=test_map))
test_map_result = SampleData()
#Waiting for server to receive the datas
# Waiting for server to receive the datas
time.sleep(1)
client2._rep_store[data_sample_key].load(target=test_map_result)
client.disconnect()
client2.disconnect()
server.stop()
self.assertEqual(test_map_result.map["toto"], test_map["toto"])
def test_client_unregister_key(self):
@ -179,13 +175,13 @@ class TestClient(unittest.TestCase):
client2 = Client(factory=factory, id="cli2_test_client_data_intergity")
client2.connect(port=5560)
test_map = {"toto":"test"}
test_map = {"toto": "test"}
# Test the key registering
data_sample_key = client.register(SampleData(map=test_map))
test_map_result = SampleData()
#Waiting for server to receive the datas
# Waiting for server to receive the datas
time.sleep(.1)
client2._rep_store[data_sample_key].load(target=test_map_result)
@ -234,7 +230,7 @@ class TestStressClient(unittest.TestCase):
while len(client2._rep_store.keys()) < 10000:
time.sleep(0.00001)
total_time+=0.00001
total_time += 0.00001
# test_num_items = len(client2._rep_store.keys())
server.stop()
@ -242,7 +238,8 @@ class TestStressClient(unittest.TestCase):
client2.disconnect()
logger.info("{} s for 10000 values".format(total_time))
self.assertLess(total_time,1)
self.assertLess(total_time, 1)
def suite():
suite = unittest.TestSuite()
@ -261,7 +258,7 @@ def suite():
return suite
if __name__ == '__main__':
runner = unittest.TextTestRunner(verbosity=2)
runner.run(suite())