Backups/multi-user
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refacor: code formatting

Browse Source
This commit is contained in:
Swann
2019-07-23 20:18:51 +02:00
parent ea645044c6
commit 7ff01273be
3 changed files with 116 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
replication.py
View File

@ -1,19 +1,14 @@
import logging
from uuid import uuid4
import json import json
try: import logging
from .libs import umsgpack
except:
# Server import
from libs import umsgpack
import zmq
import pickle import pickle
from enum import Enum from enum import Enum
from uuid import uuid4
import zmq
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class RepState(Enum): class RepState(Enum):
ADDED = 0 ADDED = 0
COMMITED = 1 COMMITED = 1
@ -32,7 +27,6 @@ class ReplicatedDataFactory(object):
self.register_type(str, RepCommand) self.register_type(str, RepCommand)
self.register_type(RepDeleteCommand, RepDeleteCommand) self.register_type(RepDeleteCommand, RepDeleteCommand)
def register_type(self, dtype, implementation): def register_type(self, dtype, implementation):
""" """
Register a new replicated datatype implementation Register a new replicated datatype implementation
@ -127,7 +121,6 @@ class ReplicatedDatablock(object):
return instance return instance
def store(self, dict, persistent=False): def store(self, dict, persistent=False):
""" """
I want to store my replicated data. Persistent means into the disk I want to store my replicated data. Persistent means into the disk
@ -142,21 +135,18 @@ class ReplicatedDatablock(object):
return self.uuid return self.uuid
def deserialize(self, data): def deserialize(self, data):
""" """
BUFFER -> JSON BUFFER -> JSON
""" """
raise NotImplementedError raise NotImplementedError
def serialize(self, data): def serialize(self, data):
""" """
JSON -> BUFFER JSON -> BUFFER
""" """
raise NotImplementedError raise NotImplementedError
def dump(self): def dump(self):
""" """
DCC -> JSON DCC -> JSON
@ -165,14 +155,12 @@ class ReplicatedDatablock(object):
return json.dumps(self.pointer) return json.dumps(self.pointer)
def load(self, target=None): def load(self, target=None):
""" """
JSON -> DCC JSON -> DCC
""" """
raise NotImplementedError raise NotImplementedError
def resolve(self): def resolve(self):
""" """
I want to resolve my orphan data to an existing one I want to resolve my orphan data to an existing one
@ -181,7 +169,6 @@ class ReplicatedDatablock(object):
""" """
raise NotImplementedError raise NotImplementedError
def __repr__(self): def __repr__(self):
return "{uuid} - owner: {owner} - type: {type}".format( return "{uuid} - owner: {owner} - type: {type}".format(
uuid=self.uuid, uuid=self.uuid,
@ -189,6 +176,7 @@ class ReplicatedDatablock(object):
type=self.str_type type=self.str_type
) )
class RepCommand(ReplicatedDatablock): class RepCommand(ReplicatedDatablock):
def serialize(self, data): def serialize(self, data):
return pickle.dumps(data) return pickle.dumps(data)
@ -199,6 +187,7 @@ class RepCommand(ReplicatedDatablock):
def load(self, target): def load(self, target):
target = self.pointer target = self.pointer
class RepDeleteCommand(ReplicatedDatablock): class RepDeleteCommand(ReplicatedDatablock):
def serialize(self, data): def serialize(self, data):
return pickle.dumps(data) return pickle.dumps(data)
@ -213,7 +202,6 @@ class RepDeleteCommand(ReplicatedDatablock):
del rep_store[self.buffer] del rep_store[self.buffer]
# class RepObject(ReplicatedDatablock): # class RepObject(ReplicatedDatablock):
# def deserialize(self): # def deserialize(self):
# try: # try:

37
replication_client.py
View File

@ -1,8 +1,10 @@
import threading
import logging import logging
import zmq import threading
import time import time
from replication import ReplicatedDatablock, RepCommand,RepDeleteCommand
import zmq
from replication import RepCommand, RepDeleteCommand, ReplicatedDatablock
from replication_graph import ReplicationGraph from replication_graph import ReplicationGraph
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -54,7 +56,8 @@ class Client(object):
assert(object) assert(object)
# Construct the coresponding replication type # Construct the coresponding replication type
new_item = self._factory.construct_from_dcc(object)(owner="client", pointer=object) new_item = self._factory.construct_from_dcc(
object)(owner="client", pointer=object)
if new_item: if new_item:
logger.info("Registering {} on {}".format(object, new_item.uuid)) logger.info("Registering {} on {}".format(object, new_item.uuid))
@ -76,7 +79,9 @@ class Client(object):
""" """
if object_uuid in self._rep_store.keys(): if object_uuid in self._rep_store.keys():
delete_command = RepDeleteCommand(owner='client', buffer=object_uuid) delete_command = RepDeleteCommand(
owner='client', buffer=object_uuid)
# remove the key from our store
delete_command.store(self._rep_store) delete_command.store(self._rep_store)
delete_command.push(self._net_client.publish) delete_command.push(self._net_client.publish)
else: else:
@ -89,6 +94,7 @@ class Client(object):
""" """
pass pass
class ClientNetService(threading.Thread): class ClientNetService(threading.Thread):
def __init__(self, store_reference=None, factory=None, id="default"): def __init__(self, store_reference=None, factory=None, id="default"):
@ -147,7 +153,6 @@ class ClientNetService(threading.Thread):
self.snapshot.send(b"SNAPSHOT_REQUEST") self.snapshot.send(b"SNAPSHOT_REQUEST")
self.state = STATE_SYNCING self.state = STATE_SYNCING
"""NET IN """NET IN
Given the net state we do something: Given the net state we do something:
SYNCING : load snapshots SYNCING : load snapshots
@ -157,7 +162,8 @@ class ClientNetService(threading.Thread):
if self.snapshot in items: if self.snapshot in items:
if self.state == STATE_SYNCING: if self.state == STATE_SYNCING:
datablock = ReplicatedDatablock.pull(self.snapshot, self._factory) datablock = ReplicatedDatablock.pull(
self.snapshot, self._factory)
if 'SNAPSHOT_END' in datablock.buffer: if 'SNAPSHOT_END' in datablock.buffer:
self.state = STATE_ACTIVE self.state = STATE_ACTIVE
@ -168,14 +174,15 @@ class ClientNetService(threading.Thread):
# We receive updates from the server ! # We receive updates from the server !
if self.subscriber in items: if self.subscriber in items:
if self.state == STATE_ACTIVE: if self.state == STATE_ACTIVE:
logger.debug("{} : Receiving changes from server".format(self._id)) logger.debug(
datablock = ReplicatedDatablock.pull(self.subscriber, self._factory) "{} : Receiving changes from server".format(self._id))
datablock = ReplicatedDatablock.pull(
self.subscriber, self._factory)
datablock.store(self._store_reference) datablock.store(self._store_reference)
if not items: if not items:
logger.error("No request ") logger.error("No request ")
self.snapshot.close() self.snapshot.close()
self.subscriber.close() self.subscriber.close()
self.publish.close() self.publish.close()
@ -192,11 +199,11 @@ class ClientNetService(threading.Thread):
self.state = 0 self.state = 0
class Server(): class Server():
def __init__(self, config=None, factory=None): def __init__(self, config=None, factory=None):
self._rep_store = {} self._rep_store = {}
self._net = ServerNetService(store_reference=self._rep_store, factory=factory) self._net = ServerNetService(
store_reference=self._rep_store, factory=factory)
def serve(self, port=5560): def serve(self, port=5560):
self._net.listen(port=port) self._net.listen(port=port)
@ -227,7 +234,6 @@ class ServerNetService(threading.Thread):
self.factory = factory self.factory = factory
self.clients = {} self.clients = {}
def listen(self, port=5560): def listen(self, port=5560):
try: try:
# Update request # Update request
@ -288,8 +294,8 @@ class ServerNetService(threading.Thread):
# Snapshot end # Snapshot end
self.snapshot.send(identity, zmq.SNDMORE) self.snapshot.send(identity, zmq.SNDMORE)
RepCommand(owner='server',pointer='SNAPSHOT_END').push(self.snapshot) RepCommand(owner='server', pointer='SNAPSHOT_END').push(
self.snapshot)
# Regular update routing (Clients / Server / Clients) # Regular update routing (Clients / Server / Clients)
if self.pull in socks: if self.pull in socks:
@ -312,7 +318,6 @@ class ServerNetService(threading.Thread):
self._exit_event.clear() self._exit_event.clear()
def stop(self): def stop(self):
self._exit_event.set() self._exit_event.set()

29
test_replication.py
View File

@ -1,17 +1,20 @@
import unittest
from replication import ReplicatedDatablock, ReplicatedDataFactory
import umsgpack
import logging
from replication_client import Client, Server
import time
import cProfile import cProfile
import logging
import re import re
import time
import unittest
logging.basicConfig() logging.basicConfig()
logging.getLogger().setLevel(logging.INFO) logging.getLogger().setLevel(logging.INFO)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
from replication import ReplicatedDatablock, ReplicatedDataFactory
from replication_client import Client, Server
class SampleData(): class SampleData():
def __init__(self, map={"sample": "data"}): def __init__(self, map={"sample": "data"}):
self.map = map self.map = map
@ -28,14 +31,12 @@ class RepSampleData(ReplicatedDatablock):
return pickle.loads(data) return pickle.loads(data)
def dump(self): def dump(self):
import json import json
output = {} output = {}
output['map'] = json.dumps(self.pointer.map) output['map'] = json.dumps(self.pointer.map)
return output return output
def load(self, target=None): def load(self, target=None):
import json import json
if target is None: if target is None:
@ -49,7 +50,8 @@ class TestDataFactory(unittest.TestCase):
factory = ReplicatedDataFactory() factory = ReplicatedDataFactory()
factory.register_type(SampleData, RepSampleData) factory.register_type(SampleData, RepSampleData)
data_sample = SampleData() data_sample = SampleData()
rep_sample = factory.construct_from_dcc(data_sample)(owner="toto", pointer=data_sample) rep_sample = factory.construct_from_dcc(
data_sample)(owner="toto", pointer=data_sample)
self.assertEqual(isinstance(rep_sample, RepSampleData), True) self.assertEqual(isinstance(rep_sample, RepSampleData), True)
@ -91,7 +93,6 @@ class TestClient(unittest.TestCase):
# Test the key registering # Test the key registering
data_sample_key = client.register(SampleData()) data_sample_key = client.register(SampleData())
client2.connect(port=5575) client2.connect(port=5575)
time.sleep(0.2) time.sleep(0.2)
rep_test_key = client2._rep_store[data_sample_key].uuid rep_test_key = client2._rep_store[data_sample_key].uuid
@ -117,7 +118,6 @@ class TestClient(unittest.TestCase):
client2 = Client(factory=factory, id="cli2_test_register_client_data") client2 = Client(factory=factory, id="cli2_test_register_client_data")
client2.connect(port=5560) client2.connect(port=5560)
# Test the key registering # Test the key registering
data_sample_key = client.register(SampleData()) data_sample_key = client.register(SampleData())
@ -125,12 +125,10 @@ class TestClient(unittest.TestCase):
# Waiting for server to receive the datas # Waiting for server to receive the datas
rep_test_key = client2._rep_store[data_sample_key].uuid rep_test_key = client2._rep_store[data_sample_key].uuid
client.disconnect() client.disconnect()
client2.disconnect() client2.disconnect()
server.stop() server.stop()
self.assertEqual(rep_test_key, data_sample_key) self.assertEqual(rep_test_key, data_sample_key)
def test_client_data_intergity(self): def test_client_data_intergity(self):
@ -157,12 +155,10 @@ class TestClient(unittest.TestCase):
client2._rep_store[data_sample_key].load(target=test_map_result) client2._rep_store[data_sample_key].load(target=test_map_result)
client.disconnect() client.disconnect()
client2.disconnect() client2.disconnect()
server.stop() server.stop()
self.assertEqual(test_map_result.map["toto"], test_map["toto"]) self.assertEqual(test_map_result.map["toto"], test_map["toto"])
def test_client_unregister_key(self): def test_client_unregister_key(self):
@ -244,6 +240,7 @@ class TestStressClient(unittest.TestCase):
self.assertLess(total_time, 1) self.assertLess(total_time, 1)
def suite(): def suite():
suite = unittest.TestSuite() suite = unittest.TestSuite()
@ -261,7 +258,7 @@ def suite():
return suite return suite
if __name__ == '__main__': if __name__ == '__main__':
runner = unittest.TextTestRunner(verbosity=2) runner = unittest.TextTestRunner(verbosity=2)
runner.run(suite()) runner.run(suite())