Backups/multi-user
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: ground work on multithreading

Browse Source
This commit is contained in:
Swann Martinez
2019-05-02 14:46:31 +02:00
parent 64a5d8e56c
commit 2e803080ed
3 changed files with 201 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

205
client.py
View File

@ -30,13 +30,20 @@ except:
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO) logging.basicConfig(level=logging.INFO)
CONNECT_TIMEOUT = 2 CONNECT_TIMEOUT = 2
WAITING_TIME = 0.001 WAITING_TIME = 0.001
SERVER_MAX = 1 SERVER_MAX = 1
stop = False stop = False
class State(Enum):
INITIAL = 1
SYNCING = 2
ACTIVE = 3
def zpipe(ctx): def zpipe(ctx):
"""build inproc pipe for talking to threads """build inproc pipe for talking to threads
@ -54,33 +61,28 @@ def zpipe(ctx):
return a, b return a, b
class State(Enum):
INITIAL = 1
SYNCING = 2
ACTIVE = 3
class RCFClient(object): class RCFClient(object):
ctx = None ctx = None
pipe = None pipe = None
net_agent = None net_agent = None
store = None
def __init__(self): def __init__(self):
self.ctx = zmq.Context() self.ctx = zmq.Context()
self.pipe, peer = zpipe(self.ctx) self.pipe, peer = zpipe(self.ctx)
self.serial_pipe, serial_peer = zpipe(self.ctx) self.store = {}
serial_in, net_in = zpipe(self.ctx)
self.tasks = queue.Queue() self.serial_product = queue.Queue()
self.serial_feed = queue.Queue()
# Database and connexion agent # Database and connexion agent
self.net_agent = threading.Thread( self.net_agent = threading.Thread(
target=rcf_client_agent, args=(self.ctx, peer, self.tasks), name="net-agent") target=rcf_client_agent, args=(self.ctx, self.store, peer, self.serial_product), name="net-agent")
self.net_agent.daemon = True self.net_agent.daemon = True
self.net_agent.start() self.net_agent.start()
# Local data translation agent # Local data translation agent
self.serial_agent = threading.Thread( self.serial_agent = threading.Thread(
target=serialization_agent, args=(self.ctx, serial_peer, self.tasks), name="serial-agent") target=serialization_agent, args=(self.serial_product, self.serial_feed), name="serial-agent")
self.serial_agent.daemon = True self.serial_agent.daemon = True
self.serial_agent.start() self.serial_agent.start()
@ -109,29 +111,32 @@ class RCFClient(object):
Sends [SET][key][value] to the agent Sends [SET][key][value] to the agent
""" """
if value:
pass
self.pipe.send_multipart( self.pipe.send_multipart(
[b"SET", umsgpack.packb(key), (umsgpack.packb(value) if value else umsgpack.packb('None')),umsgpack.packb(override)]) [b"SET", umsgpack.packb(key), (umsgpack.packb(value) if value else umsgpack.packb('None')),umsgpack.packb(override)])
else:
self.serial_feed.put(key)
# self.serial_pipe.send_multipart(
# [b"DUMP", umsgpack.packb(key)])
# self.pipe.send_multipart(
# [b"SET", umsgpack.packb(key), (umsgpack.packb(value) if value else umsgpack.packb('None')),umsgpack.packb(override)])
def add(self, key, value=None): def add(self, key, value=None):
"""Set new value in distributed hash table """Set new value in distributed hash table
Sends [SET][key][value] to the agent Sends [SET][key][value] to the agent
""" """
self.pipe.send_multipart( self.serial_feed.put(key)
[b"ADD", umsgpack.packb(key), (umsgpack.packb(value) if value else umsgpack.packb('None'))])
def get(self, key): # self.pipe.send_multipart(
"""Lookup value in distributed hash table # [b"ADD", umsgpack.packb(key), (umsgpack.packb(value) if value else umsgpack.packb('None'))])
Sends [GET][key] to the agent and waits for a value response
If there is no clone available, will eventually return None.
"""
self.pipe.send_multipart([b"GET", umsgpack.packb(key)])
try: def is_busy(self):
reply = self.pipe.recv_multipart() if self.serial_feed.qsize() == 0 and self.serial_product.qsize() == 0:
except KeyboardInterrupt: return False
return
else: else:
return umsgpack.unpackb(reply[0]) return True
def exit(self): def exit(self):
if self.net_agent.is_alive(): if self.net_agent.is_alive():
@ -143,16 +148,55 @@ class RCFClient(object):
global stop global stop
stop = True stop = True
# READ-ONLY FUNCTIONS
def get(self, key):
"""Lookup value in distributed hash table
Sends [GET][key] to the agent and waits for a value response
If there is no clone available, will eventually return None.
"""
value = []
for k in self.store.keys():
if key in k:
value.append([k, self.store.get(k).body])
return value
# if not self.is_busy():
# self.pipe.send_multipart([b"GET", umsgpack.packb(key)])
# try:
# reply = self.pipe.recv_multipart()
# except KeyboardInterrupt:
# return
# else:
# return umsgpack.unpackb(reply[0])
# else:
# return None
def list(self): def list(self):
self.pipe.send_multipart([b"LIST"]) dump_list = []
try: for k,v in self.store.items():
reply = self.pipe.recv_multipart() if 'Client' in k:
except KeyboardInterrupt: dump_list.append([k,v.id.decode()])
return
else: else:
return umsgpack.unpackb(reply[0]) try:
dump_list.append([k,v.body['id']])
except:
pass
return dump_list
# if not self.is_busy():
# self.pipe.send_multipart([b"LIST"])
# try:
# reply = self.pipe.recv_multipart()
# except KeyboardInterrupt:
# return
# else:
# return umsgpack.unpackb(reply[0])
# else:
# return None
def state(self): def state(self):
if not self.is_busy():
self.pipe.send_multipart([b"STATE"]) self.pipe.send_multipart([b"STATE"])
try: try:
reply = self.pipe.recv_multipart() reply = self.pipe.recv_multipart()
@ -160,6 +204,8 @@ class RCFClient(object):
return return
else: else:
return umsgpack.unpackb(reply[0]) return umsgpack.unpackb(reply[0])
else:
return None
class RCFServer(object): class RCFServer(object):
@ -193,10 +239,10 @@ class RCFClientAgent(object):
serial = None serial = None
serialisation_agent = None serialisation_agent = None
def __init__(self, ctx, pipe): def __init__(self, ctx, store, pipe):
self.ctx = ctx self.ctx = ctx
self.pipe = pipe self.pipe = pipe
self.property_map = {} self.property_map = store
self.id = b"test" self.id = b"test"
self.state = State.INITIAL self.state = State.INITIAL
self.admin = False self.admin = False
@ -205,8 +251,6 @@ class RCFClientAgent(object):
self.publisher.setsockopt(zmq.IDENTITY, self.id) self.publisher.setsockopt(zmq.IDENTITY, self.id)
self.publisher.setsockopt(zmq.SNDHWM, 60) self.publisher.setsockopt(zmq.SNDHWM, 60)
self.publisher.linger = 0 self.publisher.linger = 0
self.serial, peer = zpipe(self.ctx)
self.updates = queue.Queue()
# self.serial_agent = threading.Thread( # self.serial_agent = threading.Thread(
# target=serialization_agent, args=(self.ctx, peer), name="serial-agent") # target=serialization_agent, args=(self.ctx, peer), name="serial-agent")
# self.serial_agent.daemon = True # self.serial_agent.daemon = True
@ -341,10 +385,10 @@ class RCFClientAgent(object):
elif command == b"STATE": elif command == b"STATE":
self.pipe.send(umsgpack.packb(self.state.value)) self.pipe.send(umsgpack.packb(self.state.value))
def rcf_client_agent(ctx, pipe, queue): def rcf_client_agent(ctx,store, pipe, queue):
agent = RCFClientAgent(ctx, pipe) agent = RCFClientAgent(ctx,store, pipe)
server = None server = None
update_queue = queue net_feed = queue
global stop global stop
while True: while True:
if stop: if stop:
@ -399,52 +443,35 @@ def rcf_client_agent(ctx, pipe, queue):
agent.state = State.ACTIVE agent.state = State.ACTIVE
else: else:
helpers.load(rcfmsg.key, rcfmsg.body) helpers.load(rcfmsg.key, rcfmsg.body)
rcfmsg.store(agent.property_map) rcfmsg.store(agent.property_map)
logger.info("snapshot from {} stored".format(rcfmsg.id)) logger.info("snapshot from {} stored".format(rcfmsg.id))
elif agent.state == State.ACTIVE: elif agent.state == State.ACTIVE:
# IN # IN
if rcfmsg.id != agent.id: if rcfmsg.id != agent.id:
# update_queue.put((rcfmsg.key,rcfmsg.body))
# try:
# logger.info(rcfmsg.body['id'])
# except:
# pass
with lock: with lock:
helpers.load(rcfmsg.key, rcfmsg.body) helpers.load(rcfmsg.key, rcfmsg.body)
rcfmsg.store(agent.property_map) rcfmsg.store(agent.property_map)
# elif rcfmsg.id == agent.property_map[rcfmsg.key].id:
# with lock:
# helpers.load(rcfmsg.key, rcfmsg.body)
# logger.info("load")
# agent.serial.send_multipart([b"LOAD", umsgpack.packb(rcfmsg.key), umsgpack.packb(rcfmsg.body)])
# reply = agent.serial.recv_multipart()
# if reply == b"DONE":
# rcfmsg.store(agent.property_map)
# action = "update" if rcfmsg.body else "delete"
# logging.info("{}: received from {}:{},{} {}".format(rcfmsg.key,
# server.address, rcfmsg.id, server.port, action))
else: else:
logger.debug("{} nothing to do".format(agent.id)) logger.debug("{} nothing to do".format(agent.id))
# LOCAL SYNC # Serialisation thread input
# if not update_queue.empty(): if not net_feed.empty():
# key = update_queue.get() key, value = net_feed.get()
# value = helpers.dump(key) if value:
# value['id'] = agent.id.decode() # Stamp with id
# if value: value['id'] = agent.id.decode()
# rcfmsg = message.RCFMessage(
# key=key, id=agent.id, body=value)
# rcfmsg.store(agent.property_map) # Format massage
# rcfmsg.send(agent.publisher) rcfmsg = message.RCFMessage(
# else: key=key, id=agent.id, body=value)
# logger.error("Fail to dump ")
rcfmsg.store(agent.property_map)
rcfmsg.send(agent.publisher)
else:
logger.error("Fail to dump ")
@ -459,13 +486,15 @@ class SerializationAgent(object):
ctx = None ctx = None
pipe = None pipe = None
def __init__(self, ctx, pipe): def __init__(self, ctx, pipe, product, feed):
self.ctx = ctx self.ctx = ctx
self.pipe = pipe self.pipe = pipe
self.product = product
self.feed = feed
logger.info("serialisation service launched") logger.info("serialisation service launched")
def control_message(self): def control_message(self):
msg = self.pipe.recv_multipart() msg = self.pipe.recv_multipart(zmq.NOBLOCK)
command = msg.pop(0) command = msg.pop(0)
if command == b"DUMP": if command == b"DUMP":
@ -473,7 +502,8 @@ class SerializationAgent(object):
value = helpers.dump(key) value = helpers.dump(key)
self.pipe.send_multipart(umsgpack.packb(value)) self.product.put((key,value))
# self.pipe.send_multipart(umsgpack.packb(value))
elif command == b"LOAD": elif command == b"LOAD":
@ -485,25 +515,32 @@ class SerializationAgent(object):
self.pipe.send_multipart([b"DONE"]) self.pipe.send_multipart([b"DONE"])
def serialization_agent(ctx, pipe, tasks): def serialization_agent(product, feed ):
agent = SerializationAgent(ctx, pipe) # agent = SerializationAgent(ctx, pipe, feed)
global stop global stop
while True: while True:
if stop: if stop:
break break
poller = zmq.Poller()
poller.register(agent.pipe, zmq.POLLIN)
try: key = feed.get()
items = dict(poller.poll(1)) value = helpers.dump(key)
except:
raise
break
if agent.pipe in items: if value:
agent.control_message() product.put((key,value))
# poller = zmq.Poller()
# poller.register(agent.pipe, zmq.POLLIN)
# try:
# items = dict(poller.poll(1))
# except:
# raise
# break
# if agent.pipe in items:
# agent.control_message()
# TODO : Fill tasks # TODO : Fill tasks

3
draw.py
View File

@ -120,6 +120,7 @@ class HUD(object):
def draw_selected_object(self): def draw_selected_object(self):
clients = self.client.get("Client") clients = self.client.get("Client")
if clients:
for client in clients: for client in clients:
name = client[0].split('/')[1] name = client[0].split('/')[1]
local_username = bpy.context.scene.session_settings.username local_username = bpy.context.scene.session_settings.username
@ -133,6 +134,7 @@ class HUD(object):
(0, 4), (1, 5), (2, 6), (3, 7) (0, 4), (1, 5), (2, 6), (3, 7)
) )
if select_ob in bpy.data.objects.keys(): if select_ob in bpy.data.objects.keys():
ob = bpy.data.objects[select_ob] ob = bpy.data.objects[select_ob]
else: else:
@ -164,6 +166,7 @@ class HUD(object):
def draw_clients(self): def draw_clients(self):
clients = self.client.get("Client") clients = self.client.get("Client")
if clients:
for client in clients: for client in clients:
name = client[0].split('/')[1] name = client[0].split('/')[1]
local_username = bpy.context.scene.session_settings.username local_username = bpy.context.scene.session_settings.username

26
operators.py
View File

@ -106,7 +106,7 @@ def update_selected_object(context):
if len(selected_objects) > 0: if len(selected_objects) > 0:
for obj in selected_objects: for obj in selected_objects:
if obj not in client_data[0][1]['active_objects']: # if obj not in client_data[0][1]['active_objects']:
client_data[0][1]['active_objects'] = selected_objects client_data[0][1]['active_objects'] = selected_objects
client_instance.set(client_key,client_data[0][1]) client_instance.set(client_key,client_data[0][1])
@ -166,7 +166,7 @@ def init_datablocks():
for item in getattr(bpy.data, helpers.CORRESPONDANCE[datatype]): for item in getattr(bpy.data, helpers.CORRESPONDANCE[datatype]):
item.id= bpy.context.scene.session_settings.username item.id= bpy.context.scene.session_settings.username
key = "{}/{}".format(datatype, item.name) key = "{}/{}".format(datatype, item.name)
client_instance.add(key) client_instance.set(key)
def default_tick(): def default_tick():
@ -212,16 +212,16 @@ def register_ticks():
bpy.app.timers.register(draw_tick) bpy.app.timers.register(draw_tick)
bpy.app.timers.register(sync) bpy.app.timers.register(sync)
bpy.app.timers.register(default_tick) bpy.app.timers.register(default_tick)
pass
def unregister_ticks(): def unregister_ticks():
# REGISTER Updaters # REGISTER Updaters
global drawer global drawer
drawer.unregister_handlers() drawer.unregister_handlers()
bpy.app.timers.unregister(draw_tick) bpy.app.timers.unregister(draw_tick)
bpy.app.timers.unregister(sync)
bpy.app.timers.unregister(default_tick) bpy.app.timers.unregister(default_tick)
pass
# OPERATORS # OPERATORS
class session_join(bpy.types.Operator): class session_join(bpy.types.Operator):
@ -277,8 +277,14 @@ class session_refresh(bpy.types.Operator):
def execute(self, context): def execute(self, context):
global client_instance, client_keys,client_state global client_instance, client_keys,client_state
client_keys = client_instance.list() keys = client_instance.list()
client_state = client_instance.state()
if keys:
client_keys= keys
state = client_instance.state()
if state:
client_state = state
return {"FINISHED"} return {"FINISHED"}
@ -366,10 +372,10 @@ class session_create(bpy.types.Operator):
bpy.ops.session.join() bpy.ops.session.join()
# if net_settings.init_scene: if net_settings.init_scene:
# init_datablocks() init_datablocks()
client_instance.init() # client_instance.init()
net_settings.is_admin = True net_settings.is_admin = True
return {"FINISHED"} return {"FINISHED"}