diff --git a/README.md b/README.md index eb18d99..376cc66 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # Realtime Collaborative Framework experiments +## Idea + +![basic](img/basic.png) + ## Dependencies | Dependency | Version | Needed | diff --git a/img/basic.png b/img/basic.png new file mode 100644 index 0000000..48395ef Binary files /dev/null and b/img/basic.png differ diff --git a/libs/umsgpack.py b/libs/umsgpack.py new file mode 100644 index 0000000..139ab98 --- /dev/null +++ b/libs/umsgpack.py @@ -0,0 +1,1134 @@ +# u-msgpack-python v2.5.0 - v at sergeev.io +# https://github.com/vsergeev/u-msgpack-python +# +# u-msgpack-python is a lightweight MessagePack serializer and deserializer +# module, compatible with both Python 2 and 3, as well CPython and PyPy +# implementations of Python. u-msgpack-python is fully compliant with the +# latest MessagePack specification.com/msgpack/msgpack/blob/master/spec.md). In +# particular, it supports the new binary, UTF-8 string, and application ext +# types. +# +# MIT License +# +# Copyright (c) 2013-2016 vsergeev / Ivan (Vanya) A. Sergeev +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +""" +u-msgpack-python v2.5.0 - v at sergeev.io +https://github.com/vsergeev/u-msgpack-python + +u-msgpack-python is a lightweight MessagePack serializer and deserializer +module, compatible with both Python 2 and 3, as well CPython and PyPy +implementations of Python. u-msgpack-python is fully compliant with the +latest MessagePack specification.com/msgpack/msgpack/blob/master/spec.md). In +particular, it supports the new binary, UTF-8 string, and application ext +types. + +License: MIT +""" +import struct +import collections +import datetime +import sys +import io + +__version__ = "2.5.0" +"Module version string" + +version = (2, 5, 0) +"Module version tuple" + + +############################################################################## +# Ext Class +############################################################################## + +# Extension type for application-defined types and data +class Ext: + """ + The Ext class facilitates creating a serializable extension object to store + an application-defined type and data byte array. + """ + + def __init__(self, type, data): + """ + Construct a new Ext object. + + Args: + type: application-defined type integer + data: application-defined data byte array + + Example: + >>> foo = umsgpack.Ext(0x05, b"\x01\x02\x03") + >>> umsgpack.packb({u"special stuff": foo, u"awesome": True}) + '\x82\xa7awesome\xc3\xadspecial stuff\xc7\x03\x05\x01\x02\x03' + >>> bar = umsgpack.unpackb(_) + >>> print(bar["special stuff"]) + Ext Object (Type: 0x05, Data: 01 02 03) + >>> + """ + # Check type is type int + if not isinstance(type, int): + raise TypeError("ext type is not type integer") + # Check data is type bytes + elif sys.version_info[0] == 3 and not isinstance(data, bytes): + raise TypeError("ext data is not type \'bytes\'") + elif sys.version_info[0] == 2 and not isinstance(data, str): + raise TypeError("ext data is not type \'str\'") + self.type = type + self.data = data + + def __eq__(self, other): + """ + Compare this Ext object with another for equality. + """ + return (isinstance(other, self.__class__) and + self.type == other.type and + self.data == other.data) + + def __ne__(self, other): + """ + Compare this Ext object with another for inequality. + """ + return not self.__eq__(other) + + def __str__(self): + """ + String representation of this Ext object. + """ + s = "Ext Object (Type: 0x%02x, Data: " % self.type + s += " ".join(["0x%02x" % ord(self.data[i:i + 1]) + for i in xrange(min(len(self.data), 8))]) + if len(self.data) > 8: + s += " ..." + s += ")" + return s + + def __hash__(self): + """ + Provide a hash of this Ext object. + """ + return hash((self.type, self.data)) + + +class InvalidString(bytes): + """Subclass of bytes to hold invalid UTF-8 strings.""" + pass + +############################################################################## +# Exceptions +############################################################################## + + +# Base Exception classes +class PackException(Exception): + "Base class for exceptions encountered during packing." + pass + + +class UnpackException(Exception): + "Base class for exceptions encountered during unpacking." + pass + + +# Packing error +class UnsupportedTypeException(PackException): + "Object type not supported for packing." + pass + + +# Unpacking error +class InsufficientDataException(UnpackException): + "Insufficient data to unpack the serialized object." + pass + + +class InvalidStringException(UnpackException): + "Invalid UTF-8 string encountered during unpacking." + pass + + +class UnsupportedTimestampException(UnpackException): + "Unsupported timestamp format encountered during unpacking." + pass + + +class ReservedCodeException(UnpackException): + "Reserved code encountered during unpacking." + pass + + +class UnhashableKeyException(UnpackException): + """ + Unhashable key encountered during map unpacking. + The serialized map cannot be deserialized into a Python dictionary. + """ + pass + + +class DuplicateKeyException(UnpackException): + "Duplicate key encountered during map unpacking." + pass + + +# Backwards compatibility +KeyNotPrimitiveException = UnhashableKeyException +KeyDuplicateException = DuplicateKeyException + +############################################################################# +# Exported Functions and Glob +############################################################################# + +# Exported functions and variables, set up in __init() +pack = None +packb = None +unpack = None +unpackb = None +dump = None +dumps = None +load = None +loads = None + +compatibility = False +""" +Compatibility mode boolean. + +When compatibility mode is enabled, u-msgpack-python will serialize both +unicode strings and bytes into the old "raw" msgpack type, and deserialize the +"raw" msgpack type into bytes. This provides backwards compatibility with the +old MessagePack specification. + +Example: +>>> umsgpack.compatibility = True +>>> +>>> umsgpack.packb([u"some string", b"some bytes"]) +b'\x92\xabsome string\xaasome bytes' +>>> umsgpack.unpackb(_) +[b'some string', b'some bytes'] +>>> +""" + +############################################################################## +# Packing +############################################################################## + +# You may notice struct.pack("B", obj) instead of the simpler chr(obj) in the +# code below. This is to allow for seamless Python 2 and 3 compatibility, as +# chr(obj) has a str return type instead of bytes in Python 3, and +# struct.pack(...) has the right return type in both versions. + + +def _pack_integer(obj, fp, options): + if obj < 0: + if obj >= -32: + fp.write(struct.pack("b", obj)) + elif obj >= -2**(8 - 1): + fp.write(b"\xd0" + struct.pack("b", obj)) + elif obj >= -2**(16 - 1): + fp.write(b"\xd1" + struct.pack(">h", obj)) + elif obj >= -2**(32 - 1): + fp.write(b"\xd2" + struct.pack(">i", obj)) + elif obj >= -2**(64 - 1): + fp.write(b"\xd3" + struct.pack(">q", obj)) + else: + raise UnsupportedTypeException("huge signed int") + else: + if obj <= 127: + fp.write(struct.pack("B", obj)) + elif obj <= 2**8 - 1: + fp.write(b"\xcc" + struct.pack("B", obj)) + elif obj <= 2**16 - 1: + fp.write(b"\xcd" + struct.pack(">H", obj)) + elif obj <= 2**32 - 1: + fp.write(b"\xce" + struct.pack(">I", obj)) + elif obj <= 2**64 - 1: + fp.write(b"\xcf" + struct.pack(">Q", obj)) + else: + raise UnsupportedTypeException("huge unsigned int") + + +def _pack_nil(obj, fp, options): + fp.write(b"\xc0") + + +def _pack_boolean(obj, fp, options): + fp.write(b"\xc3" if obj else b"\xc2") + + +def _pack_float(obj, fp, options): + float_precision = options.get('force_float_precision', _float_precision) + + if float_precision == "double": + fp.write(b"\xcb" + struct.pack(">d", obj)) + elif float_precision == "single": + fp.write(b"\xca" + struct.pack(">f", obj)) + else: + raise ValueError("invalid float precision") + + +def _pack_string(obj, fp, options): + obj = obj.encode('utf-8') + if len(obj) <= 31: + fp.write(struct.pack("B", 0xa0 | len(obj)) + obj) + elif len(obj) <= 2**8 - 1: + fp.write(b"\xd9" + struct.pack("B", len(obj)) + obj) + elif len(obj) <= 2**16 - 1: + fp.write(b"\xda" + struct.pack(">H", len(obj)) + obj) + elif len(obj) <= 2**32 - 1: + fp.write(b"\xdb" + struct.pack(">I", len(obj)) + obj) + else: + raise UnsupportedTypeException("huge string") + + +def _pack_binary(obj, fp, options): + if len(obj) <= 2**8 - 1: + fp.write(b"\xc4" + struct.pack("B", len(obj)) + obj) + elif len(obj) <= 2**16 - 1: + fp.write(b"\xc5" + struct.pack(">H", len(obj)) + obj) + elif len(obj) <= 2**32 - 1: + fp.write(b"\xc6" + struct.pack(">I", len(obj)) + obj) + else: + raise UnsupportedTypeException("huge binary string") + + +def _pack_oldspec_raw(obj, fp, options): + if len(obj) <= 31: + fp.write(struct.pack("B", 0xa0 | len(obj)) + obj) + elif len(obj) <= 2**16 - 1: + fp.write(b"\xda" + struct.pack(">H", len(obj)) + obj) + elif len(obj) <= 2**32 - 1: + fp.write(b"\xdb" + struct.pack(">I", len(obj)) + obj) + else: + raise UnsupportedTypeException("huge raw string") + + +def _pack_ext(obj, fp, options): + if len(obj.data) == 1: + fp.write(b"\xd4" + struct.pack("B", obj.type & 0xff) + obj.data) + elif len(obj.data) == 2: + fp.write(b"\xd5" + struct.pack("B", obj.type & 0xff) + obj.data) + elif len(obj.data) == 4: + fp.write(b"\xd6" + struct.pack("B", obj.type & 0xff) + obj.data) + elif len(obj.data) == 8: + fp.write(b"\xd7" + struct.pack("B", obj.type & 0xff) + obj.data) + elif len(obj.data) == 16: + fp.write(b"\xd8" + struct.pack("B", obj.type & 0xff) + obj.data) + elif len(obj.data) <= 2**8 - 1: + fp.write(b"\xc7" + + struct.pack("BB", len(obj.data), obj.type & 0xff) + obj.data) + elif len(obj.data) <= 2**16 - 1: + fp.write(b"\xc8" + + struct.pack(">HB", len(obj.data), obj.type & 0xff) + obj.data) + elif len(obj.data) <= 2**32 - 1: + fp.write(b"\xc9" + + struct.pack(">IB", len(obj.data), obj.type & 0xff) + obj.data) + else: + raise UnsupportedTypeException("huge ext data") + + +def _pack_ext_timestamp(obj, fp, options): + delta = obj - _epoch + seconds = delta.seconds + delta.days * 86400 + microseconds = delta.microseconds + + if microseconds == 0 and 0 <= seconds <= 2**32 - 1: + # 32-bit timestamp + fp.write(b"\xd6\xff" + + struct.pack(">I", seconds)) + elif 0 <= seconds <= 2**34 - 1: + # 64-bit timestamp + value = ((microseconds * 1000) << 34) | seconds + fp.write(b"\xd7\xff" + + struct.pack(">Q", value)) + elif -2**63 <= abs(seconds) <= 2**63 - 1: + # 96-bit timestamp + fp.write(b"\xc7\x0c\xff" + + struct.pack(">I", microseconds * 1000) + + struct.pack(">q", seconds)) + else: + raise UnsupportedTypeException("huge timestamp") + + +def _pack_array(obj, fp, options): + if len(obj) <= 15: + fp.write(struct.pack("B", 0x90 | len(obj))) + elif len(obj) <= 2**16 - 1: + fp.write(b"\xdc" + struct.pack(">H", len(obj))) + elif len(obj) <= 2**32 - 1: + fp.write(b"\xdd" + struct.pack(">I", len(obj))) + else: + raise UnsupportedTypeException("huge array") + + for e in obj: + pack(e, fp, **options) + + +def _pack_map(obj, fp, options): + if len(obj) <= 15: + fp.write(struct.pack("B", 0x80 | len(obj))) + elif len(obj) <= 2**16 - 1: + fp.write(b"\xde" + struct.pack(">H", len(obj))) + elif len(obj) <= 2**32 - 1: + fp.write(b"\xdf" + struct.pack(">I", len(obj))) + else: + raise UnsupportedTypeException("huge array") + + for k, v in obj.items(): + pack(k, fp, **options) + pack(v, fp, **options) + +######################################## + + +# Pack for Python 2, with 'unicode' type, 'str' type, and 'long' type +def _pack2(obj, fp, **options): + """ + Serialize a Python object into MessagePack bytes. + + Args: + obj: a Python object + fp: a .write()-supporting file-like object + + Kwargs: + ext_handlers (dict): dictionary of Ext handlers, mapping a custom type + to a callable that packs an instance of the type + into an Ext object + force_float_precision (str): "single" to force packing floats as + IEEE-754 single-precision floats, + "double" to force packing floats as + IEEE-754 double-precision floats. + + Returns: + None. + + Raises: + UnsupportedType(PackException): + Object type not supported for packing. + + Example: + >>> f = open('test.bin', 'wb') + >>> umsgpack.pack({u"compact": True, u"schema": 0}, f) + >>> + """ + global compatibility + + ext_handlers = options.get("ext_handlers") + + if obj is None: + _pack_nil(obj, fp, options) + elif ext_handlers and obj.__class__ in ext_handlers: + _pack_ext(ext_handlers[obj.__class__](obj), fp, options) + elif isinstance(obj, bool): + _pack_boolean(obj, fp, options) + elif isinstance(obj, int) or isinstance(obj, long): + _pack_integer(obj, fp, options) + elif isinstance(obj, float): + _pack_float(obj, fp, options) + elif compatibility and isinstance(obj, unicode): + _pack_oldspec_raw(bytes(obj), fp, options) + elif compatibility and isinstance(obj, bytes): + _pack_oldspec_raw(obj, fp, options) + elif isinstance(obj, unicode): + _pack_string(obj, fp, options) + elif isinstance(obj, str): + _pack_binary(obj, fp, options) + elif isinstance(obj, list) or isinstance(obj, tuple): + _pack_array(obj, fp, options) + elif isinstance(obj, dict): + _pack_map(obj, fp, options) + elif isinstance(obj, datetime.datetime): + _pack_ext_timestamp(obj, fp, options) + elif isinstance(obj, Ext): + _pack_ext(obj, fp, options) + elif ext_handlers: + # Linear search for superclass + t = next((t for t in ext_handlers.keys() if isinstance(obj, t)), None) + if t: + _pack_ext(ext_handlers[t](obj), fp, options) + else: + raise UnsupportedTypeException( + "unsupported type: %s" % str(type(obj))) + else: + raise UnsupportedTypeException("unsupported type: %s" % str(type(obj))) + + +# Pack for Python 3, with unicode 'str' type, 'bytes' type, and no 'long' type +def _pack3(obj, fp, **options): + """ + Serialize a Python object into MessagePack bytes. + + Args: + obj: a Python object + fp: a .write()-supporting file-like object + + Kwargs: + ext_handlers (dict): dictionary of Ext handlers, mapping a custom type + to a callable that packs an instance of the type + into an Ext object + force_float_precision (str): "single" to force packing floats as + IEEE-754 single-precision floats, + "double" to force packing floats as + IEEE-754 double-precision floats. + + Returns: + None. + + Raises: + UnsupportedType(PackException): + Object type not supported for packing. + + Example: + >>> f = open('test.bin', 'wb') + >>> umsgpack.pack({u"compact": True, u"schema": 0}, f) + >>> + """ + global compatibility + + ext_handlers = options.get("ext_handlers") + + if obj is None: + _pack_nil(obj, fp, options) + elif ext_handlers and obj.__class__ in ext_handlers: + _pack_ext(ext_handlers[obj.__class__](obj), fp, options) + elif isinstance(obj, bool): + _pack_boolean(obj, fp, options) + elif isinstance(obj, int): + _pack_integer(obj, fp, options) + elif isinstance(obj, float): + _pack_float(obj, fp, options) + elif compatibility and isinstance(obj, str): + _pack_oldspec_raw(obj.encode('utf-8'), fp, options) + elif compatibility and isinstance(obj, bytes): + _pack_oldspec_raw(obj, fp, options) + elif isinstance(obj, str): + _pack_string(obj, fp, options) + elif isinstance(obj, bytes): + _pack_binary(obj, fp, options) + elif isinstance(obj, list) or isinstance(obj, tuple): + _pack_array(obj, fp, options) + elif isinstance(obj, dict): + _pack_map(obj, fp, options) + elif isinstance(obj, datetime.datetime): + _pack_ext_timestamp(obj, fp, options) + elif isinstance(obj, Ext): + _pack_ext(obj, fp, options) + elif ext_handlers: + # Linear search for superclass + t = next((t for t in ext_handlers.keys() if isinstance(obj, t)), None) + if t: + _pack_ext(ext_handlers[t](obj), fp, options) + else: + raise UnsupportedTypeException( + "unsupported type: %s" % str(type(obj))) + else: + raise UnsupportedTypeException( + "unsupported type: %s" % str(type(obj))) + + +def _packb2(obj, **options): + """ + Serialize a Python object into MessagePack bytes. + + Args: + obj: a Python object + + Kwargs: + ext_handlers (dict): dictionary of Ext handlers, mapping a custom type + to a callable that packs an instance of the type + into an Ext object + force_float_precision (str): "single" to force packing floats as + IEEE-754 single-precision floats, + "double" to force packing floats as + IEEE-754 double-precision floats. + + Returns: + A 'str' containing serialized MessagePack bytes. + + Raises: + UnsupportedType(PackException): + Object type not supported for packing. + + Example: + >>> umsgpack.packb({u"compact": True, u"schema": 0}) + '\x82\xa7compact\xc3\xa6schema\x00' + >>> + """ + fp = io.BytesIO() + _pack2(obj, fp, **options) + return fp.getvalue() + + +def _packb3(obj, **options): + """ + Serialize a Python object into MessagePack bytes. + + Args: + obj: a Python object + + Kwargs: + ext_handlers (dict): dictionary of Ext handlers, mapping a custom type + to a callable that packs an instance of the type + into an Ext object + force_float_precision (str): "single" to force packing floats as + IEEE-754 single-precision floats, + "double" to force packing floats as + IEEE-754 double-precision floats. + + Returns: + A 'bytes' containing serialized MessagePack bytes. + + Raises: + UnsupportedType(PackException): + Object type not supported for packing. + + Example: + >>> umsgpack.packb({u"compact": True, u"schema": 0}) + b'\x82\xa7compact\xc3\xa6schema\x00' + >>> + """ + fp = io.BytesIO() + _pack3(obj, fp, **options) + return fp.getvalue() + +############################################################################# +# Unpacking +############################################################################# + + +def _read_except(fp, n): + data = fp.read(n) + if len(data) < n: + raise InsufficientDataException() + return data + + +def _unpack_integer(code, fp, options): + if (ord(code) & 0xe0) == 0xe0: + return struct.unpack("b", code)[0] + elif code == b'\xd0': + return struct.unpack("b", _read_except(fp, 1))[0] + elif code == b'\xd1': + return struct.unpack(">h", _read_except(fp, 2))[0] + elif code == b'\xd2': + return struct.unpack(">i", _read_except(fp, 4))[0] + elif code == b'\xd3': + return struct.unpack(">q", _read_except(fp, 8))[0] + elif (ord(code) & 0x80) == 0x00: + return struct.unpack("B", code)[0] + elif code == b'\xcc': + return struct.unpack("B", _read_except(fp, 1))[0] + elif code == b'\xcd': + return struct.unpack(">H", _read_except(fp, 2))[0] + elif code == b'\xce': + return struct.unpack(">I", _read_except(fp, 4))[0] + elif code == b'\xcf': + return struct.unpack(">Q", _read_except(fp, 8))[0] + raise Exception("logic error, not int: 0x%02x" % ord(code)) + + +def _unpack_reserved(code, fp, options): + if code == b'\xc1': + raise ReservedCodeException( + "encountered reserved code: 0x%02x" % ord(code)) + raise Exception( + "logic error, not reserved code: 0x%02x" % ord(code)) + + +def _unpack_nil(code, fp, options): + if code == b'\xc0': + return None + raise Exception("logic error, not nil: 0x%02x" % ord(code)) + + +def _unpack_boolean(code, fp, options): + if code == b'\xc2': + return False + elif code == b'\xc3': + return True + raise Exception("logic error, not boolean: 0x%02x" % ord(code)) + + +def _unpack_float(code, fp, options): + if code == b'\xca': + return struct.unpack(">f", _read_except(fp, 4))[0] + elif code == b'\xcb': + return struct.unpack(">d", _read_except(fp, 8))[0] + raise Exception("logic error, not float: 0x%02x" % ord(code)) + + +def _unpack_string(code, fp, options): + if (ord(code) & 0xe0) == 0xa0: + length = ord(code) & ~0xe0 + elif code == b'\xd9': + length = struct.unpack("B", _read_except(fp, 1))[0] + elif code == b'\xda': + length = struct.unpack(">H", _read_except(fp, 2))[0] + elif code == b'\xdb': + length = struct.unpack(">I", _read_except(fp, 4))[0] + else: + raise Exception("logic error, not string: 0x%02x" % ord(code)) + + # Always return raw bytes in compatibility mode + global compatibility + if compatibility: + return _read_except(fp, length) + + data = _read_except(fp, length) + try: + return bytes.decode(data, 'utf-8') + except UnicodeDecodeError: + if options.get("allow_invalid_utf8"): + return InvalidString(data) + raise InvalidStringException("unpacked string is invalid utf-8") + + +def _unpack_binary(code, fp, options): + if code == b'\xc4': + length = struct.unpack("B", _read_except(fp, 1))[0] + elif code == b'\xc5': + length = struct.unpack(">H", _read_except(fp, 2))[0] + elif code == b'\xc6': + length = struct.unpack(">I", _read_except(fp, 4))[0] + else: + raise Exception("logic error, not binary: 0x%02x" % ord(code)) + + return _read_except(fp, length) + + +def _unpack_ext(code, fp, options): + if code == b'\xd4': + length = 1 + elif code == b'\xd5': + length = 2 + elif code == b'\xd6': + length = 4 + elif code == b'\xd7': + length = 8 + elif code == b'\xd8': + length = 16 + elif code == b'\xc7': + length = struct.unpack("B", _read_except(fp, 1))[0] + elif code == b'\xc8': + length = struct.unpack(">H", _read_except(fp, 2))[0] + elif code == b'\xc9': + length = struct.unpack(">I", _read_except(fp, 4))[0] + else: + raise Exception("logic error, not ext: 0x%02x" % ord(code)) + + ext_type = struct.unpack("b", _read_except(fp, 1))[0] + ext_data = _read_except(fp, length) + + # Create extension object + ext = Ext(ext_type, ext_data) + + # Unpack with ext handler, if we have one + ext_handlers = options.get("ext_handlers") + if ext_handlers and ext.type in ext_handlers: + return ext_handlers[ext.type](ext) + + # Timestamp extension + if ext.type == -1: + return _unpack_ext_timestamp(ext, options) + + return ext + + +def _unpack_ext_timestamp(ext, options): + if len(ext.data) == 4: + # 32-bit timestamp + seconds = struct.unpack(">I", ext.data)[0] + microseconds = 0 + elif len(ext.data) == 8: + # 64-bit timestamp + value = struct.unpack(">Q", ext.data)[0] + seconds = value & 0x3ffffffff + microseconds = (value >> 34) // 1000 + elif len(ext.data) == 12: + # 96-bit timestamp + seconds = struct.unpack(">q", ext.data[4:12])[0] + microseconds = struct.unpack(">I", ext.data[0:4])[0] // 1000 + else: + raise UnsupportedTimestampException( + "unsupported timestamp with data length %d" % len(ext.data)) + + return _epoch + datetime.timedelta(seconds=seconds, + microseconds=microseconds) + + +def _unpack_array(code, fp, options): + if (ord(code) & 0xf0) == 0x90: + length = (ord(code) & ~0xf0) + elif code == b'\xdc': + length = struct.unpack(">H", _read_except(fp, 2))[0] + elif code == b'\xdd': + length = struct.unpack(">I", _read_except(fp, 4))[0] + else: + raise Exception("logic error, not array: 0x%02x" % ord(code)) + + return [_unpack(fp, options) for i in xrange(length)] + + +def _deep_list_to_tuple(obj): + if isinstance(obj, list): + return tuple([_deep_list_to_tuple(e) for e in obj]) + return obj + + +def _unpack_map(code, fp, options): + if (ord(code) & 0xf0) == 0x80: + length = (ord(code) & ~0xf0) + elif code == b'\xde': + length = struct.unpack(">H", _read_except(fp, 2))[0] + elif code == b'\xdf': + length = struct.unpack(">I", _read_except(fp, 4))[0] + else: + raise Exception("logic error, not map: 0x%02x" % ord(code)) + + d = {} if not options.get('use_ordered_dict') \ + else collections.OrderedDict() + for _ in xrange(length): + # Unpack key + k = _unpack(fp, options) + + if isinstance(k, list): + # Attempt to convert list into a hashable tuple + k = _deep_list_to_tuple(k) + elif not isinstance(k, collections.Hashable): + raise UnhashableKeyException( + "encountered unhashable key: %s, %s" % (str(k), str(type(k)))) + elif k in d: + raise DuplicateKeyException( + "encountered duplicate key: %s, %s" % (str(k), str(type(k)))) + + # Unpack value + v = _unpack(fp, options) + + try: + d[k] = v + except TypeError: + raise UnhashableKeyException( + "encountered unhashable key: %s" % str(k)) + return d + + +def _unpack(fp, options): + code = _read_except(fp, 1) + return _unpack_dispatch_table[code](code, fp, options) + +######################################## + + +def _unpack2(fp, **options): + """ + Deserialize MessagePack bytes into a Python object. + + Args: + fp: a .read()-supporting file-like object + + Kwargs: + ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext + type to a callable that unpacks an instance of + Ext into an object + use_ordered_dict (bool): unpack maps into OrderedDict, instead of + unordered dict (default False) + allow_invalid_utf8 (bool): unpack invalid strings into instances of + InvalidString, for access to the bytes + (default False) + + Returns: + A Python object. + + Raises: + InsufficientDataException(UnpackException): + Insufficient data to unpack the serialized object. + InvalidStringException(UnpackException): + Invalid UTF-8 string encountered during unpacking. + UnsupportedTimestampException(UnpackException): + Unsupported timestamp format encountered during unpacking. + ReservedCodeException(UnpackException): + Reserved code encountered during unpacking. + UnhashableKeyException(UnpackException): + Unhashable key encountered during map unpacking. + The serialized map cannot be deserialized into a Python dictionary. + DuplicateKeyException(UnpackException): + Duplicate key encountered during map unpacking. + + Example: + >>> f = open('test.bin', 'rb') + >>> umsgpack.unpackb(f) + {u'compact': True, u'schema': 0} + >>> + """ + return _unpack(fp, options) + + +def _unpack3(fp, **options): + """ + Deserialize MessagePack bytes into a Python object. + + Args: + fp: a .read()-supporting file-like object + + Kwargs: + ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext + type to a callable that unpacks an instance of + Ext into an object + use_ordered_dict (bool): unpack maps into OrderedDict, instead of + unordered dict (default False) + allow_invalid_utf8 (bool): unpack invalid strings into instances of + InvalidString, for access to the bytes + (default False) + + Returns: + A Python object. + + Raises: + InsufficientDataException(UnpackException): + Insufficient data to unpack the serialized object. + InvalidStringException(UnpackException): + Invalid UTF-8 string encountered during unpacking. + UnsupportedTimestampException(UnpackException): + Unsupported timestamp format encountered during unpacking. + ReservedCodeException(UnpackException): + Reserved code encountered during unpacking. + UnhashableKeyException(UnpackException): + Unhashable key encountered during map unpacking. + The serialized map cannot be deserialized into a Python dictionary. + DuplicateKeyException(UnpackException): + Duplicate key encountered during map unpacking. + + Example: + >>> f = open('test.bin', 'rb') + >>> umsgpack.unpackb(f) + {'compact': True, 'schema': 0} + >>> + """ + return _unpack(fp, options) + + +# For Python 2, expects a str object +def _unpackb2(s, **options): + """ + Deserialize MessagePack bytes into a Python object. + + Args: + s: a 'str' or 'bytearray' containing serialized MessagePack bytes + + Kwargs: + ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext + type to a callable that unpacks an instance of + Ext into an object + use_ordered_dict (bool): unpack maps into OrderedDict, instead of + unordered dict (default False) + allow_invalid_utf8 (bool): unpack invalid strings into instances of + InvalidString, for access to the bytes + (default False) + + Returns: + A Python object. + + Raises: + TypeError: + Packed data type is neither 'str' nor 'bytearray'. + InsufficientDataException(UnpackException): + Insufficient data to unpack the serialized object. + InvalidStringException(UnpackException): + Invalid UTF-8 string encountered during unpacking. + UnsupportedTimestampException(UnpackException): + Unsupported timestamp format encountered during unpacking. + ReservedCodeException(UnpackException): + Reserved code encountered during unpacking. + UnhashableKeyException(UnpackException): + Unhashable key encountered during map unpacking. + The serialized map cannot be deserialized into a Python dictionary. + DuplicateKeyException(UnpackException): + Duplicate key encountered during map unpacking. + + Example: + >>> umsgpack.unpackb(b'\x82\xa7compact\xc3\xa6schema\x00') + {u'compact': True, u'schema': 0} + >>> + """ + if not isinstance(s, (str, bytearray)): + raise TypeError("packed data must be type 'str' or 'bytearray'") + return _unpack(io.BytesIO(s), options) + + +# For Python 3, expects a bytes object +def _unpackb3(s, **options): + """ + Deserialize MessagePack bytes into a Python object. + + Args: + s: a 'bytes' or 'bytearray' containing serialized MessagePack bytes + + Kwargs: + ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext + type to a callable that unpacks an instance of + Ext into an object + use_ordered_dict (bool): unpack maps into OrderedDict, instead of + unordered dict (default False) + allow_invalid_utf8 (bool): unpack invalid strings into instances of + InvalidString, for access to the bytes + (default False) + + Returns: + A Python object. + + Raises: + TypeError: + Packed data type is neither 'bytes' nor 'bytearray'. + InsufficientDataException(UnpackException): + Insufficient data to unpack the serialized object. + InvalidStringException(UnpackException): + Invalid UTF-8 string encountered during unpacking. + UnsupportedTimestampException(UnpackException): + Unsupported timestamp format encountered during unpacking. + ReservedCodeException(UnpackException): + Reserved code encountered during unpacking. + UnhashableKeyException(UnpackException): + Unhashable key encountered during map unpacking. + The serialized map cannot be deserialized into a Python dictionary. + DuplicateKeyException(UnpackException): + Duplicate key encountered during map unpacking. + + Example: + >>> umsgpack.unpackb(b'\x82\xa7compact\xc3\xa6schema\x00') + {'compact': True, 'schema': 0} + >>> + """ + if not isinstance(s, (bytes, bytearray)): + raise TypeError("packed data must be type 'bytes' or 'bytearray'") + return _unpack(io.BytesIO(s), options) + +############################################################################# +# Module Initialization +############################################################################# + + +def __init(): + global pack + global packb + global unpack + global unpackb + global dump + global dumps + global load + global loads + global compatibility + global _epoch + global _utc_tzinfo + global _float_precision + global _unpack_dispatch_table + global xrange + + # Compatibility mode for handling strings/bytes with the old specification + compatibility = False + + if sys.version_info[0] == 3: + _utc_tzinfo = datetime.timezone.utc + else: + _utc_tzinfo = None + + # Calculate epoch datetime + _epoch = datetime.datetime(1970, 1, 1, tzinfo=_utc_tzinfo) + + # Auto-detect system float precision + if sys.float_info.mant_dig == 53: + _float_precision = "double" + else: + _float_precision = "single" + + # Map packb and unpackb to the appropriate version + if sys.version_info[0] == 3: + pack = _pack3 + packb = _packb3 + dump = _pack3 + dumps = _packb3 + unpack = _unpack3 + unpackb = _unpackb3 + load = _unpack3 + loads = _unpackb3 + xrange = range + else: + pack = _pack2 + packb = _packb2 + dump = _pack2 + dumps = _packb2 + unpack = _unpack2 + unpackb = _unpackb2 + load = _unpack2 + loads = _unpackb2 + + # Build a dispatch table for fast lookup of unpacking function + + _unpack_dispatch_table = {} + # Fix uint + for code in range(0, 0x7f + 1): + _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer + # Fix map + for code in range(0x80, 0x8f + 1): + _unpack_dispatch_table[struct.pack("B", code)] = _unpack_map + # Fix array + for code in range(0x90, 0x9f + 1): + _unpack_dispatch_table[struct.pack("B", code)] = _unpack_array + # Fix str + for code in range(0xa0, 0xbf + 1): + _unpack_dispatch_table[struct.pack("B", code)] = _unpack_string + # Nil + _unpack_dispatch_table[b'\xc0'] = _unpack_nil + # Reserved + _unpack_dispatch_table[b'\xc1'] = _unpack_reserved + # Boolean + _unpack_dispatch_table[b'\xc2'] = _unpack_boolean + _unpack_dispatch_table[b'\xc3'] = _unpack_boolean + # Bin + for code in range(0xc4, 0xc6 + 1): + _unpack_dispatch_table[struct.pack("B", code)] = _unpack_binary + # Ext + for code in range(0xc7, 0xc9 + 1): + _unpack_dispatch_table[struct.pack("B", code)] = _unpack_ext + # Float + _unpack_dispatch_table[b'\xca'] = _unpack_float + _unpack_dispatch_table[b'\xcb'] = _unpack_float + # Uint + for code in range(0xcc, 0xcf + 1): + _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer + # Int + for code in range(0xd0, 0xd3 + 1): + _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer + # Fixext + for code in range(0xd4, 0xd8 + 1): + _unpack_dispatch_table[struct.pack("B", code)] = _unpack_ext + # String + for code in range(0xd9, 0xdb + 1): + _unpack_dispatch_table[struct.pack("B", code)] = _unpack_string + # Array + _unpack_dispatch_table[b'\xdc'] = _unpack_array + _unpack_dispatch_table[b'\xdd'] = _unpack_array + # Map + _unpack_dispatch_table[b'\xde'] = _unpack_map + _unpack_dispatch_table[b'\xdf'] = _unpack_map + # Negative fixint + for code in range(0xe0, 0xff + 1): + _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer + + +__init() diff --git a/main.py b/main.py index b0905e0..255d60a 100644 --- a/main.py +++ b/main.py @@ -5,50 +5,58 @@ import zmq import sys import argparse import time +from zmq.asyncio import Context, ZMQEventLoop import asyncio +Url = 'tcp://127.0.0.1:5555' +Ctx = Context() + + +@asyncio.coroutine +def server(): + print("Getting ready for hello world client. Ctrl-C to exit.\n") + socket = Ctx.socket(zmq.REP) + socket.bind(Url) + while True: + # Wait for next request from client + message = yield from socket.recv() + print("Received request: {}".format(message)) + # Do some "work" + yield from asyncio.sleep(1) + # Send reply back to client + message = message.decode('utf-8') + message = '{}, world'.format(message) + message = message.encode('utf-8') + print("Sending reply: {}".format(message)) + yield from socket.send(message) + # TODO: Implement a manager class for each aspect (ex: Network_Manager) # TODO: Is it right to implement server-client as ESC ?... -@asyncio.coroutine def main(): - # Argument parsing - parser = argparse.ArgumentParser( - description='Launch an instance of collaboration system') - parser.add_argument('-r', choices=list(net_components.Role), - type=net_components.Role, help='role for the instance ') - - args = parser.parse_args() - - instance_role = args.r - instance_context = zmq.Context() - - print("Starting a {} instance \n".format(instance_role)) - - # Create a World instance to hold everything: - world = esper.World() - - # Instantiate a Processor (or more), and add them to the world: - network_system = net_systems.NetworkSystem() - world.add_processor(network_system) - - # Instanciate a session entity - session = world.create_entity() - - world.add_component( - session, net_components.NetworkInterface(context=instance_context)) - world.add_component( - session, net_components.User(role=instance_role)) - - - # A dummy main loop: + args = sys.argv[1:] + if len(args) != 0: + sys.exit(__doc__) try: - while True: - # Call world.process() to run all Processors. - world.process() - time.sleep(1) + loop = asyncio.get_event_loop() + loop.run_until_complete(server()) except KeyboardInterrupt: - return + print('\nFinished (interrupted)') + sys.exit(0) + + # Socket to talk to server + print("Connecting to hello world server...") + socket = context.socket(zmq.REQ) + socket.connect("tcp://localhost:5555") + + # Do 10 requests, waiting each time for a response + for request in range(10): + print("Sending request %s ..." % request) + socket.send(b"Hello") + + # Get the reply. + message = socket.recv() + print("Received reply %s [ %s ]" % (request, message)) if __name__ == '__main__': diff --git a/main_old.py b/main_old.py new file mode 100644 index 0000000..6566bec --- /dev/null +++ b/main_old.py @@ -0,0 +1,52 @@ +import net_systems +import net_components +from libs.esper import esper +import zmq +import sys +import argparse +import time + +# TODO: Implement a manager class for each aspect (ex: Network_Manager) +# TODO: Is it right to implement server-client as ESC ?... +def main(): + # Argument parsing + parser = argparse.ArgumentParser( + description='Launch an instance of collaboration system') + parser.add_argument('-r', choices=list(net_components.Role), + type=net_components.Role, help='role for the instance ') + + args = parser.parse_args() + + instance_role = args.r + instance_context = zmq.Context() + + print("Starting a {} instance \n".format(instance_role)) + + # Create a World instance to hold everything: + world = esper.World() + + # Instantiate a Processor (or more), and add them to the world: + network_system = net_systems.NetworkSystem() + world.add_processor(network_system) + + # Instanciate a session entity + session = world.create_entity() + + world.add_component( + session, net_components.NetworkInterface(context=instance_context)) + world.add_component( + session, net_components.User(role=instance_role)) + + + # A dummy main loop: + try: + while True: + # Call world.process() to run all Processors. + world.process() + time.sleep(1) + except KeyboardInterrupt: + return + + +if __name__ == '__main__': + main() diff --git a/msgpacktest.py b/msgpacktest.py new file mode 100644 index 0000000..6ac8f44 --- /dev/null +++ b/msgpacktest.py @@ -0,0 +1,23 @@ +import sys +import os + + +thirdPartyDir = "C:\\Users\\slumber\\repos\\phd\src\\2019_rcf\\libs" + + + +if thirdPartyDir in sys.path: + print('Third party module already added') +else: + print('Adding local modules dir to the path') + sys.path.insert(0, thirdPartyDir) + + +import umsgpack +import bpy +import esper + + +#c = umsgpack.packb("test") +#print(umsgpack.unpackb(c)) + diff --git a/net_components.py b/net_components.py index d9f21e9..aa15720 100644 --- a/net_components.py +++ b/net_components.py @@ -42,3 +42,4 @@ class Function: def __init__(self, function=None): self.function = function +