From f4dc5c453b90dc387c455ccb6c56ff45ad090d75 Mon Sep 17 00:00:00 2001 From: Henrik Nordstrom Date: Fri, 11 Apr 2014 22:40:27 +0200 Subject: [PATCH 1/9] Add block height in version message. --- protocoin/serializers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/protocoin/serializers.py b/protocoin/serializers.py index 44848ad..bcfffe3 100644 --- a/protocoin/serializers.py +++ b/protocoin/serializers.py @@ -179,6 +179,7 @@ def __init__(self): self.addr_from = IPv4Address() self.nonce = random.randint(0, 2**32-1) self.user_agent = "/Perone:0.0.1/" + self.last_block = 0 class VersionSerializer(Serializer): """The version command serializer.""" @@ -190,6 +191,7 @@ class VersionSerializer(Serializer): addr_from = fields.NestedField(IPv4AddressSerializer) nonce = fields.UInt64LEField() user_agent = fields.VariableStringField() + last_block = fields.Int32LEField() class VerAck(object): """The version acknowledge (verack) command.""" From 2731e2e79cd1e14dd0f6c4df316d9ab471b2e4a5 Mon Sep 17 00:00:00 2001 From: Henrik Nordstrom Date: Fri, 11 Apr 2014 23:23:06 +0200 Subject: [PATCH 2/9] Use official name for start_height field in version message --- protocoin/serializers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/protocoin/serializers.py b/protocoin/serializers.py index bcfffe3..6fdc3e7 100644 --- a/protocoin/serializers.py +++ b/protocoin/serializers.py @@ -179,7 +179,7 @@ def __init__(self): self.addr_from = IPv4Address() self.nonce = random.randint(0, 2**32-1) self.user_agent = "/Perone:0.0.1/" - self.last_block = 0 + self.start_height = 0 class VersionSerializer(Serializer): """The version command serializer.""" @@ -191,7 +191,7 @@ class VersionSerializer(Serializer): addr_from = fields.NestedField(IPv4AddressSerializer) nonce = fields.UInt64LEField() user_agent = fields.VariableStringField() - last_block = fields.Int32LEField() + start_height = fields.Int32LEField() class VerAck(object): """The version acknowledge (verack) command.""" @@ -534,4 +534,4 @@ class GetAddrSerializer(Serializer): "headers": HeaderVectorSerializer, "mempool": MemPoolSerializer, "getaddr": GetAddrSerializer, -} \ No newline at end of file +} From e88b20e59d44b5e88c4c0c091a4a2c7c5ab2cb16 Mon Sep 17 00:00:00 2001 From: Henrik Nordstrom Date: Sat, 12 Apr 2014 07:33:53 +0200 Subject: [PATCH 3/9] StringIO needs to be imported differently in Python3 --- protocoin/clients.py | 5 ++++- protocoin/fields.py | 5 ++++- protocoin/serializers.py | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/protocoin/clients.py b/protocoin/clients.py index 8f10e67..61f4836 100644 --- a/protocoin/clients.py +++ b/protocoin/clients.py @@ -1,4 +1,7 @@ -from cStringIO import StringIO +try: + from io import StringIO +except ImportError: + from cStringIO import StringIO from .serializers import * from .exceptions import NodeDisconnectException import os diff --git a/protocoin/fields.py b/protocoin/fields.py index 14547cf..912890f 100644 --- a/protocoin/fields.py +++ b/protocoin/fields.py @@ -1,6 +1,9 @@ from .exceptions import NodeDisconnectException -from cStringIO import StringIO +try: + from io import StringIO +except ImportError: + from cStringIO import StringIO import struct import time import random diff --git a/protocoin/serializers.py b/protocoin/serializers.py index 6fdc3e7..0621640 100644 --- a/protocoin/serializers.py +++ b/protocoin/serializers.py @@ -2,7 +2,10 @@ import random import hashlib import struct -from cStringIO import StringIO +try: + from io import StringIO +except ImportError: + from cStringIO import StringIO from collections import OrderedDict from . import fields From 016260e3ebdd8d5f8a58f416e100aed8ab83d5fb Mon Sep 17 00:00:00 2001 From: Henrik Nordstrom Date: Sat, 12 Apr 2014 07:34:15 +0200 Subject: [PATCH 4/9] There is no itemlist() in python3, only item() --- protocoin/serializers.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/protocoin/serializers.py b/protocoin/serializers.py index 0621640..412d330 100644 --- a/protocoin/serializers.py +++ b/protocoin/serializers.py @@ -24,7 +24,7 @@ def get_fields(meta, bases, attrs, field_class): """This method will construct an ordered dict with all the fields present on the serializer classes.""" fields = [(field_name, attrs.pop(field_name)) - for field_name, field_value in list(attrs.iteritems()) + for field_name, field_value in list(attrs.items()) if isinstance(field_value, field_class)] for base_cls in bases[::-1]: @@ -34,9 +34,8 @@ def get_fields(meta, bases, attrs, field_class): fields.sort(key=lambda it: it[1].count) return OrderedDict(fields) -class SerializerABC(object): +class SerializerABC(object, metaclass = SerializerMeta): """The serializer abstract base class.""" - __metaclass__ = SerializerMeta class Serializer(SerializerABC): """The main serializer class, inherit from this class to @@ -54,7 +53,7 @@ def serialize(self, obj, fields=None): :param obj: The object to serializer. """ bin_data = StringIO() - for field_name, field_obj in self._fields.iteritems(): + for field_name, field_obj in self._fields.items(): if fields: if field_name not in fields: continue @@ -71,7 +70,7 @@ def deserialize(self, stream): :param stream: A file-like object (StringIO, file, socket, etc.) """ model = self.model_class() - for field_name, field_obj in self._fields.iteritems(): + for field_name, field_obj in self._fields.items(): value = field_obj.deserialize(stream) setattr(model, field_name, value) return model @@ -86,7 +85,7 @@ def __init__(self, coin="bitcoin"): def _magic_to_text(self): """Converts the magic value to a textual representation.""" - for k, v in fields.MAGIC_VALUES.iteritems(): + for k, v in fields.MAGIC_VALUES.items(): if v == self.magic: return k return "Unknown Magic" @@ -130,7 +129,7 @@ def _services_to_text(self): """Converts the services field into a textual representation.""" services = [] - for service_name, flag_mask in fields.SERVICES.iteritems(): + for service_name, flag_mask in fields.SERVICES.items(): if self.services & flag_mask: services.append(service_name) return services @@ -244,7 +243,7 @@ def __init__(self): def type_to_text(self): """Converts the inventory type to text representation.""" - for k, v in fields.INVENTORY_TYPE.iteritems(): + for k, v in fields.INVENTORY_TYPE.items(): if v == self.inv_type: return k return "Unknown Type" From b71bd9ab61463aa1ae47c9ba5ac7185d04fae7f7 Mon Sep 17 00:00:00 2001 From: Henrik Nordstrom Date: Sat, 12 Apr 2014 13:10:51 +0200 Subject: [PATCH 5/9] Convert to python3 --- protocoin/clients.py | 14 +++++++------- protocoin/fields.py | 31 +++++++++++++++---------------- protocoin/serializers.py | 12 ++++++------ 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/protocoin/clients.py b/protocoin/clients.py index 61f4836..7961d6f 100644 --- a/protocoin/clients.py +++ b/protocoin/clients.py @@ -1,7 +1,7 @@ try: - from io import StringIO + from io import BytesIO except ImportError: - from cStringIO import StringIO + from cBytesIO import BytesIO from .serializers import * from .exceptions import NodeDisconnectException import os @@ -18,7 +18,7 @@ class BitcoinBasicClient(object): def __init__(self, socket): self.socket = socket - self.buffer = StringIO() + self.buffer = BytesIO() def close_stream(self): """This method will close the socket stream.""" @@ -55,7 +55,7 @@ def receive_message(self): return # Go to the beginning of the buffer - self.buffer.reset() + self.buffer.seek(0) message_model = None message_header_serial = MessageHeaderSerializer() @@ -69,7 +69,7 @@ def receive_message(self): return payload = self.buffer.read(message_header.length) - self.buffer = StringIO() + self.buffer = BytesIO() self.handle_message_header(message_header, payload) payload_checksum = \ @@ -81,7 +81,7 @@ def receive_message(self): if message_header.command in MESSAGE_MAPPING: deserializer = MESSAGE_MAPPING[message_header.command]() - message_model = deserializer.deserialize(StringIO(payload)) + message_model = deserializer.deserialize(BytesIO(payload)) return (message_header, message_model) @@ -92,7 +92,7 @@ def send_message(self, message): :param message: The message object to send """ - bin_data = StringIO() + bin_data = BytesIO() message_header = MessageHeader(self.coin) message_header_serial = MessageHeaderSerializer() diff --git a/protocoin/fields.py b/protocoin/fields.py index 912890f..30f8eb1 100644 --- a/protocoin/fields.py +++ b/protocoin/fields.py @@ -1,9 +1,9 @@ from .exceptions import NodeDisconnectException try: - from io import StringIO + from io import BytesIO except ImportError: - from cStringIO import StringIO + from cBytesIO import BytesIO import struct import time import random @@ -160,12 +160,11 @@ def parse(self, value): def deserialize(self, stream): data = stream.read(self.length) - return data.split("\x00", 1)[0] + return data[:(data+b'\x00').index(b'\x00')].decode("utf-8") def serialize(self): - bin_data = StringIO() - bin_data.write(self.value[:self.length]) - bin_data.write("\x00" * (12 - len(self.value))) + bin_data = BytesIO() + bin_data.write(struct.pack("12s", self.value.encode("utf-8"))) return bin_data.getvalue() class NestedField(Field): @@ -214,7 +213,7 @@ def parse(self, value): self.value = value def serialize(self): - bin_data = StringIO() + bin_data = BytesIO() self.var_int.parse(len(self)) bin_data.write(self.var_int.serialize()) serializer = self.serializer_class() @@ -239,7 +238,7 @@ def __len__(self): class IPv4AddressField(Field): """An IPv4 address field without timestamp and reserved IPv6 space.""" - reserved = "\x00"*10 + "\xff"*2 + reserved = b"\x00"*10 + b"\xff"*2 def parse(self, value): self.value = value @@ -250,7 +249,7 @@ def deserialize(self, stream): return socket.inet_ntoa(addr) def serialize(self): - bin_data = StringIO() + bin_data = BytesIO() bin_data.write(self.reserved) bin_data.write(socket.inet_aton(self.value)) return bin_data.getvalue() @@ -276,12 +275,12 @@ def deserialize(self, stream): def serialize(self): if self.value < 0xFD: - return chr(self.value) + return struct.pack("B", self.value) if self.value <= 0xFFFF: - return chr(0xFD) + struct.pack(" Date: Sat, 12 Apr 2014 13:12:22 +0200 Subject: [PATCH 6/9] Drop python2 compatibility glue. Other parts are python3 dependent already. --- protocoin/clients.py | 5 +---- protocoin/fields.py | 5 +---- protocoin/serializers.py | 5 +---- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/protocoin/clients.py b/protocoin/clients.py index 7961d6f..3e345b1 100644 --- a/protocoin/clients.py +++ b/protocoin/clients.py @@ -1,7 +1,4 @@ -try: - from io import BytesIO -except ImportError: - from cBytesIO import BytesIO +from io import BytesIO from .serializers import * from .exceptions import NodeDisconnectException import os diff --git a/protocoin/fields.py b/protocoin/fields.py index 30f8eb1..e68988b 100644 --- a/protocoin/fields.py +++ b/protocoin/fields.py @@ -1,9 +1,6 @@ from .exceptions import NodeDisconnectException -try: - from io import BytesIO -except ImportError: - from cBytesIO import BytesIO +from io import BytesIO import struct import time import random diff --git a/protocoin/serializers.py b/protocoin/serializers.py index 7e219d7..a61bfa3 100644 --- a/protocoin/serializers.py +++ b/protocoin/serializers.py @@ -2,10 +2,7 @@ import random import hashlib import struct -try: - from io import BytesIO -except ImportError: - from cBytesIO import BytesIO +from io import BytesIO from collections import OrderedDict from . import fields From bf75aad29006bbd0797785c822837e22f9832ce9 Mon Sep 17 00:00:00 2001 From: Henrik Nordstrom Date: Wed, 16 Apr 2014 22:41:06 +0200 Subject: [PATCH 7/9] Preserve any unprocessed data when there is more than one message to process --- protocoin/clients.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protocoin/clients.py b/protocoin/clients.py index 3e345b1..4657a4a 100644 --- a/protocoin/clients.py +++ b/protocoin/clients.py @@ -66,7 +66,7 @@ def receive_message(self): return payload = self.buffer.read(message_header.length) - self.buffer = BytesIO() + self.buffer = BytesIO(self.buffer.read()) self.handle_message_header(message_header, payload) payload_checksum = \ From 95b41895605c1dc3f66b1d7bfe9fa55ce6617841 Mon Sep 17 00:00:00 2001 From: Henrik Nordstrom Date: Wed, 16 Apr 2014 22:41:42 +0200 Subject: [PATCH 8/9] python3: xrange() is now range() --- protocoin/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protocoin/fields.py b/protocoin/fields.py index e68988b..c8b0a0c 100644 --- a/protocoin/fields.py +++ b/protocoin/fields.py @@ -222,7 +222,7 @@ def deserialize(self, stream): count = self.var_int.deserialize(stream) items = [] serializer = self.serializer_class() - for i in xrange(count): + for i in range(count): data = serializer.deserialize(stream) items.append(data) return items From bd0a5290d8d4c8d260ccb3737b1fe50f17ae272d Mon Sep 17 00:00:00 2001 From: Henrik Nordstrom Date: Thu, 17 Apr 2014 00:10:10 +0200 Subject: [PATCH 9/9] Process received data until there is no more complete message to parse --- protocoin/clients.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/protocoin/clients.py b/protocoin/clients.py index 4657a4a..2af7591 100644 --- a/protocoin/clients.py +++ b/protocoin/clients.py @@ -66,7 +66,9 @@ def receive_message(self): return payload = self.buffer.read(message_header.length) - self.buffer = BytesIO(self.buffer.read()) + buffer = BytesIO() + buffer.write(self.buffer.read()) + self.buffer = buffer self.handle_message_header(message_header, payload) payload_checksum = \ @@ -118,21 +120,22 @@ def loop(self): raise NodeDisconnectException("Node disconnected.") self.buffer.write(data) - data = self.receive_message() - - # Check if the message is still incomplete to parse - if data is None: - continue - - # Check for the header and message - message_header, message = data - if not message: - continue - - handle_func_name = "handle_" + message_header.command - handle_func = getattr(self, handle_func_name, None) - if handle_func: - handle_func(message_header, message) + while True: + data = self.receive_message() + + # Check if the message is still incomplete to parse + if data is None: + break + + # Check for the header and message + message_header, message = data + if not message: + continue + + handle_func_name = "handle_" + message_header.command + handle_func = getattr(self, handle_func_name, None) + if handle_func: + handle_func(message_header, message) class BitcoinClient(BitcoinBasicClient): """This class implements all the protocol rules needed