From 3d559569b5b15f2e47893baba1ffe910b2f50c34 Mon Sep 17 00:00:00 2001 From: mixmix Date: Fri, 21 Mar 2025 11:31:37 +1300 Subject: [PATCH 1/2] refactor sidechain writing function --- simplechat/simplepub/replica.py | 35 ++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/simplechat/simplepub/replica.py b/simplechat/simplepub/replica.py index 24b28b7..0fff8f5 100755 --- a/simplechat/simplepub/replica.py +++ b/simplechat/simplepub/replica.py @@ -253,32 +253,40 @@ def write48(self, content, sign_fct): # publish event, returns seq or None def write(self, content, sign_fct): # publish event, returns seq or None assert os.path.getsize(self.log_fname) == self.state['max_pos'] - chunks = [] - seq = self.state['max_seq'] + 1 sz = bipf.varint_encode_to_bytes(len(content)) - payload = sz + content[:28-len(sz)] - if len(payload) != 28: - payload += bytes(28 - len(payload)) - content = content[28-len(sz):] - i = len(content) % 100 + + root_chunk = content[:28-len(sz)] # the first bytes fit on root + + # prep the chunks + content = content[28-len(sz):] # skip first bytes that were in root + i = len(content) % 100 # 240 => 240 % 100 = 40... we want to pad content to be 300 add (100-40) = 60 if i > 0: - content += bytes(100-i) - ptr = bytes(20) + pad = bytes(100-i) # pad the remaining chunk content to be multiples of 100 + content += pad + + ## we build the hash-chain of chunks from the end to the start + ptr = bytes(20) # end of chain is signalled by all-zero "hash" + chunks = [] while len(content) > 0: buf = content[-100:] + ptr chunks.append(buf) ptr = hashlib.sha256(buf).digest()[:20] content = content[:-100] chunks.reverse() - payload += ptr - nam = PFX + self.fid + seq.to_bytes(4,'big') + \ - self.state['prev'] + + # prep the root message + seq = self.state['max_seq'] + 1 + nam = PFX + self.fid + seq.to_bytes(4,'big') + self.state['prev'] dmx = hashlib.sha256(nam).digest()[:7] + payload = sz + root_chunk + ptr msg = dmx + bytes([PKTTYPE_chain20]) + payload wire = msg + sign_fct(nam + msg) + assert len(wire) == 120 assert self.verify_fct(self.fid, wire[56:], nam + wire[:56]) + chunks.insert(0, wire) + log_entry = b''.join(chunks) log_entry += self.state['max_pos'].to_bytes(4, 'big') with open(self.log_fname, 'ab') as f: @@ -286,9 +294,8 @@ def write(self, content, sign_fct): # publish event, returns seq or None self._persist_frontier(seq, self.state['max_pos'] + len(log_entry), hashlib.sha256(nam + wire).digest()[:20]) return seq - + # ---------------------------------------------------------------------- - pass # eof From 6dda49d8d3e1b83a2a7bab77b36c96155d9b0149 Mon Sep 17 00:00:00 2001 From: mixmix Date: Fri, 21 Mar 2025 11:49:39 +1300 Subject: [PATCH 2/2] tweaks --- simplechat/simplepub/replica.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/simplechat/simplepub/replica.py b/simplechat/simplepub/replica.py index 0fff8f5..d1899df 100755 --- a/simplechat/simplepub/replica.py +++ b/simplechat/simplepub/replica.py @@ -255,44 +255,50 @@ def write(self, content, sign_fct): # publish event, returns seq or None assert os.path.getsize(self.log_fname) == self.state['max_pos'] sz = bipf.varint_encode_to_bytes(len(content)) - root_chunk = content[:28-len(sz)] # the first bytes fit on root + root_content_length = 28 - len(sz) + root_content = content[:root_content_length] # the first bytes fit on root # prep the chunks - content = content[28-len(sz):] # skip first bytes that were in root - i = len(content) % 100 # 240 => 240 % 100 = 40... we want to pad content to be 300 add (100-40) = 60 + content = content[root_content_length:] # skip over root_content + chunk_content_length = 100 + # pad so all chunk content is size 100 + i = len(content) % chunk_content_length if i > 0: - pad = bytes(100-i) # pad the remaining chunk content to be multiples of 100 + pad = bytes(chunk_content_length - i) content += pad ## we build the hash-chain of chunks from the end to the start ptr = bytes(20) # end of chain is signalled by all-zero "hash" chunks = [] while len(content) > 0: - buf = content[-100:] + ptr + buf = content[-chunk_content_length:] + ptr chunks.append(buf) ptr = hashlib.sha256(buf).digest()[:20] - content = content[:-100] + content = content[:-chunk_content_length] chunks.reverse() # prep the root message seq = self.state['max_seq'] + 1 nam = PFX + self.fid + seq.to_bytes(4,'big') + self.state['prev'] dmx = hashlib.sha256(nam).digest()[:7] - payload = sz + root_chunk + ptr + payload = sz + root_content + ptr msg = dmx + bytes([PKTTYPE_chain20]) + payload wire = msg + sign_fct(nam + msg) assert len(wire) == 120 assert self.verify_fct(self.fid, wire[56:], nam + wire[:56]) + # write to log chunks.insert(0, wire) - log_entry = b''.join(chunks) log_entry += self.state['max_pos'].to_bytes(4, 'big') with open(self.log_fname, 'ab') as f: f.write(log_entry) + + # ?? self._persist_frontier(seq, self.state['max_pos'] + len(log_entry), hashlib.sha256(nam + wire).digest()[:20]) + return seq # ----------------------------------------------------------------------