Skip to content

Commit

Permalink
Fix writing on large files. Improve speed. Improve usb handling to ma…
Browse files Browse the repository at this point in the history 
…tch newer libusb. Bug fixing.
  • Loading branch information
@bkerler
bkerler committed Jun 18, 2021
1 parent 8d8eadb commit 2886810
Show file tree
Hide file tree
Showing 4 changed files with 99 additions and 96 deletions.
130 changes: 57 additions & 73 deletions Library/firehose.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,9 +201,9 @@ def decoder(self, data):

def xmlsend(self, data, skipresponse=False):
if isinstance(data, bytes) or isinstance(data, bytearray):
self.cdc.write(data, self.cfg.MaxXMLSizeInBytes)
self.cdc.write(data[:self.cfg.MaxXMLSizeInBytes])
else:
self.cdc.write(bytes(data, 'utf-8'), self.cfg.MaxXMLSizeInBytes)
self.cdc.write(bytes(data, 'utf-8')[:self.cfg.MaxXMLSizeInBytes])
# time.sleep(0.01)
rdata = bytearray()
counter = 0
Expand Down Expand Up @@ -416,13 +416,7 @@ def cmd_program(self, physical_partition_number, start_sector, filename, display
if rsp[0]:
old = 0
while bytestowrite > 0:
# todo: Why on earth doesn't MaxPayloadSizeToTargetInBytes work here ?
# Forced to use self.cfg.SECTOR_SIZE_IN_BYTES ... thus slow as hell
"""
wlen = min(bytestowrite, self.cfg.MaxPayloadSizeToTargetInBytes // self.cfg.SECTOR_SIZE_IN_BYTES \
* self.cfg.SECTOR_SIZE_IN_BYTES)
"""
wlen = min(bytestowrite, self.cfg.SECTOR_SIZE_IN_BYTES)
wlen = min(bytestowrite, self.cfg.MaxPayloadSizeToTargetInBytes)

if sparseformat:
wdata = sparse.read(wlen)
Expand All @@ -434,9 +428,8 @@ def cmd_program(self, physical_partition_number, start_sector, filename, display
filllen = (wlen // self.cfg.SECTOR_SIZE_IN_BYTES * self.cfg.SECTOR_SIZE_IN_BYTES) + \
self.cfg.SECTOR_SIZE_IN_BYTES
wdata += b"\x00" * (filllen - wlen)
wlen = len(wdata)

self.cdc.write(wdata, wlen)
self.cdc.write(wdata)

prog = round(float(total - bytestowrite) / float(total) * float(100), 1)
if prog > old:
Expand All @@ -447,7 +440,7 @@ def cmd_program(self, physical_partition_number, start_sector, filename, display
bar_length=50)
old = prog

self.cdc.write(b'', self.cfg.MaxPayloadSizeToTargetInBytes)
self.cdc.write(b'')
# time.sleep(0.2)

wd = self.wait_for_data()
Expand All @@ -467,10 +460,11 @@ def cmd_program(self, physical_partition_number, start_sector, filename, display
return True

def cmd_program_buffer(self, physical_partition_number, start_sector, wfdata, display=True):
size = len(wfdata)
bytestowrite = len(wfdata)
total = bytestowrite
# Make sure we fill data up to the sector size
num_partition_sectors = size // self.cfg.SECTOR_SIZE_IN_BYTES
if (size % self.cfg.SECTOR_SIZE_IN_BYTES) != 0:
num_partition_sectors = bytestowrite // self.cfg.SECTOR_SIZE_IN_BYTES
if (bytestowrite % self.cfg.SECTOR_SIZE_IN_BYTES) != 0:
num_partition_sectors += 1
if display:
self.info(f"\nWriting to physical partition {str(physical_partition_number)}, " +
Expand All @@ -481,66 +475,57 @@ def cmd_program_buffer(self, physical_partition_number, start_sector, wfdata, di
f" num_partition_sectors=\"{num_partition_sectors}\"" + \
f" physical_partition_number=\"{physical_partition_number}\"" + \
f" start_sector=\"{start_sector}\" "

if self.modules is not None:
data += self.modules.addprogram()
data += f"/>\n</data>"
rsp = self.xmlsend(data)
pos = 0
rsp = self.xmlsend(data, self.skipresponse)
prog = 0
# time.sleep(0.01)
if display:
print_progress(prog, 100, prefix='Progress:', suffix='Complete', bar_length=50)
print_progress(prog, 100, prefix='Progress:', suffix='Complete (Sector %d)' % start_sector,
bar_length=50)
if rsp[0]:
bytesToWrite = self.cfg.SECTOR_SIZE_IN_BYTES * num_partition_sectors
total = self.cfg.SECTOR_SIZE_IN_BYTES * num_partition_sectors
old = 0
fpos = 0
fsize = len(wfdata)
while fsize > 0:
# todo: Why on earth doesn't MaxPayloadSizeToTargetInBytes work here ?
# Forced to use self.cfg.SECTOR_SIZE_IN_BYTES ... thus slow as hell
"""
wlen = self.cfg.MaxPayloadSizeToTargetInBytes // self.cfg.SECTOR_SIZE_IN_BYTES \
* self.cfg.SECTOR_SIZE_IN_BYTES
wlen = min(fsize, wlen)
"""
wlen = min(fsize, self.cfg.SECTOR_SIZE_IN_BYTES)
wdata = wfdata[fpos:fpos + wlen]
bytesToWrite -= wlen
fsize -= wlen
pos += wlen
fpos += wlen
if (wlen % self.cfg.SECTOR_SIZE_IN_BYTES) != 0:
pos = 0
while bytestowrite > 0:
wlen = min(bytestowrite, self.cfg.MaxPayloadSizeToTargetInBytes)

wdata = data[pos:pos+wlen]
pos+=wlen
bytestowrite -= wlen

if wlen % self.cfg.SECTOR_SIZE_IN_BYTES != 0:
filllen = (wlen // self.cfg.SECTOR_SIZE_IN_BYTES * self.cfg.SECTOR_SIZE_IN_BYTES) + \
self.cfg.SECTOR_SIZE_IN_BYTES
wdata += b"\x00" * (filllen - wlen)
wlen = len(wdata)
self.cdc.write(wdata, wlen)
prog = round(float(pos) / float(total) * float(100), 1)

self.cdc.write(wdata)

prog = round(float(total - bytestowrite) / float(total) * float(100), 1)
if prog > old:
if display:
print_progress(prog, 100, prefix='Progress:', suffix='Written (Sector %d)'
% (pos // self.cfg.SECTOR_SIZE_IN_BYTES),
print_progress(prog, 100, prefix='Progress:',
suffix='Complete (Sector %d)' % ((total - bytestowrite) //
self.cfg.SECTOR_SIZE_IN_BYTES),
bar_length=50)
if display and prog != 100:
print_progress(100, 100, prefix='Progress:', suffix='Done', bar_length=50)
self.cdc.write(b'', self.cfg.MaxPayloadSizeToTargetInBytes)
old = prog

self.cdc.write(b'')
# time.sleep(0.2)

wd = self.wait_for_data()
info = self.xml.getlog(wd)
log = self.xml.getlog(wd)
rsp = self.xml.getresponse(wd)
if "value" in rsp:
if rsp["value"] != "ACK":
self.error(f"Error:")
for line in info:
for line in log:
self.error(line)
return False
else:
self.error(f"Error:{rsp}")
return False
else:
self.error(f"Error:{rsp}")
return False
if display and prog != 100:
print_progress(100, 100, prefix='Progress:', suffix='Complete', bar_length=50)
print_progress(100, 100, prefix='Progress:', suffix='Done', bar_length=50)
return True

def cmd_erase(self, physical_partition_number, start_sector, num_partition_sectors, display=True):
Expand Down Expand Up @@ -576,7 +561,7 @@ def cmd_erase(self, physical_partition_number, start_sector, num_partition_secto
wlen = bytesToWrite
"""
wlen = min(bytesToWrite, self.cfg.SECTOR_SIZE_IN_BYTES)
self.cdc.write(empty[0:wlen], self.cfg.MaxPayloadSizeToTargetInBytes)
self.cdc.write(empty[0:wlen])
prog = round(float(pos) / float(total) * float(100), 1)
if prog > old:
if display:
Expand All @@ -587,7 +572,7 @@ def cmd_erase(self, physical_partition_number, start_sector, num_partition_secto
pos += wlen
if display and prog != 100:
print_progress(100, 100, prefix='Progress:', suffix='Done', bar_length=50)
self.cdc.write(b'', self.cfg.MaxPayloadSizeToTargetInBytes)
self.cdc.write(b'')
# time.sleep(0.2)
res = self.wait_for_data()
info = self.xml.getlog(res)
Expand Down Expand Up @@ -635,12 +620,13 @@ def cmd_read(self, physical_partition_number, start_sector, num_partition_sector
prog = 0
if display:
print_progress(prog, 100, prefix='Progress:', suffix='Complete', bar_length=50)
# todo: Why on earth doesn't MaxPayloadSizeToTargetInBytes work here ?
# Forced to use self.cfg.SECTOR_SIZE_IN_BYTES ... thus slow as hell
size = min(self.cfg.SECTOR_SIZE_IN_BYTES, 1048576)
while bytesToRead > 0:
size = min(size, bytesToRead)
tmp = self.cdc.read(size)
if bytesToRead > self.cfg.MaxPayloadSizeFromTargetInBytes:
tmp = b"".join([self.cdc.read(self.cdc.EP_IN.wMaxPacketSize)
for _ in range(self.cfg.MaxPayloadSizeFromTargetInBytes//self.cdc.EP_IN.wMaxPacketSize)])
else:
size = min(self.cdc.EP_IN.wMaxPacketSize, bytesToRead)
tmp = self.cdc.read(size)
bytesToRead -= len(tmp)
wr.write(tmp)
if display:
Expand Down Expand Up @@ -699,23 +685,21 @@ def cmd_read_buffer(self, physical_partition_number, start_sector, num_partition
prog = 0
if display:
print_progress(prog, 100, prefix='Progress:', suffix='Complete', bar_length=50)
# todo: Why on earth doesn't MaxPayloadSizeToTargetInBytes work here ?
# Forced to use self.cfg.SECTOR_SIZE_IN_BYTES ... thus slow as hell
#size = min(self.cfg.SECTOR_SIZE_IN_BYTES, 1048576)
size = min(self.cfg.SECTOR_SIZE_IN_BYTES, 1048576)
while bytesToRead > 0:
size = min(size, bytesToRead)
tmp = self.cdc.read(size)
if bytesToRead > self.cfg.MaxPayloadSizeFromTargetInBytes:
tmp = b"".join([self.cdc.read(self.cdc.EP_IN.wMaxPacketSize) for _ in
range(self.cfg.MaxPayloadSizeFromTargetInBytes // self.cdc.EP_IN.wMaxPacketSize)])
else:
size = min(self.cdc.EP_IN.wMaxPacketSize, bytesToRead)
tmp = self.cdc.read(size)
bytesToRead -= len(tmp)
dataread += len(tmp)
resData += tmp
prog = round(float(dataread) / float(total) * float(100), 1)
if prog > old:
if display:
print_progress(prog, 100, prefix='Progress:', suffix='Read (Sector %d)'
% (
dataread // self.cfg.SECTOR_SIZE_IN_BYTES),
bar_length=50)
% (dataread // self.cfg.SECTOR_SIZE_IN_BYTES), bar_length=50)
old = prog
if display and prog != 100:
print_progress(100, 100, prefix='Progress:', suffix='Complete', bar_length=50)
Expand Down Expand Up @@ -1151,7 +1135,7 @@ def cmd_poke(self, address, data, filename="", info=False):
xdata = f"<?xml version=\"1.0\" ?><data><poke address64=\"{str(address + pos)}\" " + \
f"SizeInBytes=\"{str(maxsize)}\" value64=\"{content}\" /></data>\n"
try:
self.cdc.write(xdata, self.cfg.MaxXMLSizeInBytes)
self.cdc.write(xdata[:self.cfg.MaxXMLSizeInBytes])
except Exception as e: # pylint: disable=broad-except
self.debug(str(e))
pass
Expand All @@ -1162,7 +1146,7 @@ def cmd_poke(self, address, data, filename="", info=False):
tmp += self.cdc.read(self.cfg.MaxXMLSizeInBytes)
xdata = f"<?xml version=\"1.0\" ?><data><poke address64=\"{str(address + pos)}\" " + \
f"SizeInBytes=\"{str(maxsize)}\" value64=\"{content}\" /></data>\n"
self.cdc.write(xdata, self.cfg.MaxXMLSizeInBytes)
self.cdc.write(xdata[:self.cfg.MaxXMLSizeInBytes])
addrinfo = self.cdc.read(self.cfg.MaxXMLSizeInBytes)
if (b'<response' in addrinfo and 'NAK' in addrinfo) or b"Invalid parameters" in addrinfo:
self.error(f"Error:{addrinfo}")
Expand Down Expand Up @@ -1211,7 +1195,7 @@ def cmd_peek(self, address, SizeInBytes, filename="", info=False):
0xFF 0xEA 0xFE 0xFF 0xFF 0xEA 0xFE 0xFF " /></data>
'''
try:
self.cdc.write(data, self.cfg.MaxXMLSizeInBytes)
self.cdc.write(data[:self.cfg.MaxXMLSizeInBytes])
except Exception as err: # pylint: disable=broad-except
self.debug(str(err))
pass
Expand All @@ -1222,7 +1206,7 @@ def cmd_peek(self, address, SizeInBytes, filename="", info=False):
tmp += self.cdc.read(self.cfg.MaxXMLSizeInBytes)
data = f"<?xml version=\"1.0\" ?><data><peek address64=\"{hex(address)}\" " + \
f"SizeInBytes=\"{hex(SizeInBytes)}\" /></data>"
self.cdc.write(data, self.cfg.MaxXMLSizeInBytes)
self.cdc.write(data[:self.cfg.MaxXMLSizeInBytes])
addrinfo = self.cdc.read(self.cfg.MaxXMLSizeInBytes)
if (b'<response' in addrinfo and 'NAK' in addrinfo) or b"Invalid parameters" in addrinfo:
self.error(f"Error:{addrinfo}")
Expand Down
2 changes: 1 addition & 1 deletion Library/hdlc.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ def send_unframed_buf(self, outdata, prefixflag):
tmp.append(0x7E)
tmp.extend(outdata)
outdata = tmp
return self.cdc.write(outdata, MAX_PACKET_LEN)
return self.cdc.write(outdata[:MAX_PACKET_LEN])
# FlushFileBuffers(ser)

def send_cmd_base(self, outdata, prefixflag, nocrc=False):
Expand Down
8 changes: 4 additions & 4 deletions Library/sahara.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -386,12 +386,12 @@ def connect(self):
return ["nandprg", None]
else:
data = b"<?xml version=\"1.0\" ?><data><nop /></data>"
self.cdc.write(data, 0x80)
self.cdc.write(data)
res = self.cdc.read()
if res == b"":
try:
data = b"\x7E\x06\x4E\x95\x7E" # Streaming nop
self.cdc.write(data, 0x80)
self.cdc.write(data)
res = self.cdc.read()
if b"\x7E\x0D\x16\x00\x00\x00\x00" in res or b"Invalid Command" in res:
return ["nandprg", None]
Expand All @@ -407,7 +407,7 @@ def connect(self):
return ["sahara", None]
else:
data = b"\x7E\x11\x00\x12\x00\xA0\xE3\x00\x00\xC1\xE5\x01\x40\xA0\xE3\x1E\xFF\x2F\xE1\x4B\xD9\x7E"
self.cdc.write(data, 0x80)
self.cdc.write(data)
res = self.cdc.read()
if len(res) > 0 and res[1] == 0x12:
return ["nandprg", None]
Expand Down Expand Up @@ -793,7 +793,7 @@ def upload_loader(self):
while len(programmer) < data_offset + data_len:
programmer += b"\xFF"
data_to_send = programmer[data_offset:data_offset + data_len]
self.cdc.write(data_to_send, self.pktsize)
self.cdc.write(data_to_send)
datalen -= data_len
self.info("Loader uploaded.")
cmd, pkt = self.get_rsp()
Expand Down
Loading

0 comments on commit 2886810

Please sign in to comment.