Implemented stone-unpack mode

README.md

@@ -2,7 +2,7 @@
 
 ## About
 
-Cross-platform MediaTek feature phone dumping had been achieved long ago with [MTreader](https://gitlab.com/suborg/mtreader). Now it's time to do the same for Unisoc (ex-Spreadtrum) phones. And not only that, but also to be able to flash them!
+Cross-platform MediaTek feature phone dumping had been achieved long ago with [MTreader](https://gitlab.com/suborg/mtreader). Now it's time to do the same for Unisoc (ex-Spreadtrum) phones. And not only that, but also to be able to flash them and extract their compressed firmware contents!
 
 Unfortunately, the architecture of Unisoc chipset boot ROM doesn't allow us to get away without any loader blobs. So, some FDL binaries are also shipped in this repo.
 
@@ -18,7 +18,7 @@ For further dumped firmware unpacking/repacking, I recommend [bzpwork](https://g
 
 Python 3.8+ and PyUSB.
 
-## Usage
+## Usage as a flasher/dumper
 
 Run `python uniflash.py -h` to see all parameters. But there are several typical scenarios that UniFlash officially supports.
 
@@ -70,6 +70,20 @@ For single-FDL targets, the FDL file name must be `[targetname]_[load_addr]_sing
 
 For example, if you have found signed FDL loaders for Nokia 105 2019 somewhere and want to add this phone as a target, you know that they are loaded into `0x40004000` and `0x14000000` respectively, so you can rename them, for instance, to `sc6531efm_nokia105_0x40004000_fdl1.bin` and `sc6531efm_nokia105_0x14000000_fdl2.bin`, place them into `fdls/` and then just use `-t sc6531efm_nokia105` in your commands. This signed target, by the way, has already been added as an example.
 
+## Usage as a stone image unpacker
+
+The main syntax is as follows: `python uniflash.py stone-unpack [stone-file.bin] [-d target_dir]`
+
+The firmware is going to be unpacked into the following files in the target directory (specified with `-d` parameter, defaults to the same directory as the source stone image file):
+
+- `ps.bin` - protocol station image, always uncompressed, the first in the binary;
+- `kern.bin` (optional) - single-block LZMA-SPD compressed kernel partition;
+- `user.bin` - multi-block LZMA-SPD compressed user firmware partition;
+- `rsrc.bin` - multi-block LZMA-SPD compressed user resources partition;
+- `blk_xxxxxxxx` (optional) - any of the additional LZMA-SPD or standard LZMA compressed sections found in the image.
+
+LZMA-SPD, also known as LZMA-B3, is a proprietary modification of LZMA algorithm with simplified literal encoder/decoder.
+
 ## Credits
 
 Created by Luxferre in 2021. All files except the FDL blobs are public domain.

custlzma/frenchlzma.py

@@ -0,0 +1,231 @@
+from io import BytesIO
+from collections import defaultdict
+
+class LecteurDeBits:
+    
+    def __init__(self, entree : bytes):
+        self.octets = BytesIO(entree)
+        self.bits_non_lus = 0
+        self.taille_bits_non_lus = 0
+    
+    def lire_bits(self, nombre_bits : int) -> int:
+        while self.taille_bits_non_lus < nombre_bits:
+            prochain_octet = self.octets.read(1)
+            if not prochain_octet:
+                raise EOFError
+            self.bits_non_lus |= prochain_octet[0] << self.taille_bits_non_lus
+            self.taille_bits_non_lus += 8
+        masque_bits_lus = (1 << nombre_bits) - 1
+        bits_lus = self.bits_non_lus & masque_bits_lus
+        self.bits_non_lus >>= nombre_bits
+        self.taille_bits_non_lus -= nombre_bits
+        return bits_lus
+    
+    def lire_octets(self, nombre_octets : int) -> bytes:
+        self.aligner_bits_sur_octet()
+        octets_lus = self.octets.read(nombre_octets)
+        if len(octets_lus) < nombre_octets:
+            raise EOFError
+        return octets_lus
+    
+    def aligner_bits_sur_octet(self):
+        
+        self.bits_non_lus = 0
+        self.taille_bits_non_lus = 0
+
+class LecteurDeBitsRangeCode:
+    
+    def __init__(self, entree : bytes):
+        self.octets = BytesIO(entree)
+        self.taille_code = 0xffffffff
+        self.code = 0
+        assert self.octets.read(1)[0] == 0
+        
+        for position in range(4):
+            self.code = (self.code << 8) | self.octets.read(1)[0]
+        
+        assert self.code < self.taille_code
+
+    def reprendre_des_bits_si_besoin(self):
+        if self.taille_code <= 0xffffff:
+            self.taille_code <<= 8
+            self.code <<= 8
+            self.code |= self.octets.read(1)[0]
+
+class RangeDecoder:
+
+    def __init__(self, lecteur_de_bits : LecteurDeBitsRangeCode, probabilite_initiale : int = 0x400):
+        self.lecteur_de_bits : LecteurDeBitsRangeCode = lecteur_de_bits
+        self.probabilite : int = probabilite_initiale # 0 - 0x800
+    
+    def lire_bit(self, utiliser_probas = True):
+        if utiliser_probas:
+            milieu_du_code_pondere = self.lecteur_de_bits.taille_code // 0x800 * self.probabilite
+        else:
+            milieu_du_code_pondere = self.lecteur_de_bits.taille_code // 2
+            
+        if self.lecteur_de_bits.code < milieu_du_code_pondere:
+            bit_lu = 0
+            self.probabilite += (0x800 - self.probabilite) // 32
+            self.lecteur_de_bits.taille_code = milieu_du_code_pondere
+        else:
+            bit_lu = 1
+            self.probabilite -= self.probabilite // 32
+            self.lecteur_de_bits.code -= milieu_du_code_pondere
+            if utiliser_probas:
+                self.lecteur_de_bits.taille_code -= milieu_du_code_pondere
+            else:
+                self.lecteur_de_bits.taille_code = milieu_du_code_pondere
+        self.lecteur_de_bits.reprendre_des_bits_si_besoin()
+        return bit_lu
+
+class DecodeurLZMA:
+
+    def decode(self, entree : bytes) -> bytes:
+        self.flux_decompresse = b''
+        self.dernieres_distances : List[int] = [0] * 4
+        lecteur_de_bits = LecteurDeBits(entree)
+        properties = lecteur_de_bits.lire_bits(8)
+        self.literal_context_bits = properties % 9
+        literal_position_bits = (properties // 9) % 5
+        position_bits = properties // 9 // 5
+        
+        if position_bits > 4:
+            raise ValueError('LZMA invalid')
+        
+        self.taille_fenetre = lecteur_de_bits.lire_bits(32)
+        self.uncompressed_size = lecteur_de_bits.lire_bits(64)
+        lecteur_de_bits = LecteurDeBitsRangeCode(entree[13:])
+        self.nom_vers_range_decodeur : Dict[tuple, RangeDecoder] = defaultdict(lambda: RangeDecoder(lecteur_de_bits))
+        self.state = 0
+        self.masque_pos_state = (1 << position_bits) - 1
+        self.masque_lit_state = (1 << literal_position_bits) - 1
+        
+        while len(self.flux_decompresse) < self.uncompressed_size:
+            try:
+                pos_state = len(self.flux_decompresse) & self.masque_pos_state
+                bit_choice = self.nom_vers_range_decodeur[('IsMatch', self.state, pos_state)].lire_bit()
+                if bit_choice == 0:
+                    self.LITERAL()
+                elif bit_choice == 1:
+                    is_rep = self.nom_vers_range_decodeur[('IsRep', self.state)].lire_bit()
+                    if is_rep == 0:
+                        self.MATCH()
+                    elif is_rep == 1:
+                        is_rep0 = self.nom_vers_range_decodeur[('IsRepG0', self.state)].lire_bit()
+                        if is_rep0 == 0:
+                            is_rep0_long = self.nom_vers_range_decodeur[('IsRep0Long', self.state, pos_state)].lire_bit()
+                            if is_rep0_long == 0:
+                                self.SHORTREP()
+                            elif is_rep0_long == 1:
+                                self.LONGREP(0)
+                        elif is_rep0 == 1:
+                            is_rep1 = self.nom_vers_range_decodeur[('IsRepG1', self.state)].lire_bit()
+                            if is_rep1 == 0:
+                                self.LONGREP(1)
+                            elif is_rep1 == 1:
+                                is_rep2 = self.nom_vers_range_decodeur[('IsRepG2', self.state)].lire_bit()
+                                if is_rep2 == 0:
+                                    self.LONGREP(2)
+                                elif is_rep2 == 1:
+                                    self.LONGREP(3)
+            except EOFError:
+                break
+        
+        return self.flux_decompresse
+
+    def LITERAL(self):
+        dernier_octet_decompresse = self.flux_decompresse[-1] if self.flux_decompresse else 0
+        octet_lu = self.bit_tree_decode(('LiteralNormal',
+            len(self.flux_decompresse) & self.masque_lit_state, # total_pos
+            dernier_octet_decompresse >> (8 - self.literal_context_bits), # prev_byte
+        ), None, 8, use_pos_state = False)
+        self.flux_decompresse += bytes([octet_lu])
+        if self.state > 9:
+            self.state -= 6
+        elif self.state > 3:
+            self.state -= 3
+        else:
+            self.state = 0
+    
+    def MATCH(self):
+        match_len = 2 + self.len_decode('LenDecoder')
+        pos_slot = self.bit_tree_decode(('PosSlot', min(5, match_len)), None, 6, use_pos_state = False)
+        if pos_slot >= 4:
+            num_direct_bits = (pos_slot >> 1) - 1
+            distance = (2 | (pos_slot & 1)) << num_direct_bits
+            if pos_slot < 14:
+                distance += self.bit_tree_decode('SpecPos', None, num_direct_bits + (distance - pos_slot - 1),
+                    use_pos_state = False, reverse = True,
+                    debut_bit_tree = distance - pos_slot - 1)
+            else:
+                distance += self.bit_tree_decode('AlignFixed', None, num_direct_bits - 4, utiliser_probas = False) << 4
+                distance += self.bit_tree_decode('Align', None, 4, use_pos_state = False, reverse = True)
+        else:
+            distance = pos_slot
+        
+        self.dernieres_distances.append(distance)
+        if distance == 0xffffffff:
+            raise EOFError
+        assert distance < len(self.flux_decompresse)
+        assert distance < self.taille_fenetre
+        self.repeter_donnees(distance, match_len)
+        if self.state < 7:
+            self.state = 7
+        else:
+            self.state = 10
+        
+    def SHORTREP(self): # Réutiliser la dernière distance pour un octet
+        if self.state < 7:
+            self.state = 9
+        else:
+            self.state = 11
+        self.repeter_donnees(self.dernieres_distances[-1], 1)
+        
+    def LONGREP(self, num): # Réutiliser l'une des dernières distances pour une taille donnée
+        match_len = 2 + self.len_decode('RepLenDecoder')
+        self.dernieres_distances.append(self.dernieres_distances.pop(-(1 + num)))
+        distance = self.dernieres_distances[-1]
+        self.repeter_donnees(distance, match_len)
+        if self.state < 7:
+            self.state = 8
+        else:
+            self.state = 11
+    
+    def repeter_donnees(self, distance, match_len):
+        debut_slice = len(self.flux_decompresse) - (1 + distance)
+        fin_slice = match_len
+        a_repeter = self.flux_decompresse[debut_slice:debut_slice + fin_slice]
+        fin_slice -= len(self.flux_decompresse) - debut_slice
+        while fin_slice > 0:
+            a_repeter += self.flux_decompresse[debut_slice:debut_slice + fin_slice]
+            fin_slice -= min(len(self.flux_decompresse), debut_slice + fin_slice) - debut_slice
+    
+        self.flux_decompresse += a_repeter
+    
+    def len_decode(self, len_decoder_name):
+        if self.nom_vers_range_decodeur[('LenChoice', len_decoder_name)].lire_bit() == 0:
+            return self.bit_tree_decode('LenLow', len_decoder_name, 3)
+        else:
+            if self.nom_vers_range_decodeur[('LenChoice2', len_decoder_name)].lire_bit() == 0:
+                return (1 << 3) + self.bit_tree_decode('LenMid', len_decoder_name, 3)
+            else:
+                return (1 << 4) + self.bit_tree_decode('LenHigh', len_decoder_name, 8, use_pos_state = False)
+
+    def bit_tree_decode(self, bit_tree_decoder_name, len_decoder_name, num_bits,
+        use_pos_state = True, utiliser_probas = True, reverse = False,
+        bit_tree_lu = 0, debut_bit_tree = 0):
+        for position_bit in range(debut_bit_tree, num_bits):
+            bit_lu = self.nom_vers_range_decodeur[(
+                bit_tree_decoder_name,
+                len_decoder_name,
+                bit_tree_lu,
+                position_bit,
+                (len(self.flux_decompresse) & self.masque_pos_state) if use_pos_state else None,
+            )].lire_bit(utiliser_probas = utiliser_probas)
+            if not reverse:
+                bit_tree_lu <<= 1
+                bit_tree_lu |= bit_lu
+            else:
+                bit_tree_lu |= bit_lu << (position_bit - debut_bit_tree)
+        return bit_tree_lu

stoned.py

@@ -0,0 +1,156 @@
+# StoneD (Stone Depacker) - unpack Unisoc SC6531 stone images (part of UniFlash)
+# Created by Luxferre in 2021, released into public domain
+
+import os
+import sys
+import struct
+import lzma
+#from custlzma.lzma_decoder import LZMADecoder # (for LZMA_SPD decompression)
+from custlzma.frenchlzma import DecodeurLZMA
+
+# common utils
+
+def readFile(fname):
+    f = open(fname, 'rb')
+    fdata = f.read()
+    f.close()
+    return fdata
+
+def writeFile(fname, fdata):
+    outf = open(fname, 'wb')
+    outf.write(fdata)
+    outf.close()
+
+# unpack part
+
+CMP_NONE=0
+CMP_LZMA_SPRD=1
+CMP_LZMA=2
+
+def getCompType(data):
+    if (data[0] == 0x5d or data[0] == 0x67) and data[1] == 0:
+        return CMP_LZMA
+    elif data[0] == 0x5a and data[1] == 0:
+        return CMP_LZMA_SPRD
+    else:
+        return CMP_NONE
+
+def getTblOffset(blocksOffTbl, index):
+    tind = index << 2
+    return struct.unpack('<L', blocksOffTbl[tind:tind+4])[0]
+
+def unpack_block(blkData, blkPacSize, targetFile):
+    print('Extracting %s...' % targetFile)
+    blocksOffTbl = None
+    compData = blkData
+    npacHdr = blkData[:16]
+    (npacHdrMagic, npacHdrFlags, compDataSize, lzmaBlocksAmount) = struct.unpack('<LLLL', npacHdr)
+    # npacHdrMagic must be CAPN if using offsets
+    if npacHdrMagic == 0x4E504143:
+        blocksOffTbl = blkData[compDataSize:]
+        compData = blkData[getTblOffset(blocksOffTbl,0):]
+    else:
+        lzmaBlocksAmount = 1
+    cType = getCompType(compData)
+    assert cType == CMP_LZMA or cType == CMP_LZMA_SPRD, 'Only LZMA compression type is implemented as of now'
+    print('Found LZMA blocks: %d, decompressing...' % lzmaBlocksAmount)
+    dest = b''
+    inSizePure = blkPacSize * 2
+    lzmaDec = DecodeurLZMA()
+    for i in range(lzmaBlocksAmount):
+        if blocksOffTbl is not None:
+            dataOffset = getTblOffset(blocksOffTbl,i)
+        else:
+            dataOffset = 0
+        compData = blkData[dataOffset:]
+        lzData = compData[0:inSizePure]
+        if cType == CMP_LZMA_SPRD:
+            outdata = lzmaDec.decode(lzData)
+            dest += outdata
+        else:
+            dest += lzma.decompress(lzData, format=lzma.FORMAT_ALONE)
+        sys.stdout.write('.')
+        sys.stdout.flush()
+    writeFile(targetFile, dest)
+    print('\n%s decompressed!' % targetFile)
+
+def unpack_section(sectionData, targetDir):
+    bzpFileHdr = sectionData[:16]
+    (bzpFileHdrMagic, bzpType, blocksOffset, blocksAmount) = struct.unpack('<LLLL', bzpFileHdr)
+    # bzpFileHdrMagic must be DRPS or RRPS
+    assert bzpFileHdrMagic == 0x53505244 or bzpFileHdrMagic == 0x53505252, 'Invalid BZP header: 0x%X' % bzpFileHdrMagic
+    bzpSize = blocksOffset + blocksAmount * 20
+    for i in range(blocksAmount):
+        blkHdrStart = blocksOffset + i*20
+        blkHdr = sectionData[blkHdrStart:blkHdrStart+20]
+        (blkHdrMagic, blkId, blkDataOffset, blkPackedSize, blkPacSize) = struct.unpack('<LLLLL', blkHdr)
+        # blkHdrMagic must be COLB
+        assert blkHdrMagic == 0x424C4F43, 'Invalid BZP block header: 0x%X' % blkHdrMagic
+        if bzpSize < blkDataOffset + blkPackedSize:
+            bzpSize = blkDataOffset + blkPackedSize
+        if blkId == 0x494D4147: # GAMI -> kernel image
+            targetFile = targetDir + '/kern.bin'
+        elif blkId == 0x75736572: # resu -> user image
+            targetFile = targetDir + '/user.bin'
+        elif blkId == 0x7253736F: # resources
+            targetFile = targetDir + '/rsrc.bin'
+        else:
+            targetFile = targetDir + ('/blk_%X.bin' % blkId)
+        unpack_block(sectionData[blkDataOffset:], blkPacSize, targetFile)
+
+def unpack_stone(fname, targetDir):
+    fdata = readFile(fname)
+    flen = len(fdata)
+    assert flen >= 0x10, 'Input file %s is too small' % fname
+
+    # check for security header
+    sectionOffset = 0
+    if fdata[0:15] == b'SPRD-SECUREFLAG':
+        sectionOffset = 1024
+        print('Signed image detected, using section offset %d' % sectionOffset)
+
+    # look for TRAPGAMI header
+    startPos = -1
+    for i in range(flen):
+        if fdata[i:i+8] == b'TRAPGAMI':
+            startPos = i
+            break
+    assert startPos > 0, 'No stone header found in %s' % fname
+    print('Stone header found at 0x%X' % startPos)
+
+    psImageEnd = 0xffffffff # PS (protocol station) image is the first in the flash backup and not compressed
+
+    dfcStruct = fdata[startPos+8:startPos+120]
+    for i in range(0,112,4):
+        targetAddr = struct.unpack('<L', dfcStruct[i:i+4])[0]
+        if targetAddr < 0xffffffff:
+            if targetAddr < psImageEnd:
+                psImageEnd = targetAddr
+            print('Target section address found: 0x%X' % targetAddr)
+            unpack_section(fdata[sectionOffset+targetAddr:], targetDir)
+            print('Section 0x%X unpacked!' % targetAddr)
+
+    if psImageEnd > 0:
+        psPath = targetDir + '/ps.bin'
+        writeFile(psPath, fdata[:psImageEnd])
+        print('Protocol station image %s written!' % psPath)
+
+# main code start
+
+if __name__ == '__main__': # main app start
+    from argparse import ArgumentParser
+    rootdir = os.path.dirname(os.path.realpath(__file__))
+    parser = ArgumentParser(description='StoneD: an opensource Unisoc/Spreadtrum stone image unpacker', epilog='(c) Luxferre 2021 --- No rights reserved <https://unlicense.org>')
+    parser.add_argument('file', help='Stone image file to unpack')
+    parser.add_argument('-d','--directory', default=None, help='Directory where component files will be written to (defaults to the same where the main stone file resides)')
+
+    args = parser.parse_args()
+
+    imgfile = args.file
+    imgdir = os.path.dirname(os.path.realpath(imgfile))
+    if args.directory is not None:
+        imgdir = os.path.realpath(args.directory)
+
+    print('Unpacking %s to %s' % (imgfile, imgdir))
+    unpack_stone(imgfile, imgdir)
+

uniflash.py

@@ -4,6 +4,7 @@ import usb
 import sys, time
 import os
 import unicmd
+import stoned
 
 # global params
 
@@ -162,12 +163,13 @@ if __name__ == '__main__': # main app start
     from argparse import ArgumentParser
     rootdir = os.path.dirname(os.path.realpath(__file__))
     parser = ArgumentParser(description='UniFlash: an opensource Unisoc/Spreadtrum feature phone flash reader/writer', epilog='(c) Luxferre 2021 --- No rights reserved <https://unlicense.org>')
-    parser.add_argument('mode', help='Operation mode (flash/dump)')
+    parser.add_argument('mode', help='Operation mode (flash/dump/stone-unpack)')
-    parser.add_argument('file', help='File to read the flash data from or write the dump into')
+    parser.add_argument('file', help='File to read the flash data from or write the dump into, or the stone file to unpack')
     parser.add_argument('-p','--partid', type=auto_int, default=0x80000003, help='partition ID for readback (defaults to 0x80000003 that can address full flash space on SC6531E/F/M)')
     parser.add_argument('-s','--start', type=auto_int, default=0, help='start position (in the partition, defaults to 0)')
     parser.add_argument('-l', '--length', type=auto_int, default=0x400000, help='data length in bytes to read/write, defaults to 0x400000')
     parser.add_argument('-t','--target', default='sc6531efm_generic', help='Preinstalled target (defaults to sc6531efm_generic, overridable with individual FDL parameters)')
+    parser.add_argument('-d','--directory', default=None, help='Directory where component files will be written to in stone-unpack mode (defaults to the same where the main stone file resides)')
     parser.add_argument('-nr','--flash-noremap', action='store_true', help='Disable base address remapping for flashing')
     parser.add_argument('-e','--force-erase', action='store_true', help='Erase target flash memory area before flashing')
     parser.add_argument('-wf','--enable-write-flash', action='store_true', help='Send the write flash enable command before flashing (if necessary and supported)')
@@ -183,6 +185,16 @@ if __name__ == '__main__': # main app start
 
     args = parser.parse_args()
 
+    if args.mode.startswith('stone'): # stone-unpack mode
+        imgfile = args.file
+        imgdir = os.path.dirname(os.path.realpath(imgfile))
+        if args.directory is not None:
+            imgdir = os.path.realpath(args.directory)
+
+        print('Unpacking %s to %s' % (imgfile, imgdir))
+        stoned.unpack_stone(imgfile, imgdir)
+
+    else: # flash/dump mode
         is_flash = False
         if args.mode == 'flash':
             is_flash = True