From 0712c86338caab6ac83afec67c54fa9634e6383c Mon Sep 17 00:00:00 2001
From: josch <j.schauer@email.de>
Date: Sat, 15 Mar 2014 15:31:40 +0100
Subject: [PATCH] properly encode files in format 1 and 3

---
 defextract.py |  16 ++--
 lodextract.py |  30 ++++++--
 makedef.py    | 199 +++++++++++++++++++++++++++++++++++++++++++++-----
 3 files changed, 213 insertions(+), 32 deletions(-)

diff --git a/defextract.py b/defextract.py
index c5f82bd..3de5718 100644
--- a/defextract.py
+++ b/defextract.py
@@ -18,7 +18,7 @@ def get_color(fname):
 def extract_def(infile,outdir,shred=True):
     f = open(infile)
     bn = os.path.basename(infile)
-    bn = os.path.splitext(bn)[0]
+    bn = os.path.splitext(bn)[0].lower()
 
     # t - type
     # blocks - # of blocks
@@ -67,7 +67,7 @@ def extract_def(infile,outdir,shred=True):
             # lm,tm - left and top margin
             _,fmt,fw,fh,w,h,lm,tm = struct.unpack("<IIIIIIii", f.read(32))
             n = os.path.splitext(n)[0]
-            outname = "%s"%outdir+os.sep+"%02d_%s_%02d_%02d_%s_%dx%d_%dx%d.png"%(t,bn,bid,j,n,fw,fh,lm,tm)
+            outname = "%s"%outdir+os.sep+"%02d_%s_%02d_%02d_%s_%dx%d_%dx%d_%d.png"%(t,bn,bid,j,n,fw,fh,lm,tm,fmt)
             print "writing to %s"%outname
 
             if w != 0 and h != 0:
@@ -100,8 +100,8 @@ def extract_def(infile,outdir,shred=True):
                         totalrowlength=0
                         while totalrowlength<w:
                             segment, = struct.unpack("<B", f.read(1))
-                            code = segment/32
-                            length = (segment&31)+1
+                            code = segment>>5
+                            length = (segment&0x1f)+1
                             if code == 7: # raw data
                                 pixeldata += f.read(length)
                             else: # rle
@@ -113,12 +113,14 @@ def extract_def(infile,outdir,shred=True):
                     lineoffs = [struct.unpack("<"+"H"*(w/32), f.read(w/16))[0] for i in range(h)]
 
                     for lineoff in lineoffs:
-                        f.seek(offs+32+lineoff)
+                        if f.tell() != offs+32+lineoff:
+                            print "unexpected offset: %d, expected %d"%(f.tell(),offs+32+lineoff)
+                            f.seek(offs+32+lineoff)
                         totalrowlength=0
                         while totalrowlength<w:
                             segment, = struct.unpack("<B", f.read(1))
-                            code = segment/32
-                            length = (segment&31)+1
+                            code = segment>>5
+                            length = (segment&0x1f)+1
                             if code == 7: # raw data
                                 pixeldata += f.read(length)
                             else: # rle
diff --git a/lodextract.py b/lodextract.py
index dc16625..756f3ca 100644
--- a/lodextract.py
+++ b/lodextract.py
@@ -30,8 +30,9 @@ def read_pcx(data):
 def unpack_lod(infile,outdir,shred=True):
     f = open(infile)
 
-    if f.read(4) != 'LOD\0':
-        print "not LOD file"
+    header = f.read(4)
+    if header != 'LOD\0':
+        print "not LOD file: %s"%header
         return False
 
     f.seek(8)
@@ -41,7 +42,7 @@ def unpack_lod(infile,outdir,shred=True):
     files=[]
     for i in range(total):
         filename, = struct.unpack("16s", f.read(16))
-        filename = filename[:filename.index('\0')]
+        filename = filename[:filename.index('\0')].lower()
         offset,size,_,csize = struct.unpack("<IIII", f.read(16))
         files.append((filename,offset,size,csize))
 
@@ -62,9 +63,23 @@ def unpack_lod(infile,outdir,shred=True):
                     g = (crc&0xff00)>>8
                     b = crc&0xff
                     w,h = im.size
-                    im = Image.new("RGB", (w*3,h*3), (r,g,b))
+                    pixels = im.load()
+                    for i in range(w):
+                        for j in range(h):
+                            if pixels[i,j] > 7:
+                                if im.mode == 'P':
+                                    pixels[i,j] = 8+crc%248
+                                else:
+                                    pixels[i,j] = (r,g,b)
+                    im.resize((w*3,h*3))
                     draw = ImageDraw.Draw(im)
-                    draw.text((0,0),os.path.basename(filename),get_complement(r,g,b),font=font)
+                    tw,th = draw.textsize(os.path.basename(filename),font=font)
+                    tpos = ((w*3-tw)/2,(h*3-th)/2)
+                    if im.mode == 'P':
+                        # we can't really have a complement in palette mode, so just get some color
+                        draw.text(tpos,os.path.basename(filename),255,font=font)
+                    else:
+                        draw.text(tpos,os.path.basename(filename),get_complement(r,g,b),font=font)
                     im = im.resize((w,h),Image.ANTIALIAS)
                 im.save(filename, "PNG")
             else:
@@ -82,9 +97,10 @@ if __name__ == '__main__':
         print "usage: %s infile.lod ./outdir"%sys.argv[0]
         print ""
         print "usually after installing the normal way:"
-        print "    %s .vcmi/Data/H3bitmap.lod .vcmi/Mods/vcmi/Data/"
+        print "    %s .vcmi/Data/H3bitmap.lod .vcmi/Mods/vcmi/Data/"%sys.argv[0]
         print "    rm .vcmi/Data/H3bitmap.lod"
-        print "    %s .vcmi/Data/H3sprite.lod .vcmi/Mods/vcmi/Data/"
+        print "    %s .vcmi/Data/H3sprite.lod .vcmi/Mods/vcmi/Data/"%sys.argv[0]
         print "    rm .vcmi/Data/H3sprite.lod"
+        exit(1)
     ret = unpack_lod(sys.argv[1], sys.argv[2])
     exit(0 if ret else 1)
diff --git a/makedef.py b/makedef.py
index ae740c4..512c6ee 100644
--- a/makedef.py
+++ b/makedef.py
@@ -5,25 +5,129 @@ import re
 import struct
 from collections import defaultdict
 from PIL import Image
+ushrtmax = (1<<16)-1
+
+# greedy RLE
+# for each pixel, test which encoding results in the smaller size, then apply
+# that encoding and look at the next pixel after the encoded chunk
+def encode1(im):
+    pixels = im.load()
+    w,h = im.size
+    result = []
+    # these function return a tuple of the compressed string and the amount of
+    # pixels compressed
+    def rle_comp(x,y):
+        # find all pixels after the first one with the same color
+        color = pixels[x,y]
+        if color == 0xff:
+            # this color can't be run length encoded
+            return raw_comp(x,y)
+        else:
+            count = 1
+            for x in range(x+1,w):
+                if pixels[x,y] == color and count < 255:
+                    count += 1
+                else:
+                    break
+            return (struct.pack("<BB", color, count-1), count)
+    def raw_comp(x,y):
+        # read pixels until finding finding two consecutive ones with the same color
+        data = [pixels[x,y]]
+        for x in range(x+1,w):
+            color = pixels[x,y]
+            if color != data[-1] and len(data) < 255:
+                data.append(color)
+            else:
+                break
+        return (struct.pack("<BB%dB"%len(data), 0xff, len(data)-1, *data), len(data))
+    for y in range(h):
+        r = ''
+        x = 0
+        while x < w:
+            rlec, rlel = rle_comp(x, y)
+            rawc, rawl = raw_comp(x, y)
+            # the message that managed to encode more is chosen 
+            if rlel > rawl:
+                r += rlec
+                x += rlel
+            else:
+                r += rawc
+                x += rawl
+        result.append(r)
+    return result
+
+def encode3(im):
+    pixels = im.load()
+    w,h = im.size
+    result = []
+    for y in range(h):
+        r = ''
+        if pixels[0,y] < 8:
+            colors = pixels[0,y]
+            count = 1
+        else:
+            colors = [pixels[0,y]]
+            count = 0
+        for x in range(1,w):
+            color = pixels[x,y]
+            if count > 0:
+                # rle was started
+                if color == colors and count < 32:
+                    # same color again, increase count
+                    count+=1
+                else:
+                    # either new color or maximum length reached, so write current one
+                    r+=struct.pack("<B", (colors<<5) | (count-1))
+                    if color < 7:
+                        # new rle color
+                        colors = color
+                        count = 1
+                    else:
+                        # new non rle color
+                        colors = [color]
+                        count = 0
+            else:
+                # non rle was started
+                if color < 7 or len(colors) > 31:
+                    # new rle color, or maximum length reached so write current non rle
+                    r+=struct.pack("<B", (7<<5) | (len(colors)-1))
+                    r+=struct.pack("<%dB"%len(colors), *colors)
+                    if color < 7:
+                        colors = color
+                        count = 1
+                    else:
+                        colors = [color]
+                        count = 0
+                else:
+                    # new non rle color, so append it to current
+                    colors.append(color)
+        # write last color
+        if count > 0:
+            # write rle
+            r+=struct.pack("<B", (colors<<5) | (count-1))
+        else:
+            # write non rle
+            r+=struct.pack("<B", (7<<5) | (len(colors)-1))
+            r+=struct.pack("<%dB"%len(colors), *colors)
+        result.append(r)
+    return result
 
 def makedef(indir, outdir):
     infiles = defaultdict(list)
     sig = None
-    tw,th = 0,0
     # sanity checks and fill infiles dict
     for f in os.listdir(indir):
-        m = re.match('(\d+)_([A-Za-z0-9_]+)_(\d\d)_(\d\d)_([A-Za-z0-9_]+)_(\d+)x(\d+)_(\d+)x(\d+).png', f)
+        m = re.match('(\d+)_([a-z0-9_]+)_(\d\d)_(\d\d)_([A-Za-z0-9_]+)_(\d+)x(\d+)_(\d+)x(\d+)_([0-3]).png', f)
         if not m:
             continue
-        t,p,bid,j,fn,fw,fh,lm,tm = m.groups()
-        t,bid,j,fw,fh,lm,tm = int(t),int(bid),int(j),int(fw),int(fh),int(lm),int(tm)
+        t,p,bid,j,fn,fw,fh,lm,tm,fmt = m.groups()
+        t,bid,j,fw,fh,lm,tm,fmt = int(t),int(bid),int(j),int(fw),int(fh),int(lm),int(tm),int(fmt)
         im = Image.open(os.sep.join([indir,f]))
         w,h = im.size
-        tw,th = fw,fh
         if im.mode != 'P':
             print "input images must have a palette"
             return False
-        cursig =(t,p,im.getpalette())
+        cursig =(t,p,fw,fh,im.getpalette(),fmt)
         if not sig:
             sig = cursig
         else:
@@ -35,7 +139,21 @@ def makedef(indir, outdir):
         if len(fn) > 9:
             print "filename can't be longer than 9 bytes"
             return False
-        infiles[bid].append((im,t,p,j,fn,fw,fh,lm,tm))
+        if fmt == 0:
+            data = ''.join([chr(i) for i in list(im.getdata())])
+        elif fmt == 1:
+            data = encode1(im)
+        elif fmt == 2:
+            data = encode3(im)
+        elif fmt == 3:
+            if w < 16:
+                print "width must not be less than 16 for format 3"
+                return False
+            data = encode3(im)
+        else:
+            print "unknown format: %d"%fmt
+            return False
+        infiles[bid].append((im,t,p,j,fn,lm,tm,fmt,data))
 
     if len(infiles) == 0:
         print "no input files detected"
@@ -48,13 +166,15 @@ def makedef(indir, outdir):
             if k != j:
                 print "incorrect j value %d for bid %d should be %d"%(j,bid,k)
 
-    t,p,pal = cursig
-    outf = open(outdir+"/"+p+".def", "w+")
+    t,p,fw,fh,pal,fmt = cursig
+    outname = os.path.join(outdir,p)+".def"
+    print "writing to %s"%outname
+    outf = open(outname, "w+")
 
     # write the header
     # full width and height are not used and not the same for all frames
     # in some defs, so just putting the last known value
-    outf.write(struct.pack("<IIII", t,tw,th,len(infiles)))
+    outf.write(struct.pack("<IIII", t,fw,fh,len(infiles)))
     # write the palette
     outf.write(struct.pack("768B", *pal))
 
@@ -71,23 +191,66 @@ def makedef(indir, outdir):
         for _,_,_,_,fn,_,_,_,_ in l:
             outf.write(struct.pack("13s", fn+".pcx"))
         # write data offsets
-        for im,_,_,_,_,_,_,_,_ in l:
+        for im,_,_,_,_,_,_,fmt,data in l:
             outf.write(struct.pack("<I",curoffset))
             w,h = im.size
-            # every image occupies one byte per pixel plus 32 byte header
-            curoffset += w*h+32
+            # every image occupies size depending on its format plus 32 byte header
+            if fmt == 0:
+                curoffset += 32+len(data)
+            elif fmt == 1:
+                # 4*height bytes for lineoffsets
+                curoffset += 32+4*h+sum(len(d) for d in data)
+            elif fmt == 2:
+                # two bytes for the header
+                curoffset += 32+2+sum(len(d) for d in data)
+            elif fmt == 3:
+                # width/16 bytes per line as offset header
+                curoffset += 32+(w/16)*h+sum(len(d) for d in data)
 
     for bid,l in infiles.items():
-        for im,_,p,j,_,fw,fh,lm,tm in l:
+        for im,_,p,j,_,lm,tm,fmt,data in l:
             w,h = im.size
             # size
-            # format = 0 (uncompressed)
+            # format
             # full width and full height
             # width and height
             # left and top margin 
-            outf.write(struct.pack("<IIIIIIii",w*h,0,fw,fh,w,h,lm,tm))
-            buf = ''.join([chr(i) for i in list(im.getdata())])
-            outf.write(buf)
+            outf.write(struct.pack("<IIIIIIii",w*h,fmt,fw,fh,w,h,lm,tm))
+            if fmt == 0:
+                buf = ''.join([chr(i) for i in list(im.getdata())])
+                outf.write(buf)
+            elif fmt == 1:
+                lineoffs = []
+                acc = 4*h
+                for d in data:
+                    lineoffs.append(acc)
+                    acc += len(d)
+                outf.write(struct.pack("<"+"I"*h, *lineoffs))
+                for i in data:
+                    outf.write(i)
+            elif fmt == 2:
+                offs = outf.tell()-32+2
+                if offs > ushrtmax: 
+                    print "exceeding max ushort value: %d"%offs
+                    return False
+                outf.write(struct.pack("<H",offs))
+                for i in data:
+                    outf.write(i)
+            elif fmt == 3:
+                # store the same value in all w/16 blocks per line
+                lineoffs = []
+                acc = 0
+                for d in data:
+                    offs = acc+(w/16)*h
+                    if offs > ushrtmax:
+                        print "exceeding max ushort value: %d"%offs
+                        return False
+                    lineoffs.append(offs)
+                    lineoffs.extend([0 for i in range(w/32-1)])
+                    acc += len(d)
+                outf.write(struct.pack("<"+"H"*(w/32)*h, *lineoffs))
+                for i in data:
+                    outf.write(i)
     return True
 
 if __name__ == '__main__':