a bunch of more fixes

common.py

@@ -0,0 +1,20 @@
+import crcmod
+import colorsys
+
+from PIL import ImageFont
+
+font = ImageFont.truetype("/usr/share/fonts/truetype/ttf-dejavu/DejaVuSans.ttf", 24)
+
+def get_complement(r,g,b):
+    r = r/255.0
+    g = g/255.0
+    b = b/255.0
+    h,l,s = colorsys.rgb_to_hls(r, g, b)
+    if h > 0.5:
+        h -= 0.5
+    else:
+        h += 0.5
+    r,g,b = colorsys.hls_to_rgb(h, l, s)
+    return int(r*255), int(g*255), int(b*255)
+
+crc24_func = crcmod.mkCrcFun(0x1864CFBL) # polynomial from libgcrypt

defextract.py

@@ -8,8 +8,18 @@ from PIL import Image
 from collections import defaultdict
 import os
 
+def sanitize_filename(fname):
+    # find the first character outside range [32-126]
+    for i,c in enumerate(fname):
+        if ord(c) < 32 or ord(c) > 126:
+            break
+    return fname[:i]
+
 def extract_def(infile,outdir):
     f = open(infile)
+    bn = os.path.basename(infile)
+    bn = os.path.splitext(bn)[0]
+
     # t - type
     # blocks - # of blocks
     # the second and third entry are width and height which are not used
@@ -21,6 +31,7 @@ def extract_def(infile,outdir):
         palette.extend((r,g,b))
 
     offsets = defaultdict(list)
+    k = 0 # for naming bogus filename entries
     for i in range(blocks):
         # bid - block id
         # entries - number of images in this block
@@ -30,17 +41,23 @@ def extract_def(infile,outdir):
         # a list of 13 character long filenames
         for j in range(entries):
             name, = struct.unpack("13s", f.read(13))
-            name = name[:name.index('\0')]
+            name = sanitize_filename(name)
+            # if nothing remains, create bogus name
+            if len(name) == 0:
+                num = "%02d"%k
+                if len(bn)+len(num) > 9: # truncate name
+                    name = bn[:9-len(num)]+num
+                else:
+                    name = bn+num
+            k+=1
             names.append(name)
         # a list of offsets
         for n in names:
             offs, = struct.unpack("<I", f.read(4))
             offsets[bid].append((n,offs))
 
-    bn = os.path.basename(infile)
-    bn = os.path.splitext(bn)[0]
     for bid,l in offsets.items():
-        for n,offs in l:
+        for j,(n,offs) in enumerate(l):
             f.seek(offs)
             pixeldata = ""
             # first entry is the size which is unused
@@ -50,67 +67,74 @@ def extract_def(infile,outdir):
             # lm,tm - left and top margin
             _,fmt,fw,fh,w,h,lm,tm = struct.unpack("<IIIIIIii", f.read(32))
             n = os.path.splitext(n)[0]
-            outname = "%s"%outdir+os.sep+"%s_%02d_%s_%dx%d_%dx%d.png"%(bn,bid,n,fw,fh,lm,tm)
+            outname = "%s"%outdir+os.sep+"%02d_%s_%02d_%02d_%s_%dx%d_%dx%d.png"%(t,bn,bid,j,n,fw,fh,lm,tm)
             print "writing to %s"%outname
-            if fmt == 0:
-                # uncompressed, so the following *should* work but there is no
-                # test data
-                pixeldata = f.read(w*h)
-            elif fmt == 1:
-                # SGTWMTA.def and SGTWMTB.def fail here
-                # they have inconsistent left and top margins
-                # they seem to be unused
-                if lm > fw or tm > fh:
-                    print "margins (%dx%d) are higher than dimensions (%dx%d)"%(lm,tm,fw,fh)
-                    return False
-                lineoffs = struct.unpack("<"+"I"*h, f.read(4*h))
-                for lineoff in lineoffs:
-                    f.seek(offs+32+lineoff)
-                    totalrowlength=0
-                    while totalrowlength<w:
-                        code,length = struct.unpack("<BB", f.read(2))
-                        length+=1
-                        if code == 0xff: #raw data
-                            pixeldata += f.read(length)
-                        else: # rle
-                            pixeldata += length*chr(code)
-                        totalrowlength+=length
-            elif fmt == 2:
-                coff, = struct.unpack("<H", f.read(2))
-                coff += 32
-                f.seek(coff)
-                for i in range(h):
-                    totalrowlength=0
-                    while totalrowlength<w:
-                        segment, = struct.unpack("<B", f.read(1))
-                        code = segment/32
-                        length = (segment&31)+1
-                        if code == 7: # raw data
-                            pixeldata += f.read(length)
-                        else: # rle
-                            pixeldata += length*chr(code)
-                        totalrowlength+=length
-            elif fmt == 3:
-                # the first unsigned short in every w/16 byte block is the
-                # offset we want - the others have an unknown function
-                lineoffs = [struct.unpack("<"+"H"*(w/32), f.read(w/16))[0] for i in range(h)]
 
-                for lineoff in lineoffs:
+            if w != 0 and h != 0:
-                    f.seek(offs+32+lineoff)
+                if fmt == 0:
-                    totalrowlength=0
+                    pixeldata = f.read(w*h)
-                    while totalrowlength<w:
+                elif fmt == 1:
-                        segment, = struct.unpack("<B", f.read(1))
+                    # SGTWMTA.def and SGTWMTB.def fail here
-                        code = segment/32
+                    # they have inconsistent left and top margins
-                        length = (segment&31)+1
+                    # they seem to be unused
-                        if code == 7: # raw data
+                    if lm > fw or tm > fh:
-                            pixeldata += f.read(length)
+                        print "margins (%dx%d) are higher than dimensions (%dx%d)"%(lm,tm,fw,fh)
-                        else: # rle
+                        return False
-                            pixeldata += length*chr(code)
+                    lineoffs = struct.unpack("<"+"I"*h, f.read(4*h))
-                        totalrowlength+=length
+                    for lineoff in lineoffs:
-            else:
+                        f.seek(offs+32+lineoff)
-                print "unknown format: %d"%fmt
+                        totalrowlength=0
-                return False
+                        while totalrowlength<w:
-            im = Image.fromstring('P', (w,h),pixeldata)
+                            code,length = struct.unpack("<BB", f.read(2))
+                            length+=1
+                            if code == 0xff: #raw data
+                                pixeldata += f.read(length)
+                            else: # rle
+                                pixeldata += length*chr(code)
+                            totalrowlength+=length
+                elif fmt == 2:
+                    coff, = struct.unpack("<H", f.read(2))
+                    coff += 32
+                    f.seek(coff)
+                    for i in range(h):
+                        totalrowlength=0
+                        while totalrowlength<w:
+                            segment, = struct.unpack("<B", f.read(1))
+                            code = segment/32
+                            length = (segment&31)+1
+                            if code == 7: # raw data
+                                pixeldata += f.read(length)
+                            else: # rle
+                                pixeldata += length*chr(code)
+                            totalrowlength+=length
+                elif fmt == 3:
+                    # the first unsigned short in every w/16 byte block is the
+                    # offset we want - the others have an unknown function
+                    lineoffs = [struct.unpack("<"+"H"*(w/32), f.read(w/16))[0] for i in range(h)]
+
+                    for lineoff in lineoffs:
+                        f.seek(offs+32+lineoff)
+                        totalrowlength=0
+                        while totalrowlength<w:
+                            segment, = struct.unpack("<B", f.read(1))
+                            code = segment/32
+                            length = (segment&31)+1
+                            if code == 7: # raw data
+                                pixeldata += f.read(length)
+                            else: # rle
+                                pixeldata += length*chr(code)
+                            totalrowlength+=length
+                else:
+                    print "unknown format: %d"%fmt
+                    return False
+                im = Image.fromstring('P', (w,h),pixeldata)
+            else: # either width or height is zero
+                if w == 0:
+                    w = 1
+                if h == 0:
+                    h = 1
+                # TODO: encode this information correctly and dont create a fake 1px image
+                im = Image.new('P', (w,h))
             im.putpalette(palette)
             im.save(outname)
     return True
@@ -120,7 +144,7 @@ if __name__ == '__main__':
     if len(sys.argv) != 3:
         print "usage: %s input.def ./outdir"%sys.argv[0]
         print "to process all files:"
-        print "    for f in *.def; do n=`basename $f .def`; mkdir -p defs/$n; $s defextract.py $f defs/$n; done"%sys.argv[0]
+        print "    for f in *.def; do n=`basename $f .def`; mkdir -p defs/$n; %s defextract.py $f defs/$n; done"%sys.argv[0]
         exit(1)
     ret = extract_def(sys.argv[1], sys.argv[2])
     exit(0 if ret else 1)

lodextract.py

@@ -3,7 +3,9 @@
 import zlib
 import struct
 import os
-from PIL import Image
+from PIL import Image, ImageDraw
+
+from common import crc24_func, get_complement, font
 
 def is_pcx(data):
     size,width,height = struct.unpack("<III",data[:12])
@@ -25,7 +27,7 @@ def read_pcx(data):
     else:
         return None
 
-def unpack_lod(infile,outdir):
+def unpack_lod(infile,outdir,shred=True):
     f = open(infile)
 
     if f.read(4) != 'LOD\0':
@@ -54,6 +56,16 @@ def unpack_lod(infile,outdir):
         if is_pcx(data):
             im = read_pcx(data)
             if im:
+                if shred:
+                    crc = crc24_func(filename)
+                    r = crc>>16
+                    g = (crc&0xff00)>>8
+                    b = crc&0xff
+                    w,h = im.size
+                    im = Image.new("RGB", (w*3,h*3), (r,g,b))
+                    draw = ImageDraw.Draw(im)
+                    draw.text((0,0),os.path.basename(filename),get_complement(r,g,b),font=font)
+                    im = im.resize((w,h),Image.ANTIALIAS)
                 im.save(filename, "PNG")
             else:
                 return False

makedef.py

@@ -4,12 +4,18 @@ import os
 import re
 import struct
 from collections import defaultdict
-from PIL import Image
+from PIL import Image, ImageDraw
+from common import crc24_func, font
 
-def makedef(indir, outdir):
+def get_color(fname):
+    crc = crc24_func(fname)
+    # values 0-7 must not be used as they might represent transparency
+    # so we are left with 248 values 
+    return 8+crc%248
+
+def makedef(indir, outdir, shred=True):
     infiles = defaultdict(list)
     sig = None
-    fnames = set()
     # sanity checks and fill infiles dict
     for f in os.listdir(indir):
         m = re.match('(\d+)_([A-Za-z0-9_]+)_(\d\d)_(\d\d)_([A-Za-z0-9_]+)_(\d+)x(\d+)_(\d+)x(\d+).png', f)
@@ -22,7 +28,7 @@ def makedef(indir, outdir):
         if im.mode != 'P':
             print "input images must have a palette"
             return False
-        cursig =(t,p,fw,fh,im.getpalette())
+        cursig =(t,p,im.getpalette())
         if not sig:
             sig = cursig
         else:
@@ -31,30 +37,29 @@ def makedef(indir, outdir):
                 print sig
                 print cursig
                 return False
-        if fn in fnames:
-            print "duplicate filename: %s"%fn
-            return False
-        if w%16 != 0:
-            print "width must be divisible by 16"
-            return False
         if len(fn) > 9:
             print "filename can't be longer than 9 bytes"
             return False
-        fnames.add(fn)
         infiles[bid].append((im,t,p,j,fn,fw,fh,lm,tm))
 
+    if len(infiles) == 0:
+        print "no input files detected"
+        return False
+
     # check if j values for all bids are correct and sort them in j order in the process
     for bid in infiles:
         infiles[bid].sort(key=lambda t: t[3])
-        for k,(_,_,bid,j,_,_,_,_,_) in enumerate(infiles[bid]):
+        for k,(_,_,_,j,_,_,_,_,_) in enumerate(infiles[bid]):
             if k != j:
                 print "incorrect j value %d for bid %d should be %d"%(j,bid,k)
 
-    t,p,fw,fh,pal = cursig
+    t,p,pal = cursig
     outf = open(outdir+"/"+p+".def", "w+")
 
     # write the header
-    outf.write(struct.pack("<IIII", t,fw,fh,len(infiles)))
+    # full width and height are not used and not the same for all frames
+    # in some defs, so setting to zero
+    outf.write(struct.pack("<IIII", t,0,0,len(infiles)))
     # write the palette
     outf.write(struct.pack("768B", *pal))
 
@@ -65,6 +70,7 @@ def makedef(indir, outdir):
 
     for bid,l in infiles.items():
         # write bid and number of frames
+        # the last two values have unknown meaning
         outf.write(struct.pack("<IIII",bid,len(l),0,0))
         # write filenames
         for _,_,_,_,fn,_,_,_,_ in l:
@@ -73,17 +79,20 @@ def makedef(indir, outdir):
         for im,_,_,_,_,_,_,_,_ in l:
             outf.write(struct.pack("<I",curoffset))
             w,h = im.size
-            print curoffset
             # every image occupies one byte per pixel plus 32 byte header
             curoffset += w*h+32
 
     for bid,l in infiles.items():
-        for im,_,_,_,_,fw,fh,lm,tm in l:
+        for im,_,p,j,_,fw,fh,lm,tm in l:
             w,h = im.size
             outf.write(struct.pack("<IIIIIIii",w*h,0,fw,fh,w,h,lm,tm))
+            if shred:
+                im = Image.new("P", (w*3,h*3), get_color(p))
+                draw = ImageDraw.Draw(im)
+                draw.text((0,0),"%d%s"%(j,p),font=font)
+                im = im.resize((w,h),Image.ANTIALIAS)
             buf = ''.join([chr(i) for i in list(im.getdata())])
             outf.write(buf)
-            print outf.tell()
     return True
 
 if __name__ == '__main__':