Added README, definfo.py, shred.py

- added README.md
 - added definfo.py to retrieve information of DEF files
 - remove shredding option from defextract and lodextract
 - added shred.py which modifies PNG images or directories of them
 - support DEF files with different full width and height in individual
   frames (only ovslot.def)
 - removed common.py

README.md

@@ -0,0 +1,64 @@
+This is a set of scripts which shows how to unpack all bitmaps and animations
+of Heroes of Might and Magic 3 into PNG images and then back into the formats
+understood by VCMI.
+
+These scripts are probably the first open source implementation of a writer for
+the Heroes of Might and Magic 3 animation format called DEF. They are meant to
+make it possible for artists to create a free replacement for the proprietary
+assets VCMI currently needs.
+
+Install VCMI and then install original game files via any of the following methods:
+
+	vcmibuilder --cd1 /path/to/iso/or/cd --cd2 /path/to/second/cd --download
+	vcmibuilder --gog /path/to/gog.com/installer --download
+	vcmibuilder --data /path/to/h3/data --download
+
+Symlink sprites to Data directory
+
+	ln -s Data ~/.vcmi/Sprites
+
+Backup original archives:
+
+	mkdir ~/lods
+	for f in H3ab_bmp.lod H3ab_spr.lod H3bitmap.lod H3sprite.lod; do mv ~/.vcmi/Data/$f ~/lods; done
+	for f in hmm35wog.pac "wog - animated objects.pac" "wog - animated trees.pac" "wog - battle decorations.pac"; do mv ~/.vcmi/Mods/WoG/Data/"$f" ~/lods; done
+
+Extract archives:
+
+	for f in H3bitmap.lod H3sprite.lod H3ab_bmp.lod H3ab_spr.lod hmm35wog.pac "wog - animated objects.pac" "wog - animated trees.pac" "wog - battle decorations.pac"; do python lodextract.py ~/lods/"$f" ~/.vcmi/Data/; done
+
+Backup original DEFs:
+
+	mkdir ~/defs
+	mv ~/.vcmi/Data/*.def ~/defs
+	rm ~/defs/sgtwmta.def ~/defs/sgtwmtb.def # these are having higher offsets than size
+
+Extract all DEFs into JSON files and directories with PNG images:
+
+	for f in ~/defs/*; do python defextract.py $f ~/.vcmi/Data || break; done
+
+(optional) modify all frames:
+
+	for d in ~/.vcmi/Data/*.dir; do python shred.py $d || break; done
+
+(optional) modify all bitmaps:
+	for f in ~/.vcmi/Data/*.png; do python shred.py $f || break; done
+
+Repack all JSON:
+
+	for f in ~/.vcmi/Data/*.json; do python makedef.py $f ~/.vcmi/Data || break; done
+
+In case you followed the optional steps, enjoy your LSD infused game now :)
+
+After above steps you will have a mixture of DEF files as well as JSON
+files and their *.dir data directories. All parts of vcmi that support it will
+read the animations from the JSON files now. All others will fall back to
+reading the DEF files.
+
+You can now make changes to either the PNG images in the Data directory or in
+the *.dir subdirectories. If you make changes to PNG images in *.dir
+subdirectories you might have to repack them into DEF files for all animations
+which do not support JSON animations yet.
+
+I only tested these scripts on Linux because I do not own a license for Windows
+or MacOS. Patches welcome.

defextract.py

@@ -33,7 +33,7 @@ def get_color(fname):
     # so we are left with 248 values 
     return 8+crc%248
 
-def extract_def(infile,outdir,shred=True):
+def extract_def(infile,outdir):
     f = open(infile)
     bn = os.path.basename(infile)
     bn = os.path.splitext(bn)[0].lower()
@@ -63,10 +63,17 @@ def extract_def(infile,outdir,shred=True):
             offs, = struct.unpack("<I", f.read(4))
             offsets[bid].append(offs)
 
-    os.mkdir(os.path.join(outdir,"%s.dir"%bn))
+    outpath = os.path.join(outdir,"%s.dir"%bn)
+    if os.path.exists(outpath):
+        if not os.path.isdir(outpath):
+            print "output path exists and is no directory"
+            return False
+    else:
+        os.mkdir(outpath)
 
     out_json = {"sequences":[],"type":t,"format":-1}
 
+    firstfw,firstfh = -1,-1
     for bid,l in offsets.items():
         frames=[]
         for j,offs in enumerate(l):
@@ -81,6 +88,30 @@ def extract_def(infile,outdir,shred=True):
             outname = os.path.join(outdir,"%s.dir"%bn,"%02d_%02d.png"%(bid,j))
             print "writing to %s"%outname
 
+            # SGTWMTA.def and SGTWMTB.def fail here
+            # they have inconsistent left and top margins
+            # they seem to be unused
+            if lm > fw or tm > fh:
+                print "margins (%dx%d) are higher than dimensions (%dx%d)"%(lm,tm,fw,fh)
+                return False
+
+            if firstfw==-1 and firstfh == -1:
+                firstfw = fw
+                firstfh = fh
+            else:
+                if firstfw > fw:
+                    print "must enlarge width"
+                    fw = firstfw
+                if firstfw < fw:
+                    print "first with smaller than latter one"
+                    return False
+                if firstfh > fh:
+                    print "must enlarge height"
+                    fh = firstfh
+                if firstfh < fh:
+                    print "first height smaller than latter one"
+                    return False
+
             if out_json["format"] == -1:
                 out_json["format"] = fmt
             elif out_json["format"] != fmt:
@@ -93,12 +124,6 @@ def extract_def(infile,outdir,shred=True):
                 if fmt == 0:
                     pixeldata = f.read(w*h)
                 elif fmt == 1:
-                    # SGTWMTA.def and SGTWMTB.def fail here
-                    # they have inconsistent left and top margins
-                    # they seem to be unused
-                    if lm > fw or tm > fh:
-                        print "margins (%dx%d) are higher than dimensions (%dx%d)"%(lm,tm,fw,fh)
-                        return False
                     lineoffs = struct.unpack("<"+"I"*h, f.read(4*h))
                     for lineoff in lineoffs:
                         f.seek(offs+32+lineoff)
@@ -120,7 +145,6 @@ def extract_def(infile,outdir,shred=True):
                             f.seek(offs+32+lineoff)
                         totalrowlength=0
                         while totalrowlength<w:
-                            print f.tell()-32-offs
                             segment, = struct.unpack("<B", f.read(1))
                             code = segment>>5
                             length = (segment&0x1f)+1
@@ -155,32 +179,10 @@ def extract_def(infile,outdir,shred=True):
                 im = Image.fromstring('P', (w,h),pixeldata)
             else: # either width or height is zero
                 im = None
-            if im and shred:
-                #im = Image.new("P", (w*3,h*3), get_color(bn))
-                #draw = ImageDraw.Draw(im)
-                #tw,th = draw.textsize("%d%s"%(j,bn),font=font)
-                #draw.text(((w*3-tw)/2,(h*3-th)/2),"%d%s"%(j,bn),font=font)
-                #im = im.resize((w,h),Image.ANTIALIAS)
-
-                #pixels = im.load()
-                #color = get_color(bn)
-                #for i in range(w):
-                #    for j in range(h):
-                #        if pixels[i,j] > 7:
-                #            pixels[i,j] = color
-
-                color = get_color(bn)
-                import numpy as np
-                pixels = np.array(im)
-                pixels[pixels > 7] = color
-                im = Image.fromarray(pixels)
             imo = Image.new('P', (fw,fh))
             imo.putpalette(palette)
             if im:
                 imo.paste(im,(lm,tm))
-            #draw = ImageDraw.Draw(imo)
-            #tw,th = draw.textsize(bn,font=font)
-            #draw.text(((fw-tw)/2,(fh-th)/2),bn,255,font=font)
             imo.save(outname)
         out_json["sequences"].append({"group":bid,"frames":frames})
         with open(os.path.join(outdir,"%s.json"%bn),"w+") as o:

definfo.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2014  Johannes Schauer <j.schauer@email.de>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import struct
+from collections import defaultdict
+
+from common import sanitize_filename
+
+def main(infile):
+    f = open(infile)
+    t,_x,_y,blocks = struct.unpack("<IIII", f.read(16))
+    print "%d %d %d %d"%(t,_x,_y,blocks)
+    palette = []
+    for i in range(256):
+        r,g,b = struct.unpack("<BBB", f.read(3))
+        palette.extend((r,g,b))
+    print palette
+    offsets = defaultdict(list)
+    for i in range(blocks):
+        bid,entries,x_,y_ = struct.unpack("<IIII", f.read(16))
+        print bid,entries,x_,y_
+        names=[]
+        for j in range(entries):
+            name, = struct.unpack("13s", f.read(13))
+            name = sanitize_filename(name)
+            print j, name
+            names.append(name)
+        for n in names:
+            offs, = struct.unpack("<I", f.read(4))
+            offsets[bid].append((n,offs))
+    print "#\tnum\tsize\tformat\tfwidth\tfheight\twidth\theight\tlmargin\ttmargin"
+    for bid,l in offsets.items():
+        for j,(n,offs) in enumerate(l):
+            f.seek(offs)
+            s,fmt,fw,fh,w,h,lm,tm = struct.unpack("<IIIIIIii", f.read(32))
+            print "frame:\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d"%(j,s,fmt,fw,fh,w,h,lm,tm)
+
+if __name__ == '__main__':
+    import sys
+    if len(sys.argv) != 2:
+        print "usage: %s input.def"%sys.argv[0]
+        exit(1)
+    main(sys.argv[1])

lodextract.py

@@ -43,7 +43,7 @@ def read_pcx(data):
     else:
         return None
 
-def unpack_lod(infile,outdir,shred=True):
+def unpack_lod(infile,outdir):
     f = open(infile)
 
     header = f.read(4)
@@ -72,38 +72,14 @@ def unpack_lod(infile,outdir,shred=True):
             data = f.read(size)
         if is_pcx(data):
             im = read_pcx(data)
-            if im:
+            if not im:
-                if shred:
-                    crc = crc24_func(filename)
-                    r = crc>>16
-                    g = (crc&0xff00)>>8
-                    b = crc&0xff
-                    w,h = im.size
-                    pixels = im.load()
-                    for i in range(w):
-                        for j in range(h):
-                            if pixels[i,j] > 7:
-                                if im.mode == 'P':
-                                    pixels[i,j] = 8+crc%248
-                                else:
-                                    pixels[i,j] = (r,g,b)
-                    im.resize((w*3,h*3))
-                    draw = ImageDraw.Draw(im)
-                    tw,th = draw.textsize(os.path.basename(filename),font=font)
-                    tpos = ((w*3-tw)/2,(h*3-th)/2)
-                    if im.mode == 'P':
-                        # we can't really have a complement in palette mode, so just get some color
-                        draw.text(tpos,os.path.basename(filename),255,font=font)
-                    else:
-                        draw.text(tpos,os.path.basename(filename),get_complement(r,g,b),font=font)
-                    im = im.resize((w,h),Image.ANTIALIAS)
-                im.save(filename, "PNG")
-            else:
                 return False
+            filename = os.path.splitext(filename)[0]
+            filename = filename+".png"
+            im.save(filename)
         else:
-            o = open(filename,"w+")
+            with open(filename,"w+") as o:
                 o.write(data)
-            o.close()
 
     return True
 

shred.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+
+import numpy as np
+from PIL import Image
+import crcmod
+import os
+
+crc24_func = crcmod.mkCrcFun(0x1864CFBL) # polynomial from libgcrypt
+
+def handle_img(inf, color):
+    with open(inf) as f:
+        im = Image.open(f)
+        pal = im.getpalette()
+        pixels = np.array(im)
+    if pal:
+        pal[765], pal[766], pal[767] = color
+        pixels[pixels > 7] = 255
+        im = Image.fromarray(pixels)
+        im.putpalette(pal)
+    else:
+        # non-palette pictures have no transparency
+        im = Image.new('RGB', im.size, color)
+        # in case we ever want to replace colors in rgb images: 
+        #rc, gc, bc = pixels[:,:,0], pixels[:,:,1], pixels[:,:,2]
+        #mask = (rc == 0) & (gc == 255) & (bc == 255)
+        #pixels[:,:,:3][mask] = color
+    im.save(inf)
+
+def main(inf):
+    print "processing %s"%inf
+    crc = crc24_func(inf)
+    r = crc>>16
+    g = (crc&0xff00)>>8
+    b = crc&0xff
+    color = r%255,g%255,b%255 # avoid hitting special values
+    if os.path.isdir(inf):
+        for fname in os.listdir(inf):
+            fname = os.path.join(inf,fname)
+            handle_img(fname, color)
+    else:
+        handle_img(inf, color)
+    return True
+
+if __name__ == '__main__':
+    import sys
+    if len(sys.argv) != 2:
+        print "usage: %s indir/infile"
+        exit(0)
+    ret = main(sys.argv[1])
+    exit(0 if ret else 1)