From d1f101c36ac77651a187b7716d96286f92953c0a Mon Sep 17 00:00:00 2001 From: Johannes 'josch' Schauer Date: Tue, 12 Mar 2019 03:07:43 +0100 Subject: [PATCH] test.sh: replace imagemagick with custom python script to produce bit-by-bit identical results on all architectures --- magick.py | 306 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ test.sh | 58 ++++------- 2 files changed, 324 insertions(+), 40 deletions(-) create mode 100644 magick.py diff --git a/magick.py b/magick.py new file mode 100644 index 0000000..b3dcc18 --- /dev/null +++ b/magick.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python3 + +import sys +import numpy +import scipy.signal +import zlib +import struct + + +def find_closest_palette_color(color, palette): + if color.ndim == 0: + idx = (numpy.abs(palette - color)).argmin() + else: + # naive distance function by computing the euclidean distance in RGB space + idx = ((palette - color) ** 2).sum(axis=-1).argmin() + return palette[idx] + + +def floyd_steinberg(img, palette): + for y in range(img.shape[0]): + for x in range(img.shape[1]): + oldpixel = img[y, x] + newpixel = find_closest_palette_color(oldpixel, palette) + quant_error = oldpixel - newpixel + img[y, x] = newpixel + if x + 1 < img.shape[1]: + img[y, x + 1] += quant_error * 7 / 16 + if y + 1 < img.shape[0]: + img[y + 1, x - 1] += quant_error * 3 / 16 + img[y + 1, x] += quant_error * 5 / 16 + if x + 1 < img.shape[1] and y + 1 < img.shape[0]: + img[y + 1, x + 1] += quant_error * 1 / 16 + return img + + +def convolve_rgba(img, kernel): + return numpy.stack( + ( + scipy.signal.convolve2d(img[:, :, 0], kernel, "same"), + scipy.signal.convolve2d(img[:, :, 1], kernel, "same"), + scipy.signal.convolve2d(img[:, :, 2], kernel, "same"), + scipy.signal.convolve2d(img[:, :, 3], kernel, "same"), + ), + axis=-1, + ) + + +def rgb2gray(img): + result = numpy.zeros((60, 60), dtype=numpy.dtype("int64")) + for y in range(img.shape[0]): + for x in range(img.shape[1]): + clin = sum(img[y, x] * [0.2126, 0.7152, 0.0722]) / 0xFFFF + if clin <= 0.0031308: + csrgb = 12.92 * clin + else: + csrgb = 1.055 * clin ** (1 / 2.4) - 0.055 + result[y, x] = csrgb * 0xFFFF + return result + + +def palettize(img, pal): + result = numpy.zeros((img.shape[0], img.shape[1]), dtype=numpy.dtype("int64")) + for y in range(img.shape[0]): + for x in range(img.shape[1]): + for i, col in enumerate(pal): + if numpy.array_equal(img[y, x], col): + result[y, x] = i + break + else: + raise Exception() + return result + + +def write_png(data, path, bitdepth, colortype, palette=None): + with open(path, "wb") as f: + f.write(b"\x89PNG\r\n\x1A\n") + # PNG image type Colour type Allowed bit depths + # Greyscale 0 1, 2, 4, 8, 16 + # Truecolour 2 8, 16 + # Indexed-colour 3 1, 2, 4, 8 + # Greyscale with alpha 4 8, 16 + # Truecolour with alpha 6 8, 16 + block = b"IHDR" + struct.pack( + ">IIBBBBB", + data.shape[1], # width + data.shape[0], # height + bitdepth, # bitdepth + colortype, # colortype + 0, # compression + 0, # filtertype + 0, # interlaced + ) + f.write( + struct.pack(">I", len(block) - 4) + + block + + struct.pack(">I", zlib.crc32(block)) + ) + if palette is not None: + block = b"PLTE" + for col in palette: + block += struct.pack(">BBB", col[0], col[1], col[2]) + f.write( + struct.pack(">I", len(block) - 4) + + block + + struct.pack(">I", zlib.crc32(block)) + ) + raw = b"" + for y in range(data.shape[0]): + raw += b"\0" + if bitdepth == 16: + raw += data[y].astype(">u2").tobytes() + elif bitdepth == 8: + raw += data[y].astype(">u1").tobytes() + elif bitdepth in [4, 2, 1]: + valsperbyte = 8 // bitdepth + for x in range(0, data.shape[1], valsperbyte): + val = 0 + for j in range(valsperbyte): + if x + j >= data.shape[1]: + break + val |= (data[y, x + j].astype(">u2") & (2 ** bitdepth - 1)) << ( + (valsperbyte - j - 1) * bitdepth + ) + raw += struct.pack(">B", val) + else: + raise Exception() + compressed = zlib.compress(raw) + block = b"IDAT" + compressed + f.write( + struct.pack(">I", len(compressed)) + + block + + struct.pack(">I", zlib.crc32(block)) + ) + block = b"IEND" + f.write(struct.pack(">I", 0) + block + struct.pack(">I", zlib.crc32(block))) + + +def main(): + outdir = sys.argv[1] + + # create a 256 color palette by first writing 16 shades of gray + # and then writing an array of RGB colors with 6, 8 and 5 levels + # for red, green and blue, respectively + pal8 = numpy.zeros((256, 3), dtype=numpy.dtype("int64")) + i = 0 + for gray in range(15, 255, 15): + pal8[i] = [gray, gray, gray] + i += 1 + for red in 0, 0x33, 0x66, 0x99, 0xCC, 0xFF: + for green in 0, 0x24, 0x49, 0x6D, 0x92, 0xB6, 0xDB, 0xFF: + for blue in 0, 0x40, 0x80, 0xBF, 0xFF: + pal8[i] = [red, green, blue] + i += 1 + assert i == 256 + + # windows 16 color palette + pal4 = numpy.array( + [ + [0x00, 0x00, 0x00], + [0x80, 0x00, 0x00], + [0x00, 0x80, 0x00], + [0x80, 0x80, 0x00], + [0x00, 0x00, 0x80], + [0x80, 0x00, 0x80], + [0x00, 0x80, 0x80], + [0xC0, 0xC0, 0xC0], + [0x80, 0x80, 0x80], + [0xFF, 0x00, 0x00], + [0x00, 0xFF, 0x00], + [0xFF, 0x00, 0x00], + [0x00, 0xFF, 0x00], + [0xFF, 0x00, 0xFF], + [0x00, 0xFF, 0x00], + [0xFF, 0xFF, 0xFF], + ], + dtype=numpy.dtype("int64"), + ) + + # choose values slightly off red, lime and blue because otherwise + # imagemagick will classify the image as Depth: 8/1-bit + pal2 = numpy.array( + [[0, 0, 0], [0xFE, 0, 0], [0, 0xFE, 0], [0, 0, 0xFE]], + dtype=numpy.dtype("int64"), + ) + + # don't choose black and white or otherwise imagemagick will classify the + # image as bilevel with 8/1-bit depth instead of palette with 8-bit color + # don't choose gray colors or otherwise imagemagick will classify the + # image as grayscale + pal1 = numpy.array( + [[0x01, 0x02, 0x03], [0xFE, 0xFD, 0xFC]], dtype=numpy.dtype("int64") + ) + + # gaussian kernel with sigma=3 + kernel = numpy.array( + [ + [0.011362, 0.014962, 0.017649, 0.018648, 0.017649, 0.014962, 0.011362], + [0.014962, 0.019703, 0.02324, 0.024556, 0.02324, 0.019703, 0.014962], + [0.017649, 0.02324, 0.027413, 0.028964, 0.027413, 0.02324, 0.017649], + [0.018648, 0.024556, 0.028964, 0.030603, 0.028964, 0.024556, 0.018648], + [0.017649, 0.02324, 0.027413, 0.028964, 0.027413, 0.02324, 0.017649], + [0.014962, 0.019703, 0.02324, 0.024556, 0.02324, 0.019703, 0.014962], + [0.011362, 0.014962, 0.017649, 0.018648, 0.017649, 0.014962, 0.011362], + ], + numpy.float, + ) + + # constructs a 2D array of a circle with a width of 36 + circle = list() + offsets_36 = [14, 11, 9, 7, 6, 5, 4, 3, 3, 2, 2, 1, 1, 1, 0, 0, 0, 0] + for offs in offsets_36 + offsets_36[::-1]: + circle.append([0] * offs + [1] * (len(offsets_36) - offs) * 2 + [0] * offs) + + alpha = numpy.zeros((60, 60, 4), dtype=numpy.dtype("int64")) + + # draw three circles + for (xpos, ypos, color) in [ + (12, 3, [0xFFFF, 0, 0, 0xFFFF]), + (21, 21, [0, 0xFFFF, 0, 0xFFFF]), + (3, 21, [0, 0, 0xFFFF, 0xFFFF]), + ]: + for x, row in enumerate(circle): + for y, pos in enumerate(row): + if pos: + alpha[y + ypos, x + xpos] += color + alpha = numpy.clip(alpha, 0, 0xFFFF) + alpha = convolve_rgba(alpha, kernel) + + write_png(alpha, outdir + "/alpha.png", 16, 6) + + normal16 = alpha[:, :, 0:3] + write_png(normal16, outdir + "/normal16.png", 16, 2) + + write_png(normal16 / 0xFFFF * 0xFF, outdir + "/normal.png", 8, 2) + + write_png(0xFF - normal16 / 0xFFFF * 0xFF, outdir + "/inverse.png", 8, 2) + + gray16 = rgb2gray(normal16) + + write_png(gray16, outdir + "/gray16.png", 16, 0) + + write_png(gray16 / 0xFFFF * 0xFF, outdir + "/gray8.png", 8, 0) + + write_png( + floyd_steinberg(gray16, numpy.arange(16) / 0xF * 0xFFFF) / 0xFFFF * 0xF, + outdir + "/gray4.png", + 4, + 0, + ) + + write_png( + floyd_steinberg(gray16, numpy.arange(4) / 0x3 * 0xFFFF) / 0xFFFF * 0x3, + outdir + "/gray2.png", + 2, + 0, + ) + + write_png( + floyd_steinberg(gray16, numpy.arange(2) / 0x1 * 0xFFFF) / 0xFFFF * 0x1, + outdir + "/gray1.png", + 1, + 0, + ) + + write_png( + palettize( + floyd_steinberg(normal16, pal8 * 0xFFFF / 0xFF) / 0xFFFF * 0xFF, pal8 + ), + outdir + "/palette8.png", + 8, + 3, + pal8, + ) + + write_png( + palettize( + floyd_steinberg(normal16, pal4 * 0xFFFF / 0xFF) / 0xFFFF * 0xFF, pal4 + ), + outdir + "/palette4.png", + 4, + 3, + pal4, + ) + + write_png( + palettize( + floyd_steinberg(normal16, pal2 * 0xFFFF / 0xFF) / 0xFFFF * 0xFF, pal2 + ), + outdir + "/palette2.png", + 2, + 3, + pal2, + ) + + write_png( + palettize( + floyd_steinberg(normal16, pal1 * 0xFFFF / 0xFF) / 0xFFFF * 0xFF, pal1 + ), + outdir + "/palette1.png", + 1, + 3, + pal1, + ) + + +main() diff --git a/test.sh b/test.sh index 5b34a30..013a041 100755 --- a/test.sh +++ b/test.sh @@ -90,48 +90,26 @@ tempdir=$(mktemp --directory --tmpdir img2pdf.XXXXXXXXXX) trap error EXIT -# we use -strip to remove all timestamps (tIME chunk and exif data) -convert -size 60x60 \( xc:none -fill red -draw 'circle 30,21 30,3' -gaussian-blur 0x3 \) \ - \( \( xc:none -fill lime -draw 'circle 39,39 36,57' -gaussian-blur 0x3 \) \ - \( xc:none -fill blue -draw 'circle 21,39 24,57' -gaussian-blur 0x3 \) \ - -compose plus -composite \ - \) -compose plus -composite \ - -strip \ - "$tempdir/alpha.png" - -convert "$tempdir/alpha.png" -background black -alpha remove -alpha off -strip "$tempdir/normal16.png" - -convert "$tempdir/normal16.png" -depth 8 -strip "$tempdir/normal.png" - -convert "$tempdir/normal.png" -negate -strip "$tempdir/inverse.png" - -convert "$tempdir/normal16.png" -colorspace Gray -depth 16 -strip "$tempdir/gray16.png" -convert "$tempdir/normal16.png" -colorspace Gray -dither FloydSteinberg -colors 256 -depth 8 -strip "$tempdir/gray8.png" -convert "$tempdir/normal16.png" -colorspace Gray -dither FloydSteinberg -colors 16 -depth 4 -strip "$tempdir/gray4.png" -convert "$tempdir/normal16.png" -colorspace Gray -dither FloydSteinberg -colors 4 -depth 2 -strip "$tempdir/gray2.png" -convert "$tempdir/normal16.png" -colorspace Gray -dither FloydSteinberg -colors 2 -depth 1 -strip "$tempdir/gray1.png" - -# use "-define png:exclude-chunk=bkgd" because otherwise, imagemagick will -# add the background color (white) as an additional entry to the palette -convert "$tempdir/normal.png" -dither FloydSteinberg -colors 2 -define png:exclude-chunk=bkgd -strip "$tempdir/palette1.png" -convert "$tempdir/normal.png" -dither FloydSteinberg -colors 4 -define png:exclude-chunk=bkgd -strip "$tempdir/palette2.png" -convert "$tempdir/normal.png" -dither FloydSteinberg -colors 16 -define png:exclude-chunk=bkgd -strip "$tempdir/palette4.png" -convert "$tempdir/normal.png" -dither FloydSteinberg -colors 256 -define png:exclude-chunk=bkgd -strip "$tempdir/palette8.png" +# instead of using imagemagick to craft the test input, we use a custom python +# script. This is because the output of imagemagick is not bit-by-bit identical +# across versions and architectures. +# See https://gitlab.mister-muffin.de/josch/img2pdf/issues/56 +python3 magick.py "$tempdir" cat << END | ( cd "$tempdir"; md5sum --check --status - ) -a99ef2a356c315090b6939fa4ce70516 alpha.png -0df21ebbce5292654119b17f6e52bc81 gray16.png -6faee81b8db446caa5004ad71bddcb5b gray1.png -97e423da517ede069348484a1283aa6c gray2.png -cbed1b6da5183aec0b86909e82b77c41 gray4.png -c0df42fdd69ae2a16ad0c23adb39895e gray8.png -ac6bb850fb5aaee9fa7dcb67525cd0fc inverse.png -3f3f8579f5054270e79a39e7cc4e89e0 normal16.png -cbe63b21443af8321b213bde6666951f normal.png -2f00705cca05fd94406fc39ede4d7322 palette1.png -6cb250d1915c2af99c324c43ff8286eb palette2.png -ab7b3d3907a851692ee36f5349ed0b2c palette4.png -03829af4af8776adf56ba2e68f5b111e palette8.png +7ed200c092c726c68e889514fff0d8f1 alpha.png +bf56e00465b98fb738f6edd2c58dac3b gray16.png +f93c3e3c11dad3f8c11db4fd2d01c2cc gray1.png +d63167c66e8a65bd5c15f68c8d554c48 gray2.png +6bceb845d9c9946adad1526954973945 gray4.png +7ba8152b9146eb7d9d50529189baf7db gray8.png +75130ec7635919e40c7396d45899ddbe inverse.png +bd1a2c9f9dfc51a827eafa3877cf7f83 normal16.png +329eac79fec2e1bc30d7a50ba2e3f2a5 normal.png +9ffd3f592b399f9f9c23892db82369cd palette1.png +6d3a39fe5f2efea5975f048d11a5cb02 palette2.png +57add39e5c278249b64ab23314a41c39 palette4.png +192a3b298d812a156e6f248238d2bb52 palette8.png END # use img2pdfprog environment variable if it is set