Index: bitprocess.py
===================================================================
RCS file: /var/cvs/key_as_data/bitprocess.py,v
retrieving revision 1.11
retrieving revision 1.12
diff -u -r1.11 -r1.12
--- bitprocess.py	19 May 2008 20:33:57 -0000	1.11
+++ bitprocess.py	20 May 2008 21:04:56 -0000	1.12
@@ -138,7 +138,6 @@
 
 	def reorder_cols(self, i0):
 
-
 		res = dataset(self.max_x, self.max_y)
 		res.col_order = shuffle(self.col_order, i0)
 		res.row_order = copy(self.row_order)
@@ -147,6 +146,7 @@
 		isort = copy(i0)
 		isort.sort()
 
+		i0 = copy(i0)
 		for x in range(len(i0)):
 			idx[isort.pop(0)] = i0.pop(0)
 
@@ -166,6 +166,7 @@
 		isort = copy(i0)
 		isort.sort()
 
+		i0 = copy(i0)
 		for x in range(len(i0)):
 			idx[isort.pop(0)] = i0.pop(0)
 
@@ -192,7 +193,7 @@
 
 		return newd
 
-	def make_z_ratios(self, test_x, test_y, allow_err, exclude={}):
+	def make_z_ratios(self, test_x, test_y, max_height, allow_err, exclude={}):
 
 		test_height = self[test_x, test_y]
 		res = {}
@@ -210,22 +211,22 @@
 				if exclude.has_key(pi/2):
 					ex_x, ex_y = exclude[pi/2]
 					if ex_x is not None and x == ex_x and dist_x == 0:
-						print "excluding x", degrees(ang), x, y, test_x, test_y
+						print "excluding x", degrees(pi/2), x, y, test_x, test_y
 						continue
 				if exclude.has_key(0.0):
 					ex_x, ex_y = exclude[0.0]
 					if ex_y is not None and y == ex_y and dist_y == 0:
-						print "excluding y", degrees(ang), x, y, test_x, test_y
+						print "excluding y", degrees(0.0), x, y, test_x, test_y
 						continue
 
 				dist = sqrt(dist_x*dist_x + dist_y*dist_y)
-				height_diff = self[x,y] - test_height
+				height_diff = 100 * (self[x,y] - test_height) / max_height
 
-				ang = fabs(atan(height_diff / dist))
+				ang = abs(atan(height_diff / dist))
 
 				ang = float(int(ang/allow_err)*allow_err) # round to err range
 
-				#print x, y, ang
+				print test_x, test_y, x, y, self[x,y], ang
 
 				add_list_key(res, ang, (test_x, test_y, x, y))
 
@@ -233,13 +234,20 @@
 
 	def z_ratio_detect(self, allow_err, exclude={}):
 
+		max_height = 0
+		for x in range(0, self.max_x):
+			for y in range(0, self.max_y):
+				if not self.has_key((x, y)):
+					continue
+				max_height = max(self[x,y], max_height)
+
 		res = {}
 		for x in range(0, self.max_x):
 			for y in range(0, self.max_y):
 				if not self.has_key((x, y)):
 					continue
 
-				rats = self.make_z_ratios(x, y, allow_err, exclude)
+				rats = self.make_z_ratios(x, y, max_height, allow_err, exclude)
 				for ang, locs in rats.items():
 					for xy in locs:
 						add_list_key(res, ang, xy)
@@ -274,7 +282,7 @@
 			for (x1, y1, x2, y2) in vectors:
 				add_unique(res, angles, [(x1, y1)])
 				add_unique(res, angles, [(x2, y2)])
-				points = self.find_close_values(x1, y1, x2, y2, 0.5)
+				points = self.find_close_values(x1, y1, x2, y2, 0.01)
 				if not points:
 					continue
 				add_unique(res, angles, points)
@@ -320,6 +328,25 @@
 
 		return newd
 		
+	def sub_average(self, avg):
+		"""	
+		"""
+
+		newd = dataset(self.max_x, self.max_y)
+
+		for new_x in range(0, self.max_x):
+			for new_y in range(0, self.max_y):
+
+				if avg.has_key((new_x, new_y)):
+					total = avg[new_x, new_y]
+
+				newd[new_x, new_y] = self[new_x, new_y] - total
+
+		newd.row_order = copy(self.row_order)
+		newd.col_order = copy(self.col_order)
+
+		return newd
+
 	def average(self, block_x, block_y):
 
 		newd = dataset(self.max_x, self.max_y)
@@ -496,16 +523,7 @@
 		
 	os.chdir("graphs")
 
-	#fname = "rjd.71680.txt"
-	#fname = "cast-pval.256000.txt"
-	#fname = "rjd-pval.256000.txt"
-	fname = "des-pval.256000.txt"
-	#fname = "random.txt"
-	fname = "rjd.2560.txt"
-	#fname = "rjd.256000.txt"
-	#fname = "rjd-pval.255872.txt"
-	#fname = "rjd.2.5120.txt"
-	#fname = "rjd.5.256000.txt"
+	fname = "test.256.txt"
 	bitstep_x = 2
 	bitstep_y = 2
 
@@ -548,7 +566,7 @@
 	#			repeat=4, avg=avg)
 	d.write_file("remap.%dx%d.%s" % (bitstep_x, bitstep_y, fname))
 		
-	allow_err = pi/2/30
+	allow_err = pi/2/180
 	r = d.z_ratio_detect(allow_err)
 	r = theta_ratios_detect(r, allow_err)
 	#print_ratios(r, 4)
Index: data_xor.c
===================================================================
RCS file: /var/cvs/key_as_data/data_xor.c,v
retrieving revision 1.7
retrieving revision 1.8
diff -u -r1.7 -r1.8
--- data_xor.c	18 May 2008 21:05:19 -0000	1.7
+++ data_xor.c	20 May 2008 21:04:56 -0000	1.8
@@ -237,10 +237,9 @@
 	/* count the number of bits, create a p-value */
 	pval = data_test(test_data, sizeof(test_data));
 
-	if (pval < 0.01)
+	if (pval < 0.0005)
 		num_fails++;
 
-
 	/*printf("testval: %10.f pval: %f\n", testval, pval);*/
 	return num_fails;
 }
@@ -617,10 +616,13 @@
 
 	sprintf(fn, "%s-pval", fname);
 
-	if (interactive && !read_key)
+	if (interactive)
 	{
 		max_run = interactive_run;
+	}
 
+	if (interactive && !read_key)
+	{
 		for (i = 0; i < K_SZ*8; i++)
 		{
 			fscanf(stdin, "%f\n", &keyprob[i]);
@@ -640,7 +642,7 @@
 		double pval = erfc(f);
 		/*p_buckets[i] = pval;*/
 
-		printf("%.30f\n", pval);
+		printf("%d %.30f\n", i, pval);
 	}
 	exit(0);
 #endif
@@ -649,6 +651,7 @@
 
 	for (k = 0; k < max_run; k++) /* the more, the merrier */
 	{
+		int block_total_num_fails = 0;
 		if (interactive)
 		{
 			if (read_key)
@@ -715,13 +718,11 @@
 				 * p-value (see NIST SP800-22b.pdf)
 				 * fell belowout2 0.01 */
 				num_fails = analyse(baseline, input, key, xorbuf, xorkey, out);
-				/*printf("i: %d j: %d num_fails: %d/%d\n",
-						i, j, num_fails, attempts);*/
-
 				/* create a histogram of the failures, using
 				 * the key bit index and the data bit index */
 				bitbucket[i][j] += num_fails;
 				total += num_fails;
+				block_total_num_fails += num_fails;
 
 				if (num_fails)
 				{
@@ -739,7 +740,9 @@
 						pval_bitbucket[i][j] += 1;
 				}
 			}
+
 		}
+		printf("num_fails: %d\n", block_total_num_fails);
 		fprintf(stderr, "%d\n", k);
 		if ((k % RECORD_OUT != 0) || (k == 0))
 		{
@@ -796,6 +799,8 @@
 	print_sortedbucket(total);
 	print_ij_rnums();
 #endif
+	printf("done\n");
+	fflush(stdout);
 	return( 0 );
 }
 
Index: interact.py
===================================================================
RCS file: /var/cvs/key_as_data/interact.py,v
retrieving revision 1.3
diff -u -r1.3 interact.py
--- interact.py	19 May 2008 20:33:57 -0000	1.3
+++ interact.py	21 May 2008 19:15:43 -0000
@@ -3,6 +3,7 @@
 from telnetlib2 import TelnetPopen4
 from bitprocess import dataset, sorted_index, theta_ratios_detect, print_ratios
 from bitprocess import remove_row_from_ratios, remove_col_from_ratios
+from bitrotater import bitxormatrix
 from time import sleep
 from math import pi, degrees, radians
 import sys
@@ -30,7 +31,7 @@
 	return i | 1<<bit 
 
 def bit_is_set(i, bit):
-	return i & 1<<bit != 0
+	return i & (1<<bit) != 0
 
 def dump_data(d):
 
@@ -61,7 +62,7 @@
 def pre_encrypt(run_len, dt, key):
 
 	t = TelnetPopen4("./rjd_xor -k -e -i -o test -l %d" % run_len)
-	#t.set_debuglevel(1)
+	t.set_debuglevel(1)
 
 	from random import Random
 	r = Random()
@@ -81,6 +82,7 @@
 		dump_data(base)
 		baseline.append(str_to_int(base))
 
+	txt = t.read_until('done\n')
 	t.close()
 
 	return baseline
@@ -96,6 +98,7 @@
 
 	baseline = []
 	keys = []
+	num_fails = []
 
 	for d in range(run_len):
 
@@ -112,12 +115,18 @@
 		base = txt[:16]
 		print "base:",
 		dump_data(base)
+		txt = t.read_until('num_fails: ')
+		txt = t.read_until('\n').strip()
 		baseline.append(str_to_int(base))
 		keys.append(str_to_int(key))
+		num_fail = int(txt)
+		num_fails.append(num_fail)
+		print "num_fails:", num_fail
 
+	txt = t.read_until('done\n')
 	t.close()
 
-	return baseline, keys
+	return baseline, keys, num_fails
 
 def test_k(run_len, k_prob, dt, keys):
 
@@ -143,24 +152,27 @@
 		#dump_data(key)
 		print txt
 
+	txt = t.read_until('done\n')
 	t.close()
 
 def process_data(ds, row_num, col_num):
 
 	d = ds.average(16, 16)
+	#da = ds.sub_average(d)
+	da = ds.std_dev()
 	d = d.std_dev()
 
 	if col_num is not None:
 		c0 = d.select_col(col_num)
 		i0 = sorted_index(c0)
 		d = d.reorder_cols(i0)
+		#da = da.reorder_cols(i0)
 	
 	if row_num is not None:
 		r0 = d.select_row(row_num)
-		print r0
 		i0 = sorted_index(r0)
-		print i0
 		d = d.reorder_rows(i0)
+		#da = da.reorder_rows(i0)
 
 	yo = reverse_order(d.row_order)
 	xo = reverse_order(d.col_order)
@@ -181,7 +193,7 @@
 	r = theta_ratios_detect(r, allow_err)
 	r = d.find_lines(r)
 
-	return d, r
+	return d, da, r
 
 def reverse_order(order):
 	"""	the col order and row order represent where the data has been
@@ -201,14 +213,14 @@
 		ro[ksorted] = data[krev]
 	return ro
 
-def process_ratios(kk, d, r, dt, baseline, keys, min_vector_len):
+def process_ratios(kk, da, d, r, dt, blkz, min_vector_len):
 
-	kp = [0.0] * 128
+	kp = {}
+	for i in range(128):
+		kp[i] = 0.0
 
 	k_shift = 1e-5
 
-	blkz = zip(baseline, keys)
-
 	yo = reverse_order(d.row_order)
 	xo = reverse_order(d.col_order)
 
@@ -216,36 +228,70 @@
 		if len(vectors) < min_vector_len:
 			continue
 		print vectors
-		for x, y in vectors:
+		for i in range(len(vectors)-1):
+			x, y = vectors[i]
+			x2, y2 = vectors[i+1]
 
 			avgs_x = yo[x]
 			avgs_y = yo[y]
 
-			for ax in avgs_x:
-				for ay in avgs_y:
-					for (base, key) in blkz:
-						if bit_is_set(key, ay) == bit_is_set(base, ax):
-							kp[ay] += k_shift
+			avgs_x2 = yo[x2]
+			avgs_y2 = yo[y2]
+
+			for ix in range(len(avgs_x)):
+				for iy in range(len(avgs_y)):
+					ax = avgs_x[ix]
+					ay = avgs_y[iy]
+					ax2 = avgs_x2[ix]
+					ay2 = avgs_y2[iy]
+
+					for bk in blkz:
+						kdelta = k_shift #* da[ax, ay]
+						if bk.bit_eq(ay, ax) == bk.bit_eq(ay2, ax2):
+							kp[ay] += kdelta
 						else:
-							kp[ay] -= k_shift
+							kp[ay] -= kdelta
 
 		print_k_prob(kk, kp)
-	return kp
+
+	k = []
+	for i in range(128):
+		k.append(kp[i])
+	return k
 
 def print_k_prob(key, kp):
 
 	tot = 0
-	for (n, r) in zip(range(len(kp)), map(lambda x: "%.2g" % x, kp)):
+	for n in range(len(kp)):
+		kpn = kp[n]
 		bk = bit_is_set(key, n)
-		b = cmp(0.5, bk)
-		b = cmp(r, b*r)
-		print "%d %d %d %s" % (n, b, bk, repr(r)),
+		if bk:
+			b = kpn > 0
+		else:
+			b = kpn < 0
+		print "%d %d %d %.3g   " % (n, b, bk, kpn),
 		if n % 4 == 3:
 			print
 		if b:
 			tot += 1
 	print tot
 
+def print_last_kprob(key, kp):
+
+	f = open("results.txt", "w")
+	tot = 0
+	for (n, kpn) in zip(range(len(kp)), kp):
+		bk = bit_is_set(key, n)
+		if bk:
+			b = kpn > 0
+		else:
+			b = kpn < 0
+		f.write("%d %.2g (%d %d)\n" % (n, kpn, b, bk))
+		if b:
+			tot += 1
+	f.write("%d\n" % tot)
+	f.close()
+
 def test():
 
 	run_len = 512
@@ -253,7 +299,7 @@
 
 	# encrypt data with pre-arranged key, which we're going to then
 	# try and 'find'.
-	key = str_to_int('e' * 16)
+	key = str_to_int('0123456789abcdef')
 	dt_e = pre_encrypt(run_len, dt, key)
 
 	# start off with probabilities 0.5 for all key bits - i.e. we ain't
@@ -263,23 +309,30 @@
 	# ok - hand the encrypted data, and the probabilities, to the
 	# stats-analyser.  receive back each weighted-probability-generated
 	# key and the decrypted data block.
-	baseline, keys = test_decrypt(run_len, k_prob, dt_e)
+	baseline, keys, num_fails = test_decrypt(run_len, k_prob, dt_e)
+
+	blkz = []
+	for base, key in zip(baseline, keys):
+		blkz.append(bitxormatrix(key, base, 128, 128))
 
 	# now perform analysis
+	kp_sum = [0.0] * 128
 	ds = dataset()
-	ds.read_file("graphs/test-pval.%d.txt" % run_len)
-	#d, r = process_data(ds, 32, None)
-	d, r = process_data(ds, None, 16)
-
-	kp = process_ratios(key, d, r, dt, baseline, keys, 8)
-
-	print_k_prob(key, kp)
+	ds.read_file("graphs/test.%d.txt" % run_len)
+	for i in range(0, 128, 16):
 
-	#print_ratios(r, 8)
-	#remove_row_from_ratios(r, 32)
-	#remove_col_from_ratios(r, 16)
+		d, da, r = process_data(ds, None, i)
+		remove_row_from_ratios(r, i)
 
-	#test_k(run_len, k_prob, dt, keys)
+		kp = process_ratios(key, da, d, r, dt, blkz, 8)
+		kp_sum = map(lambda (x,y): x+y, zip(kp_sum, kp))
+		print_last_kprob(key, kp_sum)
+
+		d, da, r = process_data(ds, i, None)
+		remove_col_from_ratios(r, i)
+		kp = process_ratios(key, da, d, r, dt, blkz, 8)
+		kp_sum = map(lambda (x,y): x+y, zip(kp_sum, kp))
+		print_last_kprob(key, kp_sum)
 
 	
 def test_str_convert():
Index: scatter.dem
===================================================================
RCS file: /var/cvs/key_as_data/scatter.dem,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -r1.3 -r1.4
--- scatter.dem	18 May 2008 21:05:19 -0000	1.3
+++ scatter.dem	20 May 2008 21:04:56 -0000	1.4
@@ -1,5 +1,5 @@
 #
-# $Id: scatter.dem,v 1.3 2008-05-18 21:05:19 lkcl Exp $
+# $Id: scatter.dem,v 1.4 2008-05-20 21:04:56 lkcl Exp $
 #
 # Simple demo of scatter data conversion to grid data.
 #
@@ -49,7 +49,7 @@
 #splot "graphs/remap.2x2.random.txt" 
 #splot "graphs/remap.2x2.serpent-pval.256000.txt" 
 #splot "graphs/remap.2x2.cast-pval.256000.txt" 
-splot "graphs/remap.2x2.rjd.2560.txt" 
+splot "graphs/remap.2x2.test.256.txt" 
 #splot "graphs/remap.2x2.blowfisn-pval.15360.txt" 
 #splot "graphs/remap.2x2.des-pval.256000.txt" 
 #splot "graphs/remap.2x2.serpent.256000.txt"