From 708e8e07b23be2db7e1dda0a519bdceebe8f211e Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Wed, 14 Feb 2018 09:26:06 +0100 Subject: [PATCH 01/34] Test added --- src/Test.java | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 src/Test.java diff --git a/src/Test.java b/src/Test.java new file mode 100644 index 0000000..4999b48 --- /dev/null +++ b/src/Test.java @@ -0,0 +1,31 @@ +import edu.hitsz.c102c.cnn.CNN; +import edu.hitsz.c102c.cnn.Layer; +import edu.hitsz.c102c.dataset.Dataset; + +/** + *

+ * Created: 14.02.2018 09:06 + * + * @author Ralf Th. Pietsch <ratopi@abwesend.de> + */ +public class Test +{ + public static void main( final String[] args ) + { + final CNN.LayerBuilder builder = new CNN.LayerBuilder(); + builder.addLayer( Layer.buildInputLayer( new Layer.Size( 28, 28 ) ) ); + builder.addLayer( Layer.buildConvLayer( 6, new Layer.Size( 5, 5 ) ) ); + builder.addLayer( Layer.buildSampLayer( new Layer.Size( 2, 2 ) ) ); + builder.addLayer( Layer.buildConvLayer( 12, new Layer.Size( 5, 5 ) ) ); + builder.addLayer( Layer.buildSampLayer( new Layer.Size( 2, 2 ) ) ); + builder.addLayer( Layer.buildOutputLayer( 10 ) ); + final CNN cnn = new CNN( builder, 50 ); + + final String fileName = "dataset/train.format"; + final Dataset dataset = Dataset.load( fileName, ",", 784 ); + cnn.train( dataset, 100 ); + + final Dataset testset = Dataset.load( "dataset/test.format", ",", -1 ); + cnn.predict( testset, "dataset/test.predict" ); + } +} From 96f0500e597f5c18296bc38f1d321e7bc070acd5 Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Thu, 15 Feb 2018 11:16:36 +0100 Subject: [PATCH 02/34] typo : lable -> label --- src/edu/hitsz/c102c/cnn/CNN.java | 162 +++++++++++------------ src/edu/hitsz/c102c/dataset/Dataset.java | 70 +++++----- 2 files changed, 115 insertions(+), 117 deletions(-) diff --git a/src/edu/hitsz/c102c/cnn/CNN.java b/src/edu/hitsz/c102c/cnn/CNN.java index 096ba73..868cb9e 100644 --- a/src/edu/hitsz/c102c/cnn/CNN.java +++ b/src/edu/hitsz/c102c/cnn/CNN.java @@ -29,31 +29,31 @@ public class CNN implements Serializable { private static final long serialVersionUID = 337920299147929932L; private static double ALPHA = 0.85; protected static final double LAMBDA = 0; - // ĸ + // ����ĸ��� private List layers; - // + // ���� private int layerNum; - // µĴС + // �������µĴ�С private int batchSize; - // ԾÿһԪسһֵ + // �������������Ծ����ÿһ��Ԫ�س���һ��ֵ private Operator divide_batchSize; - // ԾÿһԪسalphaֵ + // �������������Ծ����ÿһ��Ԫ�س���alphaֵ private Operator multiply_alpha; - // ԾÿһԪس1-labmda*alphaֵ + // �������������Ծ����ÿһ��Ԫ�س���1-labmda*alphaֵ private Operator multiply_lambda; /** - * ʼ + * ��ʼ������ * * @param layerBuilder - * + * ����� * @param inputMapSize - * mapĴС + * ����map�Ĵ�С * @param classNum - * ĸҪݼתΪ0-classNum-1ֵ + * ���ĸ�����Ҫ�����ݼ������ת��Ϊ0-classNum-1����ֵ */ public CNN(LayerBuilder layerBuilder, final int batchSize) { layers = layerBuilder.mLayers; @@ -64,7 +64,7 @@ public CNN(LayerBuilder layerBuilder, final int batchSize) { } /** - * ʼ + * ��ʼ�������� */ private void initPerator() { divide_batchSize = new Operator() { @@ -102,19 +102,19 @@ public double process(double value) { } /** - * ѵѵ + * ��ѵ������ѵ������ * * @param trainset * @param repeat - * Ĵ + * �����Ĵ��� */ public void train(Dataset trainset, int repeat) { - // ֹͣť + // ����ֹͣ��ť new Lisenter().start(); for (int t = 0; t < repeat && !stopTrain.get(); t++) { int epochsNum = trainset.size() / batchSize; if (trainset.size() % batchSize != 0) - epochsNum++;// ȡһΣȡ + epochsNum++;// ���ȡһ�Σ�������ȡ�� Log.i(""); Log.i(t + "th iter epochsNum:" + epochsNum); int right = 0; @@ -131,7 +131,7 @@ public void train(Dataset trainset, int repeat) { Layer.prepareForNewRecord(); } - // һbatchȨ + // ����һ��batch�����Ȩ�� updateParas(); if (i % 50 == 0) { System.out.print(".."); @@ -140,7 +140,7 @@ public void train(Dataset trainset, int repeat) { } } double p = 1.0 * right / count; - if (t % 10 == 1 && p > 0.96) {//̬׼ѧϰ + if (t % 10 == 1 && p > 0.96) {//��̬����׼ѧϰ���� ALPHA = 0.001 + ALPHA * 0.9; Log.i("Set alpha = " + ALPHA); } @@ -176,7 +176,7 @@ public void run() { } /** - * + * �������� * * @param trainset * @return @@ -204,7 +204,7 @@ public double test(Dataset trainset) { } /** - * Ԥ + * Ԥ���� * * @param testset * @param fileName @@ -227,13 +227,13 @@ public void predict(Dataset testset, String fileName) { double[][] outmap = outputLayer.getMap(m); out[m] = outmap[0][0]; } - // int lable = + // int label = // Util.binaryArray2int(out); - int lable = Util.getMaxIndex(out); - // if (lable >= max) - // lable = lable - (1 << (out.length - + int label = Util.getMaxIndex(out); + // if (label >= max) + // label = label - (1 << (out.length - // 1)); - writer.write(lable + "\n"); + writer.write(label + "\n"); } writer.flush(); writer.close(); @@ -255,7 +255,7 @@ private boolean isSame(double[] output, double[] target) { } /** - * ѵһ¼ͬʱǷԤȷǰ¼ + * ѵ��һ����¼��ͬʱ�����Ƿ�Ԥ����ȷ��ǰ��¼ * * @param record * @return @@ -268,7 +268,7 @@ private boolean train(Record record) { } /* - * + * ������ */ private boolean backPropagation(Record record) { boolean result = setOutLayerErrors(record); @@ -277,7 +277,7 @@ private boolean backPropagation(Record record) { } /** - * ² + * ���²��� */ private void updateParas() { for (int l = 1; l < layerNum; l++) { @@ -296,7 +296,7 @@ private void updateParas() { } /** - * ƫ + * ����ƫ�� * * @param layer * @param lastLayer @@ -311,7 +311,7 @@ private void updateBias(final Layer layer, Layer lastLayer) { public void process(int start, int end) { for (int j = start; j < end; j++) { double[][] error = Util.sum(errors, j); - // ƫ + // ����ƫ�� double deltaBias = Util.sum(error) / batchSize; double bias = layer.getBias(j) + ALPHA * deltaBias; layer.setBias(j, bias); @@ -322,12 +322,12 @@ public void process(int start, int end) { } /** - * layerľˣȨأƫ + * ����layer��ľ���ˣ�Ȩ�أ���ƫ�� * * @param layer - * ǰ + * ��ǰ�� * @param lastLayer - * ǰһ + * ǰһ�� */ private void updateKernels(final Layer layer, final Layer lastLayer) { int mapNum = layer.getOutMapNum(); @@ -338,24 +338,24 @@ private void updateKernels(final Layer layer, final Layer lastLayer) { public void process(int start, int end) { for (int j = start; j < end; j++) { for (int i = 0; i < lastMapNum; i++) { - // batchÿ¼delta + // ��batch��ÿ����¼delta��� double[][] deltaKernel = null; for (int r = 0; r < batchSize; r++) { double[][] error = layer.getError(r, j); if (deltaKernel == null) deltaKernel = Util.convnValid( lastLayer.getMap(r, i), error); - else {// ۻ + else {// �ۻ���� deltaKernel = Util.matrixOp(Util.convnValid( lastLayer.getMap(r, i), error), deltaKernel, null, null, Util.plus); } } - // batchSize + // ����batchSize deltaKernel = Util.matrixOp(deltaKernel, divide_batchSize); - // ¾ + // ���¾���� double[][] kernel = layer.getKernel(i, j); deltaKernel = Util.matrixOp(kernel, deltaKernel, multiply_lambda, multiply_alpha, Util.plus); @@ -369,7 +369,7 @@ public void process(int start, int end) { } /** - * нIJв + * �����н�����IJв� */ private void setHiddenLayerErrors() { for (int l = layerNum - 2; l > 0; l--) { @@ -382,14 +382,14 @@ private void setHiddenLayerErrors() { case conv: setConvErrors(layer, nextLayer); break; - default:// ֻв;ҪвûввѾ + default:// ֻ�в�����;������Ҫ����в�����û�вв������Ѿ������ break; } } } /** - * òIJв + * ���ò�����IJв� * * @param layer * @param nextLayer @@ -402,11 +402,11 @@ private void setSampErrors(final Layer layer, final Layer nextLayer) { @Override public void process(int start, int end) { for (int i = start; i < end; i++) { - double[][] sum = null;// ÿһ + double[][] sum = null;// ��ÿһ������������ for (int j = 0; j < nextMapNum; j++) { double[][] nextError = nextLayer.getError(j); double[][] kernel = nextLayer.getKernel(i, j); - // Ծ˽180תȻfullģʽµþ + // �Ծ���˽���180����ת��Ȼ�����fullģʽ�µþ�� if (sum == null) sum = Util .convnFull(nextError, Util.rot180(kernel)); @@ -425,14 +425,14 @@ public void process(int start, int end) { } /** - * þIJв + * ���þ����IJв� * * @param layer * @param nextLayer */ private void setConvErrors(final Layer layer, final Layer nextLayer) { - // һΪ㣬mapͬһmapֻһһmapӣ - // ֻ轫һIJвkroneckerչõ + // ��������һ��Ϊ�����㣬�������map������ͬ����һ��mapֻ����һ���һ��map���ӣ� + // ���ֻ�轫��һ��IJв�kronecker��չ���õ������ int mapNum = layer.getOutMapNum(); new TaskManager(mapNum) { @@ -442,7 +442,7 @@ public void process(int start, int end) { Size scale = nextLayer.getScaleSize(); double[][] nextError = nextLayer.getError(m); double[][] map = layer.getMap(m); - // ˣԵڶÿԪvalue1-value + // ������ˣ����Եڶ��������ÿ��Ԫ��value����1-value���� double[][] outMatrix = Util.matrixOp(map, Util.cloneMatrix(map), null, Util.one_value, Util.multiply); @@ -459,7 +459,7 @@ public void process(int start, int end) { } /** - * IJвֵ,񾭵Ԫ٣ݲǶ߳ + * ���������IJв�ֵ,�������񾭵�Ԫ�������٣��ݲ����Ƕ��߳� * * @param record * @return @@ -479,7 +479,7 @@ private boolean setOutLayerErrors(Record record) { // (target[m] - output); // outputLayer.setError(m, 0, 0, errors); // } - // // ȷ + // // ��ȷ // if (isSame(outmaps, target)) // return true; // return false; @@ -491,8 +491,8 @@ private boolean setOutLayerErrors(Record record) { outmaps[m] = outmap[0][0]; } - int lable = record.getLable().intValue(); - target[lable] = 1; + int label = record.getLable().intValue(); + target[label] = 1; // Log.i(record.getLable() + "outmaps:" + // Util.fomart(outmaps) // + Arrays.toString(target)); @@ -500,28 +500,28 @@ private boolean setOutLayerErrors(Record record) { outputLayer.setError(m, 0, 0, outmaps[m] * (1 - outmaps[m]) * (target[m] - outmaps[m])); } - return lable == Util.getMaxIndex(outmaps); + return label == Util.getMaxIndex(outmaps); } /** - * ǰһ¼ + * ǰ�����һ����¼ * * @param record */ private void forward(Record record) { - // map + // ����������map setInLayerOutput(record); for (int l = 1; l < layers.size(); l++) { Layer layer = layers.get(l); Layer lastLayer = layers.get(l - 1); switch (layer.getType()) { - case conv:// + case conv:// ������������ setConvOutput(layer, lastLayer); break; - case samp:// + case samp:// ������������� setSampOutput(layer, lastLayer); break; - case output:// ,һľ + case output:// �������������,�������һ������ľ���� setConvOutput(layer, lastLayer); break; default: @@ -531,7 +531,7 @@ private void forward(Record record) { } /** - * ݼ¼ֵֵ + * ���ݼ�¼ֵ���������������ֵ * * @param record */ @@ -540,17 +540,17 @@ private void setInLayerOutput(Record record) { final Size mapSize = inputLayer.getMapSize(); final double[] attr = record.getAttrs(); if (attr.length != mapSize.x * mapSize.y) - throw new RuntimeException("ݼ¼ĴС붨mapСһ!"); + throw new RuntimeException("���ݼ�¼�Ĵ�С�붨���map��С��һ��!"); for (int i = 0; i < mapSize.x; i++) { for (int j = 0; j < mapSize.y; j++) { - // ¼ԵһάŪɶά + // ����¼���Ե�һά����Ū�ɶ�ά���� inputLayer.setMapValue(0, i, j, attr[mapSize.x * i + j]); } } } /* - * ֵ,ÿ̸߳һmap + * �����������ֵ,ÿ���̸߳���һ����map */ private void setConvOutput(final Layer layer, final Layer lastLayer) { int mapNum = layer.getOutMapNum(); @@ -560,7 +560,7 @@ private void setConvOutput(final Layer layer, final Layer lastLayer) { @Override public void process(int start, int end) { for (int j = start; j < end; j++) { - double[][] sum = null;// ÿһmapľ + double[][] sum = null;// ��ÿһ������map�ľ��������� for (int i = 0; i < lastMapNum; i++) { double[][] lastMap = lastLayer.getMap(i); double[][] kernel = layer.getKernel(i, j); @@ -591,7 +591,7 @@ public double process(double value) { } /** - * òֵǶԾľֵ + * ���ò���������ֵ���������ǶԾ����ľ�ֵ���� * * @param layer * @param lastLayer @@ -605,7 +605,7 @@ public void process(int start, int end) { for (int i = start; i < end; i++) { double[][] lastMap = lastLayer.getMap(i); Size scaleSize = layer.getScaleSize(); - // scaleSizeоֵ + // ��scaleSize������о�ֵ���� double[][] sampMatrix = Util .scaleMatrix(lastMap, scaleSize); layer.setMapValue(i, sampMatrix); @@ -617,7 +617,7 @@ public void process(int start, int end) { } /** - * cnnÿһIJ + * ����cnn�����ÿһ��IJ��� * * @param batchSize * * @param classNum @@ -625,7 +625,7 @@ public void process(int start, int end) { */ public void setup(int batchSize) { Layer inputLayer = layers.get(0); - // ÿһ㶼Ҫʼmap + // ÿһ�㶼��Ҫ��ʼ�����map inputLayer.initOutmaps(batchSize); for (int i = 1; i < layers.size(); i++) { Layer layer = layers.get(i); @@ -635,38 +635,38 @@ public void setup(int batchSize) { case input: break; case conv: - // mapĴС + // ����map�Ĵ�С layer.setMapSize(frontLayer.getMapSize().subtract( layer.getKernelSize(), 1)); - // ʼˣfrontMapNum*outMapNum + // ��ʼ������ˣ�����frontMapNum*outMapNum������� layer.initKernel(frontMapNum); - // ʼƫãfrontMapNum*outMapNumƫ + // ��ʼ��ƫ�ã�����frontMapNum*outMapNum��ƫ�� layer.initBias(frontMapNum); - // batchÿ¼Ҫһݲв + // batch��ÿ����¼��Ҫ����һ�ݲв� layer.initErros(batchSize); - // ÿһ㶼Ҫʼmap + // ÿһ�㶼��Ҫ��ʼ�����map layer.initOutmaps(batchSize); break; case samp: - // mapһͬ + // �������map��������һ����ͬ layer.setOutMapNum(frontMapNum); - // mapĴСһmapĴСscaleС + // ������map�Ĵ�С����һ��map�Ĵ�С����scale��С layer.setMapSize(frontLayer.getMapSize().divide( layer.getScaleSize())); - // batchÿ¼Ҫһݲв + // batch��ÿ����¼��Ҫ����һ�ݲв� layer.initErros(batchSize); - // ÿһ㶼Ҫʼmap + // ÿһ�㶼��Ҫ��ʼ�����map layer.initOutmaps(batchSize); break; case output: - // ʼȨأˣľ˴СΪһmapС + // ��ʼ��Ȩ�أ�����ˣ��������ľ���˴�СΪ��һ���map��С layer.initOutputKerkel(frontMapNum, frontLayer.getMapSize()); - // ʼƫãfrontMapNum*outMapNumƫ + // ��ʼ��ƫ�ã�����frontMapNum*outMapNum��ƫ�� layer.initBias(frontMapNum); - // batchÿ¼Ҫһݲв + // batch��ÿ����¼��Ҫ����һ�ݲв� layer.initErros(batchSize); - // ÿһ㶼Ҫʼmap + // ÿһ�㶼��Ҫ��ʼ�����map layer.initOutmaps(batchSize); break; } @@ -674,11 +674,11 @@ public void setup(int batchSize) { } /** - * ģʽ,ҪڶΪΪ + * ������ģʽ�������,Ҫ�����ڶ������Ϊ�����������Ϊ����� * * @author jiqunpeng * - * ʱ䣺2014-7-8 4:54:29 + * ����ʱ�䣺2014-7-8 ����4:54:29 */ public static class LayerBuilder { private List mLayers; @@ -699,7 +699,7 @@ public LayerBuilder addLayer(Layer layer) { } /** - * лģ + * ���л�����ģ�� * * @param fileName */ @@ -717,7 +717,7 @@ public void saveModel(String fileName) { } /** - * лģ + * �����л�����ģ�� * * @param fileName * @return diff --git a/src/edu/hitsz/c102c/dataset/Dataset.java b/src/edu/hitsz/c102c/dataset/Dataset.java index d0291c2..3f3f32d 100644 --- a/src/edu/hitsz/c102c/dataset/Dataset.java +++ b/src/edu/hitsz/c102c/dataset/Dataset.java @@ -6,15 +6,13 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.Iterator; import java.util.List; -import java.util.Map; public class Dataset { - // + // �������� private List records; - // ± + // ����±� private int lableIndex; private double maxLable = -1; @@ -50,22 +48,22 @@ public void append(Record record) { } /** - * + * ������� */ public void clear() { records.clear(); } /** - * һ¼ + * ���һ����¼ * * @param attrs - * ¼ - * @param lable - * ¼ + * ��¼������ + * @param label + * ��¼����� */ - public void append(double[] attrs, Double lable) { - records.add(new Record(attrs, lable)); + public void append(double[] attrs, Double label) { + records.add(new Record(attrs, label)); } public Iterator iter() { @@ -73,7 +71,7 @@ public Iterator iter() { } /** - * ȡindex¼ + * ��ȡ��index����¼������ * * @param index * @return @@ -87,14 +85,14 @@ public Double getLable(int index) { } /** - * ݼ + * �������ݼ� * * @param filePath - * ļ· + * �ļ�����·�� * @param tag - * ֶηָ + * �ֶηָ��� * @param lableIndex - * ±꣬0ʼ + * ����±꣬��0��ʼ * @return */ public static Dataset load(String filePath, String tag, int lableIndex) { @@ -121,34 +119,34 @@ public static Dataset load(String filePath, String tag, int lableIndex) { e.printStackTrace(); return null; } - System.out.println(":" + dataset.size()); + System.out.println("��������:" + dataset.size()); return dataset; } /** - * ݼ¼(ʵ),¼Ժ,Ϊһлһл߿ + * ���ݼ�¼(ʵ��),��¼�����Ժ�������,������Ϊ��һ�л������һ�л��߿� * * @author jiqunpeng * - * ʱ䣺2014-6-15 8:03:29 + * ����ʱ�䣺2014-6-15 ����8:03:29 */ public class Record { - // 洢 + // �洢���� private double[] attrs; - private Double lable; + private Double label; - private Record(double[] attrs, Double lable) { + private Record(double[] attrs, Double label) { this.attrs = attrs; - this.lable = lable; + this.label = label; } public Record(double[] data) { if (lableIndex == -1) attrs = data; else { - lable = data[lableIndex]; - if (lable > maxLable) - maxLable = lable; + label = data[lableIndex]; + if (label > maxLable) + maxLable = label; if (lableIndex == 0) attrs = Arrays.copyOfRange(data, 1, data.length); else @@ -157,7 +155,7 @@ public Record(double[] data) { } /** - * ü¼ + * �ü�¼������ * * @return */ @@ -169,30 +167,30 @@ public String toString() { StringBuilder sb = new StringBuilder(); sb.append("attrs:"); sb.append(Arrays.toString(attrs)); - sb.append("lable:"); - sb.append(lable); + sb.append("label:"); + sb.append(label); return sb.toString(); } /** - * ü¼ + * �ü�¼����� * * @return */ public Double getLable() { if (lableIndex == -1) return null; - return lable; + return label; } /** - * жƱ + * �������ж����Ʊ��� * * @param n * @return */ public int[] getEncodeTarget(int n) { - String binary = Integer.toBinaryString(lable.intValue()); + String binary = Integer.toBinaryString(label.intValue()); byte[] bytes = binary.getBytes(); int[] encode = new int[n]; int j = n; @@ -203,7 +201,7 @@ public int[] getEncodeTarget(int n) { } public double[] getDoubleEncodeTarget(int n) { - String binary = Integer.toBinaryString(lable.intValue()); + String binary = Integer.toBinaryString(label.intValue()); byte[] bytes = binary.getBytes(); double[] encode = new double[n]; int j = n; @@ -222,12 +220,12 @@ public static void main(String[] args) { 1 }); int[] encode = r.getEncodeTarget(4); - System.out.println(r.lable); + System.out.println(r.label); System.out.println(Arrays.toString(encode)); } /** - * ȡindex¼ + * ��ȡ��index����¼ * * @param index * @return From aa819251f289c2b070b6b0211d5b519762d27046 Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Thu, 15 Feb 2018 12:43:59 +0100 Subject: [PATCH 03/34] Refactoring - DatasetLoad separated - labelIndex from Dataset removed - typo lable -> label --- src/Test.java | 33 +-- src/edu/hitsz/c102c/cnn/CNN.java | 195 +++++++++-------- src/edu/hitsz/c102c/cnn/RunCNN.java | 27 ++- src/edu/hitsz/c102c/dataset/Dataset.java | 198 +++--------------- .../hitsz/c102c/dataset/DatasetLoader.java | 41 ++++ 5 files changed, 195 insertions(+), 299 deletions(-) create mode 100644 src/edu/hitsz/c102c/dataset/DatasetLoader.java diff --git a/src/Test.java b/src/Test.java index 4999b48..6b2d3dd 100644 --- a/src/Test.java +++ b/src/Test.java @@ -1,6 +1,9 @@ +import java.io.IOException; + import edu.hitsz.c102c.cnn.CNN; import edu.hitsz.c102c.cnn.Layer; import edu.hitsz.c102c.dataset.Dataset; +import edu.hitsz.c102c.dataset.DatasetLoader; /** *

@@ -8,24 +11,24 @@ * * @author Ralf Th. Pietsch <ratopi@abwesend.de> */ -public class Test -{ - public static void main( final String[] args ) - { +public class Test { + public static void main(final String[] args) throws IOException { + final CNN.LayerBuilder builder = new CNN.LayerBuilder(); - builder.addLayer( Layer.buildInputLayer( new Layer.Size( 28, 28 ) ) ); - builder.addLayer( Layer.buildConvLayer( 6, new Layer.Size( 5, 5 ) ) ); - builder.addLayer( Layer.buildSampLayer( new Layer.Size( 2, 2 ) ) ); - builder.addLayer( Layer.buildConvLayer( 12, new Layer.Size( 5, 5 ) ) ); - builder.addLayer( Layer.buildSampLayer( new Layer.Size( 2, 2 ) ) ); - builder.addLayer( Layer.buildOutputLayer( 10 ) ); - final CNN cnn = new CNN( builder, 50 ); + builder.addLayer(Layer.buildInputLayer(new Layer.Size(28, 28))); + builder.addLayer(Layer.buildConvLayer(6, new Layer.Size(5, 5))); + builder.addLayer(Layer.buildSampLayer(new Layer.Size(2, 2))); + builder.addLayer(Layer.buildConvLayer(12, new Layer.Size(5, 5))); + builder.addLayer(Layer.buildSampLayer(new Layer.Size(2, 2))); + builder.addLayer(Layer.buildOutputLayer(10)); + + final CNN cnn = new CNN(builder, 50); final String fileName = "dataset/train.format"; - final Dataset dataset = Dataset.load( fileName, ",", 784 ); - cnn.train( dataset, 100 ); + final Dataset dataset = DatasetLoader.load(fileName, ",", 784); + cnn.train(dataset, 100); - final Dataset testset = Dataset.load( "dataset/test.format", ",", -1 ); - cnn.predict( testset, "dataset/test.predict" ); + final Dataset testset = DatasetLoader.load("dataset/test.format", ",", -1); + cnn.predict(testset, "dataset/test.predict"); } } diff --git a/src/edu/hitsz/c102c/cnn/CNN.java b/src/edu/hitsz/c102c/cnn/CNN.java index 868cb9e..4d67421 100644 --- a/src/edu/hitsz/c102c/cnn/CNN.java +++ b/src/edu/hitsz/c102c/cnn/CNN.java @@ -24,7 +24,7 @@ public class CNN implements Serializable { /** - * + * */ private static final long serialVersionUID = 337920299147929932L; private static double ALPHA = 0.85; @@ -47,13 +47,10 @@ public class CNN implements Serializable { /** * ��ʼ������ - * - * @param layerBuilder - * ����� - * @param inputMapSize - * ����map�Ĵ�С - * @param classNum - * ���ĸ�����Ҫ�����ݼ������ת��Ϊ0-classNum-1����ֵ + * + * @param layerBuilder ����� + * @param inputMapSize ����map�Ĵ�С + * @param classNum ���ĸ�����Ҫ�����ݼ������ת��Ϊ0-classNum-1����ֵ */ public CNN(LayerBuilder layerBuilder, final int batchSize) { layers = layerBuilder.mLayers; @@ -103,10 +100,9 @@ public double process(double value) { /** * ��ѵ������ѵ������ - * + * * @param trainset - * @param repeat - * �����Ĵ��� + * @param repeat �����Ĵ��� */ public void train(Dataset trainset, int repeat) { // ����ֹͣ��ť @@ -177,7 +173,7 @@ public void run() { /** * �������� - * + * * @param trainset * @return */ @@ -189,14 +185,14 @@ public double test(Dataset trainset) { Record record = iter.next(); forward(record); Layer outputLayer = layers.get(layerNum - 1); - int mapNum = outputLayer.getOutMapNum(); + int mapNum = outputLayer.getOutMapNum(); double[] out = new double[mapNum]; for (int m = 0; m < mapNum; m++) { double[][] outmap = outputLayer.getMap(m); out[m] = outmap[0][0]; } - if (record.getLable().intValue() == Util.getMaxIndex(out)) - right++; + if (record.getLabel().intValue() == Util.getMaxIndex(out)) + right++; } double p = 1.0 * right / trainset.size(); Log.i("precision", p + ""); @@ -205,7 +201,7 @@ public double test(Dataset trainset) { /** * Ԥ���� - * + * * @param testset * @param fileName */ @@ -256,7 +252,7 @@ private boolean isSame(double[] output, double[] target) { /** * ѵ��һ����¼��ͬʱ�����Ƿ�Ԥ����ȷ��ǰ��¼ - * + * * @param record * @return */ @@ -284,20 +280,20 @@ private void updateParas() { Layer layer = layers.get(l); Layer lastLayer = layers.get(l - 1); switch (layer.getType()) { - case conv: - case output: - updateKernels(layer, lastLayer); - updateBias(layer, lastLayer); - break; - default: - break; + case conv: + case output: + updateKernels(layer, lastLayer); + updateBias(layer, lastLayer); + break; + default: + break; } } } /** * ����ƫ�� - * + * * @param layer * @param lastLayer */ @@ -323,11 +319,9 @@ public void process(int start, int end) { /** * ����layer��ľ���ˣ�Ȩ�أ���ƫ�� - * - * @param layer - * ��ǰ�� - * @param lastLayer - * ǰһ�� + * + * @param layer ��ǰ�� + * @param lastLayer ǰһ�� */ private void updateKernels(final Layer layer, final Layer lastLayer) { int mapNum = layer.getOutMapNum(); @@ -376,21 +370,21 @@ private void setHiddenLayerErrors() { Layer layer = layers.get(l); Layer nextLayer = layers.get(l + 1); switch (layer.getType()) { - case samp: - setSampErrors(layer, nextLayer); - break; - case conv: - setConvErrors(layer, nextLayer); - break; - default:// ֻ�в�����;������Ҫ����в�����û�вв������Ѿ������ - break; + case samp: + setSampErrors(layer, nextLayer); + break; + case conv: + setConvErrors(layer, nextLayer); + break; + default:// ֻ�в�����;������Ҫ����в�����û�вв������Ѿ������ + break; } } } /** * ���ò�����IJв� - * + * * @param layer * @param nextLayer */ @@ -426,7 +420,7 @@ public void process(int start, int end) { /** * ���þ����IJв� - * + * * @param layer * @param nextLayer */ @@ -460,7 +454,7 @@ public void process(int start, int end) { /** * ���������IJв�ֵ,�������񾭵�Ԫ�������٣��ݲ����Ƕ��߳� - * + * * @param record * @return */ @@ -491,7 +485,7 @@ private boolean setOutLayerErrors(Record record) { outmaps[m] = outmap[0][0]; } - int label = record.getLable().intValue(); + int label = record.getLabel().intValue(); target[label] = 1; // Log.i(record.getLable() + "outmaps:" + // Util.fomart(outmaps) @@ -505,7 +499,7 @@ private boolean setOutLayerErrors(Record record) { /** * ǰ�����һ����¼ - * + * * @param record */ private void forward(Record record) { @@ -515,24 +509,24 @@ private void forward(Record record) { Layer layer = layers.get(l); Layer lastLayer = layers.get(l - 1); switch (layer.getType()) { - case conv:// ������������ - setConvOutput(layer, lastLayer); - break; - case samp:// ������������� - setSampOutput(layer, lastLayer); - break; - case output:// �������������,�������һ������ľ���� - setConvOutput(layer, lastLayer); - break; - default: - break; + case conv:// ������������ + setConvOutput(layer, lastLayer); + break; + case samp:// ������������� + setSampOutput(layer, lastLayer); + break; + case output:// �������������,�������һ������ľ���� + setConvOutput(layer, lastLayer); + break; + default: + break; } } } /** * ���ݼ�¼ֵ���������������ֵ - * + * * @param record */ private void setInLayerOutput(Record record) { @@ -592,7 +586,7 @@ public double process(double value) { /** * ���ò���������ֵ���������ǶԾ����ľ�ֵ���� - * + * * @param layer * @param lastLayer */ @@ -618,9 +612,8 @@ public void process(int start, int end) { /** * ����cnn�����ÿһ��IJ��� - * - * @param batchSize - * * @param classNum + * + * @param batchSize * @param classNum * @param inputMapSize */ public void setup(int batchSize) { @@ -632,53 +625,53 @@ public void setup(int batchSize) { Layer frontLayer = layers.get(i - 1); int frontMapNum = frontLayer.getOutMapNum(); switch (layer.getType()) { - case input: - break; - case conv: - // ����map�Ĵ�С - layer.setMapSize(frontLayer.getMapSize().subtract( - layer.getKernelSize(), 1)); - // ��ʼ������ˣ�����frontMapNum*outMapNum������� - - layer.initKernel(frontMapNum); - // ��ʼ��ƫ�ã�����frontMapNum*outMapNum��ƫ�� - layer.initBias(frontMapNum); - // batch��ÿ����¼��Ҫ����һ�ݲв� - layer.initErros(batchSize); - // ÿһ�㶼��Ҫ��ʼ�����map - layer.initOutmaps(batchSize); - break; - case samp: - // �������map��������һ����ͬ - layer.setOutMapNum(frontMapNum); - // ������map�Ĵ�С����һ��map�Ĵ�С����scale��С - layer.setMapSize(frontLayer.getMapSize().divide( - layer.getScaleSize())); - // batch��ÿ����¼��Ҫ����һ�ݲв� - layer.initErros(batchSize); - // ÿһ�㶼��Ҫ��ʼ�����map - layer.initOutmaps(batchSize); - break; - case output: - // ��ʼ��Ȩ�أ�����ˣ��������ľ���˴�СΪ��һ���map��С - layer.initOutputKerkel(frontMapNum, frontLayer.getMapSize()); - // ��ʼ��ƫ�ã�����frontMapNum*outMapNum��ƫ�� - layer.initBias(frontMapNum); - // batch��ÿ����¼��Ҫ����һ�ݲв� - layer.initErros(batchSize); - // ÿһ�㶼��Ҫ��ʼ�����map - layer.initOutmaps(batchSize); - break; + case input: + break; + case conv: + // ����map�Ĵ�С + layer.setMapSize(frontLayer.getMapSize().subtract( + layer.getKernelSize(), 1)); + // ��ʼ������ˣ�����frontMapNum*outMapNum������� + + layer.initKernel(frontMapNum); + // ��ʼ��ƫ�ã�����frontMapNum*outMapNum��ƫ�� + layer.initBias(frontMapNum); + // batch��ÿ����¼��Ҫ����һ�ݲв� + layer.initErros(batchSize); + // ÿһ�㶼��Ҫ��ʼ�����map + layer.initOutmaps(batchSize); + break; + case samp: + // �������map��������һ����ͬ + layer.setOutMapNum(frontMapNum); + // ������map�Ĵ�С����һ��map�Ĵ�С����scale��С + layer.setMapSize(frontLayer.getMapSize().divide( + layer.getScaleSize())); + // batch��ÿ����¼��Ҫ����һ�ݲв� + layer.initErros(batchSize); + // ÿһ�㶼��Ҫ��ʼ�����map + layer.initOutmaps(batchSize); + break; + case output: + // ��ʼ��Ȩ�أ�����ˣ��������ľ���˴�СΪ��һ���map��С + layer.initOutputKerkel(frontMapNum, frontLayer.getMapSize()); + // ��ʼ��ƫ�ã�����frontMapNum*outMapNum��ƫ�� + layer.initBias(frontMapNum); + // batch��ÿ����¼��Ҫ����һ�ݲв� + layer.initErros(batchSize); + // ÿһ�㶼��Ҫ��ʼ�����map + layer.initOutmaps(batchSize); + break; } } } /** * ������ģʽ�������,Ҫ�����ڶ������Ϊ�����������Ϊ����� - * + * * @author jiqunpeng - * - * ����ʱ�䣺2014-7-8 ����4:54:29 + *

+ * ����ʱ�䣺2014-7-8 ����4:54:29 */ public static class LayerBuilder { private List mLayers; @@ -700,7 +693,7 @@ public LayerBuilder addLayer(Layer layer) { /** * ���л�����ģ�� - * + * * @param fileName */ public void saveModel(String fileName) { @@ -718,7 +711,7 @@ public void saveModel(String fileName) { /** * �����л�����ģ�� - * + * * @param fileName * @return */ diff --git a/src/edu/hitsz/c102c/cnn/RunCNN.java b/src/edu/hitsz/c102c/cnn/RunCNN.java index 7f3a1c9..edeba8c 100644 --- a/src/edu/hitsz/c102c/cnn/RunCNN.java +++ b/src/edu/hitsz/c102c/cnn/RunCNN.java @@ -1,16 +1,19 @@ package edu.hitsz.c102c.cnn; +import java.io.IOException; + import edu.hitsz.c102c.cnn.CNN.LayerBuilder; import edu.hitsz.c102c.cnn.Layer.Size; import edu.hitsz.c102c.dataset.Dataset; +import edu.hitsz.c102c.dataset.DatasetLoader; import edu.hitsz.c102c.util.ConcurenceRunner; import edu.hitsz.c102c.util.TimedTest; import edu.hitsz.c102c.util.TimedTest.TestTask; public class RunCNN { - public static void runCnn() { - //һ + public static void runCnn() throws IOException { + //����һ����������� LayerBuilder builder = new LayerBuilder(); builder.addLayer(Layer.buildInputLayer(new Size(28, 28))); builder.addLayer(Layer.buildConvLayer(6, new Size(5, 5))); @@ -19,19 +22,19 @@ public static void runCnn() { builder.addLayer(Layer.buildSampLayer(new Size(2, 2))); builder.addLayer(Layer.buildOutputLayer(10)); CNN cnn = new CNN(builder, 50); - - //ݼ + + //�������ݼ� String fileName = "dataset/train.format"; - Dataset dataset = Dataset.load(fileName, ",", 784); + Dataset dataset = DatasetLoader.load(fileName, ",", 784); cnn.train(dataset, 3);// String modelName = "model/model.cnn"; - cnn.saveModel(modelName); + cnn.saveModel(modelName); dataset.clear(); dataset = null; - - //Ԥ + + //Ԥ�� // CNN cnn = CNN.loadModel(modelName); - Dataset testset = Dataset.load("dataset/test.format", ",", -1); + Dataset testset = DatasetLoader.load("dataset/test.format", ",", -1); cnn.predict(testset, "dataset/test.predict"); } @@ -41,7 +44,11 @@ public static void main(String[] args) { @Override public void process() { - runCnn(); + try { + runCnn(); + } catch (IOException e) { + throw new RuntimeException(e); + } } }, 1).test(); ConcurenceRunner.stop(); diff --git a/src/edu/hitsz/c102c/dataset/Dataset.java b/src/edu/hitsz/c102c/dataset/Dataset.java index 3f3f32d..5dfe095 100644 --- a/src/edu/hitsz/c102c/dataset/Dataset.java +++ b/src/edu/hitsz/c102c/dataset/Dataset.java @@ -1,67 +1,43 @@ package edu.hitsz.c102c.dataset; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; import java.util.List; public class Dataset { - // �������� private List records; - // ����±� - private int lableIndex; - private double maxLable = -1; - - public Dataset(int classIndex) { - - this.lableIndex = classIndex; - records = new ArrayList(); + public Dataset() { + records = new ArrayList<>(); } - public Dataset(List datas) { + public Dataset(final List datas, final List labels) { this(); - for (double[] data : datas) { - append(new Record(data)); + + if (datas.size() != labels.size()) { + throw new IllegalArgumentException("Lengths differs: " + datas.size() + " datas and " + labels.size() + " labels"); } - } - private Dataset() { - this.lableIndex = -1; - records = new ArrayList(); + for (int i = 0; i < datas.size(); i++) { + final double[] data = datas.get(i); + final Double label = labels.get(i); + append(new Record(data, label)); + } } public int size() { return records.size(); } - public int getLableIndex() { - return lableIndex; - } - public void append(Record record) { records.add(record); } - /** - * ������� - */ public void clear() { records.clear(); } - /** - * ���һ����¼ - * - * @param attrs - * ��¼������ - * @param label - * ��¼����� - */ public void append(double[] attrs, Double label) { records.add(new Record(attrs, label)); } @@ -70,68 +46,21 @@ public Iterator iter() { return records.iterator(); } - /** - * ��ȡ��index����¼������ - * - * @param index - * @return - */ public double[] getAttrs(int index) { return records.get(index).getAttrs(); } - public Double getLable(int index) { - return records.get(index).getLable(); + public Double getLabel(int index) { + return records.get(index).getLabel(); } - /** - * �������ݼ� - * - * @param filePath - * �ļ�����·�� - * @param tag - * �ֶηָ��� - * @param lableIndex - * ����±꣬��0��ʼ - * @return - */ - public static Dataset load(String filePath, String tag, int lableIndex) { - Dataset dataset = new Dataset(); - dataset.lableIndex = lableIndex; - File file = new File(filePath); - try { - - BufferedReader in = new BufferedReader(new FileReader(file)); - String line; - while ((line = in.readLine()) != null) { - String[] datas = line.split(tag); - if (datas.length == 0) - continue; - double[] data = new double[datas.length]; - for (int i = 0; i < datas.length; i++) - data[i] = Double.parseDouble(datas[i]); - Record record = dataset.new Record(data); - dataset.append(record); - } - in.close(); - - } catch (IOException e) { - e.printStackTrace(); - return null; - } - System.out.println("��������:" + dataset.size()); - return dataset; + public Record getRecord(int index) { + return records.get(index); } - /** - * ���ݼ�¼(ʵ��),��¼�����Ժ�������,������Ϊ��һ�л������һ�л��߿� - * - * @author jiqunpeng - * - * ����ʱ�䣺2014-6-15 ����8:03:29 - */ - public class Record { - // �洢���� + // --- + + public static class Record { private double[] attrs; private Double label; @@ -140,98 +69,21 @@ private Record(double[] attrs, Double label) { this.label = label; } - public Record(double[] data) { - if (lableIndex == -1) - attrs = data; - else { - label = data[lableIndex]; - if (label > maxLable) - maxLable = label; - if (lableIndex == 0) - attrs = Arrays.copyOfRange(data, 1, data.length); - else - attrs = Arrays.copyOfRange(data, 0, data.length - 1); - } - } - - /** - * �ü�¼������ - * - * @return - */ public double[] getAttrs() { return attrs; } - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("attrs:"); - sb.append(Arrays.toString(attrs)); - sb.append("label:"); - sb.append(label); - return sb.toString(); - } - - /** - * �ü�¼����� - * - * @return - */ - public Double getLable() { - if (lableIndex == -1) - return null; + public Double getLabel() { return label; } - /** - * �������ж����Ʊ��� - * - * @param n - * @return - */ - public int[] getEncodeTarget(int n) { - String binary = Integer.toBinaryString(label.intValue()); - byte[] bytes = binary.getBytes(); - int[] encode = new int[n]; - int j = n; - for (int i = bytes.length - 1; i >= 0; i--) - encode[--j] = bytes[i] - '0'; - - return encode; - } - - public double[] getDoubleEncodeTarget(int n) { - String binary = Integer.toBinaryString(label.intValue()); - byte[] bytes = binary.getBytes(); - double[] encode = new double[n]; - int j = n; - for (int i = bytes.length - 1; i >= 0; i--) - encode[--j] = bytes[i] - '0'; - - return encode; + @Override + public String toString() { + return "Record{" + + "attrs=" + Arrays.toString(attrs) + + ", label=" + label + + '}'; } - - } - - public static void main(String[] args) { - Dataset d = new Dataset(); - d.lableIndex = 10; - Record r = d.new Record(new double[] { 3, 2, 2, 5, 4, 5, 3, 11, 3, 12, - 1 }); - int[] encode = r.getEncodeTarget(4); - - System.out.println(r.label); - System.out.println(Arrays.toString(encode)); - } - - /** - * ��ȡ��index����¼ - * - * @param index - * @return - */ - public Record getRecord(int index) { - return records.get(index); } } diff --git a/src/edu/hitsz/c102c/dataset/DatasetLoader.java b/src/edu/hitsz/c102c/dataset/DatasetLoader.java new file mode 100644 index 0000000..45caaf0 --- /dev/null +++ b/src/edu/hitsz/c102c/dataset/DatasetLoader.java @@ -0,0 +1,41 @@ +package edu.hitsz.c102c.dataset; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; + +public class DatasetLoader { + public static Dataset load(final String filePath, final String tag, final int labelIndex) throws IOException { + final Dataset dataset = new Dataset(); + + final BufferedReader in = new BufferedReader(new FileReader(filePath)); + + String line; + while ((line = in.readLine()) != null) { + + final String[] datas = line.split(tag); + + if (datas.length == 0) { + continue; + } + + final int vectorLength = labelIndex < 0 ? datas.length : datas.length - 1; + + final double[] data = new double[vectorLength]; + + for (int i = 0; i < vectorLength; i++) { + data[i] = Double.parseDouble(datas[i]); + } + + final Double label = labelIndex < 0 ? null : Double.parseDouble(datas[labelIndex]); + + dataset.append(data, label); + } + in.close(); + + System.out.println("Read " + dataset.size() + " records"); + + return dataset; + } + +} From a58fe20ecef32255c3e91aaf054bb9c4472120a2 Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Sat, 17 Feb 2018 09:47:21 +0100 Subject: [PATCH 04/34] Ignoring IDE-files --- .classpath | 6 ------ .gitignore | 4 ++++ .project | 17 ----------------- 3 files changed, 4 insertions(+), 23 deletions(-) delete mode 100644 .classpath delete mode 100644 .project diff --git a/.classpath b/.classpath deleted file mode 100644 index fb565a5..0000000 --- a/.classpath +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/.gitignore b/.gitignore index f837fa2..6264549 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,7 @@ bin/ *MyTest.java data/ model/ +/.classpath +/.project +*.iml +/.idea \ No newline at end of file diff --git a/.project b/.project deleted file mode 100644 index 8f8ffe9..0000000 --- a/.project +++ /dev/null @@ -1,17 +0,0 @@ - - - JavaCNN - - - - - - org.eclipse.jdt.core.javabuilder - - - - - - org.eclipse.jdt.core.javanature - - From 3fb2ee48ee5965d9a160b80f6fc5c21520f75ad0 Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Sat, 17 Feb 2018 10:18:50 +0100 Subject: [PATCH 05/34] Refactoring - Extracting CNNLoader - Removed unreadable comments (sorry, encoding problems) --- src/{edu/hitsz/c102c/cnn => }/RunCNN.java | 49 +-- src/Test.java | 34 -- src/edu/hitsz/c102c/cnn/CNN.java | 487 ++++++++-------------- src/edu/hitsz/c102c/cnn/CNNLoader.java | 35 ++ src/edu/hitsz/c102c/cnn/Layer.java | 224 ++++------ src/edu/hitsz/c102c/util/Log.java | 12 +- src/edu/hitsz/c102c/util/TimedTest.java | 6 +- 7 files changed, 309 insertions(+), 538 deletions(-) rename src/{edu/hitsz/c102c/cnn => }/RunCNN.java (53%) delete mode 100644 src/Test.java create mode 100644 src/edu/hitsz/c102c/cnn/CNNLoader.java diff --git a/src/edu/hitsz/c102c/cnn/RunCNN.java b/src/RunCNN.java similarity index 53% rename from src/edu/hitsz/c102c/cnn/RunCNN.java rename to src/RunCNN.java index edeba8c..fc01e2b 100644 --- a/src/edu/hitsz/c102c/cnn/RunCNN.java +++ b/src/RunCNN.java @@ -1,58 +1,51 @@ -package edu.hitsz.c102c.cnn; - import java.io.IOException; +import edu.hitsz.c102c.cnn.CNN; import edu.hitsz.c102c.cnn.CNN.LayerBuilder; +import edu.hitsz.c102c.cnn.CNNLoader; +import edu.hitsz.c102c.cnn.Layer; import edu.hitsz.c102c.cnn.Layer.Size; import edu.hitsz.c102c.dataset.Dataset; import edu.hitsz.c102c.dataset.DatasetLoader; import edu.hitsz.c102c.util.ConcurenceRunner; import edu.hitsz.c102c.util.TimedTest; -import edu.hitsz.c102c.util.TimedTest.TestTask; public class RunCNN { public static void runCnn() throws IOException { - //����һ����������� - LayerBuilder builder = new LayerBuilder(); + + final LayerBuilder builder = new LayerBuilder(); + builder.addLayer(Layer.buildInputLayer(new Size(28, 28))); builder.addLayer(Layer.buildConvLayer(6, new Size(5, 5))); builder.addLayer(Layer.buildSampLayer(new Size(2, 2))); builder.addLayer(Layer.buildConvLayer(12, new Size(5, 5))); builder.addLayer(Layer.buildSampLayer(new Size(2, 2))); builder.addLayer(Layer.buildOutputLayer(10)); - CNN cnn = new CNN(builder, 50); - - //�������ݼ� - String fileName = "dataset/train.format"; - Dataset dataset = DatasetLoader.load(fileName, ",", 784); - cnn.train(dataset, 3);// - String modelName = "model/model.cnn"; - cnn.saveModel(modelName); + + final CNN cnn = new CNN(builder, 50); + + final String fileName = "dataset/train.format"; + final Dataset dataset = DatasetLoader.load(fileName, ",", 784); + cnn.train(dataset, 100); + + CNNLoader.saveModel("model.cnn", cnn); dataset.clear(); - dataset = null; - //Ԥ�� - // CNN cnn = CNN.loadModel(modelName); - Dataset testset = DatasetLoader.load("dataset/test.format", ",", -1); + // CNN cnn = CNNLoader.loadModel(modelName); + final Dataset testset = DatasetLoader.load("dataset/test.format", ",", -1); cnn.predict(testset, "dataset/test.predict"); } public static void main(String[] args) { - - new TimedTest(new TestTask() { - - @Override - public void process() { - try { - runCnn(); - } catch (IOException e) { - throw new RuntimeException(e); - } + new TimedTest(() -> { + try { + runCnn(); + } catch (IOException e) { + throw new RuntimeException(e); } }, 1).test(); ConcurenceRunner.stop(); - } } diff --git a/src/Test.java b/src/Test.java deleted file mode 100644 index 6b2d3dd..0000000 --- a/src/Test.java +++ /dev/null @@ -1,34 +0,0 @@ -import java.io.IOException; - -import edu.hitsz.c102c.cnn.CNN; -import edu.hitsz.c102c.cnn.Layer; -import edu.hitsz.c102c.dataset.Dataset; -import edu.hitsz.c102c.dataset.DatasetLoader; - -/** - *

- * Created: 14.02.2018 09:06 - * - * @author Ralf Th. Pietsch <ratopi@abwesend.de> - */ -public class Test { - public static void main(final String[] args) throws IOException { - - final CNN.LayerBuilder builder = new CNN.LayerBuilder(); - builder.addLayer(Layer.buildInputLayer(new Layer.Size(28, 28))); - builder.addLayer(Layer.buildConvLayer(6, new Layer.Size(5, 5))); - builder.addLayer(Layer.buildSampLayer(new Layer.Size(2, 2))); - builder.addLayer(Layer.buildConvLayer(12, new Layer.Size(5, 5))); - builder.addLayer(Layer.buildSampLayer(new Layer.Size(2, 2))); - builder.addLayer(Layer.buildOutputLayer(10)); - - final CNN cnn = new CNN(builder, 50); - - final String fileName = "dataset/train.format"; - final Dataset dataset = DatasetLoader.load(fileName, ",", 784); - cnn.train(dataset, 100); - - final Dataset testset = DatasetLoader.load("dataset/test.format", ",", -1); - cnn.predict(testset, "dataset/test.predict"); - } -} diff --git a/src/edu/hitsz/c102c/cnn/CNN.java b/src/edu/hitsz/c102c/cnn/CNN.java index 4d67421..f8b392c 100644 --- a/src/edu/hitsz/c102c/cnn/CNN.java +++ b/src/edu/hitsz/c102c/cnn/CNN.java @@ -1,15 +1,10 @@ package edu.hitsz.c102c.cnn; import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; import java.io.PrintWriter; import java.io.Serializable; import java.util.ArrayList; -import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; @@ -23,35 +18,27 @@ import edu.hitsz.c102c.util.Util.Operator; public class CNN implements Serializable { - /** - * - */ + private static final long serialVersionUID = 337920299147929932L; + + private static final double LAMBDA = 0; + private static double ALPHA = 0.85; - protected static final double LAMBDA = 0; - // ����ĸ��� + + private List layers; - // ���� + private int layerNum; - // �������µĴ�С private int batchSize; - // �������������Ծ����ÿһ��Ԫ�س���һ��ֵ + private Operator divide_batchSize; - // �������������Ծ����ÿһ��Ԫ�س���alphaֵ private Operator multiply_alpha; - // �������������Ծ����ÿһ��Ԫ�س���1-labmda*alphaֵ private Operator multiply_lambda; - /** - * ��ʼ������ - * - * @param layerBuilder ����� - * @param inputMapSize ����map�Ĵ�С - * @param classNum ���ĸ�����Ҫ�����ݼ������ת��Ϊ0-classNum-1����ֵ - */ + public CNN(LayerBuilder layerBuilder, final int batchSize) { layers = layerBuilder.mLayers; layerNum = layers.size(); @@ -60,63 +47,59 @@ public CNN(LayerBuilder layerBuilder, final int batchSize) { initPerator(); } - /** - * ��ʼ�������� - */ private void initPerator() { divide_batchSize = new Operator() { - private static final long serialVersionUID = 7424011281732651055L; @Override public double process(double value) { - return value / batchSize; + return value / batchSize; // TODO: Remove division } }; - multiply_alpha = new Operator() { + multiply_alpha = new Operator() { private static final long serialVersionUID = 5761368499808006552L; @Override public double process(double value) { - return value * ALPHA; } }; - multiply_lambda = new Operator() { + multiply_lambda = new Operator() { private static final long serialVersionUID = 4499087728362870577L; @Override public double process(double value) { - return value * (1 - LAMBDA * ALPHA); } - }; } - /** - * ��ѵ������ѵ������ - * - * @param trainset - * @param repeat �����Ĵ��� - */ - public void train(Dataset trainset, int repeat) { - // ����ֹͣ��ť - new Lisenter().start(); - for (int t = 0; t < repeat && !stopTrain.get(); t++) { + + public void train(final Dataset trainset, final int iterationCount) { + new Listener().start(); + + for (int iteration = 0; iteration < iterationCount && !stopTrain.get(); iteration++) { + int epochsNum = trainset.size() / batchSize; - if (trainset.size() % batchSize != 0) - epochsNum++;// ���ȡһ�Σ�������ȡ�� - Log.i(""); - Log.i(t + "th iter epochsNum:" + epochsNum); + + if (trainset.size() % batchSize != 0) { + epochsNum++; + } + + Log.info(""); + Log.info(iteration + "th iter epochsNum:" + epochsNum); + int right = 0; int count = 0; - for (int i = 0; i < epochsNum; i++) { + + for (int epoch = 0; epoch < epochsNum; epoch++) { + int[] randPerm = Util.randomPerm(trainset.size(), batchSize); + Layer.prepareForNewBatch(); for (int index : randPerm) { @@ -127,95 +110,102 @@ public void train(Dataset trainset, int repeat) { Layer.prepareForNewRecord(); } - // ����һ��batch�����Ȩ�� updateParas(); - if (i % 50 == 0) { - System.out.print(".."); - if (i + 50 > epochsNum) + + if (epoch % 50 == 0) { + System.out.print("."); + if (epoch + 50 > epochsNum) { System.out.println(); + } } } - double p = 1.0 * right / count; - if (t % 10 == 1 && p > 0.96) {//��̬����׼ѧϰ���� + + final double p = 1.0 * right / count; + + if (iteration % 10 == 1 && p > 0.96) { ALPHA = 0.001 + ALPHA * 0.9; - Log.i("Set alpha = " + ALPHA); + Log.info("Set alpha = " + ALPHA); } - Log.i("precision " + right + "/" + count + "=" + p); + + Log.info("precision " + right + "/" + count + "=" + p); } } private static AtomicBoolean stopTrain; - static class Lisenter extends Thread { - Lisenter() { + static class Listener extends Thread { + + Listener() { setDaemon(true); stopTrain = new AtomicBoolean(false); } @Override public void run() { + System.out.println("Input & to stop train."); + while (true) { try { - int a = System.in.read(); + final int a = System.in.read(); if (a == '&') { stopTrain.compareAndSet(false, true); break; } } catch (IOException e) { - e.printStackTrace(); + throw new RuntimeException(e); } } - System.out.println("Lisenter stop"); - } + System.out.println("Listener stopped"); + } } - /** - * �������� - * - * @param trainset - * @return - */ - public double test(Dataset trainset) { + public double test(final Dataset dataset) { Layer.prepareForNewBatch(); - Iterator iter = trainset.iter(); + + final Iterator iterator = dataset.iter(); + int right = 0; - while (iter.hasNext()) { - Record record = iter.next(); + while (iterator.hasNext()) { + final Record record = iterator.next(); + forward(record); - Layer outputLayer = layers.get(layerNum - 1); - int mapNum = outputLayer.getOutMapNum(); - double[] out = new double[mapNum]; + + final Layer outputLayer = layers.get(layerNum - 1); + final int mapNum = outputLayer.getOutMapNum(); + + final double[] out = new double[mapNum]; for (int m = 0; m < mapNum; m++) { - double[][] outmap = outputLayer.getMap(m); + final double[][] outmap = outputLayer.getMap(m); out[m] = outmap[0][0]; } - if (record.getLabel().intValue() == Util.getMaxIndex(out)) + + if (record.getLabel().intValue() == Util.getMaxIndex(out)) { right++; + } } - double p = 1.0 * right / trainset.size(); - Log.i("precision", p + ""); + double p = 1.0 * right / dataset.size(); + + Log.info("precision", p + ""); + return p; } - /** - * Ԥ���� - * - * @param testset - * @param fileName - */ + // TODO: Move this method to other/new class (reduce CNN-class to the minimal CNN-logic) public void predict(Dataset testset, String fileName) { - Log.i("begin predict"); + Log.info("begin predict"); try { - int max = layers.get(layerNum - 1).getClassNum(); - PrintWriter writer = new PrintWriter(new File(fileName)); + // final int max = layers.get(layerNum - 1).getClassNum(); + final PrintWriter writer = new PrintWriter(new File(fileName)); + Layer.prepareForNewBatch(); - Iterator iter = testset.iter(); + + final Iterator iter = testset.iter(); while (iter.hasNext()) { - Record record = iter.next(); + final Record record = iter.next(); forward(record); - Layer outputLayer = layers.get(layerNum - 1); + final Layer outputLayer = layers.get(layerNum - 1); int mapNum = outputLayer.getOutMapNum(); double[] out = new double[mapNum]; @@ -236,7 +226,7 @@ public void predict(Dataset testset, String fileName) { } catch (IOException e) { throw new RuntimeException(e); } - Log.i("end predict"); + Log.info("end predict"); } private boolean isSame(double[] output, double[] target) { @@ -250,12 +240,6 @@ private boolean isSame(double[] output, double[] target) { return r; } - /** - * ѵ��һ����¼��ͬʱ�����Ƿ�Ԥ����ȷ��ǰ��¼ - * - * @param record - * @return - */ private boolean train(Record record) { forward(record); boolean result = backPropagation(record); @@ -263,18 +247,12 @@ private boolean train(Record record) { // System.exit(0); } - /* - * ������ - */ private boolean backPropagation(Record record) { boolean result = setOutLayerErrors(record); setHiddenLayerErrors(); return result; } - /** - * ���²��� - */ private void updateParas() { for (int l = 1; l < layerNum; l++) { Layer layer = layers.get(l); @@ -291,12 +269,6 @@ private void updateParas() { } } - /** - * ����ƫ�� - * - * @param layer - * @param lastLayer - */ private void updateBias(final Layer layer, Layer lastLayer) { final double[][][][] errors = layer.getErrors(); int mapNum = layer.getOutMapNum(); @@ -317,14 +289,8 @@ public void process(int start, int end) { } - /** - * ����layer��ľ���ˣ�Ȩ�أ���ƫ�� - * - * @param layer ��ǰ�� - * @param lastLayer ǰһ�� - */ private void updateKernels(final Layer layer, final Layer lastLayer) { - int mapNum = layer.getOutMapNum(); + final int mapNum = layer.getOutMapNum(); final int lastMapNum = lastLayer.getOutMapNum(); new TaskManager(mapNum) { @@ -332,27 +298,19 @@ private void updateKernels(final Layer layer, final Layer lastLayer) { public void process(int start, int end) { for (int j = start; j < end; j++) { for (int i = 0; i < lastMapNum; i++) { - // ��batch��ÿ����¼delta��� double[][] deltaKernel = null; for (int r = 0; r < batchSize; r++) { - double[][] error = layer.getError(r, j); + final double[][] error = layer.getError(r, j); if (deltaKernel == null) - deltaKernel = Util.convnValid( - lastLayer.getMap(r, i), error); - else {// �ۻ���� - deltaKernel = Util.matrixOp(Util.convnValid( - lastLayer.getMap(r, i), error), - deltaKernel, null, null, Util.plus); + deltaKernel = Util.convnValid(lastLayer.getMap(r, i), error); + else { + deltaKernel = Util.matrixOp(Util.convnValid(lastLayer.getMap(r, i), error), deltaKernel, null, null, Util.plus); } } - // ����batchSize - deltaKernel = Util.matrixOp(deltaKernel, - divide_batchSize); - // ���¾���� - double[][] kernel = layer.getKernel(i, j); - deltaKernel = Util.matrixOp(kernel, deltaKernel, - multiply_lambda, multiply_alpha, Util.plus); + deltaKernel = Util.matrixOp(deltaKernel, divide_batchSize); + final double[][] kernel = layer.getKernel(i, j); + deltaKernel = Util.matrixOp(kernel, deltaKernel, multiply_lambda, multiply_alpha, Util.plus); layer.setKernel(i, j, deltaKernel); } } @@ -362,13 +320,10 @@ public void process(int start, int end) { } - /** - * �����н�����IJв� - */ private void setHiddenLayerErrors() { for (int l = layerNum - 2; l > 0; l--) { - Layer layer = layers.get(l); - Layer nextLayer = layers.get(l + 1); + final Layer layer = layers.get(l); + final Layer nextLayer = layers.get(l + 1); switch (layer.getType()) { case samp: setSampErrors(layer, nextLayer); @@ -376,20 +331,14 @@ private void setHiddenLayerErrors() { case conv: setConvErrors(layer, nextLayer); break; - default:// ֻ�в�����;������Ҫ����в�����û�вв������Ѿ������ + default: break; } } } - /** - * ���ò�����IJв� - * - * @param layer - * @param nextLayer - */ private void setSampErrors(final Layer layer, final Layer nextLayer) { - int mapNum = layer.getOutMapNum(); + final int mapNum = layer.getOutMapNum(); final int nextMapNum = nextLayer.getOutMapNum(); new TaskManager(mapNum) { @@ -398,17 +347,12 @@ public void process(int start, int end) { for (int i = start; i < end; i++) { double[][] sum = null;// ��ÿһ������������ for (int j = 0; j < nextMapNum; j++) { - double[][] nextError = nextLayer.getError(j); - double[][] kernel = nextLayer.getKernel(i, j); - // �Ծ���˽���180����ת��Ȼ�����fullģʽ�µþ�� + final double[][] nextError = nextLayer.getError(j); + final double[][] kernel = nextLayer.getKernel(i, j); if (sum == null) - sum = Util - .convnFull(nextError, Util.rot180(kernel)); + sum = Util.convnFull(nextError, Util.rot180(kernel)); else - sum = Util.matrixOp( - Util.convnFull(nextError, - Util.rot180(kernel)), sum, null, - null, Util.plus); + sum = Util.matrixOp(Util.convnFull(nextError, Util.rot180(kernel)), sum, null, null, Util.plus); } layer.setError(i, sum); } @@ -418,46 +362,23 @@ public void process(int start, int end) { } - /** - * ���þ����IJв� - * - * @param layer - * @param nextLayer - */ private void setConvErrors(final Layer layer, final Layer nextLayer) { - // ��������һ��Ϊ�����㣬�������map������ͬ����һ��mapֻ����һ���һ��map���ӣ� - // ���ֻ�轫��һ��IJв�kronecker��չ���õ������ - int mapNum = layer.getOutMapNum(); + final int mapNum = layer.getOutMapNum(); new TaskManager(mapNum) { - @Override public void process(int start, int end) { for (int m = start; m < end; m++) { - Size scale = nextLayer.getScaleSize(); - double[][] nextError = nextLayer.getError(m); - double[][] map = layer.getMap(m); - // ������ˣ����Եڶ��������ÿ��Ԫ��value����1-value���� - double[][] outMatrix = Util.matrixOp(map, - Util.cloneMatrix(map), null, Util.one_value, - Util.multiply); - outMatrix = Util.matrixOp(outMatrix, - Util.kronecker(nextError, scale), null, null, - Util.multiply); + final Size scale = nextLayer.getScaleSize(); + final double[][] nextError = nextLayer.getError(m); + final double[][] map = layer.getMap(m); + double[][] outMatrix = Util.matrixOp(map, Util.cloneMatrix(map), null, Util.one_value, Util.multiply); + outMatrix = Util.matrixOp(outMatrix, Util.kronecker(nextError, scale), null, null, Util.multiply); layer.setError(m, outMatrix); } - } - }.start(); - } - /** - * ���������IJв�ֵ,�������񾭵�Ԫ�������٣��ݲ����Ƕ��߳� - * - * @param record - * @return - */ private boolean setOutLayerErrors(Record record) { Layer outputLayer = layers.get(layerNum - 1); @@ -478,206 +399,176 @@ private boolean setOutLayerErrors(Record record) { // return true; // return false; - double[] target = new double[mapNum]; - double[] outmaps = new double[mapNum]; + final double[] target = new double[mapNum]; + final double[] outmaps = new double[mapNum]; + for (int m = 0; m < mapNum; m++) { - double[][] outmap = outputLayer.getMap(m); + final double[][] outmap = outputLayer.getMap(m); outmaps[m] = outmap[0][0]; - } - int label = record.getLabel().intValue(); + + final int label = record.getLabel().intValue(); + target[label] = 1; + // Log.i(record.getLable() + "outmaps:" + // Util.fomart(outmaps) // + Arrays.toString(target)); + for (int m = 0; m < mapNum; m++) { - outputLayer.setError(m, 0, 0, outmaps[m] * (1 - outmaps[m]) - * (target[m] - outmaps[m])); + outputLayer.setError(m, 0, 0, outmaps[m] * (1 - outmaps[m]) * (target[m] - outmaps[m])); } + return label == Util.getMaxIndex(outmaps); } - /** - * ǰ�����һ����¼ - * - * @param record - */ private void forward(Record record) { - // ����������map setInLayerOutput(record); + for (int l = 1; l < layers.size(); l++) { - Layer layer = layers.get(l); - Layer lastLayer = layers.get(l - 1); + final Layer layer = layers.get(l); + final Layer lastLayer = layers.get(l - 1); + switch (layer.getType()) { - case conv:// ������������ + case conv: setConvOutput(layer, lastLayer); break; - case samp:// ������������� + + case samp: setSampOutput(layer, lastLayer); break; - case output:// �������������,�������һ������ľ���� + + case output: setConvOutput(layer, lastLayer); break; + default: break; } } } - /** - * ���ݼ�¼ֵ���������������ֵ - * - * @param record - */ private void setInLayerOutput(Record record) { final Layer inputLayer = layers.get(0); final Size mapSize = inputLayer.getMapSize(); + final double[] attr = record.getAttrs(); - if (attr.length != mapSize.x * mapSize.y) - throw new RuntimeException("���ݼ�¼�Ĵ�С�붨���map��С��һ��!"); + + if (attr.length != mapSize.x * mapSize.y) { + throw new RuntimeException("The size of the data record does not match the size of the map defined!"); + } + for (int i = 0; i < mapSize.x; i++) { for (int j = 0; j < mapSize.y; j++) { - // ����¼���Ե�һά����Ū�ɶ�ά���� inputLayer.setMapValue(0, i, j, attr[mapSize.x * i + j]); } } } - /* - * �����������ֵ,ÿ���̸߳���һ����map + /** + * Compute the output of the convolutional layer, each thread is responsible for part of the map */ private void setConvOutput(final Layer layer, final Layer lastLayer) { - int mapNum = layer.getOutMapNum(); + final int mapNum = layer.getOutMapNum(); final int lastMapNum = lastLayer.getOutMapNum(); new TaskManager(mapNum) { @Override public void process(int start, int end) { for (int j = start; j < end; j++) { - double[][] sum = null;// ��ÿһ������map�ľ��������� + double[][] sum = null; for (int i = 0; i < lastMapNum; i++) { double[][] lastMap = lastLayer.getMap(i); double[][] kernel = layer.getKernel(i, j); - if (sum == null) + if (sum == null) { sum = Util.convnValid(lastMap, kernel); - else - sum = Util.matrixOp( - Util.convnValid(lastMap, kernel), sum, - null, null, Util.plus); + } else { + sum = Util.matrixOp(Util.convnValid(lastMap, kernel), sum, null, null, Util.plus); + } } final double bias = layer.getBias(j); - sum = Util.matrixOp(sum, new Operator() { - private static final long serialVersionUID = 2469461972825890810L; - - @Override - public double process(double value) { - return Util.sigmod(value + bias); - } - - }); + sum = Util + .matrixOp( + sum, + new Operator() { + private static final long serialVersionUID = 2469461972825890810L; + + @Override + public double process(double value) { + return Util.sigmod(value + bias); + } + } + ); layer.setMapValue(j, sum); } } - }.start(); - } - /** - * ���ò���������ֵ���������ǶԾ����ľ�ֵ���� - * - * @param layer - * @param lastLayer - */ private void setSampOutput(final Layer layer, final Layer lastLayer) { - int lastMapNum = lastLayer.getOutMapNum(); - new TaskManager(lastMapNum) { + final int lastMapNum = lastLayer.getOutMapNum(); + new TaskManager(lastMapNum) { @Override public void process(int start, int end) { for (int i = start; i < end; i++) { - double[][] lastMap = lastLayer.getMap(i); - Size scaleSize = layer.getScaleSize(); - // ��scaleSize������о�ֵ���� - double[][] sampMatrix = Util - .scaleMatrix(lastMap, scaleSize); + final double[][] lastMap = lastLayer.getMap(i); + final Size scaleSize = layer.getScaleSize(); + final double[][] sampMatrix = Util.scaleMatrix(lastMap, scaleSize); layer.setMapValue(i, sampMatrix); } } - }.start(); } - /** - * ����cnn�����ÿһ��IJ��� - * - * @param batchSize * @param classNum - * @param inputMapSize - */ - public void setup(int batchSize) { - Layer inputLayer = layers.get(0); - // ÿһ�㶼��Ҫ��ʼ�����map + private void setup(int batchSize) { + final Layer inputLayer = layers.get(0); + inputLayer.initOutmaps(batchSize); + for (int i = 1; i < layers.size(); i++) { - Layer layer = layers.get(i); - Layer frontLayer = layers.get(i - 1); - int frontMapNum = frontLayer.getOutMapNum(); + + final Layer layer = layers.get(i); + final Layer frontLayer = layers.get(i - 1); + + final int frontMapNum = frontLayer.getOutMapNum(); switch (layer.getType()) { case input: break; case conv: - // ����map�Ĵ�С - layer.setMapSize(frontLayer.getMapSize().subtract( - layer.getKernelSize(), 1)); - // ��ʼ������ˣ�����frontMapNum*outMapNum������� - + layer.setMapSize(frontLayer.getMapSize().subtract(layer.getKernelSize(), 1)); layer.initKernel(frontMapNum); - // ��ʼ��ƫ�ã�����frontMapNum*outMapNum��ƫ�� layer.initBias(frontMapNum); - // batch��ÿ����¼��Ҫ����һ�ݲв� layer.initErros(batchSize); - // ÿһ�㶼��Ҫ��ʼ�����map layer.initOutmaps(batchSize); break; + case samp: - // �������map��������һ����ͬ layer.setOutMapNum(frontMapNum); - // ������map�Ĵ�С����һ��map�Ĵ�С����scale��С - layer.setMapSize(frontLayer.getMapSize().divide( - layer.getScaleSize())); - // batch��ÿ����¼��Ҫ����һ�ݲв� + layer.setMapSize(frontLayer.getMapSize().divide(layer.getScaleSize())); layer.initErros(batchSize); - // ÿһ�㶼��Ҫ��ʼ�����map layer.initOutmaps(batchSize); break; + case output: - // ��ʼ��Ȩ�أ�����ˣ��������ľ���˴�СΪ��һ���map��С layer.initOutputKerkel(frontMapNum, frontLayer.getMapSize()); - // ��ʼ��ƫ�ã�����frontMapNum*outMapNum��ƫ�� layer.initBias(frontMapNum); - // batch��ÿ����¼��Ҫ����һ�ݲв� layer.initErros(batchSize); - // ÿһ�㶼��Ҫ��ʼ�����map layer.initOutmaps(batchSize); break; } } } - /** - * ������ģʽ�������,Ҫ�����ڶ������Ϊ�����������Ϊ����� - * - * @author jiqunpeng - *

- * ����ʱ�䣺2014-7-8 ����4:54:29 - */ + // === inner classes === + public static class LayerBuilder { private List mLayers; public LayerBuilder() { - mLayers = new ArrayList(); + mLayers = new ArrayList<>(); } public LayerBuilder(Layer layer) { @@ -691,40 +582,4 @@ public LayerBuilder addLayer(Layer layer) { } } - /** - * ���л�����ģ�� - * - * @param fileName - */ - public void saveModel(String fileName) { - try { - ObjectOutputStream oos = new ObjectOutputStream( - new FileOutputStream(fileName)); - oos.writeObject(this); - oos.flush(); - oos.close(); - } catch (IOException e) { - e.printStackTrace(); - } - - } - - /** - * �����л�����ģ�� - * - * @param fileName - * @return - */ - public static CNN loadModel(String fileName) { - try { - ObjectInputStream in = new ObjectInputStream(new FileInputStream( - fileName)); - CNN cnn = (CNN) in.readObject(); - in.close(); - return cnn; - } catch (IOException | ClassNotFoundException e) { - e.printStackTrace(); - } - return null; - } } diff --git a/src/edu/hitsz/c102c/cnn/CNNLoader.java b/src/edu/hitsz/c102c/cnn/CNNLoader.java new file mode 100644 index 0000000..0aeb688 --- /dev/null +++ b/src/edu/hitsz/c102c/cnn/CNNLoader.java @@ -0,0 +1,35 @@ +package edu.hitsz.c102c.cnn; + +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; + +public class CNNLoader { + + public static void saveModel(final String fileName, final CNN cnn) throws IOException { + saveModel(cnn, new FileOutputStream(fileName)); + return; + } + + public static void saveModel(final CNN cnn, final FileOutputStream fileOutputStream) throws IOException { + ObjectOutputStream oos = new ObjectOutputStream(fileOutputStream); + oos.writeObject(cnn); + oos.flush(); + oos.close(); + } + + public static CNN loadModel(String fileName) throws IOException, ClassNotFoundException { + return loadModel(new FileInputStream(fileName)); + } + + public static CNN loadModel(final FileInputStream fileInputStream) throws IOException, ClassNotFoundException { + final ObjectInputStream in = new ObjectInputStream(fileInputStream); + final CNN cnn = (CNN) in.readObject(); + in.close(); + + return cnn; + } + +} diff --git a/src/edu/hitsz/c102c/cnn/Layer.java b/src/edu/hitsz/c102c/cnn/Layer.java index adf78e4..1cdc5ce 100644 --- a/src/edu/hitsz/c102c/cnn/Layer.java +++ b/src/edu/hitsz/c102c/cnn/Layer.java @@ -6,70 +6,51 @@ import edu.hitsz.c102c.util.Util; /** - * cnnIJ - * + * cnn����IJ� + * * @author jiqunpeng - * - * ʱ䣺2014-7-8 3:58:46 + *

+ * ����ʱ�䣺2014-7-8 ����3:58:46 */ public class Layer implements Serializable { /** - * + * */ private static final long serialVersionUID = -5747622503947497069L; - private LayerType type;// - private int outMapNum;// mapĸ - private Size mapSize;// mapĴС - private Size kernelSize;// ˴Сֻо - private Size scaleSize;// Сֻв - private double[][][][] kernel;// ˣֻо - private double[] bias;// ÿmapӦһƫãֻо - // batchmapoutmaps[0][0]ʾһ¼ѵµ0map + private LayerType type; + private int outMapNum; + private Size mapSize; + private Size kernelSize; + private Size scaleSize; + private double[][][][] kernel; + private double[] bias; private double[][][][] outmaps; - // вmatlab toolboxdӦ private double[][][][] errors; - private static int recordInBatch = 0;// ¼ǰѵbatchĵڼ¼ + private static int recordInBatch = 0; - private int classNum = -1;// + private int classNum = -1; private Layer() { } - /** - * ׼һbatchѵ - */ public static void prepareForNewBatch() { recordInBatch = 0; } - /** - * ׼һ¼ѵ - */ public static void prepareForNewRecord() { recordInBatch++; } - /** - * ʼ - * - * @param mapSize - * @return - */ public static Layer buildInputLayer(Size mapSize) { Layer layer = new Layer(); layer.type = LayerType.input; - layer.outMapNum = 1;// mapΪ1һͼ + layer.outMapNum = 1;// ������map����Ϊ1����һ��ͼ layer.setMapSize(mapSize);// return layer; } - /** - * - * - * @return - */ public static Layer buildConvLayer(int outMapNum, Size kernelSize) { Layer layer = new Layer(); layer.type = LayerType.conv; @@ -78,12 +59,6 @@ public static Layer buildConvLayer(int outMapNum, Size kernelSize) { return layer; } - /** - * - * - * @param scaleSize - * @return - */ public static Layer buildSampLayer(Size scaleSize) { Layer layer = new Layer(); layer.type = LayerType.samp; @@ -91,11 +66,6 @@ public static Layer buildSampLayer(Size scaleSize) { return layer; } - /** - * ,ĸԪĸ - * - * @return - */ public static Layer buildOutputLayer(int classNum) { Layer layer = new Layer(); layer.classNum = classNum; @@ -106,86 +76,44 @@ public static Layer buildOutputLayer(int classNum) { // while ((1 << outMapNum) < classNum) // outMapNum += 1; // layer.outMapNum = outMapNum; - Log.i("outMapNum:" + layer.outMapNum); + Log.info("outMapNum:" + layer.outMapNum); return layer; } - /** - * ȡmapĴС - * - * @return - */ public Size getMapSize() { return mapSize; } - /** - * mapĴС - * - * @param mapSize - */ public void setMapSize(Size mapSize) { this.mapSize = mapSize; } - /** - * ȡ - * - * @return - */ public LayerType getType() { return type; } - /** - * ȡ - * - * @return - */ - public int getOutMapNum() { return outMapNum; } - /** - * mapĸ - * - * @param outMapNum - */ public void setOutMapNum(int outMapNum) { this.outMapNum = outMapNum; } - /** - * ȡ˵ĴСֻоkernelSizeδnull - * - * @return - */ public Size getKernelSize() { return kernelSize; } - /** - * ȡСֻвscaleSizeδnull - * - * @return - */ public Size getScaleSize() { return scaleSize; } enum LayerType { - // ͣ㡢㡢㡢 input, output, conv, samp } - /** - * ˻߲scaleĴС,Բ.ͰȫԺ󲻿޸ - * - * @author jiqunpeng - * - * ʱ䣺2014-7-8 4:11:00 - */ + // --- + public static class Size implements Serializable { private static final long serialVersionUID = -209157832162004118L; @@ -204,9 +132,9 @@ public String toString() { } /** - * scaleSizeõһµSizeҪthis.xthis. - * yֱܷscaleSize.xscaleSize.y - * + * ����scaleSize�õ�һ���µ�Size��Ҫ��this.x��this. + * y�ֱܷ�scaleSize.x��scaleSize.y���� + * * @param scaleSize * @return */ @@ -214,13 +142,13 @@ public Size divide(Size scaleSize) { int x = this.x / scaleSize.x; int y = this.y / scaleSize.y; if (x * scaleSize.x != this.x || y * scaleSize.y != this.y) - throw new RuntimeException(this + "" + scaleSize); + throw new RuntimeException(this + "��������" + scaleSize); return new Size(x, y); } /** - * ȥsizeСxyֱ𸽼һֵappend - * + * ��ȥsize��С����x��y�ֱ𸽼�һ��ֵappend + * * @param size * @param append * @return @@ -233,8 +161,8 @@ public Size subtract(Size size, int append) { } /** - * ʼ - * + * �����ʼ������� + * * @param frontMapNum */ public void initKernel(int frontMapNum) { @@ -244,12 +172,12 @@ public void initKernel(int frontMapNum) { this.kernel = new double[frontMapNum][outMapNum][kernelSize.x][kernelSize.y]; for (int i = 0; i < frontMapNum; i++) for (int j = 0; j < outMapNum; j++) - kernel[i][j] = Util.randomMatrix(kernelSize.x, kernelSize.y,true); + kernel[i][j] = Util.randomMatrix(kernelSize.x, kernelSize.y, true); } /** - * ľ˵ĴСһmapС - * + * �����ľ���˵Ĵ�С����һ���map��С + * * @param frontMapNum * @param size */ @@ -261,12 +189,12 @@ public void initOutputKerkel(int frontMapNum, Size size) { this.kernel = new double[frontMapNum][outMapNum][kernelSize.x][kernelSize.y]; for (int i = 0; i < frontMapNum; i++) for (int j = 0; j < outMapNum; j++) - kernel[i][j] = Util.randomMatrix(kernelSize.x, kernelSize.y,false); + kernel[i][j] = Util.randomMatrix(kernelSize.x, kernelSize.y, false); } /** - * ʼƫ - * + * ��ʼ��ƫ�� + * * @param frontMapNum */ public void initBias(int frontMapNum) { @@ -274,8 +202,8 @@ public void initBias(int frontMapNum) { } /** - * ʼmap - * + * ��ʼ�����map + * * @param batchSize */ public void initOutmaps(int batchSize) { @@ -283,14 +211,11 @@ public void initOutmaps(int batchSize) { } /** - * mapֵ - * - * @param mapNo - * ڼmap - * @param mapX - * mapĸ - * @param mapY - * mapĿ + * ����mapֵ + * + * @param mapNo �ڼ���map + * @param mapX map�ĸ� + * @param mapY map�Ŀ� * @param value */ public void setMapValue(int mapNo, int mapX, int mapY, double value) { @@ -300,8 +225,8 @@ public void setMapValue(int mapNo, int mapX, int mapY, double value) { static int count = 0; /** - * ԾʽõmapNomapֵ - * + * �Ծ�����ʽ���õ�mapNo��map��ֵ + * * @param mapNo * @param outMatrix */ @@ -312,9 +237,9 @@ public void setMapValue(int mapNo, double[][] outMatrix) { } /** - * ȡindexmap󡣴ܿǣûзظƶ󣬶ֱӷãö - * ޸outmaps޸setMapValue(...) - * + * ��ȡ��index��map���󡣴������ܿ��ǣ�û�з��ظ��ƶ��󣬶���ֱ�ӷ������ã����ö�������� + * �����޸�outmaps�������޸������setMapValue(...) + * * @param index * @return */ @@ -323,12 +248,10 @@ public double[][] getMap(int index) { } /** - * ȡǰһimapǰjmapľ - * - * @param i - * һmap± - * @param j - * ǰmap± + * ��ȡǰһ���i��map����ǰ���j��map�ľ���� + * + * @param i ��һ���map�±� + * @param j ��ǰ���map�±� * @return */ public double[][] getKernel(int i, int j) { @@ -336,8 +259,8 @@ public double[][] getKernel(int i, int j) { } /** - * òвֵ - * + * ���òв�ֵ + * * @param mapNo * @param mapX * @param mapY @@ -348,8 +271,8 @@ public void setError(int mapNo, int mapX, int mapY, double value) { } /** - * mapʽòвֵ - * + * ��map�������ʽ���òв�ֵ + * * @param mapNo * @param matrix */ @@ -360,9 +283,9 @@ public void setError(int mapNo, double[][] matrix) { } /** - * ȡmapNomapIJв.ûзظƶ󣬶ֱӷãö - * ޸errors޸setError(...) - * + * ��ȡ��mapNo��map�IJв�.û�з��ظ��ƶ��󣬶���ֱ�ӷ������ã����ö�������� + * �����޸�errors�������޸������setError(...) + * * @param mapNo * @return */ @@ -371,8 +294,8 @@ public double[][] getError(int mapNo) { } /** - * ȡ(ÿ¼ÿmap)IJв - * + * ��ȡ����(ÿ����¼��ÿ��map)�IJв� + * * @return */ public double[][][][] getErrors() { @@ -380,8 +303,8 @@ public double[][][][] getErrors() { } /** - * ʼв - * + * ��ʼ���в����� + * * @param batchSize */ public void initErros(int batchSize) { @@ -389,7 +312,6 @@ public void initErros(int batchSize) { } /** - * * @param lastMapNo * @param mapNo * @param kernel @@ -399,8 +321,8 @@ public void setKernel(int lastMapNo, int mapNo, double[][] kernel) { } /** - * ȡmapNo - * + * ��ȡ��mapNo�� + * * @param mapNo * @return */ @@ -409,8 +331,8 @@ public double getBias(int mapNo) { } /** - * õmapNomapƫֵ - * + * ���õ�mapNo��map��ƫ��ֵ + * * @param mapNo * @param value */ @@ -419,8 +341,8 @@ public void setBias(int mapNo, double value) { } /** - * ȡbatchmap - * + * ��ȡbatch����map���� + * * @return */ @@ -429,8 +351,8 @@ public double[][][][] getMaps() { } /** - * ȡrecordId¼µmapNoIJв - * + * ��ȡ��recordId��¼�µ�mapNo�IJв� + * * @param recordId * @param mapNo * @return @@ -440,8 +362,8 @@ public double[][] getError(int recordId, int mapNo) { } /** - * ȡrecordId¼µmapNomap - * + * ��ȡ��recordId��¼�µ�mapNo�����map + * * @param recordId * @param mapNo * @return @@ -451,8 +373,8 @@ public double[][] getMap(int recordId, int mapNo) { } /** - * ȡ - * + * ��ȡ������ + * * @return */ public int getClassNum() { @@ -460,8 +382,8 @@ public int getClassNum() { } /** - * ȡеľ - * + * ��ȡ���еľ���� + * * @return */ public double[][][][] getKernel() { diff --git a/src/edu/hitsz/c102c/util/Log.java b/src/edu/hitsz/c102c/util/Log.java index 9892816..210852d 100644 --- a/src/edu/hitsz/c102c/util/Log.java +++ b/src/edu/hitsz/c102c/util/Log.java @@ -3,13 +3,13 @@ import java.io.PrintStream; public class Log { - static PrintStream stream = System.out; - - public static void i(String tag,String msg){ - stream.println(tag+"\t"+msg); + private static final PrintStream stream = System.out; + + public static void info(String tag, String msg) { + stream.println(tag + "\t" + msg); } - - public static void i(String msg){ + + public static void info(String msg) { stream.println(msg); } diff --git a/src/edu/hitsz/c102c/util/TimedTest.java b/src/edu/hitsz/c102c/util/TimedTest.java index 71125aa..008195f 100644 --- a/src/edu/hitsz/c102c/util/TimedTest.java +++ b/src/edu/hitsz/c102c/util/TimedTest.java @@ -1,11 +1,11 @@ package edu.hitsz.c102c.util; /** - * ʱIJԹ + * ��ʱ�IJ��Թ��� * * @author jiqunpeng * - * ʱ䣺2014-7-8 8:21:56 + * ����ʱ�䣺2014-7-8 ����8:21:56 */ public class TimedTest { private int repeat; @@ -26,6 +26,6 @@ public void test() { task.process(); } double cost = (System.currentTimeMillis() - t) / 1000.0; - Log.i("cost ", cost + "s"); + Log.info("cost ", cost + "s"); } } From bef562c725091656222fe5cb6e832b7b09ce0c43 Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Sat, 17 Feb 2018 11:39:50 +0100 Subject: [PATCH 06/34] Mavenized - Project is now a Maven-Project --- .gitignore | 10 ++---- pom.xml | 35 +++++++++++++++++++ src/{ => main/java}/RunCNN.java | 0 .../java}/edu/hitsz/c102c/cnn/CNN.java | 0 .../java}/edu/hitsz/c102c/cnn/CNNLoader.java | 0 .../java}/edu/hitsz/c102c/cnn/Layer.java | 0 .../edu/hitsz/c102c/dataset/Dataset.java | 0 .../hitsz/c102c/dataset/DatasetLoader.java | 0 .../hitsz/c102c/util/ConcurenceRunner.java | 0 .../java}/edu/hitsz/c102c/util/Log.java | 0 .../java}/edu/hitsz/c102c/util/TestArray.java | 0 .../java}/edu/hitsz/c102c/util/TimedTest.java | 0 .../java}/edu/hitsz/c102c/util/Util.java | 0 13 files changed, 38 insertions(+), 7 deletions(-) create mode 100644 pom.xml rename src/{ => main/java}/RunCNN.java (100%) rename src/{ => main/java}/edu/hitsz/c102c/cnn/CNN.java (100%) rename src/{ => main/java}/edu/hitsz/c102c/cnn/CNNLoader.java (100%) rename src/{ => main/java}/edu/hitsz/c102c/cnn/Layer.java (100%) rename src/{ => main/java}/edu/hitsz/c102c/dataset/Dataset.java (100%) rename src/{ => main/java}/edu/hitsz/c102c/dataset/DatasetLoader.java (100%) rename src/{ => main/java}/edu/hitsz/c102c/util/ConcurenceRunner.java (100%) rename src/{ => main/java}/edu/hitsz/c102c/util/Log.java (100%) rename src/{ => main/java}/edu/hitsz/c102c/util/TestArray.java (100%) rename src/{ => main/java}/edu/hitsz/c102c/util/TimedTest.java (100%) rename src/{ => main/java}/edu/hitsz/c102c/util/Util.java (100%) diff --git a/.gitignore b/.gitignore index 6264549..705fa9d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,5 @@ -bin/ -*.class -*MyTest.java -data/ -model/ +*.iml +/.idea /.classpath /.project -*.iml -/.idea \ No newline at end of file +/target/ \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..e4cae63 --- /dev/null +++ b/pom.xml @@ -0,0 +1,35 @@ + + + 4.0.0 + + + + javacnn + javacnn + 1-SNAPSHOT + + Implementation of a CNN in Java + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.7.0 + + 1.8 + 1.8 + -deprecation + UTF-8 + + + + + + + + diff --git a/src/RunCNN.java b/src/main/java/RunCNN.java similarity index 100% rename from src/RunCNN.java rename to src/main/java/RunCNN.java diff --git a/src/edu/hitsz/c102c/cnn/CNN.java b/src/main/java/edu/hitsz/c102c/cnn/CNN.java similarity index 100% rename from src/edu/hitsz/c102c/cnn/CNN.java rename to src/main/java/edu/hitsz/c102c/cnn/CNN.java diff --git a/src/edu/hitsz/c102c/cnn/CNNLoader.java b/src/main/java/edu/hitsz/c102c/cnn/CNNLoader.java similarity index 100% rename from src/edu/hitsz/c102c/cnn/CNNLoader.java rename to src/main/java/edu/hitsz/c102c/cnn/CNNLoader.java diff --git a/src/edu/hitsz/c102c/cnn/Layer.java b/src/main/java/edu/hitsz/c102c/cnn/Layer.java similarity index 100% rename from src/edu/hitsz/c102c/cnn/Layer.java rename to src/main/java/edu/hitsz/c102c/cnn/Layer.java diff --git a/src/edu/hitsz/c102c/dataset/Dataset.java b/src/main/java/edu/hitsz/c102c/dataset/Dataset.java similarity index 100% rename from src/edu/hitsz/c102c/dataset/Dataset.java rename to src/main/java/edu/hitsz/c102c/dataset/Dataset.java diff --git a/src/edu/hitsz/c102c/dataset/DatasetLoader.java b/src/main/java/edu/hitsz/c102c/dataset/DatasetLoader.java similarity index 100% rename from src/edu/hitsz/c102c/dataset/DatasetLoader.java rename to src/main/java/edu/hitsz/c102c/dataset/DatasetLoader.java diff --git a/src/edu/hitsz/c102c/util/ConcurenceRunner.java b/src/main/java/edu/hitsz/c102c/util/ConcurenceRunner.java similarity index 100% rename from src/edu/hitsz/c102c/util/ConcurenceRunner.java rename to src/main/java/edu/hitsz/c102c/util/ConcurenceRunner.java diff --git a/src/edu/hitsz/c102c/util/Log.java b/src/main/java/edu/hitsz/c102c/util/Log.java similarity index 100% rename from src/edu/hitsz/c102c/util/Log.java rename to src/main/java/edu/hitsz/c102c/util/Log.java diff --git a/src/edu/hitsz/c102c/util/TestArray.java b/src/main/java/edu/hitsz/c102c/util/TestArray.java similarity index 100% rename from src/edu/hitsz/c102c/util/TestArray.java rename to src/main/java/edu/hitsz/c102c/util/TestArray.java diff --git a/src/edu/hitsz/c102c/util/TimedTest.java b/src/main/java/edu/hitsz/c102c/util/TimedTest.java similarity index 100% rename from src/edu/hitsz/c102c/util/TimedTest.java rename to src/main/java/edu/hitsz/c102c/util/TimedTest.java diff --git a/src/edu/hitsz/c102c/util/Util.java b/src/main/java/edu/hitsz/c102c/util/Util.java similarity index 100% rename from src/edu/hitsz/c102c/util/Util.java rename to src/main/java/edu/hitsz/c102c/util/Util.java From 2529e4ee30eba075932e458b130ba6b752c43a53 Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Sat, 17 Feb 2018 12:01:53 +0100 Subject: [PATCH 07/34] Moved package "edu.hitsz.c102c" to "javacnn" --- src/main/java/RunCNN.java | 18 ++--- .../{edu/hitsz/c102c => javacnn}/cnn/CNN.java | 63 +++++++-------- .../c102c => javacnn}/cnn/CNNLoader.java | 2 +- .../hitsz/c102c => javacnn}/cnn/Layer.java | 6 +- .../c102c => javacnn}/dataset/Dataset.java | 2 +- .../dataset/DatasetLoader.java | 2 +- .../util/ConcurenceRunner.java | 10 +-- .../hitsz/c102c => javacnn}/util/Log.java | 2 +- .../c102c => javacnn}/util/TestArray.java | 10 +-- .../c102c => javacnn}/util/TimedTest.java | 2 +- .../hitsz/c102c => javacnn}/util/Util.java | 79 +++++++++---------- 11 files changed, 96 insertions(+), 100 deletions(-) rename src/main/java/{edu/hitsz/c102c => javacnn}/cnn/CNN.java (90%) rename src/main/java/{edu/hitsz/c102c => javacnn}/cnn/CNNLoader.java (97%) rename src/main/java/{edu/hitsz/c102c => javacnn}/cnn/Layer.java (98%) rename src/main/java/{edu/hitsz/c102c => javacnn}/dataset/Dataset.java (98%) rename src/main/java/{edu/hitsz/c102c => javacnn}/dataset/DatasetLoader.java (96%) rename src/main/java/{edu/hitsz/c102c => javacnn}/util/ConcurenceRunner.java (89%) rename src/main/java/{edu/hitsz/c102c => javacnn}/util/Log.java (89%) rename src/main/java/{edu/hitsz/c102c => javacnn}/util/TestArray.java (73%) rename src/main/java/{edu/hitsz/c102c => javacnn}/util/TimedTest.java (94%) rename src/main/java/{edu/hitsz/c102c => javacnn}/util/Util.java (84%) diff --git a/src/main/java/RunCNN.java b/src/main/java/RunCNN.java index fc01e2b..e53d0a2 100644 --- a/src/main/java/RunCNN.java +++ b/src/main/java/RunCNN.java @@ -1,14 +1,14 @@ import java.io.IOException; -import edu.hitsz.c102c.cnn.CNN; -import edu.hitsz.c102c.cnn.CNN.LayerBuilder; -import edu.hitsz.c102c.cnn.CNNLoader; -import edu.hitsz.c102c.cnn.Layer; -import edu.hitsz.c102c.cnn.Layer.Size; -import edu.hitsz.c102c.dataset.Dataset; -import edu.hitsz.c102c.dataset.DatasetLoader; -import edu.hitsz.c102c.util.ConcurenceRunner; -import edu.hitsz.c102c.util.TimedTest; +import javacnn.cnn.CNN; +import javacnn.cnn.CNN.LayerBuilder; +import javacnn.cnn.CNNLoader; +import javacnn.cnn.Layer; +import javacnn.cnn.Layer.Size; +import javacnn.dataset.Dataset; +import javacnn.dataset.DatasetLoader; +import javacnn.util.ConcurenceRunner; +import javacnn.util.TimedTest; public class RunCNN { diff --git a/src/main/java/edu/hitsz/c102c/cnn/CNN.java b/src/main/java/javacnn/cnn/CNN.java similarity index 90% rename from src/main/java/edu/hitsz/c102c/cnn/CNN.java rename to src/main/java/javacnn/cnn/CNN.java index f8b392c..38acee1 100644 --- a/src/main/java/edu/hitsz/c102c/cnn/CNN.java +++ b/src/main/java/javacnn/cnn/CNN.java @@ -1,4 +1,4 @@ -package edu.hitsz.c102c.cnn; +package javacnn.cnn; import java.io.File; import java.io.IOException; @@ -9,13 +9,10 @@ import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; -import edu.hitsz.c102c.cnn.Layer.Size; -import edu.hitsz.c102c.dataset.Dataset; -import edu.hitsz.c102c.dataset.Dataset.Record; -import edu.hitsz.c102c.util.ConcurenceRunner.TaskManager; -import edu.hitsz.c102c.util.Log; -import edu.hitsz.c102c.util.Util; -import edu.hitsz.c102c.util.Util.Operator; +import javacnn.dataset.Dataset; +import javacnn.util.Log; +import javacnn.util.Util; +import javacnn.util.ConcurenceRunner; public class CNN implements Serializable { @@ -32,11 +29,11 @@ public class CNN implements Serializable { private int batchSize; - private Operator divide_batchSize; + private Util.Operator divide_batchSize; - private Operator multiply_alpha; + private Util.Operator multiply_alpha; - private Operator multiply_lambda; + private Util.Operator multiply_lambda; public CNN(LayerBuilder layerBuilder, final int batchSize) { @@ -48,7 +45,7 @@ public CNN(LayerBuilder layerBuilder, final int batchSize) { } private void initPerator() { - divide_batchSize = new Operator() { + divide_batchSize = new Util.Operator() { private static final long serialVersionUID = 7424011281732651055L; @Override @@ -58,7 +55,7 @@ public double process(double value) { }; - multiply_alpha = new Operator() { + multiply_alpha = new Util.Operator() { private static final long serialVersionUID = 5761368499808006552L; @Override @@ -68,7 +65,7 @@ public double process(double value) { }; - multiply_lambda = new Operator() { + multiply_lambda = new Util.Operator() { private static final long serialVersionUID = 4499087728362870577L; @Override @@ -164,11 +161,11 @@ public void run() { public double test(final Dataset dataset) { Layer.prepareForNewBatch(); - final Iterator iterator = dataset.iter(); + final Iterator iterator = dataset.iter(); int right = 0; while (iterator.hasNext()) { - final Record record = iterator.next(); + final Dataset.Record record = iterator.next(); forward(record); @@ -201,9 +198,9 @@ public void predict(Dataset testset, String fileName) { Layer.prepareForNewBatch(); - final Iterator iter = testset.iter(); + final Iterator iter = testset.iter(); while (iter.hasNext()) { - final Record record = iter.next(); + final Dataset.Record record = iter.next(); forward(record); final Layer outputLayer = layers.get(layerNum - 1); @@ -240,14 +237,14 @@ private boolean isSame(double[] output, double[] target) { return r; } - private boolean train(Record record) { + private boolean train(Dataset.Record record) { forward(record); boolean result = backPropagation(record); return result; // System.exit(0); } - private boolean backPropagation(Record record) { + private boolean backPropagation(Dataset.Record record) { boolean result = setOutLayerErrors(record); setHiddenLayerErrors(); return result; @@ -273,7 +270,7 @@ private void updateBias(final Layer layer, Layer lastLayer) { final double[][][][] errors = layer.getErrors(); int mapNum = layer.getOutMapNum(); - new TaskManager(mapNum) { + new ConcurenceRunner.TaskManager(mapNum) { @Override public void process(int start, int end) { @@ -292,7 +289,7 @@ public void process(int start, int end) { private void updateKernels(final Layer layer, final Layer lastLayer) { final int mapNum = layer.getOutMapNum(); final int lastMapNum = lastLayer.getOutMapNum(); - new TaskManager(mapNum) { + new ConcurenceRunner.TaskManager(mapNum) { @Override public void process(int start, int end) { @@ -340,7 +337,7 @@ private void setHiddenLayerErrors() { private void setSampErrors(final Layer layer, final Layer nextLayer) { final int mapNum = layer.getOutMapNum(); final int nextMapNum = nextLayer.getOutMapNum(); - new TaskManager(mapNum) { + new ConcurenceRunner.TaskManager(mapNum) { @Override public void process(int start, int end) { @@ -364,11 +361,11 @@ public void process(int start, int end) { private void setConvErrors(final Layer layer, final Layer nextLayer) { final int mapNum = layer.getOutMapNum(); - new TaskManager(mapNum) { + new ConcurenceRunner.TaskManager(mapNum) { @Override public void process(int start, int end) { for (int m = start; m < end; m++) { - final Size scale = nextLayer.getScaleSize(); + final Layer.Size scale = nextLayer.getScaleSize(); final double[][] nextError = nextLayer.getError(m); final double[][] map = layer.getMap(m); double[][] outMatrix = Util.matrixOp(map, Util.cloneMatrix(map), null, Util.one_value, Util.multiply); @@ -379,7 +376,7 @@ public void process(int start, int end) { }.start(); } - private boolean setOutLayerErrors(Record record) { + private boolean setOutLayerErrors(Dataset.Record record) { Layer outputLayer = layers.get(layerNum - 1); int mapNum = outputLayer.getOutMapNum(); @@ -422,7 +419,7 @@ private boolean setOutLayerErrors(Record record) { return label == Util.getMaxIndex(outmaps); } - private void forward(Record record) { + private void forward(Dataset.Record record) { setInLayerOutput(record); for (int l = 1; l < layers.size(); l++) { @@ -448,9 +445,9 @@ private void forward(Record record) { } } - private void setInLayerOutput(Record record) { + private void setInLayerOutput(Dataset.Record record) { final Layer inputLayer = layers.get(0); - final Size mapSize = inputLayer.getMapSize(); + final Layer.Size mapSize = inputLayer.getMapSize(); final double[] attr = record.getAttrs(); @@ -471,7 +468,7 @@ private void setInLayerOutput(Record record) { private void setConvOutput(final Layer layer, final Layer lastLayer) { final int mapNum = layer.getOutMapNum(); final int lastMapNum = lastLayer.getOutMapNum(); - new TaskManager(mapNum) { + new ConcurenceRunner.TaskManager(mapNum) { @Override public void process(int start, int end) { @@ -490,7 +487,7 @@ public void process(int start, int end) { sum = Util .matrixOp( sum, - new Operator() { + new Util.Operator() { private static final long serialVersionUID = 2469461972825890810L; @Override @@ -509,12 +506,12 @@ public double process(double value) { private void setSampOutput(final Layer layer, final Layer lastLayer) { final int lastMapNum = lastLayer.getOutMapNum(); - new TaskManager(lastMapNum) { + new ConcurenceRunner.TaskManager(lastMapNum) { @Override public void process(int start, int end) { for (int i = start; i < end; i++) { final double[][] lastMap = lastLayer.getMap(i); - final Size scaleSize = layer.getScaleSize(); + final Layer.Size scaleSize = layer.getScaleSize(); final double[][] sampMatrix = Util.scaleMatrix(lastMap, scaleSize); layer.setMapValue(i, sampMatrix); } diff --git a/src/main/java/edu/hitsz/c102c/cnn/CNNLoader.java b/src/main/java/javacnn/cnn/CNNLoader.java similarity index 97% rename from src/main/java/edu/hitsz/c102c/cnn/CNNLoader.java rename to src/main/java/javacnn/cnn/CNNLoader.java index 0aeb688..bbb4ecf 100644 --- a/src/main/java/edu/hitsz/c102c/cnn/CNNLoader.java +++ b/src/main/java/javacnn/cnn/CNNLoader.java @@ -1,4 +1,4 @@ -package edu.hitsz.c102c.cnn; +package javacnn.cnn; import java.io.FileInputStream; import java.io.FileOutputStream; diff --git a/src/main/java/edu/hitsz/c102c/cnn/Layer.java b/src/main/java/javacnn/cnn/Layer.java similarity index 98% rename from src/main/java/edu/hitsz/c102c/cnn/Layer.java rename to src/main/java/javacnn/cnn/Layer.java index 1cdc5ce..f099416 100644 --- a/src/main/java/edu/hitsz/c102c/cnn/Layer.java +++ b/src/main/java/javacnn/cnn/Layer.java @@ -1,9 +1,9 @@ -package edu.hitsz.c102c.cnn; +package javacnn.cnn; import java.io.Serializable; -import edu.hitsz.c102c.util.Log; -import edu.hitsz.c102c.util.Util; +import javacnn.util.Log; +import javacnn.util.Util; /** * cnn����IJ� diff --git a/src/main/java/edu/hitsz/c102c/dataset/Dataset.java b/src/main/java/javacnn/dataset/Dataset.java similarity index 98% rename from src/main/java/edu/hitsz/c102c/dataset/Dataset.java rename to src/main/java/javacnn/dataset/Dataset.java index 5dfe095..d13ddb8 100644 --- a/src/main/java/edu/hitsz/c102c/dataset/Dataset.java +++ b/src/main/java/javacnn/dataset/Dataset.java @@ -1,4 +1,4 @@ -package edu.hitsz.c102c.dataset; +package javacnn.dataset; import java.util.ArrayList; import java.util.Arrays; diff --git a/src/main/java/edu/hitsz/c102c/dataset/DatasetLoader.java b/src/main/java/javacnn/dataset/DatasetLoader.java similarity index 96% rename from src/main/java/edu/hitsz/c102c/dataset/DatasetLoader.java rename to src/main/java/javacnn/dataset/DatasetLoader.java index 45caaf0..f5a663d 100644 --- a/src/main/java/edu/hitsz/c102c/dataset/DatasetLoader.java +++ b/src/main/java/javacnn/dataset/DatasetLoader.java @@ -1,4 +1,4 @@ -package edu.hitsz.c102c.dataset; +package javacnn.dataset; import java.io.BufferedReader; import java.io.FileReader; diff --git a/src/main/java/edu/hitsz/c102c/util/ConcurenceRunner.java b/src/main/java/javacnn/util/ConcurenceRunner.java similarity index 89% rename from src/main/java/edu/hitsz/c102c/util/ConcurenceRunner.java rename to src/main/java/javacnn/util/ConcurenceRunner.java index 7a57a66..39bbf1e 100644 --- a/src/main/java/edu/hitsz/c102c/util/ConcurenceRunner.java +++ b/src/main/java/javacnn/util/ConcurenceRunner.java @@ -1,15 +1,15 @@ -package edu.hitsz.c102c.util; +package javacnn.util; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; /** - * й + * �������й��� * * @author jiqunpeng * - * ʱ䣺2014-6-16 3:33:41 + * ����ʱ�䣺2014-6-16 ����3:33:41 */ public class ConcurenceRunner { @@ -60,7 +60,7 @@ public TaskManager(int workLength) { public void start() { int runCpu = cpuNum < workLength ? cpuNum : 1; - // Ƭȡ + // ��Ƭ��������ȡ�� final CountDownLatch gate = new CountDownLatch(runCpu); int fregLength = (workLength + runCpu - 1) / runCpu; for (int cpu = 0; cpu < runCpu; cpu++) { @@ -78,7 +78,7 @@ public void run() { }; ConcurenceRunner.run(task); } - try {// ȴ߳ + try {// �ȴ������߳����� gate.await(); } catch (InterruptedException e) { e.printStackTrace(); diff --git a/src/main/java/edu/hitsz/c102c/util/Log.java b/src/main/java/javacnn/util/Log.java similarity index 89% rename from src/main/java/edu/hitsz/c102c/util/Log.java rename to src/main/java/javacnn/util/Log.java index 210852d..555ad0e 100644 --- a/src/main/java/edu/hitsz/c102c/util/Log.java +++ b/src/main/java/javacnn/util/Log.java @@ -1,4 +1,4 @@ -package edu.hitsz.c102c.util; +package javacnn.util; import java.io.PrintStream; diff --git a/src/main/java/edu/hitsz/c102c/util/TestArray.java b/src/main/java/javacnn/util/TestArray.java similarity index 73% rename from src/main/java/edu/hitsz/c102c/util/TestArray.java rename to src/main/java/javacnn/util/TestArray.java index b946432..4ccf55f 100644 --- a/src/main/java/edu/hitsz/c102c/util/TestArray.java +++ b/src/main/java/javacnn/util/TestArray.java @@ -1,15 +1,15 @@ -package edu.hitsz.c102c.util; +package javacnn.util; import java.util.Locale; -import edu.hitsz.c102c.util.TimedTest.TestTask; +import javacnn.util.TimedTest.TestTask; /** - * ԪֱӷͨЧʣ ۣʽʲûнٶ + * ����Ԫ��ֱ�ӷ���������ͨ���������������Ч�ʣ� ���ۣ�������ʽ���ʲ�û�н����ٶ� * * @author jiqunpeng * - * ʱ䣺2014-7-9 3:18:30 + * ����ʱ�䣺2014-7-9 ����3:18:30 */ public class TestArray { double[][] data; @@ -35,7 +35,7 @@ private void useFunc() { } public static void main(String[] args) { - String a = "aAdfa_"; + String a = "aAdfa��_"; System.out.println(a.toUpperCase(Locale.CHINA)); double[][] d = new double[3][]; // d[0] = new double[] { 1,2,3 }; diff --git a/src/main/java/edu/hitsz/c102c/util/TimedTest.java b/src/main/java/javacnn/util/TimedTest.java similarity index 94% rename from src/main/java/edu/hitsz/c102c/util/TimedTest.java rename to src/main/java/javacnn/util/TimedTest.java index 008195f..ea5959c 100644 --- a/src/main/java/edu/hitsz/c102c/util/TimedTest.java +++ b/src/main/java/javacnn/util/TimedTest.java @@ -1,4 +1,4 @@ -package edu.hitsz.c102c.util; +package javacnn.util; /** * ��ʱ�IJ��Թ��� diff --git a/src/main/java/edu/hitsz/c102c/util/Util.java b/src/main/java/javacnn/util/Util.java similarity index 84% rename from src/main/java/edu/hitsz/c102c/util/Util.java rename to src/main/java/javacnn/util/Util.java index e0d0077..62b3020 100644 --- a/src/main/java/edu/hitsz/c102c/util/Util.java +++ b/src/main/java/javacnn/util/Util.java @@ -1,27 +1,26 @@ -package edu.hitsz.c102c.util; +package javacnn.util; import java.io.Serializable; import java.util.Arrays; import java.util.HashSet; import java.util.Random; import java.util.Set; -import edu.hitsz.c102c.cnn.Layer.Size; -import edu.hitsz.c102c.util.TimedTest.TestTask; +import javacnn.cnn.Layer.Size; public class Util { /** - * ӦԪʱÿԪϵIJ + * �����ӦԪ�����ʱ��ÿ��Ԫ���ϵIJ��� * * @author jiqunpeng * - * ʱ䣺2014-7-9 9:28:35 + * ����ʱ�䣺2014-7-9 ����9:28:35 */ public interface Operator extends Serializable { public double process(double value); } - // ÿԪvalue1-valueIJ + // ����ÿ��Ԫ��value������1-value�IJ��� public static final Operator one_value = new Operator() { /** * @@ -34,7 +33,7 @@ public double process(double value) { } }; - // digmod + // digmod���� public static final Operator digmod = new Operator() { /** * @@ -52,7 +51,7 @@ interface OperatorOnTwo extends Serializable { } /** - * ӦԪصļӷ + * ��������ӦԪ�صļӷ����� */ public static final OperatorOnTwo plus = new OperatorOnTwo() { /** @@ -66,7 +65,7 @@ public double process(double a, double b) { } }; /** - * ӦԪصij˷ + * ��������ӦԪ�صij˷����� */ public static OperatorOnTwo multiply = new OperatorOnTwo() { /** @@ -81,7 +80,7 @@ public double process(double a, double b) { }; /** - * ӦԪصļ + * ��������ӦԪ�صļ������� */ public static OperatorOnTwo minus = new OperatorOnTwo() { /** @@ -105,7 +104,7 @@ public static void printMatrix(double[][] matrix) { } /** - * Ծ180ת,matrixĸϸƣԭľ޸ + * �Ծ������180����ת,����matrix�ĸ����ϸ��ƣ������ԭ���ľ�������޸� * * @param matrix */ @@ -113,7 +112,7 @@ public static double[][] rot180(double[][] matrix) { matrix = cloneMatrix(matrix); int m = matrix.length; int n = matrix[0].length; - // жԳƽн + // ���жԳƽ��н��� for (int i = 0; i < m; i++) { for (int j = 0; j < n / 2; j++) { double tmp = matrix[i][j]; @@ -121,7 +120,7 @@ public static double[][] rot180(double[][] matrix) { matrix[i][n - 1 - j] = tmp; } } - // жԳƽн + // ���жԳƽ��н��� for (int j = 0; j < n; j++) { for (int i = 0; i < m / 2; i++) { double tmp = matrix[i][j]; @@ -135,7 +134,7 @@ public static double[][] rot180(double[][] matrix) { private static Random r = new Random(2); /** - * ʼ + * �����ʼ������ * * @param x * @param y @@ -147,7 +146,7 @@ public static double[][] randomMatrix(int x, int y, boolean b) { int tag = 1; for (int i = 0; i < x; i++) { for (int j = 0; j < y; j++) { - // ֵ[-0.05,0.05)֮䣬ȨسʼֵСڱ + // ���ֵ��[-0.05,0.05)֮�䣬��Ȩ�س�ʼ��ֵ��С���������ڱ������� matrix[i][j] = (r.nextDouble() - 0.05) / 10; // matrix[i][j] = tag * 0.5; // if (b) @@ -160,7 +159,7 @@ public static double[][] randomMatrix(int x, int y, boolean b) { } /** - * ʼһά + * �����ʼ��һά���� * * @param len * @return @@ -175,7 +174,7 @@ public static double[] randomArray(int len) { } /** - * еijȡbatchSize[0,size) + * ������еij����������ȡbatchSize��[0,size)���� * * @param size * @param batchSize @@ -194,7 +193,7 @@ public static int[] randomPerm(int size, int batchSize) { } /** - * ƾ + * ���ƾ��� * * @param matrix * @return @@ -214,7 +213,7 @@ public static double[][] cloneMatrix(final double[][] matrix) { } /** - * Եв + * �Ե���������в��� * * @param ma * @param operator @@ -233,15 +232,15 @@ public static double[][] matrixOp(final double[][] ma, Operator operator) { } /** - * άͬľӦԪز,õĽmbУmb[i][j] = (op_a + * ����ά����ͬ�ľ����ӦԪ�ز���,�õ��Ľ������mb�У���mb[i][j] = (op_a * ma[i][j]) op (op_b mb[i][j]) * * @param ma * @param mb * @param operatorB - * ڵmbϵIJ + * �ڵ�mb�����ϵIJ��� * @param operatorA - * maԪϵIJ + * ��ma����Ԫ���ϵIJ��� * @return * */ @@ -251,7 +250,7 @@ public static double[][] matrixOp(final double[][] ma, final double[][] mb, final int m = ma.length; int n = ma[0].length; if (m != mb.length || n != mb[0].length) - throw new RuntimeException("Сһ ma.length:" + ma.length + throw new RuntimeException("���������С��һ�� ma.length:" + ma.length + " mb.length:" + mb.length); for (int i = 0; i < m; i++) { @@ -269,7 +268,7 @@ public static double[][] matrixOp(final double[][] ma, final double[][] mb, } /** - * ڿ˻,Ծչ + * �����ڿ˻�,�Ծ��������չ * * @param matrix * @param scale @@ -293,7 +292,7 @@ public static double[][] kronecker(final double[][] matrix, final Size scale) { } /** - * ԾоֵС + * �Ծ�����о�ֵ��С * * @param matrix * @param scaleSize @@ -307,7 +306,7 @@ public static double[][] scaleMatrix(final double[][] matrix, final int sn = n / scale.y; final double[][] outMatrix = new double[sm][sn]; if (sm * scale.x != m || sn * scale.y != n) - throw new RuntimeException("scalematrix"); + throw new RuntimeException("scale��������matrix"); final int size = scale.x * scale.y; for (int i = 0; i < sm; i++) { for (int j = 0; j < sn; j++) { @@ -324,7 +323,7 @@ public static double[][] scaleMatrix(final double[][] matrix, } /** - * fullģʽľ + * ����fullģʽ�ľ�� * * @param matrix * @param kernel @@ -336,7 +335,7 @@ public static double[][] convnFull(double[][] matrix, int n = matrix[0].length; final int km = kernel.length; final int kn = kernel[0].length; - // չ + // ��չ���� final double[][] extendMatrix = new double[m + 2 * (km - 1)][n + 2 * (kn - 1)]; for (int i = 0; i < m; i++) { @@ -347,7 +346,7 @@ public static double[][] convnFull(double[][] matrix, } /** - * validģʽľ + * ����validģʽ�ľ�� * * @param matrix * @param kernel @@ -360,11 +359,11 @@ public static double[][] convnValid(final double[][] matrix, int n = matrix[0].length; final int km = kernel.length; final int kn = kernel[0].length; - // Ҫ + // ��Ҫ����������� int kns = n - kn + 1; - // Ҫ + // ��Ҫ����������� final int kms = m - km + 1; - // + // ������� final double[][] outMatrix = new double[kms][kns]; for (int i = 0; i < kms; i++) { @@ -383,7 +382,7 @@ public static double[][] convnValid(final double[][] matrix, } /** - * άľ,Ҫһάͬ + * ��ά����ľ��,����Ҫ�����������һά��ͬ * * @param matrix * @param kernel @@ -401,8 +400,8 @@ public static double[][] convnValid(final double[][][][] matrix, int kns = n - kn + 1; int khs = h - kh + 1; if (matrix.length != kernel.length) - throw new RuntimeException("ڵһάϲͬ"); - // + throw new RuntimeException("�����������ڵ�һά�ϲ�ͬ"); + // ������� final double[][][] outMatrix = new double[kms][kns][khs]; for (int i = 0; i < kms; i++) { for (int j = 0; j < kns; j++) @@ -426,10 +425,10 @@ public static double sigmod(double x) { } /** - * ԾԪ + * �Ծ���Ԫ����� * * @param error - * @return עͺܻܿ + * @return ע�������ͺܿ��ܻ���� */ public static double sum(double[][] error) { @@ -445,7 +444,7 @@ public static double sum(double[][] error) { } /** - * errors[...][j]Ԫ + * ��errors[...][j]Ԫ����� * * @param errors * @param j @@ -482,7 +481,7 @@ public static int binaryArray2int(double[] array) { } /** - * Ծ,Խ4²еľ߲2 + * ���Ծ��,���Խ����4���²������еľ����߲���2�� */ private static void testConvn() { int count = 1; @@ -608,7 +607,7 @@ public static void main(String[] args) { } /** - * ȡԪص± + * ȡ����Ԫ�ص��±� * * @param out * @return From 9a38629db5dae15a0a26579f4813ee0a27d80cc8 Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Sat, 17 Feb 2018 16:08:34 +0100 Subject: [PATCH 08/34] refactoring : introducing getOutput() --- src/main/java/javacnn/cnn/CNN.java | 59 +++++++++++++++--------------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/src/main/java/javacnn/cnn/CNN.java b/src/main/java/javacnn/cnn/CNN.java index 38acee1..8a66f09 100644 --- a/src/main/java/javacnn/cnn/CNN.java +++ b/src/main/java/javacnn/cnn/CNN.java @@ -23,34 +23,35 @@ public class CNN implements Serializable { private static double ALPHA = 0.85; - private List layers; + private final List layers; - private int layerNum; + private final int layerNum; - private int batchSize; + private final int batchSize; - private Util.Operator divide_batchSize; + private final Util.Operator divide_batchSize; - private Util.Operator multiply_alpha; + private final Util.Operator multiply_alpha; - private Util.Operator multiply_lambda; + private final Util.Operator multiply_lambda; public CNN(LayerBuilder layerBuilder, final int batchSize) { - layers = layerBuilder.mLayers; - layerNum = layers.size(); + this.layers = layerBuilder.mLayers; + this.layerNum = layers.size(); this.batchSize = batchSize; setup(batchSize); - initPerator(); - } - private void initPerator() { + // --- + + final double _1_batchSize = 1. / batchSize; + divide_batchSize = new Util.Operator() { private static final long serialVersionUID = 7424011281732651055L; @Override public double process(double value) { - return value / batchSize; // TODO: Remove division + return value * _1_batchSize; } }; @@ -169,14 +170,7 @@ public double test(final Dataset dataset) { forward(record); - final Layer outputLayer = layers.get(layerNum - 1); - final int mapNum = outputLayer.getOutMapNum(); - - final double[] out = new double[mapNum]; - for (int m = 0; m < mapNum; m++) { - final double[][] outmap = outputLayer.getMap(m); - out[m] = outmap[0][0]; - } + final double[] out = getOutput(); if (record.getLabel().intValue() == Util.getMaxIndex(out)) { right++; @@ -189,6 +183,18 @@ public double test(final Dataset dataset) { return p; } + private double[] getOutput() { + final Layer outputLayer = layers.get(layerNum - 1); + + final int mapNum = outputLayer.getOutMapNum(); + final double[] out = new double[mapNum]; + for (int m = 0; m < mapNum; m++) { + final double[][] outmap = outputLayer.getMap(m); + out[m] = outmap[0][0]; + } + return out; + } + // TODO: Move this method to other/new class (reduce CNN-class to the minimal CNN-logic) public void predict(Dataset testset, String fileName) { Log.info("begin predict"); @@ -202,17 +208,10 @@ public void predict(Dataset testset, String fileName) { while (iter.hasNext()) { final Dataset.Record record = iter.next(); forward(record); - final Layer outputLayer = layers.get(layerNum - 1); - - int mapNum = outputLayer.getOutMapNum(); - double[] out = new double[mapNum]; - for (int m = 0; m < mapNum; m++) { - double[][] outmap = outputLayer.getMap(m); - out[m] = outmap[0][0]; - } + final double[] out = getOutput(); // int label = // Util.binaryArray2int(out); - int label = Util.getMaxIndex(out); + final int label = Util.getMaxIndex(out); // if (label >= max) // label = label - (1 << (out.length - // 1)); @@ -520,7 +519,7 @@ public void process(int start, int end) { } - private void setup(int batchSize) { + private void setup(final int batchSize) { final Layer inputLayer = layers.get(0); inputLayer.initOutmaps(batchSize); From b19844e2e19f7a3b1acaf618538671dfa591c827 Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Sat, 17 Feb 2018 17:54:03 +0100 Subject: [PATCH 09/34] Removed console-Listener Renamed Dataset.iter() -> Dataset.iteration() --- src/main/java/javacnn/cnn/CNN.java | 69 +++++++--------------- src/main/java/javacnn/dataset/Dataset.java | 2 +- 2 files changed, 21 insertions(+), 50 deletions(-) diff --git a/src/main/java/javacnn/cnn/CNN.java b/src/main/java/javacnn/cnn/CNN.java index 8a66f09..d8cfc8b 100644 --- a/src/main/java/javacnn/cnn/CNN.java +++ b/src/main/java/javacnn/cnn/CNN.java @@ -7,12 +7,11 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; import javacnn.dataset.Dataset; +import javacnn.util.ConcurenceRunner; import javacnn.util.Log; import javacnn.util.Util; -import javacnn.util.ConcurenceRunner; public class CNN implements Serializable { @@ -78,9 +77,7 @@ public double process(double value) { public void train(final Dataset trainset, final int iterationCount) { - new Listener().start(); - - for (int iteration = 0; iteration < iterationCount && !stopTrain.get(); iteration++) { + for (int iteration = 0; iteration < iterationCount; iteration++) { int epochsNum = trainset.size() / batchSize; @@ -101,7 +98,7 @@ public void train(final Dataset trainset, final int iterationCount) { Layer.prepareForNewBatch(); for (int index : randPerm) { - boolean isRight = train(trainset.getRecord(index)); + final boolean isRight = train(trainset.getRecord(index)); if (isRight) right++; count++; @@ -118,64 +115,33 @@ public void train(final Dataset trainset, final int iterationCount) { } } - final double p = 1.0 * right / count; + final double precision = ((double) right) / count; - if (iteration % 10 == 1 && p > 0.96) { + if (iteration % 10 == 1 && precision > 0.96) { ALPHA = 0.001 + ALPHA * 0.9; Log.info("Set alpha = " + ALPHA); } - Log.info("precision " + right + "/" + count + "=" + p); - } - } - - private static AtomicBoolean stopTrain; - - static class Listener extends Thread { - - Listener() { - setDaemon(true); - stopTrain = new AtomicBoolean(false); - } - - @Override - public void run() { - - System.out.println("Input & to stop train."); - - while (true) { - try { - final int a = System.in.read(); - if (a == '&') { - stopTrain.compareAndSet(false, true); - break; - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - System.out.println("Listener stopped"); + Log.info("precision " + right + "/" + count + "=" + precision); } } public double test(final Dataset dataset) { Layer.prepareForNewBatch(); - final Iterator iterator = dataset.iter(); + final Iterator iterator = dataset.iterator(); int right = 0; while (iterator.hasNext()) { final Dataset.Record record = iterator.next(); - forward(record); - - final double[] out = getOutput(); + final double[] out = propagate(record); if (record.getLabel().intValue() == Util.getMaxIndex(out)) { right++; } } + double p = 1.0 * right / dataset.size(); Log.info("precision", p + ""); @@ -204,11 +170,10 @@ public void predict(Dataset testset, String fileName) { Layer.prepareForNewBatch(); - final Iterator iter = testset.iter(); + final Iterator iter = testset.iterator(); while (iter.hasNext()) { final Dataset.Record record = iter.next(); - forward(record); - final double[] out = getOutput(); + final double[] out = propagate(record); // int label = // Util.binaryArray2int(out); final int label = Util.getMaxIndex(out); @@ -375,7 +340,7 @@ public void process(int start, int end) { }.start(); } - private boolean setOutLayerErrors(Dataset.Record record) { + private boolean setOutLayerErrors(final Dataset.Record record) { Layer outputLayer = layers.get(layerNum - 1); int mapNum = outputLayer.getOutMapNum(); @@ -418,7 +383,13 @@ private boolean setOutLayerErrors(Dataset.Record record) { return label == Util.getMaxIndex(outmaps); } - private void forward(Dataset.Record record) { + public double[] propagate(final Dataset.Record record) { + forward(record); + + return getOutput(); + } + + private void forward(final Dataset.Record record) { setInLayerOutput(record); for (int l = 1; l < layers.size(); l++) { @@ -444,7 +415,7 @@ private void forward(Dataset.Record record) { } } - private void setInLayerOutput(Dataset.Record record) { + private void setInLayerOutput(final Dataset.Record record) { final Layer inputLayer = layers.get(0); final Layer.Size mapSize = inputLayer.getMapSize(); diff --git a/src/main/java/javacnn/dataset/Dataset.java b/src/main/java/javacnn/dataset/Dataset.java index d13ddb8..8a4ce41 100644 --- a/src/main/java/javacnn/dataset/Dataset.java +++ b/src/main/java/javacnn/dataset/Dataset.java @@ -42,7 +42,7 @@ public void append(double[] attrs, Double label) { records.add(new Record(attrs, label)); } - public Iterator iter() { + public Iterator iterator() { return records.iterator(); } From 7859376248c5b3390dec252e931f546e85c7eb39 Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Sat, 17 Feb 2018 22:54:56 +0100 Subject: [PATCH 10/34] Using Output/InputStream in CNNLoader --- src/main/java/javacnn/cnn/CNNLoader.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main/java/javacnn/cnn/CNNLoader.java b/src/main/java/javacnn/cnn/CNNLoader.java index bbb4ecf..b084bba 100644 --- a/src/main/java/javacnn/cnn/CNNLoader.java +++ b/src/main/java/javacnn/cnn/CNNLoader.java @@ -3,29 +3,30 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; +import java.io.OutputStream; public class CNNLoader { public static void saveModel(final String fileName, final CNN cnn) throws IOException { saveModel(cnn, new FileOutputStream(fileName)); - return; } - public static void saveModel(final CNN cnn, final FileOutputStream fileOutputStream) throws IOException { - ObjectOutputStream oos = new ObjectOutputStream(fileOutputStream); + public static void saveModel(final CNN cnn, final OutputStream outputStream) throws IOException { + ObjectOutputStream oos = new ObjectOutputStream(outputStream); oos.writeObject(cnn); oos.flush(); oos.close(); } - public static CNN loadModel(String fileName) throws IOException, ClassNotFoundException { + public static CNN loadModel(final String fileName) throws IOException, ClassNotFoundException { return loadModel(new FileInputStream(fileName)); } - public static CNN loadModel(final FileInputStream fileInputStream) throws IOException, ClassNotFoundException { - final ObjectInputStream in = new ObjectInputStream(fileInputStream); + public static CNN loadModel(final InputStream inputStream) throws IOException, ClassNotFoundException { + final ObjectInputStream in = new ObjectInputStream(inputStream); final CNN cnn = (CNN) in.readObject(); in.close(); From f9ae92c8ce775c063c2dcf27ef49f564f9d8537b Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Sat, 17 Feb 2018 23:00:50 +0100 Subject: [PATCH 11/34] release 0.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e4cae63..0ebfe84 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ javacnn javacnn - 1-SNAPSHOT + 0.1 Implementation of a CNN in Java From 57bdcfb7a788c64e2055315a9df39ce5e7df424a Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Sat, 17 Feb 2018 23:01:50 +0100 Subject: [PATCH 12/34] preparation for 0.2 in pom --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0ebfe84..1d0745c 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ javacnn javacnn - 0.1 + 0.2-SNAPSHOT Implementation of a CNN in Java From b0697d4a779b3ad2d9d2df74b6ab5180bd46e884 Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Mon, 19 Feb 2018 08:43:44 +0100 Subject: [PATCH 13/34] Introducing interface 'Process' to hide implemenetation details of ConcurenceRunner --- src/main/java/javacnn/cnn/CNN.java | 55 +++++++++------- src/main/java/javacnn/cnn/Layer.java | 14 +++-- src/main/java/javacnn/cnn/Process.java | 11 ++++ .../java/javacnn/util/ConcurenceRunner.java | 62 ++++++++----------- 4 files changed, 77 insertions(+), 65 deletions(-) create mode 100644 src/main/java/javacnn/cnn/Process.java diff --git a/src/main/java/javacnn/cnn/CNN.java b/src/main/java/javacnn/cnn/CNN.java index d8cfc8b..c93ae20 100644 --- a/src/main/java/javacnn/cnn/CNN.java +++ b/src/main/java/javacnn/cnn/CNN.java @@ -234,27 +234,28 @@ private void updateBias(final Layer layer, Layer lastLayer) { final double[][][][] errors = layer.getErrors(); int mapNum = layer.getOutMapNum(); - new ConcurenceRunner.TaskManager(mapNum) { - - @Override - public void process(int start, int end) { - for (int j = start; j < end; j++) { - double[][] error = Util.sum(errors, j); - // ����ƫ�� - double deltaBias = Util.sum(error) / batchSize; - double bias = layer.getBias(j) + ALPHA * deltaBias; - layer.setBias(j, bias); - } - } - }.start(); + final Process processor = + new Process() { + @Override + public void process(int start, int end) { + for (int j = start; j < end; j++) { + final double[][] error = Util.sum(errors, j); + // update offset + final double deltaBias = Util.sum(error) / batchSize; + final double bias = layer.getBias(j) + ALPHA * deltaBias; + layer.setBias(j, bias); + } + } + }; + ConcurenceRunner.startProcess(mapNum, processor); } private void updateKernels(final Layer layer, final Layer lastLayer) { final int mapNum = layer.getOutMapNum(); final int lastMapNum = lastLayer.getOutMapNum(); - new ConcurenceRunner.TaskManager(mapNum) { + final Process process = new Process() { @Override public void process(int start, int end) { for (int j = start; j < end; j++) { @@ -277,8 +278,9 @@ public void process(int start, int end) { } } - }.start(); + }; + ConcurenceRunner.startProcess(mapNum, process); } private void setHiddenLayerErrors() { @@ -301,8 +303,8 @@ private void setHiddenLayerErrors() { private void setSampErrors(final Layer layer, final Layer nextLayer) { final int mapNum = layer.getOutMapNum(); final int nextMapNum = nextLayer.getOutMapNum(); - new ConcurenceRunner.TaskManager(mapNum) { + final Process process = new Process() { @Override public void process(int start, int end) { for (int i = start; i < end; i++) { @@ -319,13 +321,15 @@ public void process(int start, int end) { } } - }.start(); + }; + ConcurenceRunner.startProcess(mapNum, process); } private void setConvErrors(final Layer layer, final Layer nextLayer) { final int mapNum = layer.getOutMapNum(); - new ConcurenceRunner.TaskManager(mapNum) { + + final Process process = new Process() { @Override public void process(int start, int end) { for (int m = start; m < end; m++) { @@ -337,7 +341,9 @@ public void process(int start, int end) { layer.setError(m, outMatrix); } } - }.start(); + }; + + ConcurenceRunner.startProcess(mapNum, process); } private boolean setOutLayerErrors(final Dataset.Record record) { @@ -438,8 +444,8 @@ private void setInLayerOutput(final Dataset.Record record) { private void setConvOutput(final Layer layer, final Layer lastLayer) { final int mapNum = layer.getOutMapNum(); final int lastMapNum = lastLayer.getOutMapNum(); - new ConcurenceRunner.TaskManager(mapNum) { + final Process process = new Process() { @Override public void process(int start, int end) { for (int j = start; j < end; j++) { @@ -470,13 +476,15 @@ public double process(double value) { layer.setMapValue(j, sum); } } - }.start(); + }; + + ConcurenceRunner.startProcess(mapNum, process); } private void setSampOutput(final Layer layer, final Layer lastLayer) { final int lastMapNum = lastLayer.getOutMapNum(); - new ConcurenceRunner.TaskManager(lastMapNum) { + final Process process = new Process() { @Override public void process(int start, int end) { for (int i = start; i < end; i++) { @@ -486,8 +494,9 @@ public void process(int start, int end) { layer.setMapValue(i, sampMatrix); } } - }.start(); + }; + ConcurenceRunner.startProcess(lastMapNum, process); } private void setup(final int batchSize) { diff --git a/src/main/java/javacnn/cnn/Layer.java b/src/main/java/javacnn/cnn/Layer.java index f099416..5d9599f 100644 --- a/src/main/java/javacnn/cnn/Layer.java +++ b/src/main/java/javacnn/cnn/Layer.java @@ -125,10 +125,12 @@ public Size(int x, int y) { this.y = y; } + @Override public String toString() { - StringBuilder s = new StringBuilder("Size(").append(" x = ") - .append(x).append(" y= ").append(y).append(")"); - return s.toString(); + return "Size{" + + "x=" + x + + ", y=" + y + + '}'; } /** @@ -141,8 +143,10 @@ public String toString() { public Size divide(Size scaleSize) { int x = this.x / scaleSize.x; int y = this.y / scaleSize.y; - if (x * scaleSize.x != this.x || y * scaleSize.y != this.y) - throw new RuntimeException(this + "��������" + scaleSize); + if (x * scaleSize.x != this.x || y * scaleSize.y != this.y) { + throw new RuntimeException(this + " can not be divisible " + scaleSize); + } + return new Size(x, y); } diff --git a/src/main/java/javacnn/cnn/Process.java b/src/main/java/javacnn/cnn/Process.java new file mode 100644 index 0000000..0ba7194 --- /dev/null +++ b/src/main/java/javacnn/cnn/Process.java @@ -0,0 +1,11 @@ +package javacnn.cnn; + +/** + *

+ * Created: 2018-02-19 08:29 + * + * @author Ralf Th. Pietsch <ratopi@abwesend.de> + */ +public interface Process { + void process(int start, int end); +} diff --git a/src/main/java/javacnn/util/ConcurenceRunner.java b/src/main/java/javacnn/util/ConcurenceRunner.java index 39bbf1e..883f5a6 100644 --- a/src/main/java/javacnn/util/ConcurenceRunner.java +++ b/src/main/java/javacnn/util/ConcurenceRunner.java @@ -4,20 +4,22 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import javacnn.cnn.Process; + /** - * ��������� - * + * Concurrent running tools + *

+ * Created: 2014-6-16 at 3:33:41 PM + * * @author jiqunpeng - * - * ����ʱ�䣺2014-6-16 ����3:33:41 */ public class ConcurenceRunner { private static final ExecutorService exec; public static final int cpuNum; + static { cpuNum = Runtime.getRuntime().availableProcessors(); - // cpuNum = 1; System.out.println("cpuNum:" + cpuNum); exec = Executors.newFixedThreadPool(cpuNum); } @@ -30,55 +32,43 @@ public static void stop() { exec.shutdown(); } - // public abstract static class Task implements - // Runnable { - // int start, end; - // - // public Task(int start, int end) { - // this.start = start; - // this.end = end; - // // Log.i("new Task", - // // "start "+start+" end "+end); - // } - // - // @Override - // public void run() { - // process(start, end); - // } - // - // public abstract void process(int start, int - // end); - // - // } - - public abstract static class TaskManager { + public static void startProcess(final int mapNum, final Process process) { + new TaskManager(mapNum).start(process); + } + + + private static class TaskManager { private int workLength; public TaskManager(int workLength) { this.workLength = workLength; } - public void start() { + public void start(final Process processor) { int runCpu = cpuNum < workLength ? cpuNum : 1; - // ��Ƭ��������ȡ�� + + // Fragment length rounded up final CountDownLatch gate = new CountDownLatch(runCpu); - int fregLength = (workLength + runCpu - 1) / runCpu; + + final int fregLength = (workLength + runCpu - 1) / runCpu; + for (int cpu = 0; cpu < runCpu; cpu++) { final int start = cpu * fregLength; - int tmp = (cpu + 1) * fregLength; + + final int tmp = (cpu + 1) * fregLength; final int end = tmp <= workLength ? tmp : workLength; - Runnable task = new Runnable() { + final Runnable task = new Runnable() { @Override public void run() { - process(start, end); + processor.process(start, end); gate.countDown(); } - }; + ConcurenceRunner.run(task); } - try {// �ȴ������߳����� + try {// Wait for all threads to finish running gate.await(); } catch (InterruptedException e) { e.printStackTrace(); @@ -86,8 +76,6 @@ public void run() { } } - public abstract void process(int start, int end); - } } From d1b643926dbef8f1d2fc26ec56d4ce10432bf973 Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Mon, 19 Feb 2018 08:56:19 +0100 Subject: [PATCH 14/34] Refactoring ConcurenceRunner is now object --- src/main/java/RunCNN.java | 20 +++++--------- src/main/java/javacnn/cnn/CNN.java | 20 +++++++++----- .../java/javacnn/util/ConcurenceRunner.java | 26 +++++++++---------- 3 files changed, 32 insertions(+), 34 deletions(-) diff --git a/src/main/java/RunCNN.java b/src/main/java/RunCNN.java index e53d0a2..c7a428f 100644 --- a/src/main/java/RunCNN.java +++ b/src/main/java/RunCNN.java @@ -8,11 +8,12 @@ import javacnn.dataset.Dataset; import javacnn.dataset.DatasetLoader; import javacnn.util.ConcurenceRunner; -import javacnn.util.TimedTest; public class RunCNN { - public static void runCnn() throws IOException { + public static void main(String[] args) throws IOException { + + final ConcurenceRunner concurenceRunner = new ConcurenceRunner(); final LayerBuilder builder = new LayerBuilder(); @@ -23,11 +24,11 @@ public static void runCnn() throws IOException { builder.addLayer(Layer.buildSampLayer(new Size(2, 2))); builder.addLayer(Layer.buildOutputLayer(10)); - final CNN cnn = new CNN(builder, 50); + final CNN cnn = new CNN(builder, 50, concurenceRunner); final String fileName = "dataset/train.format"; final Dataset dataset = DatasetLoader.load(fileName, ",", 784); - cnn.train(dataset, 100); + cnn.train(dataset, 5); CNNLoader.saveModel("model.cnn", cnn); dataset.clear(); @@ -35,17 +36,8 @@ public static void runCnn() throws IOException { // CNN cnn = CNNLoader.loadModel(modelName); final Dataset testset = DatasetLoader.load("dataset/test.format", ",", -1); cnn.predict(testset, "dataset/test.predict"); - } - public static void main(String[] args) { - new TimedTest(() -> { - try { - runCnn(); - } catch (IOException e) { - throw new RuntimeException(e); - } - }, 1).test(); - ConcurenceRunner.stop(); + concurenceRunner.stop(); } } diff --git a/src/main/java/javacnn/cnn/CNN.java b/src/main/java/javacnn/cnn/CNN.java index c93ae20..01c815b 100644 --- a/src/main/java/javacnn/cnn/CNN.java +++ b/src/main/java/javacnn/cnn/CNN.java @@ -22,6 +22,8 @@ public class CNN implements Serializable { private static double ALPHA = 0.85; + private final ConcurenceRunner concurenceRunner; + private final List layers; private final int layerNum; @@ -35,10 +37,14 @@ public class CNN implements Serializable { private final Util.Operator multiply_lambda; - public CNN(LayerBuilder layerBuilder, final int batchSize) { + public CNN(LayerBuilder layerBuilder, final int batchSize, final ConcurenceRunner concurenceRunner) { + this.layers = layerBuilder.mLayers; this.layerNum = layers.size(); this.batchSize = batchSize; + + this.concurenceRunner = concurenceRunner; + setup(batchSize); // --- @@ -248,7 +254,7 @@ public void process(int start, int end) { } }; - ConcurenceRunner.startProcess(mapNum, processor); + concurenceRunner.startProcess(mapNum, processor); } private void updateKernels(final Layer layer, final Layer lastLayer) { @@ -280,7 +286,7 @@ public void process(int start, int end) { } }; - ConcurenceRunner.startProcess(mapNum, process); + concurenceRunner.startProcess(mapNum, process); } private void setHiddenLayerErrors() { @@ -323,7 +329,7 @@ public void process(int start, int end) { }; - ConcurenceRunner.startProcess(mapNum, process); + concurenceRunner.startProcess(mapNum, process); } private void setConvErrors(final Layer layer, final Layer nextLayer) { @@ -343,7 +349,7 @@ public void process(int start, int end) { } }; - ConcurenceRunner.startProcess(mapNum, process); + concurenceRunner.startProcess(mapNum, process); } private boolean setOutLayerErrors(final Dataset.Record record) { @@ -478,7 +484,7 @@ public double process(double value) { } }; - ConcurenceRunner.startProcess(mapNum, process); + concurenceRunner.startProcess(mapNum, process); } private void setSampOutput(final Layer layer, final Layer lastLayer) { @@ -496,7 +502,7 @@ public void process(int start, int end) { } }; - ConcurenceRunner.startProcess(lastMapNum, process); + concurenceRunner.startProcess(lastMapNum, process); } private void setup(final int batchSize) { diff --git a/src/main/java/javacnn/util/ConcurenceRunner.java b/src/main/java/javacnn/util/ConcurenceRunner.java index 883f5a6..c13e05e 100644 --- a/src/main/java/javacnn/util/ConcurenceRunner.java +++ b/src/main/java/javacnn/util/ConcurenceRunner.java @@ -15,36 +15,36 @@ */ public class ConcurenceRunner { - private static final ExecutorService exec; - public static final int cpuNum; + private final ExecutorService exec; + private final int cpuNum; - static { + public ConcurenceRunner() { cpuNum = Runtime.getRuntime().availableProcessors(); System.out.println("cpuNum:" + cpuNum); exec = Executors.newFixedThreadPool(cpuNum); } - public static void run(Runnable task) { - exec.execute(task); - } - - public static void stop() { + public void stop() { exec.shutdown(); } - public static void startProcess(final int mapNum, final Process process) { + public void startProcess(final int mapNum, final Process process) { new TaskManager(mapNum).start(process); } + private void run(Runnable task) { + exec.execute(task); + } + - private static class TaskManager { + private class TaskManager { private int workLength; - public TaskManager(int workLength) { + private TaskManager(int workLength) { this.workLength = workLength; } - public void start(final Process processor) { + private void start(final Process processor) { int runCpu = cpuNum < workLength ? cpuNum : 1; // Fragment length rounded up @@ -66,7 +66,7 @@ public void run() { } }; - ConcurenceRunner.run(task); + ConcurenceRunner.this.run(task); } try {// Wait for all threads to finish running gate.await(); From d81b5cef05830d99f8e7ab2940fb26a11995aa5a Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Mon, 19 Feb 2018 08:59:02 +0100 Subject: [PATCH 15/34] Introducing Runner-interface --- src/main/java/javacnn/cnn/CNN.java | 20 +++++++++---------- .../java/javacnn/util/ConcurenceRunner.java | 3 ++- src/main/java/javacnn/util/Runner.java | 13 ++++++++++++ 3 files changed, 25 insertions(+), 11 deletions(-) create mode 100644 src/main/java/javacnn/util/Runner.java diff --git a/src/main/java/javacnn/cnn/CNN.java b/src/main/java/javacnn/cnn/CNN.java index 01c815b..2bd4d85 100644 --- a/src/main/java/javacnn/cnn/CNN.java +++ b/src/main/java/javacnn/cnn/CNN.java @@ -9,8 +9,8 @@ import java.util.List; import javacnn.dataset.Dataset; -import javacnn.util.ConcurenceRunner; import javacnn.util.Log; +import javacnn.util.Runner; import javacnn.util.Util; public class CNN implements Serializable { @@ -22,7 +22,7 @@ public class CNN implements Serializable { private static double ALPHA = 0.85; - private final ConcurenceRunner concurenceRunner; + private final Runner runner; private final List layers; @@ -37,13 +37,13 @@ public class CNN implements Serializable { private final Util.Operator multiply_lambda; - public CNN(LayerBuilder layerBuilder, final int batchSize, final ConcurenceRunner concurenceRunner) { + public CNN(LayerBuilder layerBuilder, final int batchSize, final Runner runner) { this.layers = layerBuilder.mLayers; this.layerNum = layers.size(); this.batchSize = batchSize; - this.concurenceRunner = concurenceRunner; + this.runner = runner; setup(batchSize); @@ -254,7 +254,7 @@ public void process(int start, int end) { } }; - concurenceRunner.startProcess(mapNum, processor); + runner.startProcess(mapNum, processor); } private void updateKernels(final Layer layer, final Layer lastLayer) { @@ -286,7 +286,7 @@ public void process(int start, int end) { } }; - concurenceRunner.startProcess(mapNum, process); + runner.startProcess(mapNum, process); } private void setHiddenLayerErrors() { @@ -329,7 +329,7 @@ public void process(int start, int end) { }; - concurenceRunner.startProcess(mapNum, process); + runner.startProcess(mapNum, process); } private void setConvErrors(final Layer layer, final Layer nextLayer) { @@ -349,7 +349,7 @@ public void process(int start, int end) { } }; - concurenceRunner.startProcess(mapNum, process); + runner.startProcess(mapNum, process); } private boolean setOutLayerErrors(final Dataset.Record record) { @@ -484,7 +484,7 @@ public double process(double value) { } }; - concurenceRunner.startProcess(mapNum, process); + runner.startProcess(mapNum, process); } private void setSampOutput(final Layer layer, final Layer lastLayer) { @@ -502,7 +502,7 @@ public void process(int start, int end) { } }; - concurenceRunner.startProcess(lastMapNum, process); + runner.startProcess(lastMapNum, process); } private void setup(final int batchSize) { diff --git a/src/main/java/javacnn/util/ConcurenceRunner.java b/src/main/java/javacnn/util/ConcurenceRunner.java index c13e05e..d89fa96 100644 --- a/src/main/java/javacnn/util/ConcurenceRunner.java +++ b/src/main/java/javacnn/util/ConcurenceRunner.java @@ -13,7 +13,7 @@ * * @author jiqunpeng */ -public class ConcurenceRunner { +public class ConcurenceRunner implements Runner { private final ExecutorService exec; private final int cpuNum; @@ -28,6 +28,7 @@ public void stop() { exec.shutdown(); } + @Override public void startProcess(final int mapNum, final Process process) { new TaskManager(mapNum).start(process); } diff --git a/src/main/java/javacnn/util/Runner.java b/src/main/java/javacnn/util/Runner.java new file mode 100644 index 0000000..b404a49 --- /dev/null +++ b/src/main/java/javacnn/util/Runner.java @@ -0,0 +1,13 @@ +package javacnn.util; + +import javacnn.cnn.Process; + +/** + *

+ * Created: 2018-02-19 08:57 + * + * @author Ralf Th. Pietsch <ratopi@abwesend.de> + */ +public interface Runner { + void startProcess(int mapNum, Process process); +} From 3ca5d5bb0dc581a939cfe1986ab2c024f21b460e Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Mon, 19 Feb 2018 09:50:28 +0100 Subject: [PATCH 16/34] - TaskManager removed in ConcurenceRunner : no longer needed - Runner is transient in CNN : for de-serialization - removed unused method ConcurenceRunner.isSame - removed code in comments - restored some comments (in english) --- src/main/java/RunCNN.java | 46 +++++----- src/main/java/javacnn/cnn/CNN.java | 86 +++++++------------ .../java/javacnn/util/ConcurenceRunner.java | 64 +++++--------- 3 files changed, 81 insertions(+), 115 deletions(-) diff --git a/src/main/java/RunCNN.java b/src/main/java/RunCNN.java index c7a428f..6cbd6d1 100644 --- a/src/main/java/RunCNN.java +++ b/src/main/java/RunCNN.java @@ -1,43 +1,49 @@ import java.io.IOException; import javacnn.cnn.CNN; -import javacnn.cnn.CNN.LayerBuilder; import javacnn.cnn.CNNLoader; import javacnn.cnn.Layer; -import javacnn.cnn.Layer.Size; import javacnn.dataset.Dataset; import javacnn.dataset.DatasetLoader; import javacnn.util.ConcurenceRunner; public class RunCNN { - public static void main(String[] args) throws IOException { + public static void main(String[] args) throws IOException, ClassNotFoundException { final ConcurenceRunner concurenceRunner = new ConcurenceRunner(); - final LayerBuilder builder = new LayerBuilder(); + try { - builder.addLayer(Layer.buildInputLayer(new Size(28, 28))); - builder.addLayer(Layer.buildConvLayer(6, new Size(5, 5))); - builder.addLayer(Layer.buildSampLayer(new Size(2, 2))); - builder.addLayer(Layer.buildConvLayer(12, new Size(5, 5))); - builder.addLayer(Layer.buildSampLayer(new Size(2, 2))); - builder.addLayer(Layer.buildOutputLayer(10)); + final CNN.LayerBuilder builder = new CNN.LayerBuilder(); - final CNN cnn = new CNN(builder, 50, concurenceRunner); + builder.addLayer(Layer.buildInputLayer(new Layer.Size(28, 28))); + builder.addLayer(Layer.buildConvLayer(6, new Layer.Size(5, 5))); + builder.addLayer(Layer.buildSampLayer(new Layer.Size(2, 2))); + builder.addLayer(Layer.buildConvLayer(12, new Layer.Size(5, 5))); + builder.addLayer(Layer.buildSampLayer(new Layer.Size(2, 2))); + builder.addLayer(Layer.buildOutputLayer(10)); - final String fileName = "dataset/train.format"; - final Dataset dataset = DatasetLoader.load(fileName, ",", 784); - cnn.train(dataset, 5); + final CNN cnn = new CNN(builder, 50, concurenceRunner); - CNNLoader.saveModel("model.cnn", cnn); - dataset.clear(); + final String fileName = "dataset/train.format"; + final Dataset dataset = DatasetLoader.load(fileName, ",", 784); + cnn.train(dataset, 5); - // CNN cnn = CNNLoader.loadModel(modelName); - final Dataset testset = DatasetLoader.load("dataset/test.format", ",", -1); - cnn.predict(testset, "dataset/test.predict"); + CNNLoader.saveModel("model.cnn", cnn); + dataset.clear(); - concurenceRunner.stop(); + /* + final CNN cnn = CNNLoader.loadModel("model.cnn"); + cnn.setRunner(concurenceRunner); + */ + + final Dataset testset = DatasetLoader.load("dataset/test.format", ",", -1); + cnn.predict(testset, "dataset/test.predict"); + + } finally { + concurenceRunner.shutdown(); + } } } diff --git a/src/main/java/javacnn/cnn/CNN.java b/src/main/java/javacnn/cnn/CNN.java index 2bd4d85..6a282f4 100644 --- a/src/main/java/javacnn/cnn/CNN.java +++ b/src/main/java/javacnn/cnn/CNN.java @@ -15,27 +15,22 @@ public class CNN implements Serializable { - private static final long serialVersionUID = 337920299147929932L; + private static final long serialVersionUID = 1L; private static final double LAMBDA = 0; - private static double ALPHA = 0.85; - private final Runner runner; - private final List layers; - private final int layerNum; - private final int batchSize; private final Util.Operator divide_batchSize; - private final Util.Operator multiply_alpha; - private final Util.Operator multiply_lambda; + private transient Runner runner; + public CNN(LayerBuilder layerBuilder, final int batchSize, final Runner runner) { @@ -81,6 +76,14 @@ public double process(double value) { }; } + public void setRunner(final Runner runner) { + this.runner = runner; + } + + private Runner getRunner() { + if (runner == null) throw new NullPointerException("'runner' is null. Set runner before start training or test!"); + return runner; + } public void train(final Dataset trainset, final int iterationCount) { for (int iteration = 0; iteration < iterationCount; iteration++) { @@ -88,7 +91,7 @@ public void train(final Dataset trainset, final int iterationCount) { int epochsNum = trainset.size() / batchSize; if (trainset.size() % batchSize != 0) { - epochsNum++; + epochsNum++; // Extract once, round up } Log.info(""); @@ -111,6 +114,7 @@ public void train(final Dataset trainset, final int iterationCount) { Layer.prepareForNewRecord(); } + // After finishing a batch update weight updateParas(); if (epoch % 50 == 0) { @@ -124,7 +128,7 @@ public void train(final Dataset trainset, final int iterationCount) { final double precision = ((double) right) / count; if (iteration % 10 == 1 && precision > 0.96) { - ALPHA = 0.001 + ALPHA * 0.9; + ALPHA = 0.001 + ALPHA * 0.9; // Adjust the quasi-learning rate dynamically Log.info("Set alpha = " + ALPHA); } @@ -196,22 +200,10 @@ public void predict(Dataset testset, String fileName) { Log.info("end predict"); } - private boolean isSame(double[] output, double[] target) { - boolean r = true; - for (int i = 0; i < output.length; i++) - if (Math.abs(output[i] - target[i]) > 0.5) { - r = false; - break; - } - - return r; - } - private boolean train(Dataset.Record record) { forward(record); - boolean result = backPropagation(record); - return result; - // System.exit(0); + + return backPropagation(record); } private boolean backPropagation(Dataset.Record record) { @@ -228,7 +220,7 @@ private void updateParas() { case conv: case output: updateKernels(layer, lastLayer); - updateBias(layer, lastLayer); + updateBias(layer); break; default: break; @@ -236,7 +228,7 @@ private void updateParas() { } } - private void updateBias(final Layer layer, Layer lastLayer) { + private void updateBias(final Layer layer) { final double[][][][] errors = layer.getErrors(); int mapNum = layer.getOutMapNum(); @@ -254,7 +246,7 @@ public void process(int start, int end) { } }; - runner.startProcess(mapNum, processor); + getRunner().startProcess(mapNum, processor); } private void updateKernels(final Layer layer, final Layer lastLayer) { @@ -286,7 +278,7 @@ public void process(int start, int end) { } }; - runner.startProcess(mapNum, process); + getRunner().startProcess(mapNum, process); } private void setHiddenLayerErrors() { @@ -329,7 +321,7 @@ public void process(int start, int end) { }; - runner.startProcess(mapNum, process); + getRunner().startProcess(mapNum, process); } private void setConvErrors(final Layer layer, final Layer nextLayer) { @@ -349,29 +341,13 @@ public void process(int start, int end) { } }; - runner.startProcess(mapNum, process); + getRunner().startProcess(mapNum, process); } private boolean setOutLayerErrors(final Dataset.Record record) { - Layer outputLayer = layers.get(layerNum - 1); - int mapNum = outputLayer.getOutMapNum(); - // double[] target = - // record.getDoubleEncodeTarget(mapNum); - // double[] outmaps = new double[mapNum]; - // for (int m = 0; m < mapNum; m++) { - // double[][] outmap = outputLayer.getMap(m); - // double output = outmap[0][0]; - // outmaps[m] = output; - // double errors = output * (1 - output) * - // (target[m] - output); - // outputLayer.setError(m, 0, 0, errors); - // } - // // ��ȷ - // if (isSame(outmaps, target)) - // return true; - // return false; - + final Layer outputLayer = layers.get(layerNum - 1); + final int mapNum = outputLayer.getOutMapNum(); final double[] target = new double[mapNum]; final double[] outmaps = new double[mapNum]; @@ -384,10 +360,6 @@ private boolean setOutLayerErrors(final Dataset.Record record) { target[label] = 1; - // Log.i(record.getLable() + "outmaps:" + - // Util.fomart(outmaps) - // + Arrays.toString(target)); - for (int m = 0; m < mapNum; m++) { outputLayer.setError(m, 0, 0, outmaps[m] * (1 - outmaps[m]) * (target[m] - outmaps[m])); } @@ -395,6 +367,12 @@ private boolean setOutLayerErrors(final Dataset.Record record) { return label == Util.getMaxIndex(outmaps); } + /** + * Propagate given Record through the network. + * Returns the result. + * @param record A Record + * @return The result of the network + */ public double[] propagate(final Dataset.Record record) { forward(record); @@ -484,7 +462,7 @@ public double process(double value) { } }; - runner.startProcess(mapNum, process); + getRunner().startProcess(mapNum, process); } private void setSampOutput(final Layer layer, final Layer lastLayer) { @@ -502,7 +480,7 @@ public void process(int start, int end) { } }; - runner.startProcess(lastMapNum, process); + getRunner().startProcess(lastMapNum, process); } private void setup(final int batchSize) { diff --git a/src/main/java/javacnn/util/ConcurenceRunner.java b/src/main/java/javacnn/util/ConcurenceRunner.java index d89fa96..4d2074d 100644 --- a/src/main/java/javacnn/util/ConcurenceRunner.java +++ b/src/main/java/javacnn/util/ConcurenceRunner.java @@ -24,59 +24,41 @@ public ConcurenceRunner() { exec = Executors.newFixedThreadPool(cpuNum); } - public void stop() { + public void shutdown() { exec.shutdown(); } @Override public void startProcess(final int mapNum, final Process process) { - new TaskManager(mapNum).start(process); - } - - private void run(Runnable task) { - exec.execute(task); - } - - - private class TaskManager { - private int workLength; + final int runCpu = cpuNum < mapNum ? cpuNum : 1; - private TaskManager(int workLength) { - this.workLength = workLength; - } - - private void start(final Process processor) { - int runCpu = cpuNum < workLength ? cpuNum : 1; - - // Fragment length rounded up - final CountDownLatch gate = new CountDownLatch(runCpu); + // Fragment length rounded up + final CountDownLatch gate = new CountDownLatch(runCpu); - final int fregLength = (workLength + runCpu - 1) / runCpu; + final int fregLength = (mapNum + runCpu - 1) / runCpu; - for (int cpu = 0; cpu < runCpu; cpu++) { - final int start = cpu * fregLength; + for (int cpu = 0; cpu < runCpu; cpu++) { + final int start = cpu * fregLength; - final int tmp = (cpu + 1) * fregLength; - final int end = tmp <= workLength ? tmp : workLength; + final int tmp = (cpu + 1) * fregLength; + final int end = tmp <= mapNum ? tmp : mapNum; - final Runnable task = new Runnable() { - @Override - public void run() { - processor.process(start, end); - gate.countDown(); - } - }; + final Runnable task = new Runnable() { + @Override + public void run() { + process.process(start, end); + gate.countDown(); + } + }; - ConcurenceRunner.this.run(task); - } - try {// Wait for all threads to finish running - gate.await(); - } catch (InterruptedException e) { - e.printStackTrace(); - throw new RuntimeException(e); - } + exec.execute(task); + } + try {// Wait for all threads to finish running + gate.await(); + } catch (InterruptedException e) { + e.printStackTrace(); + throw new RuntimeException(e); } - } } From 513b368908c7f65635765bf282ad18385ff75368 Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Mon, 19 Feb 2018 09:54:43 +0100 Subject: [PATCH 17/34] Now possible to specify count of threads in ConcurenceRunner --- .../java/javacnn/util/ConcurenceRunner.java | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/main/java/javacnn/util/ConcurenceRunner.java b/src/main/java/javacnn/util/ConcurenceRunner.java index 4d2074d..e364195 100644 --- a/src/main/java/javacnn/util/ConcurenceRunner.java +++ b/src/main/java/javacnn/util/ConcurenceRunner.java @@ -16,12 +16,23 @@ public class ConcurenceRunner implements Runner { private final ExecutorService exec; - private final int cpuNum; + private final int threadCount; + /** + * Starting ConcurrenceRunner with one thread for each CPU. + */ public ConcurenceRunner() { - cpuNum = Runtime.getRuntime().availableProcessors(); - System.out.println("cpuNum:" + cpuNum); - exec = Executors.newFixedThreadPool(cpuNum); + this(Runtime.getRuntime().availableProcessors()); + } + + /** + * Starting ConcurenceRunner with the given count of threads. + * + * @param threadCount Threads to start (must be > 0). + */ + public ConcurenceRunner(final int threadCount) { + this.threadCount = threadCount; + exec = Executors.newFixedThreadPool(this.threadCount); } public void shutdown() { @@ -30,7 +41,7 @@ public void shutdown() { @Override public void startProcess(final int mapNum, final Process process) { - final int runCpu = cpuNum < mapNum ? cpuNum : 1; + final int runCpu = threadCount < mapNum ? threadCount : 1; // Fragment length rounded up final CountDownLatch gate = new CountDownLatch(runCpu); From 0a731dddc34989f55d71dab14332b9b2036c9eda Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Mon, 19 Feb 2018 09:57:41 +0100 Subject: [PATCH 18/34] Moved RunCNN and dataset to test --- .gitignore | 4 +- dataset/readme.md | 3 - dataset/test.predict | 28000 ---------------- src/test/dataset/readme.md | 3 + {dataset => src/test/dataset}/test.format | 0 {dataset => src/test/dataset}/train.format | 0 .../java => test/java/javacnn}/RunCNN.java | 12 +- 7 files changed, 13 insertions(+), 28009 deletions(-) delete mode 100644 dataset/readme.md delete mode 100644 dataset/test.predict create mode 100644 src/test/dataset/readme.md rename {dataset => src/test/dataset}/test.format (100%) rename {dataset => src/test/dataset}/train.format (100%) rename src/{main/java => test/java/javacnn}/RunCNN.java (79%) diff --git a/.gitignore b/.gitignore index 705fa9d..9b2785d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ /.idea /.classpath /.project -/target/ \ No newline at end of file +/target/ +src/test/dataset/test.predict +src/test/model.cnn diff --git a/dataset/readme.md b/dataset/readme.md deleted file mode 100644 index 3cda064..0000000 --- a/dataset/readme.md +++ /dev/null @@ -1,3 +0,0 @@ -

The dataset is part of MNIST from kaggle Digit Recognizer competition.

-

"train.format" is the train set, which has been binarized.

-

"test.format" is the test set, which has been binarized.

\ No newline at end of file diff --git a/dataset/test.predict b/dataset/test.predict deleted file mode 100644 index c6eb184..0000000 --- a/dataset/test.predict +++ /dev/null @@ -1,28000 +0,0 @@ -2 -0 -9 -9 -3 -7 -0 -3 -0 -3 -5 -7 -4 -0 -4 -0 -3 -1 -9 -0 -9 -1 -1 -5 -7 -4 -2 -7 -7 -7 -7 -5 -4 -2 -6 -2 -5 -5 -1 -6 -7 -7 -4 -9 -6 -7 -8 -2 -6 -7 -6 -8 -8 -3 -8 -2 -1 -2 -9 -0 -4 -1 -7 -0 -0 -0 -1 -9 -0 -1 -6 -5 -8 -8 -2 -8 -9 -9 -2 -3 -5 -9 -1 -0 -9 -2 -4 -3 -6 -7 -2 -0 -6 -6 -1 -4 -3 -9 -7 -4 -0 -3 -2 -0 -7 -3 -0 -5 -0 -9 -0 -0 -6 -1 -1 -7 -1 -1 -5 -3 -3 -7 -2 -8 -6 -3 -8 -7 -8 -4 -5 -5 -6 -0 -0 -0 -3 -1 -3 -0 -4 -3 -4 -5 -5 -8 -7 -7 -2 -8 -4 -3 -5 -6 -5 -1 -7 -5 -7 -8 -3 -0 -4 -5 -1 -2 -7 -6 -5 -0 -2 -7 -9 -6 -1 -3 -7 -4 -1 -2 -4 -2 -5 -2 -4 -9 -2 -1 -6 -0 -6 -1 -4 -2 -6 -0 -9 -7 -6 -9 -1 -9 -0 -5 -9 -0 -8 -4 -6 -2 -0 -9 -3 -6 -7 -2 -1 -6 -3 -4 -2 -3 -1 -3 -2 -8 -4 -6 -1 -0 -0 -4 -9 -1 -7 -3 -2 -2 -8 -6 -8 -6 -2 -8 -5 -5 -9 -8 -3 -8 -9 -7 -1 -3 -8 -4 -5 -1 -4 -3 -6 -3 -3 -5 -7 -0 -6 -8 -5 -1 -6 -0 -6 -3 -9 -9 -1 -5 -8 -4 -0 -9 -2 -0 -5 -3 -7 -8 -9 -9 -5 -9 -7 -9 -9 -6 -3 -0 -3 -3 -6 -9 -8 -2 -6 -2 -7 -1 -4 -5 -8 -5 -9 -0 -0 -3 -8 -4 -1 -5 -9 -1 -1 -9 -8 -4 -5 -1 -5 -7 -6 -3 -1 -3 -0 -9 -0 -6 -6 -0 -6 -7 -1 -8 -6 -0 -6 -5 -2 -2 -6 -7 -7 -2 -5 -8 -9 -9 -2 -1 -9 -6 -3 -8 -9 -2 -3 -8 -1 -6 -4 -8 -9 -9 -7 -6 -9 -5 -3 -7 -6 -5 -5 -9 -2 -6 -2 -1 -3 -7 -1 -7 -9 -9 -6 -1 -1 -1 -7 -3 -9 -7 -6 -1 -1 -1 -2 -3 -5 -5 -5 -0 -4 -1 -2 -3 -1 -1 -3 -5 -9 -6 -6 -5 -3 -1 -4 -7 -4 -7 -4 -8 -5 -2 -6 -1 -3 -9 -5 -0 -8 -4 -7 -4 -9 -4 -1 -5 -3 -9 -9 -7 -6 -9 -5 -9 -2 -3 -1 -6 -6 -7 -5 -0 -5 -1 -7 -4 -4 -1 -1 -4 -9 -5 -6 -0 -1 -3 -1 -0 -4 -8 -1 -2 -7 -9 -4 -8 -3 -7 -0 -4 -2 -4 -2 -7 -6 -3 -2 -0 -6 -5 -9 -2 -1 -8 -3 -3 -0 -6 -7 -5 -8 -7 -5 -3 -1 -7 -6 -3 -7 -9 -0 -7 -7 -1 -0 -1 -1 -7 -0 -5 -3 -8 -5 -5 -6 -5 -7 -3 -0 -2 -8 -2 -0 -3 -0 -9 -2 -1 -1 -3 -0 -5 -0 -0 -7 -5 -6 -2 -0 -3 -1 -1 -6 -5 -4 -1 -1 -4 -7 -5 -3 -6 -0 -4 -8 -2 -4 -2 -5 -1 -2 -6 -9 -1 -7 -5 -8 -0 -8 -8 -4 -5 -3 -6 -6 -6 -0 -3 -1 -1 -7 -1 -6 -2 -8 -5 -6 -4 -7 -4 -3 -3 -2 -4 -7 -0 -0 -9 -8 -5 -9 -4 -0 -8 -7 -3 -6 -7 -6 -1 -8 -6 -1 -9 -7 -7 -8 -3 -0 -9 -9 -6 -7 -7 -4 -4 -1 -8 -4 -9 -0 -0 -8 -2 -4 -3 -3 -7 -2 -3 -4 -0 -4 -2 -1 -3 -3 -6 -3 -9 -4 -3 -8 -7 -7 -6 -6 -0 -6 -9 -8 -1 -1 -3 -4 -6 -9 -9 -2 -6 -0 -1 -8 -4 -3 -9 -8 -8 -4 -0 -5 -0 -6 -0 -9 -4 -6 -0 -5 -8 -1 -5 -7 -6 -2 -3 -7 -8 -9 -3 -1 -0 -1 -0 -6 -9 -7 -0 -7 -1 -3 -2 -2 -7 -1 -6 -1 -5 -4 -4 -3 -4 -3 -9 -8 -7 -8 -6 -4 -9 -4 -4 -1 -4 -7 -1 -1 -2 -2 -0 -4 -0 -4 -0 -0 -8 -1 -8 -6 -5 -0 -1 -5 -3 -4 -6 -3 -1 -1 -6 -9 -8 -3 -5 -5 -4 -5 -8 -5 -0 -4 -0 -4 -3 -1 -6 -9 -7 -1 -1 -3 -3 -1 -9 -9 -6 -9 -1 -5 -9 -7 -3 -4 -4 -0 -9 -7 -4 -3 -0 -5 -0 -1 -9 -0 -4 -4 -2 -8 -4 -5 -9 -3 -9 -6 -1 -5 -5 -1 -9 -0 -8 -4 -6 -7 -2 -1 -5 -8 -9 -7 -7 -2 -8 -1 -3 -6 -5 -0 -9 -1 -4 -2 -3 -6 -9 -2 -3 -4 -5 -4 -2 -3 -3 -1 -1 -0 -1 -4 -9 -1 -1 -2 -7 -1 -5 -4 -9 -1 -7 -6 -0 -4 -2 -9 -9 -1 -1 -5 -3 -5 -7 -9 -7 -7 -3 -2 -7 -2 -0 -4 -7 -1 -6 -4 -6 -1 -5 -7 -3 -5 -9 -4 -7 -9 -6 -6 -3 -3 -2 -1 -4 -1 -3 -7 -7 -9 -5 -6 -0 -6 -1 -0 -9 -3 -2 -9 -2 -6 -7 -5 -2 -3 -2 -8 -3 -0 -2 -7 -9 -4 -0 -0 -5 -1 -8 -8 -5 -3 -2 -9 -6 -7 -0 -8 -0 -7 -4 -3 -8 -7 -9 -7 -7 -0 -5 -3 -2 -1 -9 -0 -6 -8 -3 -6 -2 -2 -9 -0 -9 -0 -7 -1 -3 -4 -6 -3 -9 -2 -6 -3 -7 -3 -7 -2 -3 -4 -9 -5 -9 -9 -6 -2 -6 -1 -5 -5 -1 -9 -1 -8 -9 -4 -8 -3 -5 -2 -0 -1 -6 -1 -9 -6 -2 -7 -7 -6 -6 -2 -6 -3 -5 -9 -1 -1 -3 -6 -5 -0 -0 -6 -0 -9 -4 -7 -0 -5 -9 -8 -3 -7 -6 -7 -2 -6 -1 -2 -9 -3 -0 -2 -3 -7 -7 -6 -6 -3 -1 -3 -1 -0 -1 -7 -6 -8 -3 -9 -3 -4 -2 -9 -1 -8 -2 -0 -6 -4 -6 -7 -2 -4 -1 -0 -5 -2 -6 -9 -9 -8 -5 -4 -1 -6 -3 -5 -6 -1 -2 -0 -0 -8 -0 -3 -6 -8 -7 -7 -7 -0 -0 -2 -1 -2 -8 -4 -5 -5 -3 -6 -7 -8 -4 -7 -4 -9 -1 -8 -0 -9 -1 -7 -0 -6 -4 -5 -2 -4 -5 -8 -2 -9 -1 -8 -6 -2 -7 -2 -5 -3 -8 -9 -9 -0 -7 -0 -3 -0 -9 -7 -3 -3 -8 -8 -9 -3 -2 -5 -4 -6 -8 -3 -8 -1 -7 -9 -6 -4 -0 -6 -2 -8 -4 -5 -9 -6 -7 -8 -2 -0 -0 -5 -0 -5 -9 -4 -9 -9 -5 -4 -3 -0 -5 -4 -1 -9 -1 -7 -9 -9 -5 -7 -8 -6 -4 -1 -9 -3 -1 -6 -6 -0 -1 -5 -5 -7 -6 -1 -3 -1 -9 -4 -2 -2 -6 -9 -9 -8 -1 -1 -0 -1 -6 -0 -4 -0 -2 -7 -6 -1 -4 -7 -0 -7 -1 -0 -7 -1 -1 -9 -8 -0 -6 -5 -9 -8 -6 -3 -6 -6 -6 -1 -1 -4 -0 -7 -8 -0 -4 -6 -7 -5 -5 -9 -6 -2 -4 -7 -5 -9 -3 -5 -1 -8 -0 -9 -6 -8 -1 -3 -0 -3 -1 -9 -1 -4 -5 -8 -2 -2 -9 -1 -3 -3 -0 -5 -6 -1 -8 -3 -6 -7 -2 -3 -2 -9 -7 -1 -5 -9 -8 -7 -3 -8 -4 -5 -8 -2 -1 -6 -7 -6 -1 -1 -0 -5 -0 -9 -1 -7 -4 -0 -9 -7 -5 -9 -8 -8 -7 -4 -4 -3 -7 -9 -4 -7 -2 -7 -4 -1 -5 -7 -2 -9 -5 -9 -8 -4 -5 -9 -1 -5 -1 -9 -3 -7 -6 -1 -7 -9 -2 -5 -6 -1 -8 -5 -0 -8 -8 -4 -7 -1 -9 -6 -8 -9 -4 -9 -9 -6 -5 -2 -7 -9 -8 -0 -4 -9 -0 -8 -8 -9 -0 -9 -0 -9 -7 -0 -8 -5 -5 -3 -6 -2 -5 -3 -1 -7 -3 -1 -0 -6 -5 -9 -3 -3 -9 -4 -8 -8 -7 -6 -4 -4 -0 -7 -9 -9 -6 -7 -3 -8 -9 -5 -0 -8 -6 -0 -3 -0 -1 -8 -3 -8 -6 -0 -1 -3 -0 -7 -3 -6 -9 -2 -3 -1 -7 -7 -9 -5 -9 -9 -3 -1 -5 -8 -3 -2 -5 -4 -1 -8 -4 -0 -2 -0 -1 -0 -0 -7 -1 -5 -5 -5 -9 -9 -7 -9 -4 -6 -8 -1 -9 -2 -7 -4 -8 -5 -0 -5 -9 -8 -7 -5 -0 -1 -7 -9 -6 -3 -0 -8 -7 -5 -2 -6 -1 -7 -2 -3 -8 -8 -1 -4 -6 -2 -4 -0 -2 -3 -6 -3 -8 -2 -9 -1 -8 -2 -5 -8 -7 -7 -6 -2 -0 -3 -2 -5 -1 -5 -9 -3 -3 -1 -9 -3 -2 -1 -1 -2 -6 -1 -9 -4 -4 -9 -6 -6 -7 -5 -9 -6 -6 -1 -0 -7 -8 -8 -5 -3 -7 -7 -7 -1 -4 -6 -1 -0 -0 -1 -7 -7 -2 -8 -9 -8 -2 -4 -8 -1 -3 -5 -1 -3 -3 -6 -6 -5 -8 -8 -3 -4 -5 -9 -8 -2 -0 -1 -3 -3 -5 -6 -5 -3 -6 -1 -3 -3 -7 -1 -5 -6 -6 -1 -7 -4 -1 -9 -0 -2 -8 -0 -7 -1 -3 -0 -7 -0 -7 -7 -1 -5 -8 -4 -9 -7 -0 -7 -1 -5 -3 -6 -2 -4 -7 -1 -6 -5 -9 -3 -4 -1 -9 -4 -0 -3 -5 -1 -0 -0 -5 -3 -9 -4 -9 -1 -0 -0 -5 -7 -1 -6 -8 -3 -7 -0 -3 -8 -2 -8 -6 -7 -9 -1 -8 -4 -5 -3 -1 -5 -2 -2 -5 -3 -8 -8 -2 -7 -4 -1 -5 -7 -0 -6 -6 -2 -3 -2 -2 -9 -1 -9 -6 -9 -2 -2 -6 -0 -9 -0 -0 -0 -0 -3 -0 -4 -3 -6 -7 -8 -6 -0 -5 -1 -8 -5 -6 -0 -2 -7 -1 -1 -3 -9 -9 -3 -0 -8 -5 -9 -6 -8 -3 -1 -0 -0 -9 -0 -6 -6 -2 -4 -3 -0 -2 -7 -1 -5 -8 -5 -8 -8 -4 -7 -4 -3 -6 -5 -7 -3 -1 -6 -7 -1 -3 -1 -5 -3 -1 -5 -2 -2 -6 -9 -0 -2 -1 -2 -7 -4 -4 -9 -3 -0 -9 -5 -1 -9 -6 -9 -7 -6 -5 -0 -1 -1 -3 -8 -4 -0 -0 -0 -3 -7 -9 -8 -8 -9 -9 -1 -5 -0 -9 -1 -7 -2 -0 -5 -9 -0 -6 -9 -2 -4 -2 -1 -4 -4 -4 -8 -8 -6 -5 -6 -1 -1 -5 -9 -3 -9 -7 -0 -7 -6 -5 -8 -6 -6 -0 -7 -4 -5 -1 -7 -7 -3 -5 -6 -8 -5 -3 -6 -9 -5 -6 -1 -9 -5 -0 -9 -4 -9 -1 -8 -4 -0 -9 -3 -0 -9 -4 -6 -4 -0 -1 -0 -1 -7 -5 -9 -8 -7 -8 -6 -2 -0 -0 -5 -7 -9 -6 -1 -6 -7 -2 -5 -9 -7 -1 -1 -4 -1 -6 -6 -7 -0 -2 -8 -7 -0 -9 -4 -6 -2 -6 -7 -3 -2 -9 -6 -5 -2 -5 -8 -5 -6 -9 -2 -7 -7 -9 -3 -4 -9 -2 -8 -0 -2 -4 -0 -9 -4 -5 -8 -8 -9 -3 -3 -5 -9 -0 -7 -0 -5 -5 -1 -9 -1 -9 -9 -4 -6 -4 -6 -8 -9 -0 -0 -8 -2 -1 -6 -8 -2 -0 -0 -2 -1 -5 -7 -3 -3 -5 -9 -6 -2 -3 -4 -3 -7 -4 -3 -2 -6 -6 -1 -5 -8 -0 -6 -7 -6 -2 -2 -4 -0 -6 -2 -1 -5 -2 -5 -7 -5 -6 -2 -0 -6 -8 -2 -4 -4 -5 -1 -6 -8 -1 -4 -6 -6 -1 -5 -2 -0 -0 -0 -1 -1 -4 -7 -0 -3 -6 -5 -0 -5 -3 -0 -8 -9 -1 -5 -7 -3 -3 -6 -1 -6 -1 -9 -5 -2 -0 -7 -9 -0 -2 -7 -4 -4 -6 -0 -9 -6 -8 -9 -1 -7 -4 -7 -9 -9 -7 -4 -8 -5 -9 -5 -5 -1 -9 -3 -6 -4 -0 -2 -6 -1 -8 -0 -4 -2 -1 -2 -6 -7 -3 -7 -9 -1 -5 -6 -4 -0 -1 -4 -2 -1 -1 -4 -2 -2 -1 -1 -7 -9 -4 -8 -0 -8 -9 -5 -8 -9 -4 -6 -3 -3 -3 -9 -0 -3 -6 -9 -1 -2 -6 -0 -6 -0 -5 -0 -0 -1 -8 -0 -8 -2 -0 -1 -5 -3 -9 -5 -7 -6 -8 -8 -7 -9 -0 -3 -2 -6 -0 -5 -6 -9 -6 -5 -4 -2 -1 -9 -3 -6 -6 -8 -7 -2 -2 -0 -3 -4 -5 -3 -3 -1 -5 -4 -5 -9 -7 -0 -1 -6 -5 -3 -2 -6 -4 -2 -5 -9 -3 -4 -6 -3 -5 -8 -2 -8 -9 -8 -7 -3 -3 -9 -9 -4 -6 -7 -3 -6 -9 -9 -6 -6 -0 -9 -9 -9 -9 -4 -1 -7 -2 -8 -5 -0 -5 -6 -7 -1 -8 -0 -8 -0 -3 -0 -2 -4 -9 -1 -1 -6 -7 -6 -7 -5 -3 -6 -6 -3 -9 -8 -7 -8 -8 -9 -0 -0 -7 -7 -1 -6 -0 -3 -1 -9 -8 -5 -1 -7 -0 -1 -0 -5 -9 -9 -0 -1 -9 -0 -9 -9 -4 -2 -2 -5 -5 -0 -0 -5 -9 -9 -3 -4 -2 -5 -0 -0 -3 -2 -3 -5 -9 -0 -1 -9 -5 -9 -8 -2 -0 -3 -4 -4 -1 -0 -8 -3 -5 -5 -6 -9 -1 -9 -8 -7 -0 -3 -3 -9 -1 -0 -3 -3 -9 -7 -3 -1 -9 -6 -3 -3 -4 -0 -2 -1 -3 -9 -9 -1 -7 -9 -7 -9 -6 -1 -5 -6 -6 -9 -0 -4 -2 -2 -4 -3 -7 -6 -4 -1 -4 -2 -6 -1 -0 -2 -2 -1 -0 -2 -7 -2 -6 -0 -2 -0 -2 -0 -0 -5 -0 -3 -9 -0 -6 -6 -4 -1 -5 -9 -2 -4 -6 -4 -1 -1 -3 -0 -1 -1 -5 -0 -2 -9 -7 -3 -6 -8 -7 -8 -8 -7 -2 -6 -9 -4 -5 -7 -1 -3 -6 -0 -9 -9 -4 -6 -8 -6 -4 -1 -0 -3 -3 -0 -1 -8 -9 -5 -5 -7 -7 -3 -4 -6 -2 -9 -3 -0 -7 -6 -8 -8 -2 -4 -0 -6 -9 -8 -3 -9 -2 -0 -1 -6 -7 -7 -6 -4 -6 -5 -9 -6 -1 -3 -3 -5 -2 -1 -6 -4 -1 -9 -9 -6 -6 -6 -4 -5 -9 -4 -9 -9 -2 -2 -5 -1 -6 -2 -8 -7 -2 -8 -4 -7 -6 -1 -0 -1 -9 -0 -7 -4 -9 -0 -9 -4 -9 -0 -3 -6 -7 -4 -3 -7 -0 -2 -1 -5 -1 -4 -2 -4 -5 -2 -6 -6 -8 -8 -6 -7 -9 -1 -9 -1 -3 -2 -5 -9 -0 -9 -0 -1 -6 -9 -7 -7 -3 -0 -5 -3 -1 -9 -2 -6 -7 -9 -1 -0 -5 -1 -6 -8 -8 -6 -7 -3 -7 -2 -8 -9 -1 -0 -6 -1 -1 -6 -6 -9 -0 -1 -6 -9 -8 -1 -3 -7 -7 -8 -1 -4 -7 -0 -6 -0 -6 -4 -3 -1 -9 -6 -3 -0 -0 -2 -8 -9 -8 -1 -1 -8 -1 -3 -2 -3 -1 -0 -5 -9 -5 -0 -7 -1 -0 -6 -1 -5 -1 -1 -8 -2 -2 -6 -1 -7 -9 -7 -6 -7 -5 -1 -0 -6 -2 -7 -6 -4 -4 -4 -0 -8 -2 -0 -0 -6 -4 -8 -1 -9 -2 -9 -5 -1 -5 -1 -1 -4 -7 -0 -1 -2 -4 -6 -1 -9 -2 -9 -2 -9 -2 -6 -6 -5 -5 -5 -8 -5 -6 -1 -1 -6 -1 -2 -1 -9 -4 -7 -6 -7 -1 -0 -6 -4 -6 -5 -3 -3 -8 -0 -1 -1 -6 -0 -5 -1 -2 -1 -6 -4 -3 -9 -0 -4 -0 -3 -1 -4 -9 -2 -1 -7 -0 -9 -6 -8 -5 -8 -6 -0 -6 -7 -9 -9 -3 -1 -3 -9 -3 -1 -1 -2 -0 -0 -7 -7 -5 -1 -6 -0 -6 -7 -9 -3 -6 -2 -2 -2 -9 -9 -9 -9 -2 -7 -9 -2 -8 -1 -7 -7 -3 -7 -8 -7 -2 -5 -6 -1 -9 -7 -7 -7 -8 -9 -9 -7 -8 -9 -5 -2 -9 -5 -0 -7 -6 -1 -2 -6 -2 -7 -3 -5 -6 -9 -5 -8 -8 -2 -2 -9 -7 -9 -5 -1 -1 -1 -4 -5 -6 -1 -7 -0 -7 -9 -2 -7 -2 -3 -1 -5 -7 -1 -5 -1 -6 -0 -2 -6 -9 -9 -7 -9 -8 -7 -7 -7 -5 -6 -2 -9 -8 -8 -6 -7 -3 -1 -3 -9 -1 -2 -0 -2 -6 -5 -1 -0 -9 -9 -1 -6 -8 -5 -5 -6 -2 -3 -6 -1 -6 -6 -2 -0 -6 -0 -8 -1 -2 -8 -6 -4 -8 -5 -0 -4 -5 -0 -9 -7 -9 -2 -9 -2 -8 -5 -3 -5 -9 -7 -9 -5 -7 -3 -0 -2 -8 -0 -6 -6 -6 -9 -9 -3 -9 -2 -2 -3 -3 -7 -4 -5 -8 -5 -3 -6 -0 -5 -1 -2 -6 -9 -5 -5 -2 -7 -9 -1 -9 -3 -6 -8 -8 -2 -5 -8 -3 -9 -1 -6 -9 -6 -7 -5 -8 -5 -0 -8 -6 -7 -8 -1 -7 -1 -9 -3 -8 -8 -3 -9 -9 -8 -7 -6 -0 -5 -3 -0 -9 -9 -6 -8 -7 -9 -1 -2 -3 -2 -1 -5 -2 -9 -4 -4 -9 -3 -0 -0 -1 -1 -3 -1 -1 -2 -1 -6 -2 -7 -6 -0 -4 -4 -7 -0 -7 -0 -8 -7 -1 -3 -7 -8 -9 -0 -9 -0 -0 -1 -8 -8 -7 -4 -8 -3 -5 -9 -0 -3 -6 -3 -6 -3 -0 -9 -2 -6 -9 -3 -0 -5 -4 -5 -7 -7 -4 -2 -3 -9 -2 -1 -1 -0 -1 -9 -4 -2 -2 -1 -3 -2 -7 -3 -5 -1 -3 -0 -2 -1 -1 -7 -9 -1 -8 -9 -1 -6 -9 -7 -1 -8 -9 -9 -2 -0 -6 -1 -5 -2 -0 -9 -2 -5 -5 -3 -9 -0 -1 -0 -9 -6 -0 -9 -3 -7 -8 -0 -4 -0 -2 -6 -8 -2 -1 -9 -2 -1 -8 -6 -6 -1 -4 -8 -6 -2 -1 -3 -9 -8 -3 -7 -8 -7 -8 -7 -1 -5 -6 -4 -7 -5 -5 -4 -0 -8 -1 -6 -6 -3 -3 -7 -8 -8 -4 -3 -4 -3 -2 -1 -7 -0 -9 -4 -2 -8 -6 -8 -5 -1 -8 -4 -8 -1 -4 -3 -9 -1 -2 -1 -5 -3 -6 -7 -2 -4 -7 -6 -9 -1 -9 -8 -6 -8 -2 -6 -2 -0 -1 -1 -0 -6 -3 -9 -7 -3 -1 -9 -4 -6 -9 -7 -2 -7 -8 -1 -0 -5 -1 -2 -7 -6 -6 -3 -0 -4 -1 -9 -6 -3 -9 -1 -2 -4 -1 -3 -4 -0 -1 -8 -0 -2 -9 -2 -7 -9 -9 -1 -4 -7 -3 -2 -9 -9 -1 -3 -7 -8 -4 -9 -7 -9 -7 -1 -4 -4 -0 -0 -0 -3 -5 -8 -3 -0 -4 -1 -2 -8 -1 -0 -8 -8 -3 -4 -6 -9 -3 -2 -7 -2 -6 -8 -1 -0 -1 -8 -1 -3 -7 -4 -9 -1 -3 -5 -0 -3 -5 -4 -8 -6 -1 -9 -9 -6 -6 -6 -1 -9 -0 -3 -9 -8 -4 -6 -5 -6 -8 -9 -9 -5 -1 -8 -0 -2 -0 -3 -9 -5 -7 -0 -1 -0 -7 -2 -8 -1 -8 -3 -3 -7 -9 -6 -9 -9 -5 -7 -4 -6 -4 -9 -6 -0 -0 -4 -3 -5 -7 -1 -2 -1 -9 -2 -1 -3 -1 -9 -1 -8 -5 -8 -7 -9 -3 -6 -3 -6 -1 -7 -5 -8 -2 -6 -3 -7 -0 -5 -6 -7 -9 -6 -8 -6 -0 -9 -5 -9 -6 -1 -9 -5 -6 -7 -6 -2 -6 -5 -6 -6 -0 -4 -3 -9 -4 -3 -5 -0 -2 -6 -5 -0 -7 -6 -0 -6 -0 -5 -3 -6 -1 -9 -3 -8 -4 -4 -3 -4 -5 -4 -6 -0 -6 -1 -2 -0 -3 -9 -5 -0 -4 -1 -1 -3 -3 -9 -3 -3 -6 -4 -9 -0 -6 -9 -7 -9 -8 -5 -7 -9 -1 -4 -6 -5 -1 -8 -4 -9 -7 -1 -6 -8 -6 -7 -9 -4 -4 -0 -7 -7 -9 -1 -1 -6 -8 -0 -8 -3 -2 -7 -6 -7 -0 -1 -6 -5 -4 -5 -6 -2 -5 -3 -9 -5 -1 -3 -9 -6 -9 -2 -0 -6 -2 -7 -6 -0 -0 -3 -9 -1 -2 -9 -3 -0 -6 -1 -0 -9 -9 -6 -5 -1 -3 -6 -7 -1 -5 -1 -8 -1 -7 -3 -7 -9 -5 -2 -4 -9 -6 -8 -6 -6 -0 -8 -1 -7 -7 -1 -9 -0 -1 -4 -9 -1 -7 -3 -1 -6 -2 -2 -7 -2 -4 -1 -5 -6 -1 -5 -3 -2 -6 -9 -1 -7 -4 -5 -6 -5 -6 -3 -8 -1 -3 -9 -5 -6 -7 -1 -3 -3 -7 -1 -1 -8 -0 -6 -3 -9 -3 -3 -6 -2 -2 -0 -1 -0 -1 -1 -3 -7 -2 -9 -0 -3 -8 -8 -0 -1 -9 -5 -5 -2 -1 -3 -6 -0 -2 -7 -2 -9 -1 -5 -7 -1 -7 -4 -3 -9 -4 -4 -3 -2 -0 -0 -0 -3 -1 -1 -4 -3 -9 -2 -2 -2 -8 -6 -9 -4 -0 -4 -9 -1 -0 -7 -1 -2 -1 -6 -2 -4 -0 -4 -6 -0 -1 -1 -4 -2 -6 -2 -7 -7 -3 -4 -3 -9 -0 -1 -4 -8 -6 -2 -1 -1 -7 -3 -2 -9 -2 -2 -5 -2 -1 -0 -2 -5 -0 -0 -0 -9 -2 -7 -3 -0 -2 -1 -9 -0 -8 -5 -3 -8 -2 -2 -0 -2 -7 -8 -4 -0 -3 -0 -2 -2 -8 -0 -9 -2 -3 -0 -2 -9 -9 -7 -1 -4 -2 -2 -1 -6 -7 -6 -6 -3 -3 -6 -7 -3 -9 -1 -2 -3 -5 -0 -1 -0 -9 -7 -9 -0 -1 -5 -1 -4 -3 -6 -4 -8 -1 -2 -2 -6 -4 -2 -5 -8 -2 -6 -5 -8 -3 -1 -7 -4 -6 -0 -1 -5 -8 -1 -4 -0 -3 -6 -6 -1 -6 -2 -5 -9 -1 -5 -6 -6 -1 -4 -0 -3 -6 -0 -1 -6 -8 -9 -4 -7 -7 -7 -8 -8 -7 -5 -2 -7 -8 -1 -1 -9 -5 -4 -1 -0 -1 -3 -2 -3 -6 -8 -3 -1 -2 -7 -1 -7 -7 -0 -0 -1 -0 -0 -2 -0 -0 -7 -1 -7 -8 -6 -5 -7 -1 -6 -6 -9 -9 -9 -1 -3 -9 -0 -4 -9 -2 -8 -6 -2 -7 -4 -7 -5 -0 -3 -7 -5 -6 -6 -9 -0 -3 -5 -2 -8 -9 -8 -2 -2 -9 -7 -1 -7 -7 -2 -7 -3 -2 -2 -5 -4 -0 -2 -2 -8 -1 -1 -2 -9 -2 -3 -8 -1 -6 -9 -9 -4 -3 -9 -7 -9 -0 -7 -3 -5 -2 -3 -0 -1 -9 -9 -7 -9 -2 -7 -4 -7 -0 -6 -5 -1 -9 -5 -9 -7 -2 -2 -5 -6 -3 -5 -8 -2 -3 -3 -1 -4 -7 -3 -8 -4 -5 -7 -6 -1 -9 -0 -4 -8 -0 -6 -5 -6 -5 -2 -0 -5 -3 -1 -5 -1 -4 -1 -1 -1 -1 -5 -7 -1 -0 -4 -2 -9 -5 -5 -9 -3 -5 -2 -0 -0 -1 -0 -4 -2 -4 -3 -2 -7 -9 -4 -0 -1 -9 -9 -4 -9 -3 -2 -0 -8 -4 -1 -0 -9 -3 -4 -9 -6 -9 -0 -6 -1 -1 -2 -9 -7 -8 -7 -3 -7 -0 -1 -8 -6 -2 -5 -4 -8 -9 -1 -0 -2 -8 -7 -9 -4 -4 -7 -6 -9 -4 -1 -9 -5 -8 -3 -4 -2 -6 -7 -8 -7 -8 -7 -4 -9 -4 -7 -3 -6 -9 -1 -1 -1 -0 -1 -8 -1 -6 -9 -3 -3 -6 -9 -5 -8 -1 -9 -2 -3 -7 -9 -9 -2 -9 -8 -3 -4 -5 -1 -9 -9 -6 -1 -3 -5 -2 -9 -6 -5 -3 -5 -1 -5 -0 -2 -6 -8 -9 -9 -0 -6 -6 -7 -5 -8 -2 -0 -9 -5 -0 -1 -9 -7 -2 -3 -3 -0 -4 -9 -3 -1 -3 -9 -0 -3 -5 -4 -9 -4 -6 -8 -9 -6 -9 -3 -8 -6 -5 -0 -4 -8 -6 -0 -0 -9 -1 -4 -2 -5 -6 -3 -8 -0 -8 -8 -2 -4 -2 -7 -6 -9 -7 -8 -6 -9 -8 -6 -1 -5 -4 -3 -9 -9 -5 -0 -8 -2 -9 -1 -6 -9 -1 -0 -1 -8 -7 -6 -1 -6 -4 -1 -7 -5 -9 -1 -1 -2 -3 -9 -0 -7 -2 -2 -4 -1 -7 -7 -7 -8 -3 -5 -5 -6 -2 -1 -0 -8 -7 -7 -5 -1 -1 -8 -6 -5 -6 -6 -9 -8 -1 -7 -4 -9 -1 -6 -7 -3 -0 -7 -2 -1 -6 -8 -1 -3 -9 -0 -7 -5 -3 -2 -9 -0 -6 -1 -4 -0 -9 -7 -0 -6 -5 -5 -7 -0 -1 -5 -6 -1 -4 -8 -6 -8 -9 -5 -1 -0 -2 -2 -0 -7 -8 -2 -5 -1 -8 -3 -7 -6 -9 -1 -4 -9 -1 -0 -3 -3 -5 -9 -2 -7 -6 -3 -3 -8 -0 -0 -9 -7 -5 -5 -0 -3 -8 -7 -3 -1 -0 -7 -1 -5 -7 -0 -2 -5 -9 -2 -6 -1 -3 -1 -1 -3 -6 -6 -8 -4 -2 -3 -0 -1 -4 -4 -1 -1 -8 -4 -7 -6 -7 -5 -1 -9 -2 -0 -9 -1 -4 -3 -9 -4 -6 -9 -8 -5 -6 -8 -9 -7 -0 -0 -1 -0 -9 -7 -0 -2 -2 -7 -9 -6 -9 -5 -6 -1 -1 -1 -5 -8 -7 -9 -6 -2 -2 -2 -8 -5 -7 -1 -1 -1 -2 -9 -1 -8 -1 -3 -2 -1 -0 -6 -4 -7 -6 -5 -0 -1 -2 -8 -8 -4 -4 -3 -7 -4 -4 -9 -2 -8 -8 -6 -1 -5 -4 -5 -6 -8 -7 -0 -0 -1 -5 -4 -1 -5 -7 -3 -2 -0 -9 -9 -4 -9 -5 -0 -3 -6 -2 -3 -1 -7 -6 -0 -8 -2 -7 -7 -5 -8 -7 -4 -5 -0 -1 -9 -7 -4 -5 -0 -9 -8 -6 -3 -3 -1 -6 -1 -8 -0 -9 -9 -1 -1 -3 -7 -0 -0 -3 -0 -4 -9 -9 -8 -8 -0 -9 -0 -1 -6 -6 -6 -0 -2 -5 -7 -0 -6 -9 -7 -2 -2 -5 -3 -6 -6 -9 -2 -5 -4 -3 -5 -2 -4 -5 -4 -1 -7 -2 -3 -1 -9 -9 -2 -9 -0 -8 -1 -5 -7 -8 -6 -6 -8 -3 -2 -6 -7 -0 -6 -0 -2 -2 -3 -0 -6 -3 -3 -4 -2 -1 -4 -3 -3 -0 -0 -3 -2 -4 -1 -5 -9 -2 -4 -2 -5 -8 -7 -9 -9 -1 -5 -0 -0 -9 -0 -8 -8 -0 -9 -8 -6 -2 -3 -2 -0 -6 -3 -3 -1 -9 -2 -0 -5 -2 -8 -9 -4 -2 -6 -0 -2 -0 -6 -5 -3 -4 -1 -2 -2 -4 -9 -4 -3 -2 -1 -9 -1 -6 -1 -2 -3 -4 -0 -1 -9 -4 -0 -0 -4 -8 -1 -9 -2 -5 -3 -4 -3 -7 -0 -7 -9 -7 -9 -5 -9 -7 -7 -0 -4 -2 -1 -1 -8 -5 -0 -4 -9 -8 -8 -2 -4 -1 -4 -1 -4 -1 -0 -9 -9 -5 -6 -9 -9 -1 -2 -6 -0 -3 -0 -3 -3 -0 -4 -0 -1 -0 -8 -4 -9 -3 -7 -1 -4 -1 -7 -6 -0 -7 -7 -1 -5 -0 -3 -6 -0 -2 -8 -5 -3 -5 -7 -7 -2 -9 -6 -7 -8 -1 -5 -7 -3 -1 -7 -3 -1 -8 -0 -1 -0 -4 -3 -2 -7 -9 -7 -9 -3 -6 -2 -3 -4 -1 -3 -0 -9 -9 -2 -6 -4 -0 -0 -7 -5 -1 -8 -3 -8 -1 -6 -5 -8 -9 -2 -2 -0 -7 -1 -6 -2 -1 -6 -8 -4 -3 -5 -9 -1 -0 -5 -3 -1 -9 -2 -1 -8 -1 -3 -6 -0 -1 -2 -4 -1 -3 -7 -4 -9 -4 -2 -0 -6 -3 -5 -0 -4 -3 -6 -1 -7 -6 -2 -7 -1 -7 -0 -6 -9 -6 -5 -1 -3 -6 -8 -1 -9 -0 -6 -6 -9 -3 -6 -8 -2 -9 -0 -6 -1 -8 -1 -1 -0 -0 -9 -9 -9 -4 -8 -4 -1 -6 -4 -9 -1 -8 -9 -2 -3 -4 -6 -1 -0 -1 -6 -7 -9 -0 -0 -8 -5 -4 -2 -5 -2 -8 -4 -6 -9 -7 -1 -4 -3 -5 -2 -9 -3 -3 -5 -3 -0 -0 -1 -9 -2 -5 -3 -7 -6 -5 -9 -6 -5 -0 -9 -1 -3 -8 -5 -7 -7 -0 -7 -7 -1 -5 -6 -7 -0 -4 -3 -0 -0 -0 -0 -7 -6 -8 -2 -6 -7 -1 -7 -1 -0 -2 -3 -5 -1 -8 -5 -4 -7 -6 -2 -6 -6 -8 -3 -8 -5 -1 -5 -2 -6 -5 -1 -5 -8 -3 -5 -5 -3 -2 -3 -5 -5 -8 -6 -7 -0 -3 -8 -8 -4 -2 -6 -4 -5 -8 -3 -0 -9 -1 -1 -3 -0 -9 -6 -7 -8 -2 -5 -3 -6 -2 -0 -2 -5 -1 -2 -9 -9 -0 -2 -5 -5 -6 -1 -4 -1 -3 -1 -5 -5 -0 -1 -9 -3 -9 -5 -0 -7 -9 -8 -3 -0 -1 -3 -3 -8 -1 -3 -5 -7 -0 -1 -6 -4 -8 -5 -0 -6 -0 -4 -5 -8 -2 -1 -5 -4 -0 -1 -4 -1 -7 -3 -9 -2 -9 -1 -2 -1 -7 -6 -6 -1 -5 -9 -8 -9 -0 -2 -3 -1 -7 -3 -1 -5 -2 -7 -3 -2 -6 -9 -7 -4 -0 -5 -1 -5 -9 -1 -0 -9 -4 -5 -9 -8 -7 -1 -3 -3 -2 -4 -1 -1 -8 -0 -5 -2 -8 -9 -0 -9 -7 -0 -9 -9 -2 -7 -6 -1 -7 -1 -6 -0 -2 -9 -9 -9 -7 -8 -2 -4 -0 -9 -3 -5 -9 -4 -2 -0 -3 -2 -0 -2 -9 -4 -8 -5 -5 -6 -9 -6 -8 -4 -7 -6 -7 -4 -9 -4 -2 -1 -0 -8 -1 -7 -6 -0 -1 -9 -3 -6 -5 -5 -0 -7 -1 -3 -6 -9 -3 -3 -4 -8 -4 -7 -9 -1 -2 -9 -6 -7 -4 -8 -9 -3 -9 -5 -6 -7 -9 -2 -2 -1 -8 -3 -2 -7 -1 -7 -1 -2 -8 -7 -9 -8 -0 -9 -4 -0 -0 -2 -2 -4 -2 -0 -7 -6 -3 -2 -7 -4 -1 -2 -7 -4 -9 -2 -1 -7 -5 -8 -0 -5 -7 -9 -7 -1 -5 -5 -4 -2 -4 -7 -9 -9 -6 -4 -5 -4 -9 -1 -9 -6 -5 -6 -2 -8 -6 -8 -3 -8 -9 -6 -8 -8 -6 -4 -0 -2 -6 -1 -3 -8 -9 -8 -7 -7 -7 -8 -0 -7 -9 -5 -0 -3 -7 -4 -6 -4 -7 -2 -9 -6 -4 -0 -4 -6 -1 -8 -4 -9 -9 -4 -3 -6 -9 -0 -4 -1 -8 -5 -7 -8 -5 -2 -8 -8 -7 -0 -5 -0 -7 -2 -3 -4 -7 -8 -1 -5 -5 -9 -5 -2 -5 -0 -5 -6 -9 -7 -4 -1 -1 -9 -1 -9 -9 -4 -8 -4 -3 -7 -6 -2 -0 -1 -8 -9 -6 -7 -9 -5 -7 -5 -3 -1 -6 -0 -3 -2 -4 -5 -1 -5 -1 -5 -3 -5 -8 -5 -8 -5 -2 -7 -0 -1 -1 -1 -9 -0 -9 -9 -2 -0 -7 -5 -1 -3 -4 -7 -8 -0 -1 -2 -9 -3 -1 -6 -2 -0 -5 -0 -1 -6 -1 -9 -7 -7 -8 -6 -8 -4 -0 -7 -1 -0 -1 -3 -4 -9 -0 -9 -4 -5 -1 -6 -9 -4 -6 -6 -6 -1 -1 -3 -5 -8 -7 -4 -8 -6 -9 -8 -1 -1 -1 -3 -3 -5 -6 -5 -6 -2 -8 -7 -9 -1 -1 -9 -9 -0 -6 -6 -2 -3 -8 -3 -0 -0 -5 -0 -9 -2 -1 -3 -3 -5 -1 -8 -9 -9 -1 -5 -6 -9 -1 -7 -2 -9 -1 -6 -3 -5 -0 -7 -7 -6 -5 -1 -8 -1 -3 -4 -5 -0 -7 -4 -2 -8 -0 -4 -5 -1 -9 -7 -4 -2 -8 -8 -5 -8 -6 -8 -7 -9 -9 -7 -9 -4 -2 -6 -9 -1 -9 -9 -0 -6 -8 -3 -0 -1 -4 -6 -6 -8 -1 -9 -4 -3 -7 -2 -8 -5 -1 -9 -6 -8 -0 -0 -1 -7 -9 -4 -9 -6 -0 -7 -3 -9 -7 -2 -7 -4 -2 -0 -5 -2 -6 -4 -1 -7 -3 -0 -4 -3 -1 -6 -1 -4 -9 -2 -1 -0 -3 -3 -9 -0 -9 -6 -6 -0 -2 -7 -3 -0 -5 -8 -8 -5 -3 -5 -0 -8 -0 -7 -3 -3 -2 -5 -1 -3 -6 -5 -9 -7 -2 -2 -9 -0 -8 -8 -0 -5 -0 -8 -0 -4 -1 -6 -7 -7 -8 -1 -0 -2 -6 -4 -6 -8 -8 -6 -6 -4 -9 -4 -5 -5 -6 -3 -1 -7 -6 -1 -9 -9 -0 -6 -6 -5 -2 -0 -8 -9 -3 -1 -8 -1 -0 -3 -0 -6 -6 -8 -6 -2 -3 -2 -9 -2 -0 -6 -0 -2 -1 -2 -6 -6 -1 -7 -7 -0 -2 -9 -4 -1 -0 -0 -7 -5 -8 -5 -6 -1 -7 -0 -6 -6 -5 -9 -3 -6 -4 -2 -1 -1 -7 -1 -5 -9 -3 -3 -7 -6 -1 -6 -2 -1 -2 -5 -7 -0 -5 -2 -6 -0 -9 -1 -0 -5 -5 -2 -4 -6 -3 -0 -9 -4 -9 -4 -7 -0 -0 -4 -0 -9 -2 -7 -1 -1 -2 -1 -1 -6 -3 -2 -5 -5 -1 -8 -2 -0 -3 -8 -4 -1 -6 -8 -5 -8 -5 -3 -5 -8 -6 -7 -1 -6 -1 -9 -3 -9 -3 -9 -3 -1 -9 -8 -8 -3 -1 -7 -8 -6 -5 -4 -6 -0 -0 -2 -0 -0 -0 -2 -6 -7 -5 -1 -0 -0 -0 -9 -3 -8 -0 -9 -5 -9 -4 -2 -3 -2 -7 -2 -0 -2 -6 -2 -2 -8 -8 -8 -5 -1 -2 -2 -6 -7 -0 -2 -2 -1 -4 -3 -4 -1 -9 -7 -0 -5 -6 -5 -1 -8 -5 -3 -3 -9 -0 -6 -6 -9 -2 -1 -0 -4 -8 -4 -5 -7 -9 -2 -2 -2 -1 -1 -4 -4 -7 -8 -6 -8 -6 -5 -5 -8 -1 -8 -2 -9 -1 -9 -7 -2 -4 -4 -4 -9 -9 -3 -0 -5 -5 -5 -5 -2 -7 -0 -0 -1 -3 -5 -3 -0 -0 -0 -3 -1 -4 -0 -2 -6 -9 -7 -6 -9 -6 -6 -9 -0 -2 -2 -4 -1 -9 -2 -7 -2 -5 -9 -4 -6 -2 -7 -6 -8 -9 -1 -8 -8 -0 -2 -4 -9 -9 -1 -7 -1 -9 -9 -2 -1 -4 -5 -2 -7 -6 -0 -0 -3 -0 -6 -6 -8 -0 -0 -3 -6 -7 -8 -0 -8 -6 -4 -5 -8 -1 -5 -0 -6 -6 -9 -5 -1 -6 -4 -3 -0 -7 -2 -5 -5 -9 -9 -5 -6 -9 -0 -1 -9 -2 -9 -5 -6 -4 -9 -7 -6 -2 -8 -0 -4 -1 -8 -2 -1 -3 -4 -1 -7 -7 -2 -9 -3 -8 -6 -3 -2 -2 -6 -3 -5 -7 -1 -1 -1 -7 -7 -5 -2 -9 -7 -9 -1 -5 -6 -4 -4 -4 -1 -9 -0 -5 -4 -3 -4 -6 -6 -3 -8 -6 -7 -4 -9 -5 -9 -0 -0 -0 -8 -6 -4 -2 -3 -0 -9 -8 -5 -9 -8 -6 -7 -6 -1 -4 -0 -1 -6 -1 -4 -5 -6 -1 -1 -9 -1 -5 -7 -5 -9 -2 -5 -1 -0 -6 -7 -1 -1 -6 -4 -0 -5 -6 -0 -8 -0 -7 -4 -9 -9 -0 -2 -2 -0 -3 -0 -5 -1 -0 -1 -1 -2 -8 -2 -6 -2 -6 -2 -0 -2 -9 -7 -3 -3 -8 -5 -0 -6 -6 -9 -5 -3 -2 -7 -3 -4 -7 -7 -5 -8 -1 -1 -6 -0 -5 -7 -5 -9 -5 -0 -4 -3 -7 -6 -3 -5 -4 -5 -1 -1 -9 -4 -8 -3 -7 -3 -0 -9 -4 -7 -8 -4 -8 -1 -1 -0 -5 -6 -7 -9 -5 -6 -1 -0 -6 -0 -2 -1 -7 -9 -0 -3 -2 -8 -7 -0 -1 -3 -4 -9 -0 -7 -6 -1 -3 -9 -2 -3 -1 -4 -1 -1 -7 -4 -4 -5 -1 -2 -8 -9 -6 -6 -5 -4 -0 -6 -1 -3 -5 -2 -8 -1 -9 -0 -5 -3 -3 -9 -2 -9 -1 -7 -4 -5 -0 -5 -1 -9 -1 -1 -9 -9 -9 -6 -6 -1 -4 -5 -4 -8 -7 -6 -1 -1 -0 -4 -6 -2 -9 -9 -5 -3 -6 -4 -3 -3 -8 -2 -2 -1 -5 -8 -7 -5 -2 -8 -0 -7 -2 -5 -5 -6 -4 -8 -7 -3 -8 -2 -1 -3 -9 -5 -0 -4 -0 -6 -5 -5 -0 -0 -6 -7 -7 -1 -1 -1 -5 -6 -6 -5 -1 -4 -9 -5 -0 -4 -9 -1 -7 -7 -4 -6 -9 -3 -0 -0 -7 -8 -0 -8 -3 -0 -5 -8 -5 -1 -6 -0 -9 -0 -9 -5 -3 -0 -1 -7 -1 -1 -4 -4 -0 -4 -1 -7 -1 -5 -5 -4 -5 -6 -8 -3 -3 -1 -1 -8 -0 -9 -8 -2 -7 -2 -9 -8 -9 -5 -4 -8 -2 -2 -3 -2 -6 -8 -9 -1 -1 -9 -5 -9 -1 -7 -2 -7 -9 -1 -2 -9 -7 -5 -5 -5 -7 -4 -3 -5 -9 -7 -8 -0 -6 -1 -4 -8 -2 -3 -1 -8 -2 -9 -7 -9 -7 -3 -3 -2 -9 -2 -7 -6 -1 -9 -9 -2 -1 -2 -5 -8 -1 -6 -3 -0 -9 -1 -2 -6 -5 -5 -8 -4 -2 -6 -1 -6 -1 -0 -4 -9 -0 -1 -4 -1 -2 -1 -6 -0 -7 -6 -1 -0 -0 -1 -8 -3 -0 -6 -7 -0 -3 -2 -0 -1 -7 -8 -4 -4 -6 -1 -6 -2 -0 -0 -3 -9 -9 -8 -7 -5 -8 -0 -1 -0 -0 -0 -5 -6 -2 -2 -5 -3 -8 -8 -1 -9 -2 -7 -4 -0 -5 -3 -8 -3 -6 -9 -9 -3 -4 -8 -9 -4 -3 -2 -9 -2 -3 -7 -3 -7 -7 -7 -0 -9 -9 -4 -2 -3 -6 -6 -7 -9 -3 -9 -7 -1 -4 -9 -0 -9 -9 -8 -0 -0 -9 -9 -7 -3 -9 -4 -4 -9 -2 -1 -9 -9 -6 -9 -1 -9 -6 -4 -7 -7 -7 -2 -7 -6 -2 -9 -6 -6 -7 -3 -9 -8 -1 -0 -9 -2 -8 -0 -3 -1 -4 -5 -2 -1 -9 -4 -4 -1 -0 -9 -1 -9 -7 -4 -5 -9 -3 -8 -5 -7 -0 -1 -4 -1 -8 -1 -6 -6 -9 -6 -8 -8 -1 -6 -2 -7 -8 -0 -2 -0 -4 -5 -5 -8 -1 -7 -6 -5 -4 -6 -0 -2 -2 -1 -5 -7 -2 -4 -6 -4 -4 -9 -2 -2 -7 -6 -1 -7 -1 -6 -3 -9 -7 -5 -1 -7 -9 -2 -2 -5 -7 -3 -5 -7 -6 -2 -8 -7 -2 -8 -9 -9 -2 -9 -2 -4 -4 -1 -6 -0 -6 -9 -6 -5 -6 -9 -6 -1 -2 -2 -5 -0 -9 -4 -1 -9 -4 -3 -6 -9 -6 -5 -7 -0 -3 -0 -0 -2 -7 -0 -7 -8 -7 -2 -7 -5 -8 -0 -7 -2 -9 -6 -2 -0 -9 -7 -3 -5 -0 -5 -9 -0 -9 -6 -9 -1 -9 -1 -0 -0 -8 -8 -9 -8 -3 -9 -7 -7 -8 -2 -5 -4 -7 -5 -5 -7 -1 -1 -9 -2 -7 -9 -5 -7 -6 -1 -2 -2 -9 -1 -4 -6 -5 -4 -4 -8 -4 -7 -1 -2 -0 -7 -6 -7 -6 -7 -1 -5 -2 -9 -2 -7 -6 -6 -6 -1 -1 -4 -6 -9 -4 -2 -1 -0 -5 -1 -5 -7 -4 -6 -2 -3 -5 -2 -5 -9 -1 -7 -3 -8 -0 -1 -6 -5 -9 -7 -7 -0 -9 -6 -3 -8 -9 -0 -1 -2 -1 -1 -3 -5 -9 -9 -8 -7 -1 -9 -9 -3 -5 -7 -1 -4 -4 -7 -4 -3 -9 -1 -9 -1 -3 -7 -3 -1 -4 -7 -3 -5 -6 -5 -6 -1 -8 -9 -4 -7 -1 -1 -8 -2 -6 -4 -6 -6 -0 -6 -2 -3 -3 -4 -3 -5 -6 -8 -2 -4 -3 -8 -0 -1 -0 -9 -3 -4 -2 -3 -0 -2 -5 -8 -1 -7 -5 -0 -9 -7 -7 -7 -1 -0 -4 -9 -4 -1 -6 -7 -4 -4 -6 -0 -7 -2 -2 -3 -9 -1 -1 -2 -2 -2 -3 -8 -0 -5 -2 -6 -2 -1 -1 -8 -7 -8 -2 -3 -8 -7 -1 -6 -3 -7 -0 -5 -6 -6 -7 -4 -2 -2 -1 -7 -3 -4 -3 -9 -8 -8 -0 -2 -3 -9 -1 -3 -6 -3 -8 -7 -9 -2 -6 -3 -6 -9 -2 -2 -5 -9 -3 -7 -2 -3 -6 -9 -1 -9 -6 -4 -5 -6 -5 -0 -7 -3 -9 -9 -1 -6 -9 -1 -6 -9 -3 -5 -2 -2 -7 -9 -7 -2 -9 -0 -9 -8 -2 -1 -9 -1 -2 -5 -0 -9 -0 -5 -3 -8 -9 -5 -8 -6 -1 -9 -4 -1 -6 -6 -9 -6 -2 -7 -3 -8 -2 -4 -9 -1 -6 -4 -1 -1 -2 -3 -7 -5 -5 -2 -5 -3 -2 -1 -5 -8 -0 -4 -9 -2 -4 -6 -2 -8 -9 -9 -5 -8 -0 -5 -0 -0 -6 -6 -2 -6 -2 -0 -7 -1 -2 -8 -2 -9 -1 -6 -4 -3 -0 -0 -1 -5 -9 -3 -1 -0 -2 -5 -6 -8 -4 -4 -3 -4 -9 -7 -6 -6 -8 -4 -8 -3 -3 -8 -4 -1 -8 -2 -5 -5 -2 -9 -0 -0 -4 -1 -5 -7 -7 -3 -3 -6 -2 -6 -5 -1 -7 -3 -1 -7 -7 -9 -8 -7 -3 -3 -3 -1 -6 -1 -9 -5 -9 -3 -0 -0 -2 -0 -8 -0 -0 -2 -5 -5 -8 -5 -9 -7 -8 -9 -1 -5 -9 -7 -9 -7 -3 -1 -4 -3 -8 -4 -0 -7 -0 -6 -0 -9 -8 -1 -5 -3 -5 -6 -8 -7 -9 -1 -4 -4 -0 -2 -0 -4 -5 -5 -8 -8 -8 -9 -6 -6 -5 -1 -5 -7 -7 -5 -2 -2 -2 -3 -8 -6 -4 -8 -4 -1 -2 -7 -5 -5 -0 -4 -6 -6 -1 -4 -4 -6 -5 -5 -5 -9 -4 -6 -1 -3 -0 -2 -9 -5 -2 -3 -3 -2 -8 -1 -5 -6 -3 -4 -6 -4 -6 -3 -0 -0 -5 -5 -0 -7 -9 -1 -3 -1 -4 -8 -7 -5 -8 -0 -0 -0 -0 -6 -9 -7 -9 -9 -9 -0 -2 -4 -0 -7 -6 -1 -1 -4 -5 -3 -0 -8 -7 -8 -1 -3 -4 -1 -5 -2 -3 -0 -7 -1 -0 -3 -8 -5 -0 -3 -1 -5 -4 -5 -3 -6 -1 -6 -6 -1 -4 -7 -3 -7 -8 -6 -0 -5 -6 -6 -9 -5 -5 -2 -2 -6 -1 -5 -6 -0 -3 -2 -6 -6 -1 -5 -9 -5 -4 -6 -1 -1 -7 -2 -0 -5 -8 -1 -0 -4 -8 -6 -1 -9 -5 -1 -9 -6 -6 -7 -6 -7 -3 -1 -1 -1 -6 -4 -8 -1 -4 -4 -1 -4 -1 -1 -7 -9 -6 -3 -8 -6 -6 -6 -2 -5 -9 -4 -5 -1 -6 -6 -3 -0 -7 -9 -9 -6 -1 -1 -5 -1 -8 -1 -6 -9 -2 -7 -7 -0 -1 -9 -3 -9 -3 -2 -4 -9 -8 -7 -7 -1 -1 -6 -9 -5 -3 -5 -2 -8 -8 -9 -2 -8 -3 -9 -0 -8 -6 -6 -5 -0 -6 -7 -8 -9 -7 -4 -2 -7 -2 -8 -0 -0 -4 -2 -9 -0 -1 -9 -9 -9 -1 -7 -0 -7 -0 -3 -1 -6 -8 -1 -9 -4 -0 -7 -0 -9 -8 -1 -8 -8 -1 -2 -6 -5 -1 -0 -2 -6 -0 -3 -7 -0 -5 -3 -7 -7 -0 -5 -5 -2 -5 -1 -2 -7 -2 -7 -5 -6 -7 -6 -2 -4 -0 -1 -8 -6 -6 -9 -4 -6 -6 -2 -4 -4 -0 -1 -7 -9 -1 -8 -3 -3 -1 -4 -5 -2 -2 -7 -4 -1 -7 -4 -8 -9 -6 -8 -7 -8 -7 -0 -9 -4 -0 -2 -5 -0 -3 -6 -9 -4 -1 -0 -1 -2 -8 -1 -6 -0 -1 -2 -6 -3 -3 -2 -0 -6 -9 -2 -1 -3 -2 -8 -8 -0 -6 -1 -1 -8 -7 -4 -8 -3 -1 -9 -5 -5 -1 -8 -5 -0 -5 -9 -9 -5 -4 -5 -7 -3 -5 -7 -8 -0 -2 -1 -4 -0 -9 -1 -7 -9 -1 -6 -7 -1 -3 -5 -4 -4 -4 -0 -3 -5 -1 -4 -2 -1 -4 -0 -1 -1 -7 -5 -7 -1 -3 -3 -0 -4 -9 -4 -6 -8 -2 -9 -5 -1 -9 -5 -0 -8 -2 -5 -4 -3 -6 -7 -2 -5 -3 -6 -0 -1 -4 -9 -3 -6 -6 -5 -1 -9 -3 -0 -3 -8 -9 -7 -1 -3 -0 -5 -7 -9 -0 -4 -5 -9 -8 -2 -8 -3 -1 -5 -8 -3 -7 -5 -0 -3 -5 -4 -4 -2 -5 -9 -6 -6 -0 -1 -4 -1 -2 -6 -5 -0 -3 -8 -3 -0 -1 -1 -2 -9 -6 -1 -1 -0 -7 -2 -8 -4 -7 -2 -1 -6 -1 -8 -6 -7 -7 -6 -3 -2 -4 -1 -4 -5 -8 -3 -5 -4 -8 -4 -6 -6 -1 -7 -5 -6 -1 -3 -2 -2 -7 -0 -0 -2 -9 -1 -3 -3 -3 -3 -9 -3 -4 -1 -2 -6 -8 -1 -9 -5 -4 -7 -9 -0 -7 -2 -9 -9 -1 -4 -0 -6 -7 -8 -7 -4 -7 -1 -9 -4 -9 -6 -5 -1 -9 -3 -6 -6 -5 -2 -8 -3 -8 -2 -4 -0 -6 -0 -1 -5 -9 -0 -1 -6 -5 -2 -3 -0 -3 -0 -0 -8 -4 -5 -0 -5 -3 -0 -2 -3 -1 -2 -4 -3 -0 -2 -0 -7 -9 -3 -3 -9 -6 -5 -9 -8 -1 -8 -0 -6 -5 -1 -9 -5 -1 -7 -1 -9 -6 -2 -1 -1 -0 -5 -6 -4 -1 -9 -5 -3 -4 -5 -3 -8 -5 -3 -7 -2 -2 -2 -9 -8 -3 -0 -6 -7 -6 -2 -8 -6 -2 -1 -5 -7 -2 -1 -2 -3 -9 -4 -1 -9 -2 -8 -4 -6 -7 -8 -4 -9 -9 -4 -5 -1 -2 -0 -2 -9 -6 -1 -5 -0 -9 -5 -4 -7 -5 -8 -1 -5 -1 -1 -0 -5 -4 -2 -0 -0 -9 -0 -2 -7 -7 -5 -2 -5 -7 -6 -3 -7 -5 -4 -0 -8 -6 -3 -7 -3 -1 -6 -2 -4 -4 -4 -0 -5 -1 -8 -8 -3 -4 -5 -0 -2 -2 -0 -1 -1 -6 -1 -4 -3 -1 -5 -9 -9 -2 -5 -0 -0 -2 -4 -2 -8 -2 -8 -4 -9 -4 -4 -8 -0 -5 -8 -5 -9 -9 -4 -4 -9 -2 -4 -8 -3 -0 -9 -8 -7 -6 -2 -4 -2 -6 -2 -2 -7 -1 -1 -0 -2 -9 -9 -0 -5 -4 -7 -0 -5 -9 -4 -1 -5 -6 -6 -8 -1 -2 -4 -3 -3 -2 -9 -7 -1 -4 -9 -6 -9 -0 -2 -4 -4 -5 -7 -8 -9 -4 -2 -4 -1 -7 -8 -2 -8 -0 -1 -2 -9 -9 -7 -0 -1 -2 -6 -9 -4 -2 -1 -4 -5 -0 -8 -6 -1 -9 -5 -1 -6 -2 -1 -8 -0 -5 -1 -9 -9 -6 -6 -8 -1 -1 -7 -9 -4 -7 -7 -3 -4 -6 -0 -6 -3 -0 -6 -3 -0 -1 -3 -2 -6 -3 -2 -0 -5 -9 -1 -3 -9 -0 -2 -0 -5 -8 -9 -6 -6 -5 -2 -9 -7 -6 -0 -7 -9 -1 -3 -9 -1 -7 -6 -4 -9 -7 -6 -0 -0 -4 -8 -4 -9 -8 -3 -5 -1 -7 -7 -1 -4 -1 -6 -3 -9 -7 -6 -6 -1 -9 -4 -0 -0 -4 -4 -8 -3 -1 -7 -5 -8 -3 -7 -4 -1 -0 -3 -2 -9 -3 -2 -2 -6 -0 -9 -9 -8 -3 -1 -8 -6 -4 -2 -2 -1 -7 -3 -1 -9 -5 -0 -6 -8 -1 -7 -8 -3 -7 -6 -7 -7 -3 -9 -2 -4 -5 -7 -7 -9 -0 -9 -1 -8 -1 -8 -9 -4 -1 -9 -9 -6 -1 -7 -5 -3 -5 -3 -1 -8 -0 -0 -1 -0 -3 -6 -6 -1 -9 -9 -0 -1 -0 -2 -0 -6 -7 -7 -5 -3 -1 -8 -9 -7 -2 -4 -6 -2 -5 -0 -6 -5 -6 -8 -3 -4 -1 -1 -4 -6 -8 -7 -1 -8 -6 -8 -1 -2 -3 -3 -2 -3 -3 -6 -8 -2 -7 -6 -9 -8 -0 -9 -7 -7 -7 -8 -3 -1 -2 -3 -1 -4 -3 -6 -4 -8 -3 -1 -3 -6 -9 -6 -9 -0 -3 -9 -0 -7 -7 -3 -0 -0 -9 -3 -5 -7 -9 -8 -6 -0 -7 -7 -2 -8 -7 -9 -1 -2 -5 -6 -0 -7 -8 -1 -0 -3 -6 -9 -9 -5 -0 -3 -0 -1 -1 -6 -2 -5 -9 -5 -4 -0 -5 -5 -1 -2 -7 -6 -5 -3 -0 -5 -6 -9 -0 -7 -8 -1 -7 -1 -8 -7 -2 -7 -9 -7 -3 -2 -1 -2 -8 -4 -9 -2 -1 -3 -1 -8 -1 -6 -2 -7 -3 -9 -9 -4 -1 -6 -3 -1 -1 -7 -5 -3 -5 -2 -2 -9 -7 -1 -3 -9 -1 -1 -1 -8 -1 -6 -2 -6 -6 -1 -8 -2 -9 -7 -0 -3 -5 -7 -8 -7 -8 -7 -9 -3 -7 -1 -8 -5 -4 -7 -5 -3 -0 -5 -9 -7 -0 -7 -7 -4 -4 -5 -8 -9 -2 -9 -7 -5 -1 -4 -9 -9 -6 -4 -2 -0 -2 -6 -2 -2 -8 -4 -1 -6 -1 -2 -2 -1 -3 -1 -2 -4 -6 -4 -3 -5 -5 -5 -1 -0 -1 -1 -9 -1 -7 -2 -6 -9 -6 -5 -2 -5 -4 -8 -9 -3 -4 -8 -9 -7 -3 -5 -6 -5 -9 -8 -5 -7 -0 -4 -5 -1 -0 -1 -6 -9 -2 -1 -9 -7 -0 -9 -3 -1 -6 -3 -6 -0 -5 -2 -6 -5 -7 -9 -0 -2 -0 -8 -4 -0 -9 -9 -6 -0 -7 -5 -0 -1 -9 -6 -1 -1 -2 -7 -0 -3 -3 -8 -9 -9 -8 -4 -5 -4 -7 -9 -8 -6 -9 -8 -3 -0 -9 -6 -7 -9 -3 -5 -8 -1 -2 -7 -8 -7 -1 -6 -8 -3 -1 -6 -8 -7 -6 -9 -1 -5 -9 -0 -9 -4 -3 -3 -3 -4 -1 -9 -3 -0 -5 -2 -0 -6 -1 -6 -7 -9 -0 -4 -5 -4 -3 -5 -9 -4 -8 -7 -2 -8 -2 -6 -7 -9 -7 -6 -4 -6 -3 -4 -5 -7 -4 -2 -2 -9 -6 -9 -0 -5 -0 -0 -3 -3 -5 -3 -7 -6 -2 -2 -4 -9 -0 -5 -3 -1 -8 -5 -9 -9 -7 -1 -4 -3 -6 -7 -8 -1 -6 -4 -1 -4 -9 -7 -1 -3 -0 -8 -7 -8 -7 -1 -6 -9 -9 -8 -3 -1 -6 -8 -5 -0 -8 -4 -1 -6 -8 -1 -9 -4 -3 -8 -4 -0 -9 -9 -4 -9 -3 -7 -2 -8 -0 -4 -3 -7 -4 -4 -6 -5 -3 -0 -5 -3 -1 -4 -6 -5 -5 -9 -3 -1 -1 -3 -5 -0 -9 -2 -3 -5 -9 -2 -4 -3 -1 -0 -7 -7 -5 -6 -0 -4 -8 -8 -2 -0 -3 -7 -7 -0 -8 -3 -8 -2 -0 -0 -9 -5 -7 -2 -0 -1 -1 -3 -9 -2 -4 -4 -1 -4 -9 -2 -9 -1 -1 -9 -0 -2 -7 -9 -7 -9 -6 -7 -7 -7 -6 -7 -9 -4 -7 -1 -1 -6 -1 -1 -1 -1 -6 -3 -1 -1 -5 -6 -1 -3 -8 -1 -6 -9 -3 -9 -9 -5 -0 -5 -9 -6 -3 -0 -2 -2 -2 -4 -2 -0 -6 -7 -7 -0 -4 -7 -3 -5 -8 -7 -4 -0 -7 -4 -2 -4 -2 -9 -0 -6 -0 -4 -4 -7 -8 -0 -9 -9 -6 -1 -6 -6 -9 -9 -1 -3 -8 -0 -9 -4 -4 -9 -0 -0 -8 -1 -1 -0 -2 -5 -2 -0 -9 -2 -3 -6 -2 -0 -5 -9 -2 -9 -4 -1 -5 -1 -9 -1 -6 -4 -2 -1 -5 -9 -6 -0 -4 -4 -1 -3 -3 -3 -0 -4 -1 -5 -6 -1 -2 -0 -7 -8 -6 -4 -1 -2 -8 -4 -4 -8 -2 -1 -1 -9 -5 -1 -9 -0 -2 -7 -1 -6 -0 -7 -4 -4 -4 -5 -1 -7 -9 -6 -4 -5 -6 -9 -1 -4 -7 -9 -7 -8 -0 -8 -6 -9 -2 -7 -1 -2 -8 -1 -1 -9 -5 -8 -8 -6 -0 -7 -6 -4 -6 -3 -2 -6 -0 -9 -1 -6 -5 -2 -7 -3 -9 -7 -1 -0 -6 -9 -7 -6 -4 -0 -5 -0 -9 -7 -1 -3 -6 -4 -4 -9 -7 -7 -4 -2 -2 -1 -1 -5 -3 -5 -9 -7 -9 -3 -4 -8 -7 -7 -1 -1 -9 -9 -4 -4 -3 -0 -3 -9 -8 -1 -6 -4 -0 -6 -2 -9 -0 -6 -3 -6 -6 -6 -5 -1 -0 -9 -3 -6 -3 -8 -0 -1 -2 -0 -3 -2 -5 -7 -9 -1 -9 -8 -9 -4 -8 -9 -8 -8 -9 -1 -5 -1 -1 -6 -7 -0 -2 -8 -7 -4 -5 -5 -4 -6 -8 -1 -4 -0 -9 -4 -3 -7 -9 -3 -0 -2 -9 -6 -3 -3 -1 -2 -3 -4 -6 -7 -9 -3 -6 -7 -9 -3 -3 -1 -2 -4 -5 -9 -9 -0 -1 -9 -7 -6 -8 -7 -8 -0 -6 -7 -9 -6 -7 -1 -2 -7 -8 -6 -0 -8 -1 -1 -7 -7 -3 -5 -3 -7 -6 -4 -0 -4 -1 -9 -1 -5 -2 -1 -6 -1 -0 -9 -6 -6 -0 -2 -1 -7 -9 -5 -8 -8 -7 -8 -3 -0 -2 -1 -9 -9 -9 -1 -3 -3 -6 -9 -0 -3 -7 -4 -1 -5 -1 -7 -5 -4 -2 -5 -0 -6 -4 -1 -9 -7 -9 -6 -7 -8 -0 -2 -3 -9 -8 -7 -2 -3 -4 -9 -5 -6 -8 -6 -6 -1 -2 -8 -1 -7 -6 -9 -1 -8 -7 -6 -2 -2 -7 -3 -0 -4 -6 -8 -0 -1 -7 -3 -2 -0 -8 -8 -3 -5 -4 -1 -8 -6 -4 -1 -0 -1 -1 -0 -2 -4 -7 -6 -4 -3 -8 -6 -4 -6 -9 -1 -4 -6 -1 -6 -9 -3 -7 -4 -9 -1 -3 -8 -2 -9 -4 -9 -5 -8 -8 -9 -2 -6 -1 -5 -5 -3 -6 -7 -0 -3 -4 -2 -6 -7 -6 -1 -5 -7 -7 -2 -7 -3 -3 -3 -9 -1 -2 -1 -1 -1 -1 -6 -0 -6 -9 -6 -9 -6 -6 -1 -1 -0 -0 -7 -6 -9 -5 -1 -7 -6 -9 -7 -4 -8 -2 -3 -5 -8 -6 -2 -2 -1 -0 -8 -9 -2 -4 -4 -0 -0 -9 -6 -8 -0 -3 -9 -7 -9 -7 -9 -7 -3 -7 -2 -8 -2 -3 -8 -6 -1 -2 -7 -2 -1 -2 -7 -1 -4 -0 -5 -3 -5 -5 -3 -3 -8 -6 -9 -9 -0 -2 -5 -7 -9 -6 -8 -6 -2 -3 -1 -7 -3 -8 -4 -9 -9 -0 -3 -8 -7 -4 -8 -2 -8 -7 -7 -5 -7 -8 -6 -9 -5 -1 -5 -9 -9 -0 -4 -9 -7 -4 -7 -9 -0 -1 -4 -3 -5 -9 -0 -9 -4 -4 -8 -6 -1 -2 -1 -1 -9 -2 -7 -7 -0 -1 -5 -1 -6 -9 -8 -3 -4 -0 -3 -1 -0 -5 -3 -9 -8 -1 -7 -8 -3 -6 -8 -1 -1 -7 -1 -4 -5 -0 -8 -9 -6 -0 -2 -7 -1 -6 -1 -0 -1 -2 -1 -0 -5 -4 -5 -1 -2 -1 -1 -2 -7 -3 -3 -6 -0 -2 -8 -5 -9 -8 -9 -3 -5 -3 -4 -0 -6 -6 -3 -5 -4 -6 -9 -1 -2 -3 -9 -0 -3 -4 -9 -6 -8 -6 -9 -3 -8 -4 -8 -4 -1 -1 -7 -3 -8 -1 -2 -3 -1 -9 -4 -7 -2 -4 -0 -1 -4 -3 -6 -2 -4 -1 -5 -9 -7 -1 -7 -6 -6 -1 -6 -7 -4 -8 -2 -1 -0 -9 -6 -2 -5 -5 -3 -9 -0 -6 -2 -5 -6 -0 -3 -4 -3 -6 -7 -6 -0 -3 -5 -5 -8 -4 -0 -6 -9 -5 -6 -0 -1 -7 -0 -7 -5 -6 -4 -3 -4 -4 -6 -2 -2 -4 -7 -7 -0 -7 -8 -7 -4 -9 -2 -4 -7 -1 -3 -9 -4 -0 -1 -2 -9 -7 -9 -9 -3 -4 -5 -5 -1 -4 -5 -6 -1 -2 -8 -8 -4 -6 -5 -9 -6 -1 -0 -5 -4 -0 -3 -1 -1 -5 -5 -3 -1 -2 -4 -6 -9 -9 -4 -1 -7 -9 -0 -4 -1 -1 -9 -3 -6 -2 -2 -1 -6 -7 -7 -1 -7 -3 -0 -1 -1 -1 -3 -1 -1 -5 -2 -7 -1 -2 -1 -8 -8 -0 -7 -5 -5 -4 -0 -8 -7 -7 -2 -4 -5 -8 -2 -0 -7 -4 -1 -1 -1 -1 -5 -4 -2 -1 -0 -2 -4 -7 -0 -2 -8 -0 -2 -1 -9 -7 -3 -3 -4 -4 -1 -1 -4 -2 -3 -1 -0 -9 -8 -5 -9 -8 -6 -1 -3 -1 -6 -3 -7 -6 -1 -7 -3 -3 -5 -7 -7 -6 -0 -8 -1 -7 -9 -7 -3 -2 -1 -3 -9 -4 -5 -1 -9 -7 -7 -2 -3 -4 -8 -0 -9 -4 -1 -1 -7 -2 -6 -6 -4 -1 -0 -9 -2 -1 -3 -7 -9 -0 -2 -8 -5 -8 -9 -9 -7 -5 -6 -8 -7 -4 -7 -0 -1 -7 -3 -5 -6 -9 -9 -5 -3 -1 -2 -0 -8 -1 -4 -1 -1 -1 -8 -9 -9 -7 -9 -6 -9 -6 -2 -4 -4 -2 -1 -8 -7 -5 -4 -6 -2 -1 -6 -7 -1 -9 -0 -0 -5 -8 -9 -7 -2 -0 -5 -9 -2 -2 -4 -2 -4 -3 -0 -1 -8 -5 -1 -8 -2 -1 -3 -2 -4 -0 -4 -1 -8 -3 -7 -9 -5 -8 -8 -6 -4 -5 -7 -4 -9 -6 -8 -8 -6 -6 -5 -7 -9 -9 -4 -2 -3 -6 -6 -0 -0 -6 -6 -9 -1 -4 -4 -7 -3 -7 -3 -2 -7 -8 -4 -5 -6 -2 -0 -3 -9 -5 -0 -0 -7 -8 -8 -3 -5 -8 -6 -3 -7 -5 -0 -2 -7 -8 -6 -1 -5 -3 -4 -4 -3 -4 -1 -3 -3 -8 -2 -1 -5 -7 -1 -7 -4 -9 -3 -1 -7 -8 -3 -6 -8 -9 -5 -7 -2 -8 -8 -8 -7 -2 -9 -7 -8 -1 -7 -8 -2 -0 -5 -1 -0 -6 -9 -6 -4 -9 -6 -9 -8 -3 -1 -6 -9 -9 -4 -9 -6 -8 -5 -8 -1 -9 -3 -9 -3 -1 -8 -0 -5 -2 -7 -1 -0 -5 -2 -4 -9 -3 -8 -9 -6 -6 -2 -8 -6 -7 -0 -0 -3 -3 -6 -5 -1 -1 -0 -9 -3 -1 -5 -8 -3 -2 -6 -8 -3 -8 -7 -4 -2 -9 -7 -3 -9 -8 -2 -3 -5 -1 -0 -9 -9 -2 -3 -7 -7 -9 -7 -8 -6 -0 -5 -4 -3 -7 -4 -1 -3 -4 -8 -7 -0 -0 -5 -6 -4 -5 -1 -9 -9 -2 -7 -7 -9 -2 -9 -8 -2 -1 -8 -2 -3 -5 -5 -3 -2 -0 -9 -8 -6 -1 -2 -0 -1 -1 -5 -6 -8 -8 -1 -3 -9 -0 -0 -3 -2 -4 -8 -9 -8 -4 -1 -9 -0 -9 -9 -2 -9 -4 -4 -7 -9 -9 -1 -2 -0 -5 -3 -9 -6 -7 -2 -5 -4 -4 -0 -1 -0 -9 -4 -0 -1 -9 -6 -0 -6 -1 -9 -3 -7 -8 -2 -7 -9 -2 -8 -7 -5 -6 -4 -9 -8 -7 -0 -1 -1 -6 -9 -6 -3 -2 -6 -8 -5 -5 -5 -2 -8 -3 -6 -2 -6 -9 -6 -5 -2 -0 -5 -8 -7 -4 -9 -7 -0 -2 -1 -9 -0 -8 -1 -0 -6 -2 -5 -0 -9 -0 -8 -5 -4 -6 -5 -3 -8 -7 -0 -9 -1 -4 -1 -5 -2 -2 -4 -2 -9 -8 -7 -1 -8 -3 -4 -7 -1 -9 -0 -9 -8 -7 -9 -4 -1 -9 -7 -0 -0 -5 -7 -4 -0 -6 -5 -4 -2 -5 -5 -5 -7 -1 -5 -3 -1 -8 -0 -9 -4 -0 -1 -6 -8 -0 -6 -2 -3 -4 -4 -2 -5 -6 -2 -1 -6 -1 -5 -9 -1 -7 -9 -0 -7 -9 -4 -2 -6 -0 -6 -6 -1 -7 -2 -8 -0 -6 -0 -1 -9 -9 -3 -7 -0 -2 -4 -6 -2 -1 -2 -1 -7 -6 -9 -0 -0 -1 -4 -8 -7 -3 -8 -6 -9 -3 -1 -3 -7 -9 -6 -0 -1 -7 -1 -1 -4 -1 -6 -6 -3 -1 -8 -1 -9 -6 -2 -1 -0 -1 -6 -7 -9 -0 -5 -1 -3 -6 -6 -1 -4 -9 -4 -5 -5 -2 -6 -6 -3 -4 -3 -4 -7 -0 -7 -0 -8 -2 -0 -0 -7 -7 -1 -0 -5 -2 -2 -3 -9 -1 -7 -6 -8 -3 -2 -0 -1 -5 -8 -2 -6 -0 -0 -5 -9 -0 -5 -6 -6 -4 -2 -7 -1 -2 -9 -5 -8 -7 -4 -9 -2 -8 -9 -8 -3 -3 -7 -3 -2 -0 -4 -9 -0 -3 -7 -6 -2 -0 -0 -7 -2 -1 -7 -1 -0 -8 -6 -0 -5 -7 -9 -8 -1 -5 -1 -9 -6 -3 -2 -0 -6 -8 -7 -0 -3 -9 -9 -0 -9 -5 -7 -6 -2 -2 -5 -3 -8 -5 -2 -1 -1 -5 -4 -6 -0 -2 -1 -2 -0 -7 -1 -1 -1 -7 -9 -0 -5 -9 -0 -8 -1 -7 -4 -2 -7 -1 -9 -3 -3 -7 -9 -4 -2 -7 -3 -7 -2 -5 -9 -0 -2 -6 -8 -7 -8 -8 -7 -0 -2 -9 -4 -0 -1 -7 -6 -7 -2 -3 -7 -6 -9 -9 -0 -3 -6 -4 -4 -2 -3 -1 -7 -6 -3 -0 -7 -0 -2 -7 -7 -7 -0 -0 -1 -9 -0 -6 -9 -7 -8 -1 -5 -1 -1 -2 -3 -9 -0 -0 -5 -2 -2 -3 -0 -9 -9 -9 -8 -2 -5 -1 -1 -5 -2 -8 -1 -5 -1 -8 -5 -3 -6 -7 -9 -1 -8 -8 -2 -1 -8 -8 -0 -9 -0 -2 -3 -5 -6 -0 -0 -0 -8 -0 -6 -3 -3 -7 -2 -5 -6 -8 -7 -5 -8 -0 -2 -7 -4 -0 -1 -3 -1 -0 -0 -0 -9 -5 -3 -0 -3 -7 -0 -6 -8 -9 -2 -6 -7 -7 -3 -1 -8 -7 -6 -3 -2 -6 -4 -9 -5 -8 -2 -7 -3 -2 -4 -8 -4 -9 -4 -1 -1 -6 -8 -9 -5 -1 -7 -3 -5 -0 -1 -3 -4 -5 -6 -2 -1 -9 -4 -9 -6 -7 -0 -1 -6 -8 -4 -3 -5 -4 -1 -6 -4 -5 -1 -2 -6 -9 -2 -4 -7 -0 -2 -1 -8 -1 -7 -7 -5 -3 -2 -8 -7 -5 -1 -4 -9 -4 -1 -7 -9 -7 -0 -6 -0 -2 -8 -2 -9 -4 -3 -0 -3 -8 -9 -1 -3 -5 -9 -2 -9 -1 -8 -1 -6 -0 -4 -8 -0 -9 -1 -0 -5 -9 -7 -7 -5 -8 -4 -7 -9 -7 -9 -6 -9 -5 -8 -4 -0 -0 -1 -9 -6 -6 -1 -6 -0 -2 -1 -8 -5 -7 -9 -4 -1 -4 -6 -2 -8 -0 -6 -5 -6 -1 -1 -3 -1 -7 -0 -3 -3 -5 -8 -7 -2 -3 -3 -7 -3 -9 -9 -8 -6 -6 -0 -5 -5 -9 -3 -0 -6 -1 -1 -3 -2 -6 -1 -5 -3 -2 -3 -8 -0 -0 -0 -1 -5 -2 -8 -7 -1 -0 -4 -9 -2 -3 -9 -7 -4 -4 -9 -6 -9 -9 -3 -9 -8 -1 -0 -6 -9 -1 -9 -4 -0 -9 -5 -0 -8 -9 -9 -4 -3 -1 -5 -5 -0 -6 -6 -6 -5 -6 -4 -4 -0 -4 -8 -5 -9 -4 -6 -4 -5 -8 -5 -7 -4 -1 -1 -6 -8 -3 -1 -0 -3 -5 -3 -7 -7 -1 -9 -9 -7 -3 -5 -9 -6 -2 -5 -2 -2 -8 -6 -5 -0 -7 -7 -3 -9 -8 -5 -0 -8 -9 -0 -6 -3 -3 -5 -9 -2 -6 -1 -3 -1 -8 -8 -1 -2 -0 -3 -5 -0 -1 -8 -0 -3 -3 -9 -8 -4 -1 -1 -3 -5 -1 -7 -4 -1 -5 -9 -9 -0 -7 -7 -6 -4 -1 -6 -1 -7 -8 -2 -1 -9 -4 -0 -9 -9 -3 -4 -2 -4 -7 -3 -9 -7 -1 -6 -5 -7 -8 -0 -1 -9 -6 -2 -6 -1 -0 -2 -4 -9 -6 -1 -1 -2 -8 -2 -6 -4 -7 -7 -1 -6 -9 -2 -9 -2 -7 -8 -6 -6 -4 -6 -5 -5 -2 -5 -7 -6 -7 -7 -9 -3 -0 -8 -9 -4 -5 -3 -8 -1 -3 -0 -9 -5 -1 -2 -0 -6 -3 -5 -9 -1 -5 -9 -9 -8 -0 -4 -5 -6 -6 -0 -6 -5 -1 -2 -6 -0 -2 -2 -9 -8 -1 -6 -2 -1 -9 -4 -8 -4 -1 -9 -9 -7 -1 -6 -7 -9 -9 -2 -4 -4 -7 -5 -9 -8 -6 -3 -6 -7 -2 -4 -6 -1 -4 -1 -9 -1 -6 -5 -6 -8 -5 -5 -9 -6 -0 -6 -9 -2 -5 -7 -6 -5 -2 -7 -7 -3 -5 -6 -6 -5 -9 -9 -5 -4 -0 -5 -5 -1 -8 -1 -2 -1 -1 -2 -3 -6 -8 -2 -1 -2 -5 -9 -2 -1 -2 -8 -6 -8 -8 -6 -0 -0 -7 -6 -2 -4 -5 -2 -9 -7 -5 -4 -6 -8 -1 -8 -5 -9 -1 -9 -3 -0 -9 -8 -8 -0 -5 -7 -3 -1 -8 -9 -3 -7 -0 -5 -7 -1 -1 -8 -8 -0 -4 -9 -6 -1 -8 -9 -6 -6 -7 -2 -9 -0 -8 -3 -3 -1 -9 -7 -6 -1 -3 -0 -2 -8 -2 -6 -5 -9 -7 -1 -7 -1 -5 -5 -1 -6 -1 -7 -2 -9 -9 -8 -1 -9 -5 -1 -4 -5 -2 -5 -1 -9 -0 -7 -1 -7 -3 -0 -6 -0 -6 -8 -5 -6 -1 -1 -1 -0 -7 -3 -3 -7 -4 -6 -7 -3 -1 -6 -1 -8 -9 -7 -6 -9 -1 -1 -4 -9 -1 -4 -1 -8 -2 -3 -6 -7 -1 -3 -4 -1 -0 -0 -8 -9 -6 -3 -0 -5 -5 -9 -0 -6 -4 -6 -0 -2 -0 -9 -1 -9 -7 -3 -2 -9 -3 -7 -4 -3 -8 -7 -5 -7 -8 -6 -0 -1 -6 -3 -9 -5 -0 -6 -7 -3 -1 -5 -5 -9 -8 -8 -1 -5 -1 -3 -7 -1 -2 -9 -5 -4 -6 -7 -3 -2 -0 -9 -5 -0 -2 -9 -0 -1 -1 -0 -5 -5 -3 -6 -9 -6 -1 -1 -7 -3 -0 -6 -7 -5 -0 -1 -1 -1 -5 -4 -5 -2 -4 -2 -8 -9 -1 -3 -5 -1 -7 -6 -5 -1 -1 -9 -4 -0 -7 -8 -5 -9 -2 -9 -2 -1 -5 -1 -0 -4 -3 -9 -8 -7 -0 -4 -9 -8 -2 -7 -7 -0 -1 -7 -4 -5 -7 -0 -3 -9 -0 -6 -4 -3 -7 -3 -7 -4 -9 -9 -7 -9 -1 -3 -6 -5 -8 -1 -0 -8 -9 -0 -1 -0 -8 -7 -5 -4 -4 -1 -2 -2 -7 -1 -2 -0 -2 -0 -6 -8 -7 -1 -3 -3 -1 -3 -8 -0 -5 -8 -1 -1 -9 -9 -1 -1 -3 -2 -2 -3 -4 -0 -1 -5 -6 -3 -3 -3 -9 -7 -1 -0 -5 -8 -7 -3 -9 -1 -9 -2 -5 -8 -9 -6 -5 -7 -2 -6 -2 -9 -2 -2 -0 -2 -0 -5 -1 -9 -8 -1 -5 -7 -1 -8 -4 -1 -9 -3 -1 -1 -8 -6 -4 -2 -0 -1 -0 -7 -2 -7 -8 -3 -1 -7 -1 -2 -4 -3 -1 -6 -5 -0 -7 -9 -1 -5 -9 -7 -5 -3 -9 -7 -2 -2 -9 -6 -1 -1 -3 -9 -0 -0 -9 -6 -0 -6 -4 -2 -9 -5 -3 -6 -6 -1 -1 -2 -6 -1 -6 -7 -2 -0 -1 -8 -9 -1 -7 -8 -7 -2 -7 -0 -0 -1 -5 -5 -8 -9 -4 -5 -4 -9 -9 -2 -0 -3 -1 -7 -1 -1 -4 -7 -2 -4 -3 -3 -5 -3 -8 -0 -8 -9 -6 -1 -9 -0 -8 -6 -6 -5 -9 -0 -2 -2 -9 -9 -8 -5 -7 -1 -8 -5 -1 -0 -4 -2 -4 -7 -3 -1 -9 -5 -0 -9 -9 -1 -2 -4 -1 -6 -9 -6 -3 -6 -4 -5 -9 -9 -0 -7 -7 -6 -6 -6 -0 -7 -4 -5 -6 -3 -9 -0 -0 -4 -2 -1 -5 -3 -0 -1 -6 -6 -7 -1 -2 -0 -9 -2 -4 -1 -5 -4 -9 -1 -6 -1 -7 -2 -1 -5 -0 -7 -1 -3 -2 -1 -1 -8 -4 -9 -5 -4 -0 -4 -6 -2 -0 -1 -6 -9 -6 -5 -8 -2 -5 -2 -1 -7 -5 -0 -0 -8 -8 -0 -0 -7 -3 -1 -3 -9 -5 -1 -3 -7 -8 -9 -0 -8 -2 -8 -0 -7 -1 -7 -6 -3 -0 -6 -4 -3 -9 -6 -7 -4 -2 -0 -9 -2 -2 -6 -1 -1 -6 -4 -5 -2 -3 -6 -9 -9 -5 -5 -2 -4 -2 -1 -5 -3 -8 -6 -9 -2 -1 -1 -4 -6 -5 -1 -1 -7 -3 -8 -3 -8 -1 -1 -9 -8 -1 -6 -0 -4 -2 -2 -8 -5 -3 -4 -3 -2 -7 -0 -3 -9 -5 -4 -7 -4 -8 -0 -6 -9 -3 -1 -5 -4 -5 -0 -1 -7 -0 -4 -5 -3 -8 -0 -6 -1 -3 -6 -9 -6 -5 -9 -7 -1 -5 -6 -5 -2 -5 -0 -6 -2 -1 -9 -9 -1 -6 -3 -3 -3 -7 -2 -9 -8 -5 -0 -1 -6 -6 -0 -1 -1 -8 -7 -5 -4 -9 -7 -0 -8 -8 -6 -7 -1 -4 -3 -6 -8 -8 -1 -1 -6 -9 -8 -1 -1 -1 -6 -1 -6 -9 -3 -5 -0 -5 -7 -7 -5 -7 -5 -5 -6 -0 -0 -1 -3 -4 -1 -8 -8 -7 -7 -2 -8 -6 -4 -6 -4 -8 -0 -6 -6 -1 -5 -1 -0 -6 -1 -6 -7 -1 -3 -2 -4 -0 -7 -9 -3 -4 -4 -3 -2 -5 -0 -9 -5 -5 -6 -8 -5 -6 -4 -9 -8 -6 -0 -9 -9 -1 -7 -3 -1 -7 -9 -9 -9 -2 -8 -0 -5 -1 -9 -3 -2 -4 -3 -1 -5 -8 -1 -4 -4 -1 -9 -7 -5 -6 -3 -6 -0 -3 -9 -1 -9 -4 -5 -1 -3 -9 -9 -4 -3 -2 -8 -1 -8 -1 -4 -1 -9 -0 -4 -0 -9 -0 -4 -1 -5 -0 -8 -0 -9 -0 -8 -1 -8 -7 -1 -8 -1 -6 -4 -0 -9 -6 -0 -7 -9 -2 -2 -5 -1 -4 -3 -3 -6 -9 -1 -0 -7 -2 -5 -4 -7 -8 -1 -9 -9 -3 -4 -3 -3 -8 -1 -7 -0 -8 -5 -7 -2 -1 -8 -6 -1 -7 -7 -9 -1 -0 -6 -5 -0 -2 -3 -9 -1 -7 -0 -8 -6 -1 -4 -9 -5 -7 -2 -4 -2 -3 -8 -2 -1 -7 -5 -1 -4 -5 -1 -8 -1 -7 -4 -4 -1 -3 -1 -0 -5 -3 -9 -7 -6 -9 -1 -1 -5 -1 -8 -4 -2 -5 -5 -8 -7 -2 -4 -0 -7 -9 -8 -6 -3 -8 -7 -4 -6 -2 -6 -4 -2 -2 -8 -8 -3 -1 -9 -1 -4 -0 -2 -1 -0 -0 -0 -9 -2 -3 -8 -3 -1 -7 -3 -0 -2 -0 -6 -0 -2 -7 -4 -5 -6 -2 -8 -9 -3 -0 -0 -0 -7 -9 -1 -6 -8 -1 -4 -2 -1 -2 -0 -6 -7 -4 -3 -5 -2 -0 -0 -4 -4 -9 -8 -5 -1 -9 -1 -2 -1 -5 -4 -9 -9 -6 -1 -8 -1 -5 -9 -9 -5 -6 -7 -6 -3 -8 -3 -4 -2 -3 -6 -3 -6 -5 -3 -4 -2 -6 -0 -1 -9 -1 -7 -7 -2 -7 -2 -9 -1 -0 -4 -6 -2 -5 -6 -0 -3 -1 -4 -8 -2 -1 -5 -5 -7 -4 -3 -0 -3 -7 -0 -2 -8 -6 -1 -1 -4 -0 -9 -0 -4 -1 -5 -3 -9 -4 -6 -5 -0 -5 -0 -1 -4 -8 -1 -0 -5 -8 -9 -6 -5 -4 -5 -5 -8 -3 -1 -9 -9 -2 -0 -3 -7 -3 -4 -3 -0 -6 -6 -5 -9 -3 -9 -2 -5 -9 -6 -3 -6 -9 -8 -2 -6 -3 -4 -1 -1 -3 -5 -8 -0 -4 -5 -4 -0 -2 -0 -6 -9 -1 -2 -3 -1 -1 -0 -1 -4 -4 -0 -6 -7 -1 -7 -2 -3 -6 -4 -3 -1 -8 -3 -9 -0 -2 -9 -9 -1 -3 -8 -6 -3 -7 -8 -0 -2 -9 -9 -9 -1 -0 -1 -5 -5 -4 -2 -4 -5 -3 -8 -6 -7 -9 -5 -2 -5 -1 -1 -2 -5 -0 -6 -9 -8 -8 -7 -1 -9 -9 -9 -2 -1 -3 -9 -6 -3 -4 -6 -1 -0 -4 -9 -1 -5 -3 -7 -9 -7 -6 -7 -6 -5 -9 -9 -6 -6 -9 -0 -4 -7 -7 -8 -9 -3 -0 -3 -2 -2 -9 -3 -0 -7 -9 -6 -1 -0 -8 -3 -3 -5 -2 -1 -2 -2 -2 -9 -1 -8 -3 -3 -5 -7 -6 -2 -2 -0 -1 -7 -5 -3 -7 -0 -1 -1 -9 -9 -9 -1 -1 -4 -3 -7 -2 -7 -3 -4 -7 -1 -0 -5 -6 -9 -7 -7 -0 -3 -9 -5 -4 -4 -5 -9 -9 -9 -4 -5 -2 -7 -3 -0 -3 -3 -8 -1 -4 -8 -1 -2 -0 -1 -6 -9 -0 -7 -7 -3 -6 -1 -0 -6 -5 -9 -3 -4 -7 -5 -6 -4 -2 -2 -2 -5 -3 -7 -8 -0 -8 -6 -1 -6 -4 -4 -0 -9 -0 -3 -5 -1 -7 -1 -4 -1 -8 -4 -9 -8 -0 -3 -5 -7 -4 -4 -6 -6 -8 -6 -9 -3 -1 -1 -6 -1 -9 -9 -3 -7 -7 -2 -7 -2 -2 -0 -6 -7 -0 -1 -5 -6 -2 -6 -2 -5 -5 -7 -6 -9 -7 -9 -4 -1 -3 -2 -6 -1 -5 -9 -9 -7 -8 -7 -0 -1 -4 -5 -7 -6 -5 -6 -0 -4 -7 -5 -1 -6 -1 -1 -9 -9 -8 -3 -5 -1 -7 -4 -4 -2 -9 -9 -7 -1 -7 -5 -2 -6 -2 -1 -4 -1 -5 -3 -9 -7 -9 -4 -3 -2 -6 -9 -1 -3 -5 -7 -0 -3 -9 -3 -5 -9 -1 -6 -8 -7 -2 -0 -0 -5 -2 -1 -8 -4 -4 -4 -4 -2 -4 -5 -9 -3 -9 -6 -7 -4 -0 -3 -4 -8 -7 -6 -1 -3 -2 -1 -0 -3 -0 -3 -5 -1 -4 -2 -8 -8 -0 -2 -8 -3 -9 -5 -9 -7 -8 -2 -6 -1 -2 -4 -1 -3 -4 -3 -2 -0 -1 -8 -4 -6 -8 -8 -7 -8 -5 -1 -2 -2 -0 -7 -6 -9 -1 -8 -7 -6 -2 -4 -6 -0 -2 -3 -5 -0 -5 -7 -4 -1 -7 -8 -9 -9 -1 -5 -0 -2 -0 -3 -3 -6 -4 -9 -4 -8 -6 -5 -2 -9 -0 -2 -1 -9 -8 -2 -5 -4 -7 -4 -8 -3 -4 -5 -2 -6 -4 -1 -3 -0 -2 -1 -2 -7 -2 -2 -6 -6 -0 -1 -4 -6 -1 -0 -2 -9 -8 -6 -1 -1 -9 -5 -6 -7 -6 -7 -5 -0 -3 -9 -9 -2 -6 -6 -9 -5 -9 -3 -2 -7 -4 -1 -4 -8 -7 -8 -4 -1 -8 -5 -6 -0 -8 -9 -3 -2 -6 -4 -6 -6 -5 -1 -8 -6 -8 -3 -7 -1 -1 -3 -6 -9 -4 -6 -0 -3 -9 -2 -6 -3 -7 -4 -3 -4 -1 -7 -2 -5 -6 -9 -6 -6 -3 -9 -4 -1 -8 -2 -3 -6 -0 -7 -7 -7 -3 -1 -0 -1 -0 -6 -0 -6 -7 -5 -5 -1 -9 -2 -5 -3 -3 -3 -7 -1 -0 -9 -2 -3 -2 -9 -3 -1 -8 -0 -1 -3 -8 -5 -0 -1 -9 -8 -1 -0 -1 -2 -6 -8 -2 -7 -4 -7 -7 -0 -1 -9 -9 -6 -1 -6 -7 -1 -3 -5 -3 -9 -0 -8 -2 -9 -4 -9 -2 -6 -1 -7 -9 -9 -0 -9 -7 -1 -9 -2 -7 -4 -6 -3 -3 -9 -7 -8 -1 -1 -0 -7 -9 -2 -5 -5 -5 -6 -5 -0 -9 -5 -5 -0 -0 -1 -7 -0 -1 -6 -9 -6 -2 -6 -9 -0 -0 -5 -3 -4 -6 -4 -2 -0 -7 -1 -2 -6 -2 -2 -6 -7 -5 -0 -2 -8 -4 -0 -9 -2 -6 -5 -9 -7 -8 -9 -5 -3 -9 -1 -1 -2 -5 -9 -4 -2 -5 -0 -1 -4 -0 -9 -6 -6 -0 -7 -8 -9 -7 -5 -3 -8 -9 -9 -1 -1 -6 -3 -3 -5 -0 -1 -1 -7 -8 -5 -7 -6 -5 -5 -9 -9 -0 -9 -1 -1 -6 -0 -4 -8 -2 -6 -7 -9 -5 -9 -6 -9 -7 -1 -8 -8 -4 -7 -2 -3 -7 -9 -0 -0 -1 -9 -5 -2 -8 -9 -1 -4 -5 -7 -8 -5 -5 -6 -0 -6 -8 -9 -2 -2 -0 -8 -8 -7 -7 -5 -7 -2 -7 -1 -4 -6 -8 -4 -9 -5 -1 -1 -7 -6 -9 -8 -1 -6 -6 -4 -3 -3 -0 -5 -0 -3 -1 -6 -2 -6 -1 -6 -7 -6 -0 -6 -4 -8 -4 -6 -7 -4 -8 -8 -7 -4 -3 -1 -0 -9 -4 -0 -1 -8 -9 -6 -0 -6 -9 -9 -1 -9 -4 -2 -0 -9 -9 -1 -2 -9 -6 -5 -8 -1 -7 -5 -7 -2 -5 -8 -0 -0 -6 -5 -9 -3 -5 -4 -0 -3 -8 -3 -6 -9 -0 -0 -5 -8 -9 -6 -7 -2 -2 -8 -9 -6 -1 -6 -4 -4 -6 -6 -4 -0 -0 -9 -6 -4 -9 -9 -3 -0 -3 -1 -2 -1 -5 -1 -9 -4 -5 -3 -1 -9 -8 -2 -0 -2 -9 -2 -3 -3 -7 -6 -9 -6 -1 -3 -0 -1 -3 -7 -7 -9 -6 -9 -4 -5 -5 -6 -8 -3 -8 -3 -5 -7 -6 -3 -7 -7 -5 -4 -5 -1 -2 -8 -8 -7 -1 -5 -6 -5 -2 -5 -5 -1 -5 -6 -1 -4 -3 -0 -0 -3 -4 -2 -9 -0 -5 -2 -9 -3 -0 -6 -3 -2 -9 -6 -1 -8 -5 -7 -0 -5 -4 -1 -8 -8 -2 -9 -2 -0 -1 -4 -9 -7 -6 -3 -8 -4 -5 -0 -6 -6 -0 -5 -9 -0 -7 -4 -8 -9 -5 -9 -7 -9 -0 -1 -1 -1 -2 -0 -7 -5 -0 -6 -0 -8 -7 -1 -9 -8 -5 -2 -6 -1 -6 -9 -2 -3 -9 -8 -8 -9 -0 -2 -9 -4 -8 -5 -1 -0 -9 -0 -1 -7 -9 -4 -0 -1 -3 -9 -8 -9 -4 -5 -0 -1 -5 -8 -8 -1 -6 -9 -7 -0 -1 -7 -5 -6 -4 -6 -1 -9 -9 -9 -3 -9 -6 -9 -0 -0 -1 -5 -9 -3 -3 -2 -1 -1 -2 -4 -3 -1 -8 -5 -2 -3 -3 -5 -4 -8 -0 -5 -3 -5 -9 -3 -5 -8 -5 -1 -6 -0 -8 -9 -8 -0 -4 -1 -0 -1 -3 -1 -8 -6 -3 -1 -1 -0 -5 -8 -6 -9 -5 -1 -6 -1 -1 -6 -4 -3 -8 -6 -9 -4 -2 -8 -6 -7 -2 -7 -7 -6 -8 -2 -2 -0 -7 -0 -7 -5 -0 -9 -9 -6 -4 -5 -6 -1 -2 -0 -3 -9 -7 -6 -0 -3 -4 -1 -7 -7 -7 -1 -0 -0 -6 -0 -6 -5 -9 -7 -8 -2 -5 -1 -1 -9 -3 -8 -8 -9 -1 -2 -6 -6 -6 -9 -0 -3 -8 -6 -1 -5 -6 -6 -4 -3 -4 -9 -0 -0 -1 -0 -9 -2 -2 -1 -7 -5 -4 -1 -6 -0 -3 -8 -9 -9 -7 -8 -2 -7 -8 -5 -0 -2 -1 -8 -9 -1 -6 -9 -3 -9 -9 -1 -8 -7 -9 -1 -2 -2 -5 -6 -5 -8 -7 -3 -6 -1 -2 -6 -8 -1 -8 -6 -7 -4 -1 -1 -3 -4 -0 -9 -1 -1 -0 -1 -8 -1 -3 -9 -0 -3 -8 -5 -9 -0 -9 -8 -7 -7 -1 -0 -1 -0 -3 -4 -9 -6 -3 -0 -2 -4 -9 -7 -1 -9 -1 -9 -3 -8 -2 -0 -4 -7 -3 -2 -6 -7 -1 -5 -1 -1 -2 -4 -0 -0 -4 -8 -8 -5 -2 -5 -8 -5 -9 -0 -3 -2 -4 -8 -2 -0 -2 -7 -5 -1 -6 -8 -9 -1 -0 -7 -2 -6 -1 -1 -6 -0 -2 -9 -6 -0 -0 -8 -7 -4 -3 -6 -8 -1 -6 -7 -4 -6 -9 -3 -2 -0 -9 -8 -6 -6 -3 -0 -2 -0 -3 -0 -7 -2 -2 -1 -8 -4 -7 -9 -6 -7 -3 -1 -3 -7 -0 -3 -7 -2 -2 -2 -0 -7 -9 -1 -2 -0 -6 -6 -2 -3 -6 -7 -4 -9 -4 -5 -8 -9 -3 -7 -4 -2 -9 -9 -8 -6 -7 -0 -3 -0 -9 -1 -3 -1 -6 -5 -0 -6 -0 -1 -9 -6 -9 -0 -8 -8 -7 -4 -0 -9 -9 -6 -8 -0 -2 -4 -8 -4 -4 -1 -7 -1 -1 -9 -2 -2 -7 -6 -8 -0 -2 -0 -6 -0 -1 -9 -0 -3 -9 -9 -7 -9 -5 -0 -3 -8 -9 -8 -0 -1 -4 -6 -7 -9 -2 -7 -9 -7 -6 -5 -8 -2 -1 -4 -9 -7 -5 -8 -1 -2 -1 -1 -7 -1 -9 -6 -0 -1 -9 -9 -6 -0 -1 -8 -1 -0 -7 -4 -6 -6 -2 -9 -2 -1 -3 -5 -1 -9 -3 -1 -9 -3 -4 -1 -1 -8 -9 -9 -7 -5 -4 -1 -6 -0 -1 -2 -6 -0 -3 -0 -6 -8 -7 -2 -0 -3 -9 -7 -8 -4 -6 -4 -0 -5 -8 -9 -3 -3 -0 -1 -2 -1 -3 -8 -7 -2 -8 -2 -8 -0 -4 -6 -4 -2 -1 -6 -1 -4 -4 -1 -1 -6 -5 -7 -1 -1 -6 -1 -5 -9 -4 -3 -3 -3 -9 -0 -7 -4 -1 -3 -3 -1 -1 -6 -5 -7 -3 -9 -5 -4 -5 -6 -4 -1 -5 -9 -3 -9 -3 -8 -0 -8 -9 -6 -8 -5 -7 -9 -0 -9 -7 -2 -9 -1 -3 -5 -1 -4 -7 -3 -8 -1 -5 -7 -5 -0 -2 -7 -3 -9 -0 -1 -3 -2 -2 -2 -4 -0 -0 -4 -9 -5 -9 -1 -7 -0 -1 -4 -0 -8 -3 -0 -4 -0 -9 -6 -9 -0 -1 -0 -7 -3 -5 -6 -8 -2 -9 -1 -6 -1 -5 -1 -9 -1 -9 -5 -8 -9 -9 -6 -3 -8 -7 -5 -0 -3 -9 -3 -7 -1 -8 -7 -9 -3 -3 -9 -7 -0 -6 -3 -9 -4 -7 -6 -4 -3 -5 -5 -9 -7 -4 -4 -1 -5 -1 -0 -3 -9 -9 -1 -6 -5 -6 -4 -3 -3 -3 -3 -7 -9 -7 -5 -1 -2 -2 -0 -9 -9 -1 -4 -5 -1 -9 -4 -2 -1 -2 -8 -0 -0 -2 -8 -3 -2 -8 -8 -3 -4 -4 -1 -8 -1 -2 -0 -6 -4 -6 -9 -6 -1 -9 -0 -3 -5 -5 -5 -2 -8 -5 -8 -1 -2 -8 -9 -4 -4 -0 -9 -8 -0 -3 -4 -2 -7 -7 -0 -7 -6 -8 -9 -9 -0 -0 -9 -2 -3 -1 -9 -2 -9 -2 -0 -6 -9 -3 -7 -9 -7 -5 -2 -5 -8 -7 -1 -1 -2 -1 -4 -7 -4 -8 -7 -4 -0 -4 -0 -5 -0 -0 -2 -6 -1 -0 -8 -5 -5 -6 -2 -2 -1 -3 -5 -8 -3 -3 -6 -7 -3 -6 -4 -5 -9 -6 -2 -7 -9 -8 -2 -7 -3 -9 -7 -8 -1 -9 -8 -3 -6 -9 -2 -2 -6 -2 -8 -0 -7 -7 -8 -9 -6 -6 -9 -3 -6 -8 -0 -4 -3 -8 -2 -5 -0 -6 -1 -1 -9 -2 -7 -1 -5 -7 -3 -9 -1 -1 -3 -3 -7 -9 -2 -1 -3 -3 -8 -3 -9 -4 -2 -1 -4 -6 -7 -6 -6 -2 -6 -4 -1 -9 -7 -1 -6 -6 -9 -3 -5 -4 -3 -7 -3 -0 -8 -4 -6 -9 -0 -2 -9 -8 -1 -7 -2 -5 -3 -3 -7 -9 -3 -9 -5 -9 -9 -2 -8 -2 -6 -1 -1 -8 -2 -2 -9 -6 -9 -8 -5 -1 -8 -1 -8 -0 -6 -0 -3 -9 -3 -3 -6 -9 -3 -2 -0 -8 -3 -0 -1 -9 -6 -0 -8 -0 -1 -5 -7 -1 -3 -5 -5 -8 -0 -6 -2 -9 -6 -8 -2 -0 -7 -1 -2 -3 -1 -2 -4 -3 -8 -7 -3 -5 -0 -9 -4 -2 -2 -2 -9 -5 -9 -6 -8 -2 -2 -2 -1 -1 -2 -2 -8 -8 -5 -3 -9 -9 -0 -3 -1 -5 -3 -3 -0 -7 -8 -9 -5 -6 -0 -1 -7 -7 -6 -1 -3 -1 -2 -1 -4 -0 -9 -7 -0 -3 -7 -9 -8 -9 -6 -3 -1 -5 -8 -1 -9 -1 -0 -6 -6 -2 -5 -6 -1 -7 -1 -7 -0 -3 -3 -7 -9 -2 -0 -6 -8 -0 -1 -1 -1 -7 -7 -3 -3 -6 -4 -6 -9 -6 -0 -9 -1 -3 -3 -6 -7 -2 -1 -7 -5 -7 -6 -6 -7 -8 -1 -6 -3 -1 -0 -5 -9 -9 -9 -4 -6 -6 -1 -7 -8 -4 -4 -1 -3 -2 -9 -6 -8 -5 -9 -0 -2 -2 -7 -1 -3 -0 -6 -2 -6 -7 -9 -9 -3 -3 -9 -2 -1 -6 -5 -4 -0 -2 -4 -6 -2 -1 -6 -7 -5 -1 -9 -9 -5 -9 -4 -1 -5 -8 -1 -3 -5 -2 -7 -1 -2 -7 -1 -3 -4 -6 -9 -4 -9 -0 -9 -6 -9 -0 -8 -5 -9 -0 -3 -3 -0 -6 -8 -9 -0 -5 -9 -7 -6 -7 -2 -7 -7 -7 -1 -4 -8 -9 -6 -0 -2 -1 -5 -7 -9 -9 -0 -2 -0 -0 -2 -3 -6 -0 -2 -3 -2 -9 -1 -9 -2 -6 -4 -0 -9 -2 -4 -3 -1 -0 -2 -7 -6 -2 -5 -3 -7 -9 -3 -4 -0 -8 -9 -8 -7 -5 -1 -3 -5 -7 -9 -3 -3 -5 -7 -1 -2 -7 -5 -1 -5 -2 -5 -9 -8 -1 -9 -6 -7 -7 -2 -8 -3 -1 -7 -5 -9 -2 -6 -7 -3 -5 -4 -0 -7 -2 -4 -8 -2 -5 -1 -9 -8 -5 -1 -6 -3 -4 -9 -2 -6 -8 -4 -7 -1 -4 -4 -5 -6 -9 -4 -4 -2 -6 -3 -2 -7 -7 -2 -7 -1 -1 -7 -4 -0 -6 -2 -3 -0 -9 -9 -2 -7 -1 -6 -4 -1 -6 -7 -4 -1 -8 -7 -5 -4 -9 -2 -1 -4 -6 -6 -2 -7 -9 -5 -3 -4 -4 -8 -5 -7 -4 -5 -6 -6 -1 -3 -5 -6 -6 -2 -8 -2 -6 -8 -5 -3 -0 -0 -9 -8 -0 -1 -2 -9 -5 -3 -0 -9 -2 -9 -6 -1 -9 -7 -5 -8 -7 -0 -6 -3 -5 -3 -7 -0 -0 -9 -1 -6 -5 -1 -1 -5 -3 -4 -0 -3 -3 -1 -2 -8 -2 -7 -8 -9 -9 -7 -9 -1 -8 -8 -4 -4 -8 -4 -8 -5 -5 -7 -4 -9 -0 -3 -6 -1 -2 -4 -6 -2 -1 -1 -0 -7 -1 -5 -9 -3 -0 -5 -1 -7 -6 -1 -5 -2 -1 -1 -3 -7 -0 -8 -2 -4 -2 -5 -0 -3 -5 -9 -1 -3 -9 -5 -0 -0 -3 -6 -9 -0 -4 -5 -4 -1 -6 -2 -8 -1 -1 -4 -3 -8 -0 -7 -6 -6 -8 -5 -0 -5 -5 -6 -8 -4 -1 -2 -0 -3 -9 -6 -7 -8 -3 -9 -7 -8 -9 -6 -1 -3 -3 -4 -8 -3 -0 -1 -9 -1 -3 -0 -8 -6 -5 -4 -1 -7 -8 -3 -1 -9 -7 -8 -2 -6 -1 -0 -4 -0 -3 -5 -2 -9 -8 -1 -9 -2 -5 -2 -0 -9 -8 -3 -9 -0 -0 -6 -6 -7 -2 -7 -2 -3 -7 -9 -8 -3 -8 -8 -9 -6 -1 -8 -7 -2 -9 -6 -6 -4 -3 -1 -1 -4 -2 -1 -7 -5 -4 -9 -9 -2 -4 -5 -9 -4 -7 -1 -6 -6 -6 -2 -8 -4 -1 -5 -3 -7 -8 -3 -1 -3 -5 -0 -3 -6 -5 -7 -9 -9 -3 -0 -2 -5 -7 -8 -4 -7 -3 -7 -6 -6 -8 -3 -1 -4 -2 -2 -4 -9 -5 -1 -6 -6 -6 -7 -7 -8 -9 -0 -3 -4 -5 -0 -1 -6 -2 -1 -0 -9 -3 -8 -5 -5 -8 -8 -7 -0 -5 -7 -9 -6 -5 -6 -2 -2 -5 -8 -8 -5 -2 -9 -4 -6 -3 -8 -4 -4 -6 -6 -0 -4 -2 -5 -5 -1 -3 -9 -6 -2 -4 -5 -1 -1 -2 -7 -3 -6 -4 -2 -2 -4 -0 -1 -6 -6 -1 -7 -7 -2 -6 -5 -7 -4 -2 -8 -4 -9 -9 -2 -6 -1 -8 -1 -5 -0 -9 -3 -4 -2 -9 -9 -8 -3 -3 -4 -6 -9 -2 -6 -3 -3 -4 -4 -1 -1 -9 -5 -5 -1 -6 -1 -0 -0 -2 -7 -1 -1 -3 -0 -5 -4 -2 -6 -7 -8 -5 -9 -9 -6 -2 -2 -3 -6 -6 -2 -9 -0 -3 -6 -9 -5 -7 -1 -1 -7 -8 -2 -9 -7 -7 -9 -9 -5 -6 -9 -9 -1 -5 -5 -1 -8 -0 -7 -8 -0 -5 -9 -6 -8 -6 -2 -6 -6 -7 -0 -0 -5 -9 -1 -7 -1 -1 -3 -1 -1 -5 -5 -8 -3 -2 -3 -2 -9 -4 -6 -3 -7 -1 -5 -7 -8 -4 -8 -3 -0 -3 -9 -8 -0 -2 -8 -7 -9 -6 -3 -4 -9 -3 -6 -1 -7 -5 -9 -4 -6 -9 -2 -1 -7 -5 -7 -6 -4 -3 -6 -6 -9 -6 -5 -1 -5 -9 -6 -5 -0 -4 -1 -4 -2 -7 -8 -9 -8 -8 -0 -6 -2 -4 -9 -9 -9 -3 -3 -5 -3 -1 -0 -2 -7 -4 -1 -1 -8 -2 -9 -7 -2 -5 -0 -6 -8 -1 -5 -8 -4 -4 -8 -5 -3 -2 -1 -6 -7 -4 -4 -9 -2 -5 -8 -5 -3 -7 -7 -3 -3 -5 -7 -0 -7 -9 -9 -1 -6 -1 -9 -1 -9 -9 -7 -2 -2 -9 -7 -8 -0 -9 -9 -6 -7 -3 -5 -7 -7 -5 -9 -9 -0 -7 -8 -0 -6 -6 -5 -1 -8 -5 -9 -9 -5 -2 -8 -6 -8 -6 -7 -7 -7 -4 -4 -4 -1 -0 -6 -1 -6 -7 -1 -1 -6 -1 -7 -3 -7 -2 -6 -2 -9 -5 -0 -3 -2 -7 -6 -6 -9 -9 -1 -9 -2 -2 -6 -0 -3 -3 -6 -6 -1 -6 -1 -9 -8 -0 -0 -9 -3 -3 -9 -9 -3 -3 -5 -9 -1 -1 -8 -5 -8 -6 -2 -0 -6 -8 -0 -0 -2 -5 -5 -1 -1 -1 -8 -2 -4 -2 -1 -7 -9 -2 -9 -6 -2 -8 -2 -6 -5 -8 -8 -5 -0 -0 -0 -3 -2 -7 -7 -7 -2 -2 -1 -8 -1 -4 -4 -2 -7 -6 -8 -9 -7 -2 -7 -7 -5 -1 -8 -5 -5 -9 -3 -9 -7 -6 -0 -4 -8 -3 -6 -2 -4 -8 -3 -4 -5 -0 -9 -9 -0 -3 -5 -1 -1 -1 -9 -4 -8 -0 -8 -9 -2 -5 -2 -5 -1 -7 -0 -0 -2 -8 -1 -0 -4 -3 -9 -2 -5 -0 -7 -3 -6 -1 -7 -5 -3 -3 -2 -1 -6 -2 -4 -5 -6 -0 -6 -2 -1 -4 -7 -7 -0 -2 -1 -1 -5 -3 -7 -4 -0 -2 -4 -0 -2 -4 -2 -9 -3 -5 -5 -7 -6 -2 -6 -0 -5 -7 -9 -6 -4 -3 -2 -8 -2 -9 -5 -2 -5 -6 -0 -3 -0 -5 -5 -1 -5 -1 -9 -4 -5 -8 -1 -1 -5 -8 -4 -1 -7 -0 -4 -4 -1 -8 -7 -7 -4 -9 -3 -6 -3 -6 -2 -2 -8 -3 -1 -7 -3 -1 -4 -5 -4 -0 -8 -2 -8 -8 -3 -2 -8 -5 -4 -6 -3 -3 -3 -7 -1 -9 -6 -6 -7 -5 -9 -0 -2 -1 -4 -2 -4 -5 -1 -1 -6 -9 -7 -6 -7 -6 -9 -8 -4 -7 -9 -1 -4 -6 -2 -4 -3 -8 -8 -9 -7 -6 -3 -9 -1 -6 -3 -9 -0 -9 -6 -4 -2 -5 -6 -3 -5 -2 -0 -5 -7 -3 -9 -5 -5 -0 -0 -9 -3 -1 -9 -1 -1 -1 -4 -8 -6 -4 -4 -3 -0 -8 -9 -1 -4 -2 -4 -0 -1 -0 -3 -9 -0 -8 -8 -7 -0 -8 -8 -9 -6 -9 -5 -6 -6 -5 -9 -9 -6 -4 -2 -1 -5 -5 -2 -7 -0 -1 -0 -0 -9 -3 -2 -1 -6 -6 -7 -7 -5 -5 -2 -8 -3 -3 -2 -5 -8 -3 -3 -9 -9 -6 -4 -6 -1 -0 -9 -3 -6 -7 -5 -3 -2 -7 -7 -5 -6 -2 -3 -3 -6 -4 -6 -1 -2 -1 -7 -2 -2 -6 -6 -4 -4 -8 -6 -9 -5 -8 -5 -2 -8 -9 -8 -0 -9 -6 -9 -3 -2 -4 -0 -6 -6 -2 -4 -7 -8 -1 -3 -8 -1 -2 -7 -1 -3 -7 -9 -9 -4 -3 -4 -3 -7 -1 -2 -6 -8 -5 -1 -2 -1 -1 -0 -5 -6 -5 -2 -6 -2 -2 -3 -6 -9 -7 -4 -9 -0 -1 -1 -1 -1 -3 -5 -5 -1 -2 -3 -1 -2 -6 -1 -9 -0 -8 -0 -2 -1 -4 -4 -5 -7 -0 -8 -7 -8 -1 -3 -7 -2 -2 -2 -4 -2 -9 -3 -9 -2 -8 -3 -2 -5 -9 -4 -3 -7 -1 -0 -6 -9 -0 -9 -2 -0 -0 -5 -0 -5 -6 -0 -2 -7 -5 -0 -7 -4 -9 -9 -3 -6 -2 -9 -1 -3 -2 -3 -0 -0 -6 -5 -1 -7 -0 -6 -2 -0 -9 -9 -1 -5 -3 -5 -8 -9 -6 -7 -1 -5 -0 -1 -0 -0 -6 -4 -7 -7 -4 -9 -3 -8 -8 -9 -6 -7 -0 -1 -1 -3 -8 -7 -7 -4 -2 -1 -5 -7 -1 -7 -8 -2 -8 -0 -8 -3 -1 -0 -1 -3 -3 -6 -6 -6 -6 -1 -0 -8 -3 -7 -0 -5 -9 -6 -0 -2 -2 -5 -6 -1 -7 -5 -1 -7 -8 -1 -0 -4 -9 -3 -0 -2 -1 -3 -8 -3 -5 -7 -2 -8 -2 -1 -0 -2 -4 -5 -6 -7 -3 -8 -5 -0 -3 -5 -8 -1 -9 -4 -5 -8 -3 -9 -6 -1 -1 -6 -6 -4 -4 -5 -0 -2 -4 -2 -5 -9 -6 -8 -8 -5 -9 -0 -3 -0 -9 -2 -6 -3 -9 -3 -9 -6 -8 -9 -5 -3 -7 -2 -7 -1 -7 -4 -6 -7 -4 -6 -1 -5 -7 -9 -5 -6 -1 -2 -9 -3 -1 -8 -5 -1 -4 -1 -1 -8 -5 -8 -2 -2 -5 -2 -7 -5 -3 -6 -7 -8 -9 -6 -6 -1 -0 -9 -0 -0 -9 -3 -4 -7 -5 -5 -6 -9 -9 -2 -3 -9 -7 -3 -1 -6 -0 -0 -0 -2 -8 -6 -3 -6 -6 -6 -0 -2 -7 -3 -8 -1 -9 -6 -7 -9 -6 -7 -0 -1 -6 -4 -8 -4 -8 -5 -3 -5 -5 -8 -2 -9 -3 -2 -4 -7 -4 -2 -6 -3 -7 -6 -2 -0 -3 -4 -5 -6 -8 -9 -1 -4 -5 -1 -9 -1 -5 -2 -6 -7 -7 -4 -6 -5 -4 -6 -0 -0 -7 -0 -5 -8 -3 -6 -3 -8 -8 -3 -6 -1 -7 -9 -9 -1 -8 -8 -1 -4 -1 -9 -5 -9 -1 -9 -8 -2 -6 -4 -8 -5 -1 -7 -9 -8 -1 -6 -9 -5 -6 -1 -1 -2 -1 -0 -7 -0 -7 -1 -4 -4 -2 -6 -3 -1 -9 -3 -6 -5 -5 -5 -7 -6 -5 -0 -7 -8 -3 -5 -4 -3 -4 -3 -7 -9 -6 -2 -6 -0 -2 -5 -1 -1 -1 -2 -5 -3 -9 -9 -8 -9 -6 -9 -7 -5 -5 -2 -0 -1 -7 -4 -7 -9 -3 -2 -3 -5 -1 -9 -5 -6 -9 -6 -5 -1 -4 -8 -2 -8 -9 -6 -6 -8 -3 -3 -4 -2 -2 -1 -0 -1 -8 -0 -7 -1 -9 -3 -7 -1 -0 -4 -3 -6 -7 -8 -3 -2 -7 -7 -6 -5 -4 -7 -6 -9 -8 -6 -1 -1 -2 -4 -6 -1 -3 -1 -8 -1 -9 -4 -3 -9 -3 -5 -4 -3 -7 -9 -9 -4 -0 -8 -2 -5 -8 -9 -0 -9 -1 -9 -4 -3 -6 -8 -9 -3 -9 -2 -7 -4 -0 -8 -9 -3 -9 -7 -5 -0 -2 -5 -2 -2 -8 -2 -0 -5 -4 -7 -3 -2 -4 -3 -0 -1 -2 -5 -1 -3 -1 -2 -4 -7 -3 -6 -2 -4 -9 -3 -7 -3 -6 -1 -5 -2 -5 -1 -5 -6 -8 -2 -2 -2 -1 -5 -6 -3 -9 -1 -7 -8 -0 -6 -7 -4 -6 -0 -3 -8 -6 -1 -6 -4 -7 -6 -7 -3 -4 -1 -7 -4 -4 -6 -6 -9 -5 -7 -9 -3 -8 -2 -5 -4 -5 -4 -0 -0 -9 -7 -7 -5 -7 -4 -5 -9 -7 -9 -9 -1 -8 -9 -9 -5 -7 -1 -2 -0 -1 -9 -6 -5 -9 -0 -4 -8 -0 -5 -9 -3 -3 -7 -7 -0 -7 -0 -5 -4 -2 -5 -7 -0 -3 -4 -4 -1 -0 -0 -9 -9 -6 -4 -4 -1 -1 -0 -0 -9 -9 -7 -4 -7 -2 -1 -8 -4 -4 -4 -5 -6 -6 -2 -1 -1 -4 -2 -1 -8 -7 -1 -7 -8 -3 -3 -3 -0 -0 -5 -1 -2 -9 -9 -0 -0 -8 -7 -6 -6 -1 -9 -7 -2 -9 -7 -8 -1 -9 -1 -5 -2 -8 -8 -9 -9 -3 -8 -5 -5 -3 -6 -8 -3 -2 -6 -1 -4 -9 -6 -4 -8 -4 -3 -1 -8 -1 -3 -4 -5 -6 -5 -1 -9 -2 -6 -5 -1 -4 -7 -3 -0 -0 -6 -1 -0 -7 -7 -5 -8 -7 -4 -7 -9 -0 -0 -8 -1 -3 -9 -2 -1 -4 -7 -7 -1 -1 -8 -2 -9 -0 -0 -3 -1 -7 -6 -6 -3 -6 -1 -4 -8 -9 -4 -9 -3 -6 -6 -5 -8 -8 -8 -8 -9 -3 -8 -3 -7 -0 -0 -4 -0 -7 -7 -4 -8 -9 -2 -8 -3 -3 -1 -5 -5 -1 -1 -5 -1 -9 -3 -3 -1 -4 -7 -1 -4 -7 -3 -5 -1 -6 -9 -5 -3 -8 -2 -8 -8 -0 -5 -6 -0 -6 -0 -0 -3 -6 -2 -2 -6 -8 -1 -7 -1 -5 -3 -4 -6 -2 -7 -6 -8 -1 -2 -4 -1 -2 -3 -5 -6 -7 -8 -9 -4 -4 -6 -2 -7 -8 -5 -1 -7 -1 -6 -2 -1 -1 -2 -2 -8 -4 -0 -7 -9 -1 -7 -6 -7 -1 -2 -2 -3 -5 -2 -4 -1 -9 -4 -9 -8 -0 -4 -3 -9 -3 -9 -1 -4 -7 -5 -8 -6 -9 -9 -9 -9 -4 -1 -4 -3 -3 -5 -6 -6 -0 -2 -9 -2 -0 -4 -2 -6 -9 -9 -6 -3 -6 -2 -1 -5 -6 -8 -0 -2 -8 -4 -5 -6 -7 -5 -2 -6 -8 -0 -5 -1 -8 -9 -7 -3 -0 -6 -8 -3 -4 -5 -3 -2 -4 -6 -9 -4 -8 -2 -8 -9 -9 -0 -6 -0 -2 -1 -1 -4 -1 -7 -2 -3 -2 -1 -9 -6 -0 -3 -8 -3 -3 -9 -2 -0 -7 -7 -2 -0 -8 -6 -1 -1 -1 -6 -1 -2 -4 -1 -8 -0 -9 -8 -3 -5 -0 -2 -1 -8 -8 -1 -9 -7 -8 -9 -7 -9 -9 -4 -3 -1 -3 -7 -0 -8 -9 -1 -7 -0 -4 -0 -5 -0 -0 -5 -7 -6 -3 -2 -9 -8 -2 -9 -7 -2 -8 -1 -9 -1 -4 -9 -0 -7 -6 -8 -2 -1 -9 -1 -9 -0 -5 -5 -3 -6 -2 -6 -5 -0 -6 -4 -0 -1 -6 -1 -2 -6 -4 -6 -4 -9 -6 -9 -5 -9 -8 -5 -6 -0 -0 -1 -4 -9 -1 -3 -4 -1 -6 -4 -5 -1 -9 -8 -4 -6 -4 -1 -8 -2 -3 -7 -2 -8 -3 -3 -1 -6 -3 -1 -4 -2 -2 -1 -6 -7 -9 -2 -0 -0 -7 -4 -5 -4 -4 -5 -8 -6 -1 -8 -1 -8 -4 -6 -4 -1 -2 -9 -9 -6 -6 -9 -7 -6 -4 -6 -3 -9 -5 -0 -8 -6 -2 -1 -7 -6 -4 -8 -3 -4 -1 -4 -6 -1 -9 -1 -8 -0 -6 -2 -1 -2 -0 -9 -8 -6 -2 -4 -4 -7 -7 -8 -8 -3 -7 -6 -3 -5 -9 -5 -2 -7 -4 -6 -2 -3 -3 -0 -1 -3 -8 -8 -9 -0 -0 -6 -8 -3 -8 -9 -2 -1 -6 -3 -6 -9 -3 -1 -9 -7 -4 -8 -7 -9 -4 -8 -0 -3 -8 -7 -1 -3 -0 -5 -4 -4 -2 -4 -9 -5 -4 -1 -9 -3 -9 -7 -6 -6 -1 -5 -2 -9 -7 -7 -4 -9 -5 -7 -8 -5 -7 -6 -7 -4 -3 -0 -4 -7 -5 -8 -5 -1 -0 -0 -4 -7 -9 -0 -7 -2 -8 -8 -0 -1 -9 -5 -9 -5 -5 -4 -7 -4 -9 -8 -9 -1 -3 -6 -4 -0 -5 -7 -7 -2 -0 -2 -4 -7 -2 -2 -1 -4 -4 -6 -8 -6 -5 -0 -3 -8 -4 -6 -7 -6 -2 -1 -8 -4 -5 -2 -1 -0 -0 -7 -4 -0 -9 -9 -1 -7 -2 -1 -5 -5 -1 -8 -4 -5 -5 -7 -6 -8 -9 -0 -6 -6 -1 -7 -5 -4 -3 -8 -6 -9 -2 -1 -1 -4 -9 -5 -8 -1 -0 -6 -0 -0 -4 -2 -3 -6 -7 -4 -1 -8 -4 -1 -2 -0 -8 -6 -1 -5 -2 -2 -5 -7 -4 -6 -9 -6 -1 -9 -0 -8 -9 -4 -4 -7 -1 -3 -1 -4 -8 -9 -5 -9 -5 -6 -4 -9 -4 -7 -0 -1 -6 -0 -1 -8 -0 -6 -6 -1 -7 -3 -7 -9 -0 -9 -6 -9 -7 -3 -0 -2 -1 -7 -0 -0 -2 -1 -1 -6 -6 -7 -0 -2 -1 -8 -5 -5 -4 -3 -6 -1 -4 -0 -2 -4 -0 -1 -5 -4 -1 -0 -8 -2 -7 -4 -3 -3 -3 -1 -9 -2 -0 -6 -0 -6 -7 -9 -3 -7 -9 -9 -5 -7 -1 -3 -6 -9 -1 -7 -2 -7 -3 -9 -3 -7 -1 -4 -9 -3 -1 -5 -3 -1 -4 -1 -0 -9 -7 -3 -2 -3 -7 -7 -8 -7 -7 -6 -4 -7 -2 -6 -5 -6 -4 -0 -4 -8 -1 -1 -3 -9 -0 -1 -2 -2 -5 -8 -3 -4 -8 -7 -4 -2 -3 -1 -3 -8 -4 -9 -4 -0 -9 -6 -9 -1 -0 -4 -9 -1 -0 -2 -1 -7 -3 -1 -6 -4 -7 -8 -7 -8 -5 -2 -4 -7 -7 -2 -2 -8 -4 -1 -7 -5 -3 -2 -1 -6 -5 -1 -9 -8 -1 -2 -6 -1 -8 -2 -5 -9 -2 -2 -4 -0 -2 -0 -2 -9 -9 -4 -4 -7 -7 -5 -8 -9 -1 -5 -0 -6 -8 -1 -0 -0 -9 -4 -9 -2 -5 -6 -2 -9 -1 -9 -1 -8 -3 -0 -6 -6 -9 -2 -1 -4 -5 -1 -2 -2 -1 -3 -0 -5 -0 -7 -9 -5 -8 -7 -2 -2 -6 -6 -8 -8 -0 -0 -4 -9 -5 -1 -5 -3 -2 -9 -2 -2 -9 -6 -8 -2 -0 -0 -9 -2 -3 -1 -5 -0 -8 -3 -2 -8 -2 -4 -6 -3 -2 -6 -5 -0 -7 -5 -8 -0 -2 -9 -4 -5 -6 -0 -9 -9 -9 -4 -7 -9 -5 -9 -2 -1 -6 -1 -5 -9 -4 -1 -1 -5 -1 -3 -4 -4 -0 -9 -0 -4 -9 -3 -2 -3 -0 -4 -7 -1 -1 -5 -6 -1 -7 -0 -3 -6 -1 -9 -0 -3 -5 -3 -1 -2 -0 -5 -9 -1 -9 -7 -0 -6 -7 -1 -8 -8 -7 -2 -1 -5 -6 -0 -3 -6 -0 -1 -3 -6 -4 -9 -7 -0 -4 -4 -1 -7 -0 -3 -8 -6 -8 -1 -4 -0 -7 -2 -8 -8 -8 -8 -1 -9 -9 -0 -6 -0 -5 -5 -7 -3 -8 -3 -7 -1 -6 -2 -9 -6 -5 -0 -4 -7 -7 -8 -9 -0 -6 -3 -5 -2 -6 -0 -0 -9 -7 -9 -2 -4 -5 -0 -6 -4 -1 -2 -9 -8 -7 -7 -8 -8 -9 -1 -9 -7 -6 -7 -3 -3 -2 -4 -3 -9 -8 -6 -2 -5 -7 -9 -9 -5 -4 -1 -2 -0 -1 -4 -8 -2 -2 -5 -7 -1 -8 -6 -7 -4 -0 -3 -2 -1 -4 -3 -5 -9 -0 -7 -1 -1 -3 -9 -8 -9 -1 -7 -9 -3 -3 -0 -6 -6 -6 -2 -2 -9 -5 -2 -4 -6 -5 -3 -9 -2 -5 -1 -7 -5 -6 -8 -5 -9 -6 -8 -1 -0 -1 -8 -2 -8 -9 -2 -9 -6 -0 -5 -3 -6 -3 -9 -2 -0 -9 -9 -0 -3 -1 -4 -7 -8 -2 -9 -5 -5 -2 -6 -0 -9 -7 -6 -2 -1 -3 -4 -4 -7 -7 -8 -9 -6 -5 -8 -9 -9 -9 -4 -0 -3 -2 -6 -3 -5 -4 -9 -5 -4 -4 -1 -4 -2 -0 -6 -8 -5 -9 -7 -4 -7 -9 -5 -3 -3 -0 -2 -3 -4 -4 -2 -6 -8 -0 -5 -9 -1 -7 -5 -3 -1 -6 -1 -6 -1 -6 -4 -3 -9 -4 -6 -3 -1 -1 -6 -9 -3 -7 -2 -3 -5 -5 -5 -6 -0 -6 -1 -7 -5 -8 -0 -8 -3 -1 -3 -9 -2 -7 -0 -8 -0 -0 -3 -0 -6 -5 -1 -5 -6 -2 -9 -7 -5 -6 -4 -3 -6 -2 -1 -9 -3 -5 -9 -4 -3 -6 -8 -1 -1 -5 -8 -3 -4 -2 -2 -1 -1 -4 -2 -8 -1 -8 -9 -0 -8 -1 -8 -1 -9 -9 -0 -0 -4 -7 -1 -0 -8 -1 -3 -9 -9 -7 -1 -2 -9 -5 -6 -1 -3 -8 -6 -1 -9 -3 -8 -2 -5 -7 -1 -9 -0 -0 -7 -9 -5 -9 -5 -6 -3 -6 -8 -4 -4 -9 -5 -8 -8 -5 -1 -9 -6 -7 -0 -4 -9 -5 -2 -9 -2 -1 -0 -9 -7 -7 -7 -6 -0 -1 -9 -6 -0 -1 -9 -1 -2 -9 -3 -1 -0 -3 -6 -5 -7 -6 -6 -8 -1 -8 -1 -7 -3 -3 -6 -1 -5 -5 -0 -1 -2 -9 -1 -4 -1 -9 -6 -1 -7 -5 -7 -6 -3 -2 -4 -8 -7 -8 -0 -2 -1 -6 -5 -4 -0 -0 -4 -1 -1 -1 -1 -4 -8 -7 -9 -2 -9 -1 -6 -4 -1 -2 -2 -1 -1 -7 -3 -3 -4 -7 -9 -7 -0 -0 -4 -1 -1 -3 -6 -5 -0 -1 -4 -6 -6 -7 -8 -9 -7 -5 -2 -6 -1 -1 -3 -6 -8 -0 -2 -5 -7 -8 -5 -0 -9 -4 -4 -3 -1 -2 -9 -1 -3 -7 -6 -9 -1 -9 -3 -7 -1 -7 -1 -9 -3 -6 -0 -1 -5 -6 -6 -2 -3 -9 -7 -6 -6 -9 -7 -6 -6 -2 -4 -1 -8 -1 -2 -3 -7 -9 -5 -9 -4 -1 -0 -3 -1 -2 -7 -5 -0 -2 -4 -1 -8 -5 -4 -0 -0 -0 -2 -7 -0 -8 -8 -4 -6 -3 -4 -0 -6 -9 -1 -7 -8 -3 -1 -6 -3 -5 -9 -9 -1 -6 -1 -1 -4 -0 -9 -2 -4 -7 -9 -2 -6 -3 -0 -6 -6 -1 -6 -9 -9 -3 -7 -5 -5 -7 -5 -0 -9 -4 -1 -8 -8 -1 -1 -4 -7 -8 -6 -1 -2 -9 -8 -1 -5 -3 -2 -7 -5 -1 -0 -1 -0 -3 -3 -1 -5 -8 -8 -9 -3 -1 -3 -8 -6 -5 -1 -6 -9 -2 -8 -7 -9 -0 -8 -5 -4 -9 -5 -5 -8 -9 -8 -9 -9 -9 -1 -0 -9 -6 -7 -0 -6 -1 -3 -0 -6 -8 -2 -0 -1 -5 -6 -9 -8 -2 -5 -3 -3 -8 -1 -2 -3 -2 -0 -5 -2 -5 -1 -1 -9 -3 -9 -1 -4 -7 -9 -4 -7 -6 -1 -9 -7 -7 -1 -5 -9 -2 -8 -6 -7 -7 -1 -5 -4 -4 -0 -9 -7 -1 -6 -3 -0 -2 -7 -2 -7 -1 -3 -2 -4 -2 -1 -6 -8 -5 -1 -0 -5 -7 -1 -7 -3 -9 -8 -0 -9 -2 -9 -9 -6 -6 -3 -2 -6 -9 -5 -5 -4 -1 -1 -4 -6 -0 -1 -5 -8 -0 -0 -3 -9 -8 -1 -0 -9 -2 -6 -2 -0 -6 -8 -7 -0 -7 -5 -7 -5 -1 -1 -5 -7 -7 -7 -3 -4 -1 -0 -6 -9 -9 -6 -9 -1 -7 -4 -5 -8 -9 -9 -1 -1 -9 -9 -1 -7 -7 -6 -5 -5 -2 -7 -1 -3 -6 -8 -5 -2 -7 -1 -4 -0 -8 -5 -8 -5 -1 -2 -4 -5 -5 -2 -1 -3 -5 -2 -2 -9 -4 -9 -9 -5 -9 -1 -3 -2 -9 -6 -2 -2 -0 -7 -5 -7 -3 -4 -9 -4 -1 -2 -9 -2 -2 -8 -1 -0 -0 -9 -0 -6 -2 -4 -1 -0 -9 -7 -1 -1 -6 -9 -0 -8 -2 -9 -5 -0 -9 -3 -0 -7 -2 -7 -6 -0 -1 -1 -1 -7 -6 -1 -7 -7 -3 -8 -5 -0 -9 -1 -0 -8 -3 -9 -9 -5 -2 -6 -5 -0 -5 -2 -7 -3 -3 -4 -4 -7 -2 -1 -9 -3 -4 -5 -2 -7 -4 -5 -5 -7 -4 -1 -8 -4 -3 -7 -1 -2 -3 -0 -9 -1 -4 -3 -9 -9 -1 -7 -5 -6 -8 -4 -1 -5 -5 -2 -2 -2 -8 -4 -0 -7 -6 -4 -3 -2 -3 -1 -7 -6 -7 -6 -8 -9 -0 -2 -3 -6 -9 -6 -9 -4 -5 -9 -1 -9 -7 -1 -5 -9 -7 -2 -7 -6 -7 -8 -1 -6 -1 -9 -9 -7 -2 -1 -1 -0 -5 -5 -5 -9 -1 -4 -9 -2 -1 -7 -6 -6 -2 -4 -2 -3 -1 -5 -2 -9 -0 -2 -7 -3 -7 -8 -2 -0 -7 -2 -7 -2 -9 -1 -2 -0 -2 -9 -3 -0 -6 -2 -8 -0 -3 -7 -6 -1 -9 -0 -3 -1 -4 -5 -7 -9 -3 -1 -9 -5 -0 -4 -7 -2 -2 -1 -9 -5 -7 -9 -6 -2 -6 -7 -7 -8 -0 -5 -2 -7 -0 -2 -0 -2 -0 -2 -4 -2 -5 -9 -7 -5 -2 -7 -5 -1 -4 -2 -5 -6 -1 -9 -8 -8 -9 -4 -1 -5 -0 -8 -6 -6 -6 -5 -0 -0 -6 -9 -5 -4 -4 -8 -2 -2 -2 -1 -7 -7 -8 -0 -0 -1 -6 -5 -4 -2 -0 -2 -5 -7 -8 -5 -2 -4 -4 -9 -4 -4 -6 -6 -9 -0 -3 -9 -1 -3 -2 -4 -0 -0 -1 -1 -7 -6 -2 -8 -1 -2 -7 -5 -1 -8 -0 -1 -5 -7 -8 -5 -9 -1 -2 -7 -7 -0 -0 -1 -7 -3 -0 -3 -1 -4 -1 -3 -8 -3 -7 -3 -9 -1 -7 -7 -1 -2 -6 -5 -8 -0 -3 -2 -9 -6 -5 -4 -6 -7 -0 -3 -4 -6 -3 -5 -2 -7 -7 -3 -5 -4 -9 -0 -3 -1 -5 -3 -1 -2 -0 -2 -4 -8 -9 -4 -2 -9 -7 -9 -9 -5 -3 -4 -2 -6 -6 -5 -1 -9 -9 -7 -1 -7 -7 -4 -4 -8 -8 -7 -6 -1 -4 -4 -9 -0 -9 -6 -7 -0 -9 -9 -1 -9 -6 -7 -5 -7 -6 -0 -5 -9 -9 -9 -9 -2 -4 -5 -7 -0 -8 -6 -1 -8 -4 -6 -2 -7 -2 -7 -9 -9 -1 -4 -7 -1 -7 -6 -6 -1 -9 -0 -1 -3 -0 -1 -3 -2 -6 -5 -3 -9 -3 -9 -9 -3 -3 -0 -7 -5 -9 -6 -6 -4 -6 -5 -7 -3 -6 -1 -6 -8 -6 -7 -4 -1 -6 -0 -4 -2 -2 -5 -8 -4 -1 -9 -2 -9 -9 -7 -9 -1 -5 -7 -1 -8 -8 -6 -6 -7 -6 -0 -7 -6 -3 -1 -0 -7 -9 -5 -7 -8 -1 -0 -9 -9 -9 -0 -0 -7 -7 -4 -9 -7 -1 -5 -7 -1 -3 -3 -0 -3 -9 -1 -7 -0 -2 -0 -1 -5 -0 -3 -7 -9 -2 -4 -4 -9 -8 -8 -0 -4 -3 -6 -1 -9 -4 -8 -9 -6 -8 -9 -4 -7 -1 -5 -9 -2 -3 -8 -6 -5 -4 -5 -0 -1 -9 -0 -5 -8 -1 -1 -5 -4 -8 -2 -5 -1 -5 -1 -1 -8 -8 -0 -8 -9 -8 -9 -3 -2 -0 -8 -9 -9 -0 -6 -0 -1 -9 -9 -9 -4 -1 -4 -9 -1 -6 -5 -1 -8 -3 -1 -7 -9 -6 -3 -9 -5 -7 -7 -5 -4 -8 -4 -5 -2 -4 -1 -9 -3 -1 -1 -3 -7 -9 -0 -4 -9 -4 -9 -0 -0 -3 -1 -4 -2 -0 -4 -1 -0 -9 -1 -9 -4 -9 -1 -4 -7 -8 -8 -1 -5 -9 -3 -0 -5 -9 -5 -9 -5 -0 -9 -5 -1 -9 -8 -5 -9 -2 -1 -9 -9 -6 -3 -8 -1 -7 -9 -7 -6 -3 -3 -1 -3 -1 -0 -1 -8 -1 -1 -9 -1 -5 -0 -6 -9 -0 -0 -6 -9 -0 -1 -7 -2 -7 -6 -7 -7 -7 -7 -2 -3 -8 -1 -0 -9 -0 -2 -0 -7 -3 -7 -4 -5 -7 -6 -1 -1 -6 -9 -6 -7 -2 -6 -1 -0 -8 -2 -9 -8 -4 -5 -2 -6 -2 -1 -5 -3 -1 -1 -6 -3 -8 -2 -6 -7 -6 -0 -9 -5 -2 -9 -6 -0 -5 -8 -0 -1 -0 -8 -7 -2 -9 -6 -9 -0 -2 -4 -1 -2 -0 -1 -4 -7 -6 -6 -9 -1 -9 -0 -7 -1 -1 -8 -8 -5 -2 -9 -1 -2 -0 -0 -7 -5 -9 -0 -5 -2 -6 -7 -1 -3 -2 -1 -9 -9 -6 -0 -9 -1 -8 -8 -8 -1 -8 -8 -3 -4 -1 -5 -6 -2 -1 -8 -6 -9 -6 -3 -6 -8 -1 -5 -0 -5 -0 -0 -9 -8 -1 -9 -0 -1 -8 -9 -2 -5 -9 -0 -1 -4 -2 -3 -4 -5 -0 -0 -3 -0 -6 -6 -1 -9 -1 -4 -9 -8 -2 -0 -6 -1 -5 -6 -4 -9 -4 -1 -0 -4 -1 -7 -0 -7 -3 -6 -9 -9 -6 -2 -6 -8 -5 -0 -0 -3 -1 -0 -0 -2 -5 -5 -1 -8 -1 -9 -1 -6 -9 -9 -9 -3 -2 -0 -9 -3 -1 -6 -2 -3 -6 -1 -1 -6 -7 -9 -3 -9 -5 -5 -8 -6 -3 -9 -4 -0 -3 -7 -8 -5 -3 -1 -5 -0 -7 -9 -0 -4 -0 -0 -4 -8 -5 -1 -9 -7 -0 -0 -7 -4 -5 -8 -6 -9 -6 -7 -7 -1 -6 -8 -3 -9 -7 -6 -3 -0 -1 -9 -6 -9 -3 -9 -2 -3 -0 -1 -2 -8 -2 -1 -3 -9 -0 -3 -8 -9 -7 -7 -9 -2 -9 -7 -5 -2 -4 -5 -5 -7 -4 -5 -9 -4 -7 -5 -6 -0 -3 -6 -5 -0 -4 -8 -3 -9 -2 -7 -0 -7 -4 -2 -9 -3 -2 -0 -6 -2 -4 -0 -1 -4 -0 -6 -7 -0 -0 -5 -8 -2 -1 -3 -7 -1 -2 -6 -3 -9 -7 -0 -3 -9 -5 -4 -4 -9 -2 -8 -0 -0 -9 -7 -3 -3 -5 -9 -2 -7 -2 -9 -5 -1 -1 -3 -6 -1 -2 -2 -5 -4 -2 -1 -3 -7 -3 -7 -5 -6 -9 -7 -4 -9 -0 -8 -8 -6 -7 -1 -5 -7 -9 -9 -0 -9 -3 -6 -1 -7 -2 -4 -3 -3 -9 -7 -3 -6 -0 -0 -1 -3 -0 -5 -2 -8 -9 -7 -6 -2 -5 -8 -0 -9 -7 -6 -7 -6 -2 -3 -3 -7 -7 -3 -2 -0 -3 -4 -6 -8 -2 -9 -1 -1 -6 -9 -1 -1 -8 -1 -8 -3 -0 -6 -3 -2 -4 -3 -9 -2 -0 -9 -5 -9 -6 -2 -7 -1 -8 -0 -3 -1 -9 -1 -0 -1 -9 -5 -9 -2 -2 -2 -1 -5 -9 -2 -2 -6 -8 -3 -9 -1 -9 -1 -8 -3 -2 -9 -7 -9 -1 -2 -2 -6 -1 -1 -9 -1 -5 -2 -7 -3 -0 -5 -6 -8 -3 -6 -2 -7 -9 -0 -8 -1 -7 -4 -6 -9 -8 -3 -7 -5 -2 -7 -6 -6 -7 -7 -6 -6 -2 -0 -8 -9 -6 -4 -7 -3 -5 -6 -8 -8 -8 -0 -6 -6 -0 -0 -0 -1 -0 -6 -3 -4 -2 -0 -6 -2 -0 -9 -8 -1 -6 -2 -5 -1 -9 -8 -8 -9 -0 -8 -1 -6 -4 -8 -5 -1 -5 -0 -6 -1 -0 -2 -6 -7 -5 -5 -4 -2 -7 -5 -5 -2 -9 -3 -1 -9 -3 -6 -1 -7 -1 -3 -4 -6 -7 -9 -7 -1 -8 -0 -1 -8 -5 -2 -1 -1 -9 -4 -9 -7 -4 -1 -5 -3 -0 -1 -0 -6 -7 -3 -7 -5 -0 -7 -7 -7 -7 -1 -4 -9 -6 -9 -9 -5 -7 -1 -5 -8 -9 -4 -1 -6 -7 -3 -4 -4 -9 -5 -6 -7 -4 -7 -1 -9 -0 -4 -5 -6 -3 -5 -3 -7 -1 -5 -7 -9 -7 -0 -3 -1 -7 -2 -7 -0 -1 -5 -2 -6 -7 -5 -7 -9 -6 -7 -2 -8 -2 -5 -6 -6 -5 -9 -7 -8 -2 -4 -9 -6 -4 -4 -9 -6 -7 -1 -8 -6 -0 -3 -2 -7 -1 -9 -5 -2 -3 -5 -5 -7 -9 -7 -9 -2 -7 -8 -5 -2 -7 -5 -0 -1 -1 -8 -0 -8 -5 -6 -3 -0 -8 -4 -9 -6 -4 -1 -0 -9 -3 -8 -4 -2 -8 -1 -1 -6 -5 -1 -2 -7 -8 -6 -2 -8 -7 -8 -9 -7 -7 -0 -2 -3 -0 -8 -9 -8 -4 -9 -2 -9 -7 -7 -9 -1 -3 -3 -3 -8 -0 -6 -2 -7 -7 -2 -5 -2 -9 -8 -0 -3 -3 -4 -4 -3 -2 -5 -2 -8 -9 -7 -9 -6 -1 -0 -4 -5 -5 -2 -7 -4 -9 -4 -6 -9 -7 -9 -2 -1 -8 -8 -3 -5 -2 -9 -1 -2 -7 -1 -4 -4 -6 -5 -7 -2 -6 -4 -3 -1 -6 -9 -0 -4 -8 -1 -1 -0 -0 -9 -6 -1 -2 -4 -0 -7 -3 -2 -1 -0 -3 -2 -3 -9 -7 -4 -0 -7 -5 -5 -4 -1 -4 -1 -9 -0 -0 -3 -1 -0 -6 -1 -1 -7 -7 -6 -5 -0 -5 -5 -9 -2 -2 -5 -9 -8 -5 -2 -9 -8 -6 -7 -1 -0 -9 -3 -5 -1 -9 -1 -3 -1 -8 -2 -5 -3 -4 -7 -3 -9 -6 -7 -0 -1 -6 -8 -3 -3 -9 -6 -3 -1 -3 -7 -1 -7 -1 -1 -9 -4 -1 -3 -6 -1 -5 -1 -5 -2 -7 -2 -6 -1 -8 -9 -4 -4 -1 -6 -6 -9 -4 -0 -2 -3 -3 -1 -7 -6 -0 -7 -8 -7 -1 -2 -3 -2 -6 -7 -2 -9 -8 -7 -6 -1 -6 -0 -3 -6 -4 -4 -9 -7 -7 -1 -5 -4 -9 -2 -6 -9 -1 -1 -9 -7 -6 -7 -2 -9 -2 -7 -2 -7 -8 -8 -6 -3 -6 -7 -6 -2 -1 -9 -8 -1 -2 -0 -6 -5 -9 -5 -9 -5 -1 -5 -1 -7 -3 -8 -4 -8 -5 -2 -9 -7 -7 -3 -1 -1 -8 -9 -1 -8 -3 -1 -9 -0 -0 -7 -4 -5 -3 -7 -9 -2 -1 -2 -9 -3 -4 -4 -7 -1 -7 -3 -3 -7 -7 -2 -9 -2 -4 -2 -0 -4 -8 -8 -2 -1 -3 -9 -5 -2 -0 -2 -9 -0 -7 -6 -5 -9 -1 -1 -9 -5 -0 -4 -5 -9 -6 -2 -0 -9 -6 -7 -9 -8 -2 -9 -5 -8 -0 -6 -0 -9 -9 -9 -7 -3 -4 -7 -8 -1 -4 -6 -7 -1 -0 -3 -1 -8 -4 -3 -2 -9 -6 -1 -7 -6 -4 -9 -4 -9 -4 -1 -3 -7 -7 -1 -4 -6 -9 -0 -7 -0 -3 -4 -0 -2 -0 -2 -7 -2 -8 -0 -9 -1 -4 -2 -0 -5 -1 -0 -6 -3 -5 -7 -3 -6 -6 -5 -1 -0 -3 -6 -4 -4 -4 -7 -9 -5 -3 -7 -7 -1 -4 -0 -4 -9 -6 -3 -9 -6 -6 -0 -9 -9 -4 -1 -9 -2 -8 -4 -5 -1 -5 -3 -1 -9 -6 -7 -2 -3 -6 -9 -4 -1 -2 -1 -6 -6 -5 -3 -0 -1 -7 -1 -8 -1 -9 -0 -7 -9 -8 -1 -4 -7 -7 -7 -3 -2 -6 -7 -6 -0 -3 -4 -7 -7 -9 -4 -2 -1 -8 -6 -7 -6 -0 -7 -4 -9 -0 -2 -9 -3 -2 -9 -2 -2 -1 -0 -7 -3 -3 -2 -1 -4 -0 -2 -9 -8 -6 -1 -1 -3 -1 -9 -1 -3 -9 -2 -6 -6 -2 -1 -7 -9 -6 -4 -6 -4 -0 -4 -6 -0 -5 -2 -7 -1 -3 -9 -3 -2 -4 -2 -6 -8 -8 -3 -6 -0 -2 -1 -6 -9 -5 -0 -6 -5 -1 -7 -8 -5 -1 -1 -8 -9 -8 -1 -9 -6 -9 -1 -5 -2 -4 -3 -8 -0 -8 -1 -9 -6 -9 -7 -9 -8 -7 -3 -2 -5 -5 -9 -1 -9 -8 -9 -4 -7 -9 -6 -3 -9 -7 -8 -2 -8 -1 -7 -1 -0 -4 -6 -8 -1 -9 -7 -0 -8 -4 -0 -5 -8 -6 -0 -7 -7 -5 -5 -6 -1 -6 -8 -0 -0 -3 -6 -1 -6 -0 -7 -2 -9 -1 -6 -9 -7 -5 -4 -3 -8 -9 -7 -2 -8 -6 -9 -8 -8 -1 -5 -4 -9 -1 -0 -2 -2 -0 -6 -1 -3 -4 -1 -0 -8 -0 -8 -1 -9 -4 -7 -9 -1 -7 -9 -1 -8 -7 -9 -8 -1 -2 -7 -4 -6 -3 -0 -5 -7 -9 -3 -1 -3 -0 -9 -8 -9 -6 -8 -1 -0 -6 -8 -1 -1 -0 -3 -4 -3 -2 -1 -9 -1 -1 -6 -9 -2 -1 -0 -2 -9 -1 -1 -4 -8 -4 -3 -7 -1 -1 -9 -9 -0 -1 -9 -1 -6 -3 -1 -4 -7 -8 -0 -0 -9 -3 -8 -1 -1 -2 -6 -7 -7 -0 -4 -5 -2 -1 -9 -1 -2 -1 -3 -9 -0 -2 -1 -9 -0 -8 -6 -0 -8 -2 -0 -0 -5 -9 -9 -6 -1 -1 -4 -5 -5 -9 -8 -7 -6 -8 -5 -2 -0 -3 -8 -8 -2 -0 -2 -4 -7 -2 -4 -2 -7 -5 -3 -7 -2 -8 -0 -1 -8 -7 -9 -7 -0 -9 -4 -2 -0 -7 -6 -3 -5 -1 -9 -9 -8 -2 -4 -6 -2 -2 -9 -3 -2 -9 -1 -5 -9 -9 -5 -9 -7 -6 -1 -7 -6 -3 -0 -4 -3 -8 -4 -7 -3 -6 -1 -3 -0 -5 -1 -6 -3 -9 -0 -1 -2 -0 -6 -7 -2 -0 -1 -4 -2 -6 -7 -6 -9 -0 -3 -7 -4 -6 -1 -1 -2 -9 -0 -0 -7 -3 -3 -2 -1 -7 -9 -0 -5 -1 -2 -6 -7 -4 -2 -2 -6 -1 -1 -4 -9 -1 -4 -3 -4 -8 -1 -7 -0 -7 -0 -8 -6 -1 -3 -0 -7 -3 -7 -1 -1 -7 -6 -4 -9 -3 -6 -0 -6 -2 -9 -2 -6 -1 -9 -1 -7 -0 -7 -3 -5 -3 -0 -7 -9 -8 -2 -6 -3 -9 -1 -4 -1 -2 -6 -6 -4 -1 -7 -1 -4 -8 -0 -9 -8 -9 -7 -4 -1 -5 -0 -2 -1 -8 -7 -3 -2 -6 -4 -8 -3 -1 -6 -9 -8 -2 -7 -0 -1 -9 -3 -7 -7 -7 -9 -9 -5 -2 -3 -5 -6 -0 -2 -1 -3 -7 -2 -1 -3 -2 -6 -7 -2 -0 -1 -8 -2 -9 -1 -9 -6 -6 -8 -8 -1 -1 -3 -1 -7 -5 -4 -1 -8 -1 -6 -6 -6 -1 -2 -6 -9 -3 -3 -1 -0 -5 -2 -8 -3 -5 -7 -4 -8 -6 -1 -1 -4 -5 -8 -7 -4 -2 -2 -0 -9 -9 -1 -1 -9 -9 -2 -1 -2 -1 -6 -4 -3 -1 -6 -9 -2 -2 -0 -1 -1 -0 -4 -6 -4 -8 -0 -9 -1 -4 -0 -2 -5 -6 -2 -2 -7 -3 -1 -8 -5 -2 -9 -8 -0 -4 -1 -6 -8 -8 -2 -6 -2 -4 -9 -9 -4 -7 -6 -0 -0 -2 -4 -8 -3 -1 -0 -8 -9 -5 -4 -0 -3 -8 -3 -4 -4 -6 -8 -6 -4 -0 -2 -6 -9 -5 -0 -3 -4 -3 -0 -1 -9 -1 -7 -1 -3 -0 -8 -4 -7 -0 -2 -6 -1 -3 -6 -0 -4 -6 -6 -7 -6 -1 -0 -9 -9 -0 -2 -8 -9 -3 -6 -1 -6 -2 -3 -7 -5 -8 -8 -3 -3 -0 -9 -6 -2 -7 -0 -9 -9 -1 -7 -6 -6 -1 -4 -8 -8 -9 -8 -9 -7 -1 -6 -2 -7 -9 -8 -8 -0 -1 -0 -0 -2 -4 -4 -9 -1 -0 -1 -6 -2 -5 -6 -3 -7 -6 -8 -0 -4 -7 -6 -9 -1 -5 -4 -7 -2 -9 -7 -4 -2 -3 -4 -9 -8 -0 -6 -6 -9 -4 -5 -2 -4 -6 -8 -1 -6 -9 -6 -5 -6 -2 -7 -6 -9 -6 -7 -5 -6 -0 -3 -9 -7 -6 -6 -4 -6 -1 -6 -0 -1 -8 -3 -1 -1 -7 -9 -3 -6 -8 -5 -2 -9 -1 -4 -1 -4 -5 -9 -6 -6 -5 -1 -3 -3 -1 -4 -2 -1 -5 -5 -0 -7 -3 -9 -2 -8 -0 -9 -8 -6 -7 -4 -6 -0 -7 -2 -8 -4 -3 -0 -1 -3 -9 -1 -0 -9 -5 -8 -0 -0 -1 -4 -7 -8 -7 -8 -3 -5 -0 -4 -4 -5 -3 -4 -9 -7 -1 -3 -5 -2 -5 -7 -0 -1 -2 -7 -9 -5 -8 -1 -1 -7 -0 -1 -6 -5 -1 -7 -5 -1 -5 -7 -5 -0 -9 -5 -7 -4 -2 -7 -3 -1 -9 -6 -3 -6 -3 -1 -6 -9 -7 -6 -2 -3 -1 -7 -9 -9 -5 -2 -2 -4 -0 -7 -6 -6 -1 -1 -1 -8 -7 -1 -7 -2 -7 -9 -2 -2 -4 -0 -0 -1 -2 -4 -9 -2 -7 -9 -2 -9 -8 -2 -4 -3 -9 -8 -8 -3 -6 -3 -3 -7 -6 -8 -0 -2 -9 -7 -1 -1 -9 -3 -0 -6 -9 -3 -8 -5 -0 -5 -9 -1 -1 -6 -4 -2 -7 -2 -3 -6 -4 -1 -9 -0 -3 -8 -5 -1 -5 -9 -3 -8 -1 -5 -0 -8 -3 -8 -9 -0 -9 -0 -9 -0 -0 -9 -9 -3 -1 -7 -7 -4 -0 -2 -6 -2 -4 -0 -1 -5 -0 -7 -8 -9 -7 -7 -1 -1 -2 -4 -8 -7 -1 -1 -6 -2 -3 -2 -8 -8 -9 -5 -9 -4 -9 -6 -6 -9 -7 -9 -5 -0 -6 -3 -0 -1 -5 -6 -8 -9 -9 -9 -4 -5 -3 -7 -3 -9 -7 -5 -0 -6 -4 -2 -9 -0 -8 -4 -7 -6 -2 -1 -7 -6 -3 -6 -9 -7 -9 -3 -4 -6 -6 -4 -3 -8 -1 -9 -9 -1 -3 -0 -6 -7 -6 -1 -1 -2 -6 -9 -8 -6 -9 -1 -0 -7 -5 -1 -9 -3 -1 -2 -2 -0 -8 -7 -2 -6 -2 -7 -8 -8 -1 -2 -9 -8 -2 -7 -0 -0 -3 -4 -9 -2 -0 -3 -8 -3 -1 -7 -9 -3 -3 -2 -5 -0 -1 -3 -5 -2 -8 -9 -9 -9 -2 -4 -1 -4 -1 -6 -6 -0 -6 -4 -9 -1 -6 -4 -9 -0 -1 -9 -9 -9 -3 -9 -9 -4 -4 -5 -6 -0 -0 -7 -7 -0 -6 -8 -1 -9 -1 -1 -4 -7 -4 -2 -4 -1 -5 -4 -7 -6 -9 -4 -2 -3 -9 -7 -1 -0 -5 -5 -1 -3 -7 -4 -1 -5 -8 -5 -6 -5 -0 -0 -3 -1 -9 -0 -5 -9 -6 -7 -4 -6 -9 -5 -4 -7 -8 -8 -0 -5 -2 -4 -6 -0 -0 -6 -5 -0 -5 -1 -5 -7 -7 -2 -7 -0 -3 -0 -3 -5 -3 -6 -9 -6 -0 -9 -5 -5 -0 -2 -7 -6 -6 -4 -7 -2 -7 -1 -1 -2 -9 -6 -1 -0 -2 -2 -8 -1 -3 -5 -2 -5 -9 -8 -5 -6 -6 -1 -0 -1 -6 -9 -5 -2 -7 -2 -3 -4 -1 -0 -9 -4 -0 -7 -8 -1 -0 -1 -5 -6 -8 -0 -3 -5 -7 -5 -5 -8 -7 -2 -3 -6 -1 -1 -9 -5 -3 -1 -3 -3 -6 -8 -3 -5 -0 -7 -1 -5 -8 -5 -6 -6 -1 -3 -2 -9 -1 -1 -9 -8 -5 -9 -5 -5 -1 -7 -1 -1 -6 -5 -7 -2 -3 -2 -5 -7 -3 -9 -2 -3 -5 -1 -6 -4 -7 -7 -0 -3 -6 -7 -1 -9 -6 -3 -7 -0 -6 -9 -1 -2 -1 -1 -2 -9 -8 -3 -2 -0 -2 -3 -7 -6 -9 -6 -6 -6 -4 -6 -1 -0 -2 -3 -0 -4 -7 -6 -8 -8 -7 -2 -6 -2 -1 -2 -9 -7 -0 -0 -1 -0 -0 -0 -3 -8 -1 -8 -0 -1 -9 -3 -0 -9 -1 -5 -5 -5 -1 -3 -8 -7 -3 -7 -0 -6 -6 -6 -7 -4 -1 -7 -5 -2 -1 -4 -7 -2 -4 -0 -5 -4 -9 -5 -1 -5 -5 -8 -7 -7 -9 -1 -5 -4 -6 -7 -2 -7 -1 -5 -9 -9 -7 -9 -1 -1 -0 -6 -6 -1 -4 -1 -0 -0 -7 -1 -2 -8 -9 -2 -6 -9 -2 -7 -0 -8 -0 -0 -0 -7 -3 -3 -4 -1 -9 -0 -2 -0 -0 -2 -2 -1 -3 -5 -6 -8 -7 -2 -4 -3 -9 -6 -4 -9 -5 -6 -1 -2 -3 -1 -1 -2 -2 -9 -2 -7 -0 -6 -9 -0 -1 -0 -8 -5 -6 -4 -2 -0 -9 -9 -6 -1 -6 -7 -9 -0 -1 -3 -1 -2 -7 -8 -9 -1 -9 -1 -0 -7 -6 -5 -0 -5 -3 -5 -5 -8 -7 -3 -3 -2 -3 -3 -6 -6 -3 -1 -1 -4 -6 -1 -5 -3 -9 -5 -4 -1 -6 -7 -0 -2 -0 -8 -1 -9 -5 -4 -8 -3 -3 -1 -8 -2 -2 -0 -8 -1 -6 -8 -1 -0 -3 -0 -4 -2 -4 -8 -1 -5 -2 -3 -6 -5 -8 -5 -8 -0 -1 -9 -8 -0 -8 -0 -5 -6 -1 -4 -8 -7 -1 -3 -4 -0 -2 -6 -3 -0 -3 -8 -4 -5 -9 -1 -2 -0 -8 -9 -2 -3 -3 -7 -5 -5 -3 -1 -6 -0 -5 -9 -5 -1 -5 -5 -6 -1 -7 -0 -1 -1 -6 -8 -5 -9 -0 -6 -5 -0 -8 -2 -1 -3 -0 -5 -6 -5 -6 -1 -8 -0 -0 -5 -9 -2 -9 -5 -2 -2 -9 -9 -8 -8 -3 -4 -7 -4 -8 -5 -3 -7 -7 -1 -6 -7 -7 -0 -1 -2 -2 -3 -5 -6 -1 -7 -7 -9 -3 -4 -2 -8 -0 -6 -8 -9 -1 -7 -8 -7 -8 -7 -1 -0 -9 -6 -3 -7 -5 -7 -2 -5 -5 -6 -6 -4 -8 -5 -7 -6 -6 -2 -6 -4 -1 -9 -6 -6 -2 -1 -6 -6 -1 -5 -8 -0 -2 -9 -6 -4 -6 -3 -1 -1 -6 -6 -9 -5 -7 -9 -6 -5 -1 -5 -7 -5 -3 -9 -6 -6 -3 -1 -6 -8 -6 -0 -6 -2 -4 -6 -2 -2 -3 -4 -6 -6 -7 -5 -3 -0 -5 -1 -3 -1 -8 -8 -9 -8 -6 -1 -1 -2 -3 -8 -8 -9 -1 -4 -1 -0 -0 -5 -0 -7 -9 -9 -2 -1 -6 -9 -9 -9 -1 -3 -1 -4 -0 -4 -9 -0 -4 -1 -6 -5 -4 -1 -5 -9 -1 -7 -5 -8 -2 -3 -0 -9 -8 -9 -2 -4 -6 -1 -9 -9 -0 -1 -4 -1 -7 -1 -9 -1 -1 -4 -5 -0 -7 -3 -5 -2 -0 -1 -7 -1 -6 -8 -6 -7 -7 -3 -3 -8 -7 -7 -1 -3 -2 -0 -5 -0 -4 -2 -8 -6 -4 -7 -8 -1 -4 -1 -7 -1 -4 -0 -5 -3 -6 -6 -0 -6 -8 -2 -1 -9 -8 -2 -0 -5 -7 -0 -7 -6 -1 -6 -6 -1 -2 -1 -0 -1 -3 -0 -5 -8 -2 -7 -6 -2 -1 -4 -0 -5 -1 -3 -1 -6 -1 -9 -6 -2 -8 -0 -2 -0 -0 -6 -3 -4 -5 -4 -0 -0 -1 -4 -0 -2 -8 -4 -0 -5 -6 -6 -9 -1 -6 -4 -4 -7 -2 -1 -5 -2 -6 -6 -3 -1 -8 -1 -8 -1 -5 -5 -2 -1 -9 -9 -9 -5 -3 -7 -9 -0 -8 -2 -3 -4 -8 -5 -9 -0 -7 -2 -1 -9 -9 -7 -2 -9 -7 -4 -5 -5 -4 -0 -2 -2 -5 -4 -5 -3 -7 -6 -1 -2 -8 -2 -0 -8 -9 -9 -6 -5 -4 -9 -6 -5 -1 -6 -2 -7 -9 -7 -6 -4 -1 -5 -3 -9 -7 -6 -1 -8 -3 -5 -0 -5 -9 -9 -9 -5 -6 -3 -3 -1 -0 -4 -8 -9 -4 -1 -3 -4 -7 -4 -5 -9 -4 -7 -7 -8 -6 -4 -9 -0 -0 -0 -7 -7 -0 -7 -0 -6 -0 -0 -0 -1 -5 -9 -6 -8 -1 -5 -6 -9 -1 -3 -1 -0 -2 -6 -5 -4 -2 -1 -9 -6 -7 -7 -8 -3 -8 -4 -4 -9 -3 -0 -7 -1 -1 -0 -5 -6 -9 -1 -0 -3 -3 -1 -7 -9 -9 -5 -1 -7 -7 -7 -7 -2 -7 -6 -6 -5 -3 -8 -1 -1 -1 -3 -9 -6 -6 -1 -5 -7 -4 -6 -8 -1 -7 -1 -5 -9 -0 -6 -3 -1 -2 -3 -3 -9 -8 -3 -9 -7 -2 -5 -4 -3 -9 -1 -1 -3 -1 -7 -5 -0 -2 -4 -5 -8 -4 -2 -8 -2 -3 -0 -3 -5 -7 -6 -8 -8 -5 -8 -1 -9 -2 -8 -7 -1 -9 -7 -0 -6 -6 -2 -1 -7 -3 -1 -5 -1 -6 -2 -0 -1 -0 -0 -2 -3 -9 -9 -0 -8 -6 -5 -8 -5 -0 -2 -7 -0 -4 -2 -1 -3 -4 -4 -5 -3 -1 -0 -1 -2 -0 -1 -8 -7 -0 -4 -3 -2 -2 -2 -3 -0 -3 -9 -3 -0 -3 -4 -9 -8 -7 -8 -4 -7 -9 -2 -6 -9 -6 -4 -7 -6 -1 -1 -6 -2 -7 -9 -5 -3 -1 -9 -0 -2 -4 -6 -7 -6 -7 -3 -7 -1 -4 -7 -7 -6 -1 -0 -5 -8 -7 -5 -4 -0 -2 -9 -2 -0 -0 -1 -0 -8 -2 -6 -2 -4 -7 -5 -3 -8 -0 -1 -4 -7 -4 -4 -5 -6 -9 -4 -9 -1 -3 -4 -9 -1 -1 -9 -1 -5 -4 -1 -8 -5 -2 -6 -3 -4 -6 -0 -6 -4 -4 -3 -3 -4 -4 -2 -8 -4 -0 -6 -8 -3 -7 -9 -6 -5 -9 -5 -3 -2 -5 -1 -3 -0 -9 -3 -0 -6 -2 -1 -6 -6 -2 -6 -5 -8 -7 -2 -4 -1 -9 -9 -7 -0 -6 -8 -3 -7 -0 -4 -4 -9 -8 -0 -0 -6 -7 -1 -1 -4 -9 -9 -8 -4 -8 -1 -0 -2 -5 -7 -4 -7 -6 -3 -7 -8 -2 -1 -3 -6 -8 -9 -6 -4 -5 -1 -2 -3 -1 -7 -6 -7 -9 -5 -1 -2 -9 -9 -3 -2 -1 -8 -0 -2 -1 -4 -6 -3 -3 -2 -5 -3 -0 -9 -4 -8 -1 -6 -6 -3 -8 -2 -1 -4 -0 -8 -5 -1 -0 -7 -1 -4 -2 -5 -4 -3 -6 -2 -2 -6 -0 -5 -2 -1 -3 -1 -9 -9 -4 -6 -9 -8 -0 -9 -2 -0 -3 -5 -9 -5 -9 -0 -2 -9 -4 -9 -8 -4 -8 -9 -5 -4 -0 -5 -1 -9 -1 -5 -2 -1 -8 -3 -9 -6 -1 -4 -6 -6 -0 -9 -1 -3 -7 -7 -4 -6 -8 -2 -8 -8 -3 -1 -2 -6 -2 -1 -0 -0 -6 -2 -5 -4 -6 -2 -8 -9 -8 -8 -9 -0 -2 -2 -9 -1 -1 -1 -6 -9 -2 -7 -8 -0 -5 -2 -0 -6 -1 -3 -8 -4 -0 -7 -1 -9 -6 -4 -0 -7 -3 -8 -1 -8 -6 -7 -1 -0 -3 -0 -9 -2 -0 -2 -8 -0 -5 -0 -3 -6 -1 -4 -9 -1 -5 -2 -3 -0 -7 -2 -9 -2 -6 -7 -0 -4 -3 -7 -3 -0 -4 -9 -6 -9 -9 -9 -5 -3 -3 -4 -1 -8 -8 -1 -4 -8 -7 -4 -3 -1 -4 -8 -6 -4 -3 -4 -8 -5 -6 -8 -9 -5 -5 -8 -7 -4 -0 -0 -0 -5 -3 -9 -8 -9 -3 -2 -7 -1 -0 -7 -1 -1 -7 -9 -0 -0 -5 -1 -8 -9 -7 -4 -2 -4 -7 -1 -0 -8 -9 -2 -9 -5 -0 -7 -0 -7 -9 -8 -3 -3 -4 -5 -1 -6 -9 -7 -1 -2 -2 -9 -6 -6 -1 -9 -2 -0 -5 -8 -1 -6 -5 -6 -3 -7 -7 -6 -7 -6 -9 -5 -3 -1 -1 -6 -4 -3 -6 -1 -4 -1 -4 -1 -6 -1 -9 -7 -7 -7 -1 -3 -9 -0 -7 -4 -6 -7 -7 -3 -7 -6 -2 -5 -8 -9 -3 -3 -4 -5 -6 -1 -9 -0 -7 -3 -5 -1 -5 -9 -0 -4 -2 -8 -6 -6 -3 -9 -0 -8 -0 -9 -9 -7 -8 -1 -8 -0 -9 -7 -1 -9 -2 -5 -1 -3 -0 -2 -7 -6 -2 -3 -9 -7 -2 -7 -0 -7 -7 -6 -0 -6 -8 -4 -7 -5 -3 -2 -3 -2 -2 -3 -4 -4 -4 -7 -4 -0 -7 -9 -6 -2 -3 -2 -5 -3 -9 -6 -6 -7 -1 -0 -6 -7 -9 -0 -2 -7 -9 -2 -9 -5 -0 -9 -5 -7 -1 -2 -5 -3 -8 -0 -0 -2 -4 -1 -7 -8 -3 -3 -1 -5 -2 -6 -0 -3 -5 -8 -0 -5 -2 -1 -5 -6 -3 -6 -4 -8 -0 -5 -4 -0 -6 -0 -5 -9 -3 -3 -0 -1 -7 -2 -2 -2 -8 -9 -0 -8 -0 -9 -8 -2 -9 -4 -3 -2 -9 -4 -9 -4 -1 -7 -2 -1 -3 -3 -7 -1 -1 -7 -7 -3 -9 -3 -1 -8 -3 -5 -1 -4 -6 -9 -4 -0 -3 -6 -3 -6 -6 -2 -9 -7 -9 -7 -8 -6 -6 -0 -5 -1 -2 -1 -4 -6 -2 -0 -2 -2 -2 -8 -8 -8 -0 -1 -0 -1 -7 -6 -9 -4 -9 -9 -4 -9 -8 -9 -7 -9 -6 -3 -9 -5 -0 -8 -7 -0 -6 -8 -1 -5 -7 -1 -4 -7 -8 -8 -3 -3 -2 -3 -6 -3 -1 -5 -5 -2 -5 -2 -4 -2 -9 -7 -9 -9 -2 -6 -9 -6 -0 -0 -9 -0 -8 -2 -1 -7 -6 -2 -0 -1 -6 -2 -3 -8 -9 -1 -2 -5 -8 -2 -0 -7 -1 -1 -1 -0 -8 -6 -7 -6 -4 -7 -1 -2 -0 -3 -4 -2 -7 -7 -1 -3 -4 -1 -3 -1 -3 -6 -6 -9 -5 -2 -8 -5 -5 -3 -6 -9 -6 -2 -0 -2 -7 -1 -9 -8 -2 -4 -6 -6 -3 -6 -8 -2 -9 -6 -1 -8 -4 -3 -6 -6 -1 -1 -0 -2 -1 -3 -9 -9 -4 -3 -9 -2 -7 -2 -3 -2 -0 -3 -3 -9 -0 -4 -0 -6 -9 -5 -3 -4 -4 -5 -0 -1 -5 -9 -4 -3 -3 -4 -0 -1 -0 -5 -8 -4 -0 -3 -5 -8 -6 -9 -4 -4 -7 -9 -3 -5 -0 -9 -2 -3 -6 -9 -8 -2 -5 -8 -7 -0 -2 -2 -5 -6 -7 -2 -2 -2 -1 -8 -5 -7 -2 -2 -7 -3 -7 -7 -6 -6 -9 -4 -1 -5 -8 -9 -0 -8 -8 -2 -3 -6 -2 -9 -8 -0 -1 -0 -5 -4 -5 -7 -0 -6 -6 -2 -5 -2 -2 -6 -2 -4 -0 -7 -6 -2 -1 -2 -6 -4 -4 -5 -2 -6 -9 -9 -6 -2 -4 -9 -8 -2 -4 -9 -6 -0 -1 -1 -0 -9 -8 -6 -4 -3 -7 -4 -7 -2 -1 -9 -1 -8 -3 -0 -4 -8 -0 -9 -0 -0 -3 -2 -7 -1 -4 -2 -8 -4 -5 -8 -5 -5 -1 -0 -3 -2 -4 -5 -2 -7 -9 -9 -1 -4 -2 -3 -9 -9 -2 -9 -1 -6 -6 -9 -6 -1 -9 -0 -9 -9 -3 -9 -6 -9 -9 -0 -7 -0 -0 -2 -8 -9 -1 -1 -3 -7 -0 -6 -7 -7 -0 -0 -6 -0 -3 -7 -5 -0 -6 -1 -2 -6 -3 -9 -1 -7 -8 -0 -9 -3 -8 -0 -1 -0 -7 -3 -0 -6 -2 -8 -2 -5 -4 -9 -6 -3 -4 -3 -9 -7 -1 -3 -2 -6 -2 -5 -5 -2 -0 -6 -8 -6 -7 -0 -8 -9 -4 -0 -2 -1 -5 -1 -6 -8 -1 -0 -2 -5 -2 -9 -8 -5 -3 -6 -9 -7 -2 -1 -9 -3 -5 -4 -4 -8 -0 -9 -5 -0 -2 -6 -0 -2 -6 -3 -3 -6 -2 -8 -1 -2 -6 -9 -1 -2 -5 -5 -9 -0 -0 -1 -8 -2 -0 -7 -0 -9 -2 -0 -4 -9 -0 -6 -0 -3 -1 -5 -2 -2 -6 -2 -7 -6 -0 -0 -5 -1 -6 -3 -7 -8 -1 -1 -5 -2 -0 -6 -5 -5 -5 -3 -7 -7 -1 -5 -8 -1 -8 -3 -3 -6 -8 -6 -0 -7 -8 -6 -2 -5 -1 -5 -0 -9 -8 -0 -7 -5 -8 -2 -0 -0 -5 -2 -2 -8 -3 -1 -3 -5 -8 -1 -0 -6 -8 -3 -1 -6 -0 -0 -2 -1 -8 -2 -6 -9 -6 -5 -7 -4 -3 -7 -2 -1 -9 -9 -0 -0 -3 -6 -2 -6 -9 -2 -1 -4 -3 -0 -5 -1 -8 -7 -8 -7 -1 -3 -5 -1 -1 -4 -2 -3 -4 -9 -2 -0 -1 -5 -3 -1 -9 -0 -6 -0 -7 -1 -7 -1 -2 -7 -4 -0 -9 -6 -2 -6 -3 -2 -6 -7 -5 -1 -0 -4 -3 -3 -0 -1 -4 -3 -1 -3 -5 -5 -7 -6 -9 -6 -7 -5 -6 -9 -8 -5 -1 -3 -0 -6 -7 -8 -7 -6 -7 -4 -2 -6 -3 -7 -3 -6 -1 -1 -2 -7 -8 -7 -9 -4 -1 -9 -8 -7 -3 -8 -8 -5 -0 -2 -7 -8 -9 -9 -7 -7 -2 -3 -1 -9 -8 -6 -9 -0 -4 -6 -1 -5 -3 -3 -3 -0 -1 -8 -1 -4 -6 -8 -7 -6 -8 -0 -5 -3 -4 -6 -3 -0 -2 -9 -4 -9 -1 -5 -1 -0 -7 -4 -4 -1 -8 -4 -5 -5 -9 -9 -2 -2 -2 -4 -4 -9 -6 -2 -5 -2 -2 -0 -2 -7 -9 -5 -0 -7 -0 -8 -0 -1 -7 -8 -2 -9 -4 -0 -8 -9 -4 -5 -3 -5 -0 -3 -3 -8 -0 -7 -0 -6 -7 -2 -0 -3 -6 -2 -1 -8 -6 -0 -1 -3 -8 -0 -3 -6 -3 -9 -1 -1 -2 -6 -8 -8 -9 -4 -9 -5 -9 -9 -6 -6 -4 -9 -0 -4 -0 -1 -3 -0 -0 -1 -9 -7 -2 -1 -6 -7 -2 -2 -1 -8 -1 -8 -9 -6 -2 -9 -9 -7 -4 -0 -9 -6 -8 -9 -9 -9 -7 -9 -5 -4 -7 -0 -9 -6 -4 -9 -4 -0 -7 -7 -9 -0 -0 -5 -6 -7 -2 -1 -3 -3 -8 -0 -1 -3 -8 -4 -3 -1 -2 -9 -9 -4 -5 -1 -4 -0 -3 -8 -8 -7 -1 -1 -6 -8 -2 -8 -6 -3 -9 -5 -9 -7 -2 -3 -1 -2 -0 -6 -3 -1 -9 -5 -2 -7 -3 -0 -1 -1 -9 -5 -2 -2 -6 -0 -6 -5 -8 -4 -3 -3 -0 -2 -9 -5 -7 -4 -7 -2 -5 -9 -5 -4 -7 -8 -2 -3 -2 -3 -6 -9 -2 -6 -3 -4 -1 -6 -1 -9 -0 -0 -6 -2 -9 -0 -9 -7 -8 -0 -8 -6 -2 -3 -5 -0 -5 -4 -2 -6 -0 -9 -8 -5 -9 -6 -1 -9 -3 -7 -1 -3 -7 -4 -1 -1 -9 -6 -9 -5 -8 -0 -2 -5 -3 -8 -0 -1 -4 -1 -6 -6 -9 -5 -8 -0 -7 -3 -4 -9 -2 -9 -9 -3 -1 -0 -8 -5 -3 -2 -4 -5 -2 -1 -3 -9 -8 -4 -4 -7 -9 -3 -7 -0 -7 -7 -9 -9 -8 -7 -1 -2 -0 -0 -0 -7 -6 -6 -8 -0 -1 -3 -0 -2 -6 -3 -6 -8 -0 -5 -6 -2 -0 -2 -9 -5 -0 -2 -4 -3 -9 -9 -1 -9 -5 -4 -4 -2 -0 -1 -7 -6 -6 -7 -0 -6 -4 -9 -8 -8 -9 -3 -3 -1 -0 -4 -4 -7 -0 -5 -7 -6 -7 -8 -1 -3 -4 -6 -7 -0 -4 -3 -2 -6 -3 -7 -3 -7 -1 -9 -1 -3 -9 -0 -1 -7 -9 -8 -4 -3 -5 -4 -0 -7 -2 -2 -6 -9 -5 -3 -9 -9 -8 -1 -6 -1 -7 -8 -3 -1 -1 -2 -8 -4 -5 -6 -1 -7 -0 -0 -1 -8 -7 -4 -9 -9 -7 -0 -5 -1 -5 -8 -6 -8 -5 -6 -2 -6 -1 -1 -9 -6 -4 -3 -5 -1 -8 -5 -7 -9 -7 -2 -7 -1 -5 -9 -1 -3 -9 -5 -0 -4 -6 -3 -5 -3 -3 -6 -1 -0 -2 -3 -2 -2 -6 -2 -0 -1 -2 -7 -8 -5 -6 -2 -8 -4 -6 -8 -5 -8 -6 -9 -8 -7 -1 -6 -7 -3 -7 -7 -4 -0 -1 -3 -9 -2 -0 -0 -5 -4 -7 -1 -6 -9 -7 -8 -4 -3 -3 -1 -3 -1 -5 -6 -0 -3 -5 -0 -8 -7 -7 -8 -6 -3 -6 -9 -5 -9 -2 -1 -2 -3 -5 -2 -2 -1 -1 -3 -3 -5 -8 -7 -9 -3 -4 -2 -7 -1 -7 -8 -2 -2 -3 -9 -5 -5 -7 -7 -3 -9 -0 -5 -9 -6 -6 -9 -2 -6 -3 -9 -2 -0 -7 -0 -7 -7 -1 -2 -0 -3 -2 -9 -0 -3 -3 -6 -3 -0 -5 -2 -6 -6 -7 -8 -0 -1 -5 -7 -1 -2 -9 -2 -5 -3 -6 -3 -6 -1 -0 -4 -5 -1 -9 -4 -6 -0 -5 -8 -4 -0 -4 -5 -3 -1 -5 -1 -5 -5 -2 -1 -0 -9 -6 -7 -5 -9 -9 -3 -4 -7 -7 -5 -6 -3 -2 -0 -4 -4 -0 -8 -5 -9 -1 -5 -5 -5 -7 -8 -1 -0 -7 -0 -4 -4 -1 -5 -6 -4 -2 -8 -2 -6 -8 -1 -2 -3 -8 -9 -1 -3 -9 -9 -6 -7 -1 -2 -7 -7 -3 -3 -1 -1 -9 -7 -2 -1 -8 -9 -2 -7 -8 -4 -0 -4 -3 -5 -8 -9 -3 -3 -2 -9 -0 -7 -7 -1 -9 -0 -4 -2 -2 -8 -9 -4 -3 -9 -4 -6 -3 -6 -0 -5 -1 -3 -0 -1 -2 -5 -0 -7 -1 -7 -2 -7 -8 -7 -4 -9 -1 -5 -0 -8 -2 -9 -2 -1 -0 -1 -2 -6 -9 -2 -5 -3 -5 -3 -4 -9 -1 -4 -7 -1 -8 -1 -8 -8 -9 -4 -6 -1 -9 -9 -3 -4 -1 -2 -4 -3 -5 -9 -0 -7 -2 -8 -2 -9 -9 -9 -9 -0 -7 -1 -2 -3 -3 -1 -9 -9 -8 -8 -3 -5 -1 -2 -1 -7 -2 -0 -7 -2 -1 -1 -7 -0 -6 -4 -0 -8 -1 -8 -7 -2 -4 -7 -3 -0 -9 -6 -7 -5 -9 -7 -4 -5 -2 -9 -6 -7 -1 -3 -7 -0 -3 -1 -4 -7 -5 -2 -0 -4 -4 -9 -7 -4 -9 -0 -1 -0 -2 -1 -6 -6 -3 -6 -4 -9 -2 -2 -3 -1 -1 -0 -7 -1 -2 -7 -9 -0 -9 -0 -6 -7 -1 -0 -9 -7 -5 -0 -8 -4 -8 -5 -0 -0 -0 -3 -6 -5 -0 -8 -2 -7 -6 -0 -8 -5 -0 -2 -6 -9 -4 -2 -0 -9 -3 -8 -1 -3 -6 -7 -6 -7 -0 -3 -1 -9 -2 -1 -9 -7 -3 -0 -6 -4 -2 -2 -5 -3 -7 -8 -8 -5 -7 -4 -2 -7 -4 -9 -0 -5 -9 -9 -9 -2 -8 -8 -8 -0 -3 -1 -1 -1 -0 -9 -0 -4 -1 -7 -4 -7 -6 -1 -1 -4 -3 -7 -1 -4 -6 -9 -5 -8 -9 -3 -2 -9 -1 -5 -3 -9 -6 -3 -6 -7 -9 -1 -7 -3 -2 -5 -6 -9 -7 -2 -6 -2 -8 -6 -5 -7 -7 -3 -5 -9 -0 -0 -4 -2 -2 -5 -1 -9 -5 -9 -6 -8 -0 -8 -5 -3 -7 -6 -1 -9 -3 -5 -8 -0 -9 -4 -0 -7 -5 -6 -1 -4 -6 -5 -0 -4 -9 -7 -1 -9 -6 -6 -6 -1 -4 -8 -1 -7 -2 -4 -9 -7 -5 -0 -0 -0 -8 -0 -7 -7 -0 -2 -6 -9 -0 -9 -8 -7 -2 -9 -7 -3 -9 -3 -9 -9 -0 -7 -5 -2 -4 -5 -8 -7 -8 -9 -1 -2 -2 -7 -7 -4 -7 -0 -6 -5 -9 -7 -2 -0 -0 -1 -2 -9 -1 -3 -4 -0 -6 -4 -9 -5 -3 -0 -2 -0 -1 -6 -5 -8 -0 -5 -7 -6 -4 -7 -0 -2 -4 -2 -1 -9 -3 -9 -0 -1 -0 -5 -7 -6 -5 -0 -5 -6 -1 -0 -0 -0 -9 -0 -3 -1 -7 -3 -7 -9 -8 -4 -4 -8 -3 -3 -0 -6 -4 -1 -0 -0 -2 -2 -2 -1 -1 -7 -3 -7 -6 -8 -6 -3 -4 -6 -4 -9 -3 -3 -9 -2 -0 -1 -3 -2 -5 -1 -2 -4 -0 -3 -6 -2 -2 -1 -6 -9 -0 -6 -1 -2 -4 -1 -3 -9 -7 -6 -0 -4 -5 -3 -3 -7 -2 -3 -4 -4 -8 -4 -6 -2 -1 -7 -0 -5 -8 -1 -2 -8 -3 -5 -1 -6 -9 -0 -9 -4 -9 -1 -0 -7 -0 -9 -3 -7 -5 -6 -9 -9 -9 -6 -4 -8 -9 -7 -9 -7 -5 -7 -1 -2 -6 -7 -6 -0 -9 -7 -5 -9 -7 -1 -2 -6 -0 -5 -6 -6 -1 -0 -0 -0 -4 -9 -8 -9 -6 -8 -0 -2 -8 -4 -9 -7 -4 -1 -1 -7 -6 -1 -5 -7 -6 -2 -2 -3 -0 -7 -9 -0 -7 -2 -6 -0 -3 -6 -1 -2 -3 -3 -1 -9 -0 -5 -6 -2 -3 -6 -1 -4 -0 -0 -4 -6 -9 -8 -7 -4 -8 -4 -3 -9 -7 -8 -9 -8 -3 -6 -7 -3 -0 -1 -2 -3 -8 -2 -0 -8 -0 -5 -4 -2 -9 -9 -4 -0 -7 -3 -2 -2 -9 -4 -5 -1 -0 -5 -9 -6 -1 -5 -0 -7 -6 -4 -2 -5 -3 -7 -9 -3 -9 -1 -8 -1 -6 -8 -3 -4 -9 -0 -9 -1 -9 -4 -9 -6 -2 -6 -0 -0 -6 -0 -8 -7 -8 -9 -9 -0 -7 -7 -4 -2 -6 -0 -2 -6 -9 -5 -2 -1 -1 -1 -0 -1 -0 -8 -0 -6 -6 -4 -2 -8 -1 -9 -3 -9 -7 -1 -5 -4 -6 -2 -1 -7 -6 -8 -0 -7 -7 -0 -7 -2 -6 -9 -3 -6 -7 -6 -5 -9 -2 -9 -4 -4 -1 -0 -1 -4 -5 -4 -4 -6 -8 -5 -2 -3 -6 -2 -8 -5 -3 -3 -0 -2 -3 -3 -3 -9 -0 -3 -0 -3 -0 -3 -2 -9 -7 -5 -4 -9 -2 -4 -0 -1 -0 -6 -5 -0 -2 -8 -1 -3 -5 -5 -8 -9 -2 -7 -1 -6 -6 -6 -1 -4 -0 -4 -9 -6 -1 -8 -0 -9 -4 -5 -1 -6 -1 -1 -0 -9 -5 -9 -7 -9 -1 -6 -0 -0 -1 -7 -5 -8 -9 -7 -9 -9 -4 -6 -7 -7 -3 -6 -8 -0 -2 -0 -9 -1 -6 -6 -9 -6 -1 -8 -8 -1 -2 -8 -6 -1 -9 -7 -5 -2 -6 -0 -1 -7 -7 -5 -7 -3 -1 -6 -1 -0 -2 -5 -4 -2 -7 -3 -7 -3 -6 -4 -9 -5 -3 -1 -4 -6 -3 -9 -7 -8 -1 -9 -3 -1 -0 -5 -8 -5 -7 -8 -4 -1 -5 -4 -7 -5 -8 -3 -2 -1 -9 -2 -2 -5 -6 -1 -4 -8 -1 -1 -8 -1 -4 -9 -8 -3 -6 -8 -1 -5 -3 -7 -2 -4 -2 -5 -0 -4 -2 -1 -8 -4 -5 -4 -1 -6 -5 -5 -8 -2 -2 -9 -0 -7 -3 -8 -2 -8 -1 -4 -7 -4 -6 -7 -4 -4 -1 -5 -7 -9 -2 -6 -1 -0 -7 -1 -0 -9 -3 -7 -6 -7 -2 -1 -6 -7 -4 -9 -1 -7 -8 -5 -6 -7 -6 -6 -9 -7 -3 -9 -6 -0 -0 -6 -9 -2 -1 -4 -6 -3 -0 -9 -9 -6 -4 -7 -6 -0 -2 -9 -0 -1 -1 -0 -5 -9 -1 -1 -9 -2 -2 -4 -1 -0 -8 -3 -1 -5 -7 -2 -9 -8 -8 -0 -7 -9 -9 -2 -4 -1 -6 -9 -4 -4 -3 -6 -1 -3 -2 -3 -7 -5 -4 -3 -4 -2 -0 -7 -1 -6 -8 -1 -2 -1 -3 -0 -1 -0 -9 -9 -4 -5 -5 -2 -1 -8 -3 -8 -7 -8 -4 -2 -0 -6 -4 -5 -3 -2 -8 -0 -2 -7 -5 -7 -8 -5 -7 -6 -6 -3 -8 -5 -7 -1 -9 -6 -0 -5 -0 -7 -7 -9 -7 -8 -5 -5 -2 -1 -8 -9 -1 -7 -3 -5 -0 -0 -9 -6 -2 -6 -6 -2 -4 -9 -8 -6 -6 -2 -5 -8 -9 -5 -6 -2 -7 -1 -7 -8 -5 -3 -1 -3 -5 -9 -6 -4 -9 -1 -8 -0 -8 -6 -7 -9 -7 -2 -1 -7 -6 -3 -1 -7 -4 -0 -8 -3 -9 -1 -2 -9 -5 -1 -0 -7 -6 -7 -8 -4 -5 -3 -0 -8 -2 -8 -0 -5 -0 -5 -0 -1 -9 -2 -3 -7 -3 -9 -9 -3 -9 -1 -0 -2 -9 -3 -3 -2 -6 -6 -8 -7 -7 -0 -9 -1 -7 -3 -3 -7 -5 -5 -0 -4 -9 -8 -2 -1 -9 -3 -3 -4 -2 -4 -3 -9 -1 -4 -0 -2 -7 -5 -3 -1 -6 -1 -5 -4 -8 -7 -4 -2 -5 -1 -7 -9 -8 -0 -5 -7 -4 -8 -2 -5 -9 -3 -4 -5 -6 -7 -5 -9 -8 -3 -7 -7 -3 -2 -8 -8 -2 -0 -1 -2 -0 -4 -1 -9 -0 -1 -4 -3 -1 -9 -4 -4 -5 -4 -5 -8 -0 -5 -0 -0 -9 -3 -2 -5 -9 -7 -4 -8 -7 -8 -0 -9 -6 -3 -3 -0 -7 -6 -6 -8 -8 -2 -1 -4 -9 -6 -4 -7 -5 -0 -4 -2 -9 -8 -8 -6 -6 -8 -3 -2 -0 -9 -1 -5 -3 -1 -0 -4 -9 -5 -9 -6 -3 -0 -9 -6 -7 -2 -7 -3 -0 -3 -3 -1 -8 -7 -2 -8 -7 -1 -0 -0 -6 -2 -0 -1 -0 -5 -9 -2 -5 -9 -9 -4 -0 -4 -0 -8 -4 -3 -1 -6 -4 -4 -1 -0 -9 -8 -8 -1 -1 -9 -8 -6 -4 -1 -8 -8 -1 -5 -1 -6 -3 -3 -5 -5 -4 -1 -2 -3 -6 -9 -8 -6 -1 -3 -4 -7 -9 -1 -5 -7 -3 -0 -6 -3 -4 -8 -8 -8 -6 -1 -0 -8 -8 -3 -2 -1 -6 -9 -8 -9 -0 -9 -1 -5 -9 -2 -6 -2 -2 -2 -5 -1 -9 -4 -0 -1 -1 -3 -4 -0 -6 -9 -8 -1 -7 -6 -4 -0 -6 -3 -0 -2 -5 -1 -5 -4 -4 -7 -1 -1 -0 -7 -7 -0 -4 -9 -7 -6 -5 -9 -1 -9 -6 -7 -9 -9 -2 -1 -2 -6 -0 -5 -1 -3 -1 -7 -2 -3 -5 -4 -1 -9 -3 -6 -0 -3 -3 -5 -0 -4 -8 -0 -3 -6 -0 -1 -9 -3 -1 -1 -0 -4 -5 -2 -2 -1 -6 -7 -6 -1 -9 -7 -9 -7 -3 -9 -2 diff --git a/src/test/dataset/readme.md b/src/test/dataset/readme.md new file mode 100644 index 0000000..01c8d6d --- /dev/null +++ b/src/test/dataset/readme.md @@ -0,0 +1,3 @@ +The dataset is part of MNIST from kaggle Digit Recognizer competition: +* "train.format" is the train set, which has been binarized. +* "test.format" is the test set, which has been binarized. diff --git a/dataset/test.format b/src/test/dataset/test.format similarity index 100% rename from dataset/test.format rename to src/test/dataset/test.format diff --git a/dataset/train.format b/src/test/dataset/train.format similarity index 100% rename from dataset/train.format rename to src/test/dataset/train.format diff --git a/src/main/java/RunCNN.java b/src/test/java/javacnn/RunCNN.java similarity index 79% rename from src/main/java/RunCNN.java rename to src/test/java/javacnn/RunCNN.java index 6cbd6d1..047e855 100644 --- a/src/main/java/RunCNN.java +++ b/src/test/java/javacnn/RunCNN.java @@ -1,3 +1,5 @@ +package javacnn; + import java.io.IOException; import javacnn.cnn.CNN; @@ -26,11 +28,11 @@ public static void main(String[] args) throws IOException, ClassNotFoundExceptio final CNN cnn = new CNN(builder, 50, concurenceRunner); - final String fileName = "dataset/train.format"; + final String fileName = "src/test/dataset/train.format"; final Dataset dataset = DatasetLoader.load(fileName, ",", 784); - cnn.train(dataset, 5); + cnn.train(dataset, 1); - CNNLoader.saveModel("model.cnn", cnn); + CNNLoader.saveModel("src/test/model.cnn", cnn); dataset.clear(); /* @@ -38,8 +40,8 @@ public static void main(String[] args) throws IOException, ClassNotFoundExceptio cnn.setRunner(concurenceRunner); */ - final Dataset testset = DatasetLoader.load("dataset/test.format", ",", -1); - cnn.predict(testset, "dataset/test.predict"); + final Dataset testset = DatasetLoader.load("src/test/dataset/test.format", ",", -1); + cnn.predict(testset, "src/test/dataset/test.predict"); } finally { concurenceRunner.shutdown(); From 59eb0cb551ee99ec4a817a90744780b8aa88edad Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Mon, 19 Feb 2018 10:09:19 +0100 Subject: [PATCH 19/34] release 0.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1d0745c..2881609 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ javacnn javacnn - 0.2-SNAPSHOT + 0.2 Implementation of a CNN in Java From e1f7804be7f19649e7227803fe418c9de9ba8daf Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Mon, 19 Feb 2018 10:15:58 +0100 Subject: [PATCH 20/34] preparation for release 0.3 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2881609..8292b2e 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ javacnn javacnn - 0.2 + 0.3-SNAPSHOT Implementation of a CNN in Java From 70b8ab83cd72e5a25d734fea6b62c44c744064ca Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Mon, 19 Feb 2018 11:00:57 +0100 Subject: [PATCH 21/34] removed System.out from CNN-class --- src/main/java/javacnn/cnn/CNN.java | 35 +++++++++----- .../javacnn/util/DotProgressIndicator.java | 47 +++++++++++++++++++ .../java/javacnn/util/ProgressIndicator.java | 14 ++++++ src/test/java/javacnn/RunCNN.java | 2 +- 4 files changed, 85 insertions(+), 13 deletions(-) create mode 100644 src/main/java/javacnn/util/DotProgressIndicator.java create mode 100644 src/main/java/javacnn/util/ProgressIndicator.java diff --git a/src/main/java/javacnn/cnn/CNN.java b/src/main/java/javacnn/cnn/CNN.java index 6a282f4..81f9701 100644 --- a/src/main/java/javacnn/cnn/CNN.java +++ b/src/main/java/javacnn/cnn/CNN.java @@ -9,7 +9,9 @@ import java.util.List; import javacnn.dataset.Dataset; +import javacnn.util.DotProgressIndicator; import javacnn.util.Log; +import javacnn.util.ProgressIndicator; import javacnn.util.Runner; import javacnn.util.Util; @@ -31,6 +33,7 @@ public class CNN implements Serializable { private transient Runner runner; + private ProgressIndicator progressIndicator = new DotProgressIndicator(); public CNN(LayerBuilder layerBuilder, final int batchSize, final Runner runner) { @@ -76,6 +79,8 @@ public double process(double value) { }; } + // === simple getters and setters === + public void setRunner(final Runner runner) { this.runner = runner; } @@ -85,14 +90,23 @@ private Runner getRunner() { return runner; } + public ProgressIndicator getProgressIndicator() { + return progressIndicator; + } + + public void setProgressIndicator(final ProgressIndicator progressIndicator) { + this.progressIndicator = progressIndicator; + } + + // === business logic === + public void train(final Dataset trainset, final int iterationCount) { for (int iteration = 0; iteration < iterationCount; iteration++) { - int epochsNum = trainset.size() / batchSize; + progressIndicator.start(); - if (trainset.size() % batchSize != 0) { - epochsNum++; // Extract once, round up - } + // separate trainset in batches of batchsize ... and round up the result + final int epochsNum = (trainset.size() + batchSize - 1) / batchSize; Log.info(""); Log.info(iteration + "th iter epochsNum:" + epochsNum); @@ -108,8 +122,7 @@ public void train(final Dataset trainset, final int iterationCount) { for (int index : randPerm) { final boolean isRight = train(trainset.getRecord(index)); - if (isRight) - right++; + if (isRight) right++; count++; Layer.prepareForNewRecord(); } @@ -117,14 +130,11 @@ public void train(final Dataset trainset, final int iterationCount) { // After finishing a batch update weight updateParas(); - if (epoch % 50 == 0) { - System.out.print("."); - if (epoch + 50 > epochsNum) { - System.out.println(); - } - } + progressIndicator.progress(); } + progressIndicator.finished(); + final double precision = ((double) right) / count; if (iteration % 10 == 1 && precision > 0.96) { @@ -370,6 +380,7 @@ private boolean setOutLayerErrors(final Dataset.Record record) { /** * Propagate given Record through the network. * Returns the result. + * * @param record A Record * @return The result of the network */ diff --git a/src/main/java/javacnn/util/DotProgressIndicator.java b/src/main/java/javacnn/util/DotProgressIndicator.java new file mode 100644 index 0000000..5996de8 --- /dev/null +++ b/src/main/java/javacnn/util/DotProgressIndicator.java @@ -0,0 +1,47 @@ +package javacnn.util; + +import java.io.Serializable; + +/** + *

+ * Created: 19.02.2018 10:53 + * + * @author Ralf Th. Pietsch <ratopi@abwesend.de> + */ +public class DotProgressIndicator implements ProgressIndicator, Serializable { + + private static final long serialVersionUID = 1L; + + private int cycle; + + private int count = 0; + + + public DotProgressIndicator() { + this(50); + } + + public DotProgressIndicator(final int cycle) { + this.cycle = cycle; + } + + + @Override + public void start() { + count = 0; + } + + @Override + public void progress() { + count++; + if (count > cycle) { + System.out.print("."); + count = 0; + } + } + + @Override + public void finished() { + System.out.println(); + } +} diff --git a/src/main/java/javacnn/util/ProgressIndicator.java b/src/main/java/javacnn/util/ProgressIndicator.java new file mode 100644 index 0000000..1bb4af4 --- /dev/null +++ b/src/main/java/javacnn/util/ProgressIndicator.java @@ -0,0 +1,14 @@ +package javacnn.util; + +/** + * Interface for feedback progress of any kind + *

+ * Created: 19.02.2018 10:52 + * + * @author Ralf Th. Pietsch <ratopi@abwesend.de> + */ +public interface ProgressIndicator { + void start(); + void progress(); + void finished(); +} diff --git a/src/test/java/javacnn/RunCNN.java b/src/test/java/javacnn/RunCNN.java index 047e855..4cf1906 100644 --- a/src/test/java/javacnn/RunCNN.java +++ b/src/test/java/javacnn/RunCNN.java @@ -30,7 +30,7 @@ public static void main(String[] args) throws IOException, ClassNotFoundExceptio final String fileName = "src/test/dataset/train.format"; final Dataset dataset = DatasetLoader.load(fileName, ",", 784); - cnn.train(dataset, 1); + cnn.train(dataset, 3); CNNLoader.saveModel("src/test/model.cnn", cnn); dataset.clear(); From d0b56a178437a28810516302cc562ac6ff4d5cf8 Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Tue, 20 Feb 2018 11:11:00 +0100 Subject: [PATCH 22/34] - CNN.propagate with double[]-input introduced - constructor of Dataset.Record is now public --- src/main/java/javacnn/cnn/CNN.java | 13 +++++++++++++ src/main/java/javacnn/dataset/Dataset.java | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/java/javacnn/cnn/CNN.java b/src/main/java/javacnn/cnn/CNN.java index 81f9701..774f048 100644 --- a/src/main/java/javacnn/cnn/CNN.java +++ b/src/main/java/javacnn/cnn/CNN.java @@ -377,6 +377,19 @@ private boolean setOutLayerErrors(final Dataset.Record record) { return label == Util.getMaxIndex(outmaps); } + /** + * Propagate given values through the network. + * Returns the result. + * + * @param inputs A vector of input values + * @return The result of the network + */ + public double[] propagate(final double[] inputs) { + final Dataset.Record record = new Dataset.Record(inputs, -1.); + + return propagate(record); + } + /** * Propagate given Record through the network. * Returns the result. diff --git a/src/main/java/javacnn/dataset/Dataset.java b/src/main/java/javacnn/dataset/Dataset.java index 8a4ce41..877d47b 100644 --- a/src/main/java/javacnn/dataset/Dataset.java +++ b/src/main/java/javacnn/dataset/Dataset.java @@ -64,7 +64,7 @@ public static class Record { private double[] attrs; private Double label; - private Record(double[] attrs, Double label) { + public Record(double[] attrs, Double label) { this.attrs = attrs; this.label = label; } From 0950f6a019b4a9d7f2ad233ed19a206cb8531631 Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Tue, 20 Feb 2018 11:11:59 +0100 Subject: [PATCH 23/34] Implemented the DirectRunner Now possible to use CNN without threads --- src/main/java/javacnn/util/DirectRunner.java | 29 ++++++++++++++++++++ src/test/java/javacnn/RunCNN.java | 1 + 2 files changed, 30 insertions(+) create mode 100644 src/main/java/javacnn/util/DirectRunner.java diff --git a/src/main/java/javacnn/util/DirectRunner.java b/src/main/java/javacnn/util/DirectRunner.java new file mode 100644 index 0000000..ad8bd4c --- /dev/null +++ b/src/main/java/javacnn/util/DirectRunner.java @@ -0,0 +1,29 @@ +package javacnn.util; + +import javacnn.cnn.Process; + +/** + *

+ * Created: 20.02.2018 11:03 + * + * @author Ralf Th. Pietsch <ratopi@abwesend.de> + */ +public class DirectRunner implements Runner { + + @Override + public void startProcess(final int mapNum, final Process process) { + final int runCpu = 1; + + // Fragment length rounded up + final int fregLength = (mapNum + runCpu - 1) / runCpu; + + for (int cpu = 0; cpu < runCpu; cpu++) { + final int start = cpu * fregLength; + + final int tmp = (cpu + 1) * fregLength; + final int end = tmp <= mapNum ? tmp : mapNum; + + process.process(start, end); + } + } +} diff --git a/src/test/java/javacnn/RunCNN.java b/src/test/java/javacnn/RunCNN.java index 4cf1906..ffafbe2 100644 --- a/src/test/java/javacnn/RunCNN.java +++ b/src/test/java/javacnn/RunCNN.java @@ -27,6 +27,7 @@ public static void main(String[] args) throws IOException, ClassNotFoundExceptio builder.addLayer(Layer.buildOutputLayer(10)); final CNN cnn = new CNN(builder, 50, concurenceRunner); + // final CNN cnn = new CNN(builder, 50, new DirectRunner()); final String fileName = "src/test/dataset/train.format"; final Dataset dataset = DatasetLoader.load(fileName, ",", 784); From 3a2511fc05a5d059029e1c626c3f128b52c7b129 Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Tue, 20 Feb 2018 11:13:17 +0100 Subject: [PATCH 24/34] release 0.3 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8292b2e..97dbb25 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ javacnn javacnn - 0.3-SNAPSHOT + 0.3 Implementation of a CNN in Java From 9ad86d7595f3904d9f561c490df62ea0e18d1321 Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Tue, 20 Feb 2018 11:13:35 +0100 Subject: [PATCH 25/34] preparation for next release (0.4) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 97dbb25..4d3644d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ javacnn javacnn - 0.3 + 0.4-SNAPSHOT Implementation of a CNN in Java From bfe2ca8b00e2d0e5b9379c522104ae0f6b0a0b7d Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Wed, 21 Feb 2018 07:55:40 +0100 Subject: [PATCH 26/34] A propagation method useful for production ;-) --- src/main/java/javacnn/cnn/CNN.java | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/main/java/javacnn/cnn/CNN.java b/src/main/java/javacnn/cnn/CNN.java index 774f048..ee602c7 100644 --- a/src/main/java/javacnn/cnn/CNN.java +++ b/src/main/java/javacnn/cnn/CNN.java @@ -377,6 +377,27 @@ private boolean setOutLayerErrors(final Dataset.Record record) { return label == Util.getMaxIndex(outmaps); } + /** + * Propagate given values through the network. + * Returns the results. + * For each input it returns the set of output values. + * + * @param inputs A list of vectors of input values + * @return A list of results of the network corresponding to each input vector + */ + public double[][] propagate(final double[][] inputs) { + final double[][] results = new double[inputs.length][]; + + int index = 0; + for (final double[] input : inputs) { + final Dataset.Record record = new Dataset.Record(input, -1.); + results[index] = propagate(record); + index++; + } + + return results; + } + /** * Propagate given values through the network. * Returns the result. From 9461325334e0c049c43074b4b8531051bcdbf809 Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Wed, 21 Feb 2018 08:27:14 +0100 Subject: [PATCH 27/34] readme revised --- readme-cn.md | 25 ------------------------- readme.md | 25 +++++++++++++++---------- 2 files changed, 15 insertions(+), 35 deletions(-) delete mode 100644 readme-cn.md diff --git a/readme-cn.md b/readme-cn.md deleted file mode 100644 index 49fb496..0000000 --- a/readme-cn.md +++ /dev/null @@ -1,25 +0,0 @@ -# JavaCNN -一个卷积神经网络的java实现. 仿Matlab toolbox(https://github.com/rasmusbergpalm/DeepLearnToolbox )实现的，同时进行了部分改进，使得卷积核和采样块可以为矩形而不仅仅是正方形。更多细节，请查看http://www.cnblogs.com/fengfenggirl/p/cnn_implement.html -## 创建一个卷积神经网络 - - LayerBuilder builder = new LayerBuilder(); - builder.addLayer(Layer.buildInputLayer(new Size(28, 28))); - builder.addLayer(Layer.buildConvLayer(6, new Size(5, 5))); - builder.addLayer(Layer.buildSampLayer(new Size(2, 2))); - builder.addLayer(Layer.buildConvLayer(12, new Size(5, 5))); - builder.addLayer(Layer.buildSampLayer(new Size(2, 2))); - builder.addLayer(Layer.buildOutputLayer(10)); - CNN cnn = new CNN(builder, 50); - -## 在 MNIST 数据集上测试 - - String fileName = "data/train.format"; - Dataset dataset = Dataset.load(fileName, ",", 784); - cnn.train(dataset, 100); - Dataset testset = Dataset.load("data/test.format", ",", -1); - cnn.predict(testset, "data/test.predict"); - -迭代100次，四核CPU大约需要运行一个小时后，正确率97.8% - -##Lisence - MIT \ No newline at end of file diff --git a/readme.md b/readme.md index 329ada9..9c578d6 100644 --- a/readme.md +++ b/readme.md @@ -1,5 +1,10 @@ # JavaCNN -A Java implement of Convolutional Neural Network. Learn from DeepLearnToolbox(https://github.com/rasmusbergpalm/DeepLearnToolbox) more detail. see here(http://www.cnblogs.com/fengfenggirl/p/cnn_implement.html) + +A Java implement of Convolutional Neural Network. +This is a fork of https://github.com/BigPeng/JavaCNN refactored for use in production. + +Original ideas are take from the DeepLearnToolbox (https://github.com/rasmusbergpalm/DeepLearnToolbox). + ## Build a CNN LayerBuilder builder = new LayerBuilder(); @@ -12,14 +17,14 @@ A Java implement of Convolutional Neural Network. Learn from DeepLearnToolbox(ht CNN cnn = new CNN(builder, 50); ## Run on MNIST dataset - - String fileName = "data/train.format"; - Dataset dataset = Dataset.load(fileName, ",", 784); - cnn.train(dataset, 100); - Dataset testset = Dataset.load("data/test.format", ",", -1); - cnn.predict(testset, "data/test.predict"); -It takes a about an hour to complete 100 iteration and get a precison of 97.8% +For running on MNIST dataset see project https://github.com/ratopi/javacnn.mnist. + +## Source Code + +Get the source code from github: + + git clone https://github.com/ratopi/JavaCNN.git -##Lisence - MIT \ No newline at end of file +##License + MIT From e85331e1724daf47890da9b0f4e458b8f6fa6f41 Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Wed, 21 Feb 2018 14:17:46 +0100 Subject: [PATCH 28/34] - ALPHA is now member of CNN - Renamed LayerBuilder.mLayers -> LayerBulder.layerList - RunCNN : Example uses return value of addLayer() --- src/main/java/javacnn/cnn/CNN.java | 16 ++++++++-------- src/test/java/javacnn/RunCNN.java | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/main/java/javacnn/cnn/CNN.java b/src/main/java/javacnn/cnn/CNN.java index ee602c7..71e951d 100644 --- a/src/main/java/javacnn/cnn/CNN.java +++ b/src/main/java/javacnn/cnn/CNN.java @@ -17,11 +17,11 @@ public class CNN implements Serializable { - private static final long serialVersionUID = 1L; + private static final long serialVersionUID = 3L; private static final double LAMBDA = 0; - private static double ALPHA = 0.85; + private double ALPHA = 0.85; private final List layers; private final int layerNum; @@ -33,11 +33,11 @@ public class CNN implements Serializable { private transient Runner runner; - private ProgressIndicator progressIndicator = new DotProgressIndicator(); + private transient ProgressIndicator progressIndicator = new DotProgressIndicator(); public CNN(LayerBuilder layerBuilder, final int batchSize, final Runner runner) { - this.layers = layerBuilder.mLayers; + this.layers = layerBuilder.layerList; this.layerNum = layers.size(); this.batchSize = batchSize; @@ -570,19 +570,19 @@ private void setup(final int batchSize) { // === inner classes === public static class LayerBuilder { - private List mLayers; + private List layerList; public LayerBuilder() { - mLayers = new ArrayList<>(); + layerList = new ArrayList<>(); } public LayerBuilder(Layer layer) { this(); - mLayers.add(layer); + layerList.add(layer); } public LayerBuilder addLayer(Layer layer) { - mLayers.add(layer); + layerList.add(layer); return this; } } diff --git a/src/test/java/javacnn/RunCNN.java b/src/test/java/javacnn/RunCNN.java index ffafbe2..cae9051 100644 --- a/src/test/java/javacnn/RunCNN.java +++ b/src/test/java/javacnn/RunCNN.java @@ -17,14 +17,14 @@ public static void main(String[] args) throws IOException, ClassNotFoundExceptio try { - final CNN.LayerBuilder builder = new CNN.LayerBuilder(); - - builder.addLayer(Layer.buildInputLayer(new Layer.Size(28, 28))); - builder.addLayer(Layer.buildConvLayer(6, new Layer.Size(5, 5))); - builder.addLayer(Layer.buildSampLayer(new Layer.Size(2, 2))); - builder.addLayer(Layer.buildConvLayer(12, new Layer.Size(5, 5))); - builder.addLayer(Layer.buildSampLayer(new Layer.Size(2, 2))); - builder.addLayer(Layer.buildOutputLayer(10)); + final CNN.LayerBuilder builder = + new CNN.LayerBuilder() + .addLayer(Layer.buildInputLayer(new Layer.Size(28, 28))) + .addLayer(Layer.buildConvLayer(6, new Layer.Size(5, 5))) + .addLayer(Layer.buildSampLayer(new Layer.Size(2, 2))) + .addLayer(Layer.buildConvLayer(12, new Layer.Size(5, 5))) + .addLayer(Layer.buildSampLayer(new Layer.Size(2, 2))) + .addLayer(Layer.buildOutputLayer(10)); final CNN cnn = new CNN(builder, 50, concurenceRunner); // final CNN cnn = new CNN(builder, 50, new DirectRunner()); From 40c1fcea7fc6e45a1290164c4aae1d986a351053 Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Wed, 21 Feb 2018 15:23:41 +0100 Subject: [PATCH 29/34] Log is dis-/enable-able --- src/main/java/javacnn/cnn/CNN.java | 3 +-- src/main/java/javacnn/util/Log.java | 10 ++++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/main/java/javacnn/cnn/CNN.java b/src/main/java/javacnn/cnn/CNN.java index 71e951d..8f5d84e 100644 --- a/src/main/java/javacnn/cnn/CNN.java +++ b/src/main/java/javacnn/cnn/CNN.java @@ -17,7 +17,7 @@ public class CNN implements Serializable { - private static final long serialVersionUID = 3L; + private static final long serialVersionUID = 5L; private static final double LAMBDA = 0; @@ -108,7 +108,6 @@ public void train(final Dataset trainset, final int iterationCount) { // separate trainset in batches of batchsize ... and round up the result final int epochsNum = (trainset.size() + batchSize - 1) / batchSize; - Log.info(""); Log.info(iteration + "th iter epochsNum:" + epochsNum); int right = 0; diff --git a/src/main/java/javacnn/util/Log.java b/src/main/java/javacnn/util/Log.java index 555ad0e..683cc86 100644 --- a/src/main/java/javacnn/util/Log.java +++ b/src/main/java/javacnn/util/Log.java @@ -5,12 +5,18 @@ public class Log { private static final PrintStream stream = System.out; + private static boolean on = false; + + public static void switchOn() { + on = true; + } + public static void info(String tag, String msg) { - stream.println(tag + "\t" + msg); + if (on) stream.println(tag + "\t" + msg); } public static void info(String msg) { - stream.println(msg); + if (on) stream.println(msg); } } From cc73be0113883cb4822bc923665b5be5ebe294da Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Wed, 21 Feb 2018 20:59:00 +0100 Subject: [PATCH 30/34] release 0.4 --- pom.xml | 2 +- src/main/java/javacnn/cnn/CNN.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 4d3644d..340fb83 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ javacnn javacnn - 0.4-SNAPSHOT + 0.4 Implementation of a CNN in Java diff --git a/src/main/java/javacnn/cnn/CNN.java b/src/main/java/javacnn/cnn/CNN.java index 8f5d84e..631362c 100644 --- a/src/main/java/javacnn/cnn/CNN.java +++ b/src/main/java/javacnn/cnn/CNN.java @@ -479,8 +479,8 @@ public void process(int start, int end) { for (int j = start; j < end; j++) { double[][] sum = null; for (int i = 0; i < lastMapNum; i++) { - double[][] lastMap = lastLayer.getMap(i); - double[][] kernel = layer.getKernel(i, j); + final double[][] lastMap = lastLayer.getMap(i); + final double[][] kernel = layer.getKernel(i, j); if (sum == null) { sum = Util.convnValid(lastMap, kernel); } else { From bc0a926badaba2dd531155af11f779e98b4b2477 Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Wed, 21 Feb 2018 20:59:48 +0100 Subject: [PATCH 31/34] preparation for release 0.5 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 340fb83..c5fff6c 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ javacnn javacnn - 0.4 + 0.5-SNAPSHOT Implementation of a CNN in Java From ca3f5b6ca277fbf80184a79ead1bf10c4f8b00f2 Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Thu, 1 Mar 2018 23:44:10 +0100 Subject: [PATCH 32/34] removed eclipse settings some typos and formatting in readme --- .settings/org.eclipse.jdt.core.prefs | 11 ----------- readme.md | 13 +++++++++---- 2 files changed, 9 insertions(+), 15 deletions(-) delete mode 100644 .settings/org.eclipse.jdt.core.prefs diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs deleted file mode 100644 index 7341ab1..0000000 --- a/.settings/org.eclipse.jdt.core.prefs +++ /dev/null @@ -1,11 +0,0 @@ -eclipse.preferences.version=1 -org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 -org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=1.7 -org.eclipse.jdt.core.compiler.debug.lineNumber=generate -org.eclipse.jdt.core.compiler.debug.localVariable=generate -org.eclipse.jdt.core.compiler.debug.sourceFile=generate -org.eclipse.jdt.core.compiler.problem.assertIdentifier=error -org.eclipse.jdt.core.compiler.problem.enumIdentifier=error -org.eclipse.jdt.core.compiler.source=1.7 diff --git a/readme.md b/readme.md index 9c578d6..2609954 100644 --- a/readme.md +++ b/readme.md @@ -1,10 +1,11 @@ # JavaCNN -A Java implement of Convolutional Neural Network. +A Java implementation of Convolutional Neural Network. This is a fork of https://github.com/BigPeng/JavaCNN refactored for use in production. Original ideas are take from the DeepLearnToolbox (https://github.com/rasmusbergpalm/DeepLearnToolbox). + ## Build a CNN LayerBuilder builder = new LayerBuilder(); @@ -15,16 +16,20 @@ Original ideas are take from the DeepLearnToolbox (https://github.com/rasmusberg builder.addLayer(Layer.buildSampLayer(new Size(2, 2))); builder.addLayer(Layer.buildOutputLayer(10)); CNN cnn = new CNN(builder, 50); - + + ## Run on MNIST dataset For running on MNIST dataset see project https://github.com/ratopi/javacnn.mnist. + ## Source Code Get the source code from github: git clone https://github.com/ratopi/JavaCNN.git -##License - MIT + +## License + +MIT From 5a918bec54a1032ffa1340180e9ef60cf2cfb79b Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Thu, 1 Mar 2018 23:46:16 +0100 Subject: [PATCH 33/34] readme.md ... --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 2609954..104022e 100644 --- a/readme.md +++ b/readme.md @@ -1,7 +1,7 @@ # JavaCNN A Java implementation of Convolutional Neural Network. -This is a fork of https://github.com/BigPeng/JavaCNN refactored for use in production. +This is a mavenized fork of https://github.com/BigPeng/JavaCNN refactored for the intention to use it in productive environments. Original ideas are take from the DeepLearnToolbox (https://github.com/rasmusbergpalm/DeepLearnToolbox). From 3edc1edeac4b0d802aa25197778b37da7f07bdae Mon Sep 17 00:00:00 2001 From: "Ralf Th. Pietsch" Date: Thu, 8 Mar 2018 14:23:29 +0100 Subject: [PATCH 34/34] get it with maven! --- readme.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/readme.md b/readme.md index 104022e..6520b53 100644 --- a/readme.md +++ b/readme.md @@ -6,6 +6,35 @@ This is a mavenized fork of https://github.com/BigPeng/JavaCNN refactored for th Original ideas are take from the DeepLearnToolbox (https://github.com/rasmusbergpalm/DeepLearnToolbox). +## Include in your project + +If you use maven, it's simple: Just add the dependency + + + + + javacnn + javacnn + 0.4 + + + + +and ratopi's repository + + + + ratopi.de releases + http://ratopi.github.io/maven/releases/ + + false + + + + +to your project's pom.xml. + + ## Build a CNN LayerBuilder builder = new LayerBuilder();