001 /*
002 Copyright (c) 1996-2012, Damon Hart-Davis
003 All rights reserved.
004
005 Redistribution and use in source and binary forms, with or without
006 modification, are permitted provided that the following conditions are
007 met:
008
009 * Redistributions of source code must retain the above copyright
010 notice, this list of conditions and the following disclaimer.
011
012 * Redistributions in binary form must reproduce the above copyright
013 notice, this list of conditions and the following disclaimer in the
014 documentation and/or other materials provided with the
015 distribution.
016
017 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
018 IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
019 TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
020 PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
021 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
022 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
023 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
024 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
025 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
026 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
027 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
028 */
029
030 /*
031 * Created by IntelliJ IDEA.
032 * User: Administrator
033 * Date: 28-Dec-02
034 * Time: 22:24:51
035 */
036 package org.hd.d.pg2k.test.dev;
037
038 import java.awt.image.BufferedImage;
039 import java.awt.image.RenderedImage;
040 import java.io.ByteArrayInputStream;
041 import java.util.Collections;
042 import java.util.HashMap;
043 import java.util.List;
044 import java.util.Map;
045 import java.util.Random;
046 import java.util.TreeSet;
047 import java.util.concurrent.Callable;
048
049 import junit.framework.TestCase;
050
051 import org.hd.d.pg2k.ai.scorer.AbstractScorer;
052 import org.hd.d.pg2k.ai.scorer.ScoreAndConf;
053 import org.hd.d.pg2k.ai.scorer.ScorerCacheIF;
054 import org.hd.d.pg2k.ai.scorer.ScorerCacheImpl;
055 import org.hd.d.pg2k.ai.scorer.ScorerCreator;
056 import org.hd.d.pg2k.ai.scorer.ScorerIF;
057 import org.hd.d.pg2k.ai.scorer.fixed.FixedScore;
058 import org.hd.d.pg2k.ai.scorer.fixed.NoConfidence;
059 import org.hd.d.pg2k.ai.scorer.parameterised.LocalSampler;
060 import org.hd.d.pg2k.ai.scorer.parameterised.SimpleExposure;
061 import org.hd.d.pg2k.svrCore.AbstractSimpleLogger;
062 import org.hd.d.pg2k.svrCore.AllExhibitProperties;
063 import org.hd.d.pg2k.svrCore.ExhibitStaticAttr;
064 import org.hd.d.pg2k.svrCore.ExhibitThumbnails;
065 import org.hd.d.pg2k.svrCore.Name;
066 import org.hd.d.pg2k.svrCore.MIME.ExhibitMIME;
067 import org.hd.d.pg2k.svrCore.MIME.ExhibitMIME.ExhibitTypeParameters;
068 import org.hd.d.pg2k.svrCore.datasource.ExhibitDataFileSource;
069 import org.hd.d.pg2k.svrCore.props.GenProps;
070
071 /**Tests of AI-based image scoring.
072 */
073 public final class ScorerTest extends TestCase
074 {
075 public ScorerTest(final String name)
076 {
077 super(name);
078 }
079
080 // /**Do any setup needed for the tests. */
081 // protected void setUp()
082 // {
083 // }
084
085 // /**Do any clearup needed after the tests. */
086 // protected void tearDown()
087 // {
088 // // cleanup code
089 // }
090
091 /**Trivial RenderedImage factory. */
092 private static Callable<RenderedImage> makeRIFactory(final RenderedImage ri)
093 { return(new Callable<RenderedImage>() { public RenderedImage call() { return(ri); } }); }
094
095 /**Test that simplest "fixed" image scorers return sensible/expected (fixed) values. */
096 public static final void testFixedImgScorers()
097 throws Exception
098 {
099 // Construct simple instances of specific Scorers here.
100 final NoConfidence noConfidence = new NoConfidence();
101 final SimpleExposure simpleExposure = (new SimpleExposure());
102 final LocalSampler localSampler = new LocalSampler();
103 final ScorerIF scorers[] = { noConfidence, simpleExposure, localSampler };
104
105 final ScoreAndConf sc00 = new ScoreAndConf(0, 0);
106 // Should always return a fixed (0-value, 0-confidence) pair.
107 assertEquals(noConfidence.computeScoreAndConfidence(null, (Name.ExhibitFull)null), sc00);
108
109 // Attempt to do simple exposure score on null factory should throw IAE.
110 try
111 {
112 simpleExposure.computeScoreAndConfidenceOnStillImage(null, null);
113 fail("A null image factory should have been rejected");
114 }
115 catch(final IllegalArgumentException e) { /* Correctly rejected null/bad image. */ }
116
117 // Attempt to do simple exposure on null image should return 'NO_OPINION'.
118 assertSame(ScoreAndConf.NO_OPINION, simpleExposure.computeScoreAndConfidenceOnStillImage(null, makeRIFactory(null)));
119
120 if(!Main.isAccessToFilesystem())
121 {
122 System.err.println("WARNING: no access to filesystem: skipping some image-scorer tests...");
123 return;
124 }
125
126 // Try to load the exhibit data, including the specific test samples that we use.
127 final ExhibitDataFileSource efds = new ExhibitDataFileSource(null);
128 final AllExhibitProperties aep = efds.getAllExhibitProperties(-1);
129 assert(aep.aeid.length > 0);
130
131 // Short names of specific image exhibits that we use.
132 final String snWD1 = "water-drops-1-AJHD.jpg"; // Good: very popular water-drop image...
133 final String snWD7 = "water-drops-7-AJHD.jpg"; // Good: very popular water-drop image...
134 final String sn1pxTransparent = "transparent-single-pixel-ANON.gif"; // Poor: "info-free" image.
135
136 // Check that all test images are present,
137 // and make a map from short name to RenderedImage.
138 final AllExhibitProperties.ExhibitDataSource eds = efds.makeExhibitDataSource();
139 final String allTestImages[] = { snWD1, snWD7, sn1pxTransparent };
140 final Map<String, BufferedImage> images = new HashMap<String, BufferedImage>(1+2*allTestImages.length);
141 final Map<String, BufferedImage> stdTns = new HashMap<String, BufferedImage>(1+2*allTestImages.length);
142 final List<Name.ExhibitFull> allExhibitsSorted = aep.aeid.getAllExhibitNamesSorted();
143 final String aesAsString = allExhibitsSorted.toString();
144 for(final String sn : allTestImages)
145 {
146 final Name.ExhibitFull fullName = aep.aeid.getFullName(sn);
147 assertNotNull("Specific test images must be present: " + aesAsString, fullName);
148 final ExhibitTypeParameters exhibitType = ExhibitMIME.getInputFileType(fullName);
149 final ExhibitStaticAttr esa = aep.aeid.getStaticAttr(fullName);
150 final BufferedImage bi = exhibitType.handler.decodeImage(eds.getInputStream(esa));
151 assertNotNull("Decoded exhibit image must not be null", bi);
152 images.put(sn, bi);
153 if(exhibitType.handler.canMakeThumbnails())
154 {
155 final ExhibitThumbnails tns = exhibitType.handler.makeThumbnails(esa, eds, aep, true);
156 if((tns != null) && (tns.getStandard() != null))
157 {
158 final BufferedImage tnbi = exhibitType.handler.decodeImage(new ByteArrayInputStream(tns.getStandard().toByteArrray()));
159 if(tnbi != null) { stdTns.put(sn, tnbi); }
160 }
161 }
162 }
163
164 // Output should be (0, 0) for 1-pixel transparent (source) image
165 // for simple exposure measure.
166 assertEquals(sc00, simpleExposure.computeScoreAndConfidenceOnStillImage(null, makeRIFactory(images.get(sn1pxTransparent))));
167 // Output should be "good" for selected example "good" images.
168 assertEquals(Boolean.TRUE, simpleExposure.computeScoreAndConfidenceOnStillImage(null, makeRIFactory(images.get(snWD1))).isGood());
169 assertEquals(Boolean.TRUE, simpleExposure.computeScoreAndConfidenceOnStillImage(null, makeRIFactory(images.get(snWD7))).isGood());
170
171 // Output should be (0, 0) for 1-pixel transparent (source) image
172 // for default local sampler measure.
173 assertEquals(sc00, localSampler.computeScoreAndConfidenceOnStillImage(null, makeRIFactory(images.get(sn1pxTransparent))));
174 // Output should be "good" for selected example "good" images.
175 assertEquals(Boolean.TRUE, localSampler.computeScoreAndConfidenceOnStillImage(null, makeRIFactory(images.get(snWD1))).isGood());
176 // assertEquals(Boolean.TRUE, localSampler.computeScoreAndConfidence(images.get(snWD7)).isGood());
177
178 // We can try running all scorers on the underlying images
179 // and the (larger) thumbnails where available,
180 // at least to make sure that they don't go bang.
181 for(final String sn : allTestImages)
182 {
183 // Output should always be the same (0, 0) for the "NoConfidence" scorer.
184 // assertEquals(noConfidence.computeScoreAndConfidence(images.get(sn)), sc00);
185 final BufferedImage tn = stdTns.get(sn);
186 // assertEquals(noConfidence.computeScoreAndConfidence(tn), sc00);
187
188 // Check that nothing bad happens with "SimpleExposure" scorer.
189 final ScoreAndConf seSC = simpleExposure.computeScoreAndConfidenceOnStillImage(null, makeRIFactory(images.get(sn)));
190 Main.getOut().println("SimpleExposure score for "+sn+" "+seSC);
191 assertNotNull(seSC);
192 if(tn != null)
193 {
194 final ScoreAndConf seSCtn = simpleExposure.computeScoreAndConfidenceOnStillImage(null, makeRIFactory(tn));
195 Main.getOut().println("SimpleExposure score for "+sn+" thumbnail "+seSCtn);
196 assertNotNull(seSCtn);
197 }
198
199 // Check that nothing bad happens with "LocalSampler" scorer.
200 final ScoreAndConf lsSC = localSampler.computeScoreAndConfidenceOnStillImage(null, makeRIFactory(images.get(sn)));
201 Main.getOut().println("LocalSampler score for "+sn+" "+lsSC);
202 assertNotNull(lsSC);
203 if(tn != null)
204 {
205 final ScoreAndConf lsSCtn = localSampler.computeScoreAndConfidenceOnStillImage(null, makeRIFactory(tn));
206 Main.getOut().println("LocalSampler score for "+sn+" thumbnail "+lsSCtn);
207 assertNotNull(lsSCtn);
208 }
209 }
210
211 // Check that we can (safely) perturb all our scorers.
212 for(final ScorerIF sc : scorers)
213 {
214 final String perturbed = sc.createPerturbedVariant().getNameAndParameters();
215 assertTrue("Perturbed Scorer name-and-parameter always starts with base name",
216 perturbed.startsWith(sc.getBaseName()));
217 Main.getOut().println("[Sample Scorer name: "+perturbed+".]");
218 }
219
220 // Now run all tests on all exhibits for which we can generate a large thumbnail
221 // to ensure that we get no crashes.
222 // This is optional, and usually disabled for speed.
223 if(false)
224 {
225 Main.getOut().println("Testing on full exhibit set...");
226 for(final ExhibitStaticAttr esa : new TreeSet<ExhibitStaticAttr>(aep.aeid.getAllStaticAttrs()))
227 {
228 final Name.ExhibitFull fullName = esa.getExhibitFullName();
229 final ExhibitTypeParameters exhibitType = ExhibitMIME.getInputFileType(fullName);
230 if(!exhibitType.canPossiblyCreateThumbnailOfSameMIMEType()) { continue; }
231 final BufferedImage bi;
232 try { bi = exhibitType.handler.decodeImage(eds.getInputStream(esa)); }
233 catch(final Exception e) { Main.getErr().println("WARNING: error decoding image "+fullName+": " + e.getMessage()); continue; } // Absorb decoding errors.
234 if(bi == null) { continue; }
235 final ExhibitThumbnails tns = exhibitType.handler.makeThumbnails(esa, eds, aep, true);
236 if((tns == null) || (tns.getStandard() == null)) { continue; }
237 final BufferedImage tnbi = exhibitType.handler.decodeImage(new ByteArrayInputStream(tns.getStandard().toByteArrray()));
238 if(tnbi == null) { continue; }
239
240 // Output should always be the same (0, 0) for the "NoConfidence" scorer.
241 // assertEquals(noConfidence.computeScoreAndConfidence(tnbi), sc00);
242
243 // Output should be non-null and there should be no failures for "SimpleExposure".
244 final ScoreAndConf seSCtn = simpleExposure.computeScoreAndConfidenceOnStillImage(null, makeRIFactory(tnbi));
245 Main.getOut().println("SimpleExposure score for "+fullName+" thumbnail "+seSCtn);
246 assertNotNull(seSCtn);
247 }
248 }
249 }
250
251 /**Test the Scorer marking algorithm against calibration data.
252 */
253 public void testCalibration()
254 throws Exception
255 {
256 // With no inputs our calibration should definitely return no confidence.
257 assertEquals("With no inputs, the output must be zero confidence",
258 ScoreAndConf.NO_OPINION, ScorerCreator.computeWeighting(
259 Collections.<Name.ExhibitShort, ScoreAndConf>emptyMap(),
260 Collections.<Name.ExhibitShort, ScoreAndConf>emptyMap()));
261
262 final ScoreAndConf sacMAXMAX = new ScoreAndConf(ScoreAndConf.MAX, ScoreAndConf.MAX);
263 final Map<Name.ExhibitShort, ScoreAndConf> singletonMapMAXMAX = Collections.<Name.ExhibitShort, ScoreAndConf>singletonMap(Name.ExhibitFull.create("a/a-A.a").getShortName(), sacMAXMAX);
264 // With identical non-zero-confidence inputs, the calibration should be perfect.
265 assertEquals("With identical inputs, the output must be full confidence",
266 sacMAXMAX, ScorerCreator.computeWeighting(
267 singletonMapMAXMAX,
268 singletonMapMAXMAX));
269 final ScoreAndConf sacMINMAX = new ScoreAndConf(-ScoreAndConf.MAX, ScoreAndConf.MAX);
270 final Map<Name.ExhibitShort, ScoreAndConf> singletonMapMINMAX = Collections.<Name.ExhibitShort, ScoreAndConf>singletonMap(Name.ExhibitFull.create("a/a-A.a").getShortName(), sacMINMAX);
271 // With identical non-zero-confidence inputs, the calibration should be perfect.
272 assertEquals("With identical inputs, the output must be full confidence",
273 sacMAXMAX, ScorerCreator.computeWeighting(
274 singletonMapMINMAX,
275 singletonMapMINMAX));
276
277 // Opposite calibration values and Scorer results should give maximum -ve mark.
278 assertEquals("With opposite inputs, the output must be full confidence but -MAX score",
279 sacMINMAX, ScorerCreator.computeWeighting(
280 singletonMapMINMAX,
281 singletonMapMAXMAX));
282 assertEquals("With opposite inputs, the output must be full confidence but -MAX score",
283 sacMINMAX, ScorerCreator.computeWeighting(
284 singletonMapMAXMAX,
285 singletonMapMINMAX));
286
287 // TODO: Test calibration process.
288 }
289
290 /**Test some features of ScoreAndConf.
291 * In particular test conversion to and from the String representation.
292 */
293 public void testScoreAndConf()
294 {
295 assertEquals("Must be able to pass simple 'no confidence' value correctly",
296 ScoreAndConf.NO_OPINION, ScoreAndConf.fromString(ScoreAndConf.NO_OPINION.toString()));
297 // Try a number of random values.
298 for(int i = 100; --i >= 0; )
299 {
300 final int score = -ScoreAndConf.MAX + rnd.nextInt(2*ScoreAndConf.MAX+1);
301 final int conf = rnd.nextInt(ScoreAndConf.MAX+1);
302 final ScoreAndConf sac = new ScoreAndConf(score, conf);
303 assertEquals("must be able to parse any ScoreAndConf toString() output correctly",
304 sac, ScoreAndConf.fromString(sac.toString()));
305 }
306 }
307
308 /**Test that we can safely canonicalise various Scorers values.
309 */
310 public static void testCanonicalisation()
311 {
312 // Test Scorer name-and-parameter values that should canonicalise to themselves.
313 final String testScorersIdentity[] =
314 {
315 (new NoConfidence()).getNameAndParameters(), // No parameters...
316 (new SimpleExposure()).createPerturbedVariant().getNameAndParameters(),
317 (new LocalSampler()).createPerturbedVariant().getNameAndParameters(),
318 (new LocalSampler()).createPerturbedVariant().getNameAndParameters(),
319 (new LocalSampler()).createPerturbedVariant().getNameAndParameters(),
320 (new LocalSampler()).createPerturbedVariant().getNameAndParameters(),
321 (new LocalSampler()).createPerturbedVariant().getNameAndParameters(),
322 (new LocalSampler()).createPerturbedVariant().getNameAndParameters(),
323 (new LocalSampler()).createPerturbedVariant().getNameAndParameters(),
324 (new LocalSampler()).createPerturbedVariant().getNameAndParameters(),
325 (new LocalSampler()).createPerturbedVariant().getNameAndParameters(),
326 (new LocalSampler()).createPerturbedVariant().getNameAndParameters(),
327 };
328
329 // Used for converting from names to instances.
330 final ScorerCacheIF cache = new ScorerCacheImpl(new SimpleCacheTest.DummyDataSource(new AllExhibitProperties(), new GenProps()),
331 new AbstractSimpleLogger() {
332 public void log(final String message) { Main.getOut().println(message); }
333 });
334
335 for(final String snp : testScorersIdentity)
336 {
337 final ScorerIF sc = cache.getScorerInstance(snp);
338 assertNotNull("must be able to create Scorer instance", sc);
339 final String canonicalised = AbstractScorer.canonicalise(sc);
340 assertEquals("canonicalisation must not change this Scorer values", snp, canonicalised);
341 }
342 }
343
344 /**Test that our similarity measures for parameters and Scorers are working.
345 */
346 public static void testSimilarity()
347 {
348 // TODO: test enum and int parameter instances...
349
350 assertTrue("Identical (fixed, parameterless) Scorers must be 'very similar'",
351 AbstractScorer.verySimilar(new NoConfidence(), new NoConfidence()));
352 assertTrue("Identical (default, parameterisable) Scorers must be 'very similar'",
353 AbstractScorer.verySimilar(new FixedScore(), new FixedScore()));
354
355 assertTrue("Close (non-default, parameterisable) Scorers must be 'very similar'",
356 AbstractScorer.verySimilar(new FixedScore("FixedScore:resultScore=71"), new FixedScore("FixedScore:resultScore=72")));
357 assertFalse("Distant (non-default, parameterisable) Scorers must NOT be 'very similar'",
358 AbstractScorer.verySimilar(new FixedScore("FixedScore:resultScore=7"), new FixedScore("FixedScore:resultScore=72")));
359 }
360
361
362 /**Private source of OK pseudo-random numbers. */
363 private static final Random rnd = new Random();
364 }