001 /*
002 Copyright (c) 1996-2011, Damon Hart-Davis
003 All rights reserved.
004
005 Redistribution and use in source and binary forms, with or without
006 modification, are permitted provided that the following conditions are
007 met:
008
009 * Redistributions of source code must retain the above copyright
010 notice, this list of conditions and the following disclaimer.
011
012 * Redistributions in binary form must reproduce the above copyright
013 notice, this list of conditions and the following disclaimer in the
014 documentation and/or other materials provided with the
015 distribution.
016
017 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
018 IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
019 TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
020 PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
021 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
022 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
023 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
024 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
025 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
026 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
027 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
028 */
029
030 package org.hd.d.pg2k.ai.scorer;
031
032 import java.awt.Point;
033 import java.awt.image.BufferedImage;
034 import java.awt.image.RenderedImage;
035 import java.io.ByteArrayInputStream;
036 import java.io.IOException;
037 import java.io.InterruptedIOException;
038 import java.util.Arrays;
039 import java.util.HashSet;
040 import java.util.List;
041 import java.util.Random;
042 import java.util.Set;
043 import java.util.concurrent.Callable;
044 import java.util.concurrent.atomic.AtomicReference;
045
046 import org.hd.d.pg2k.svrCore.ExhibitStaticAttr;
047 import org.hd.d.pg2k.svrCore.ExhibitThumbnails;
048 import org.hd.d.pg2k.svrCore.ImageUtils;
049 import org.hd.d.pg2k.svrCore.MemoryTools;
050 import org.hd.d.pg2k.svrCore.MemoryTools.SoftReferenceMap;
051 import org.hd.d.pg2k.svrCore.Name;
052 import org.hd.d.pg2k.svrCore.MIME.AbstractImageHandler;
053 import org.hd.d.pg2k.svrCore.MIME.ExhibitMIME;
054 import org.hd.d.pg2k.svrCore.MIME.ExhibitMIME.ExhibitTypeParameters;
055 import org.hd.d.pg2k.svrCore.datasource.SimpleExhibitPipelineIF;
056
057
058 /**Base interface to compute the score and confidence for a 2D still image.
059 * All methods in this interface are guaranteed to be "safe"
060 * in so far as they will complete in "reasonable" time
061 * with reasonable heap memory (and other resource, eg stack)
062 * and without doing anything that wouldn't be allowed in
063 * a minimal Applet/JWS sandbox.
064 * <p>
065 * All classes implementing this interface should be completely thread-safe
066 * in their implementation of the computeScoreAndConfidence() method,
067 * and preferably purely functional (no visible side-effects),
068 * with as many concurrent threads as required safely doing separate computations
069 * in any one instance.
070 * <p>
071 * Classes implementing this interface should, where possible,
072 * do their calculations using integer arithmetic,
073 * since FPUs to support float/double calculations may be a scarce resource
074 * on newer highly-threaded CPUs such as Sun's Niagara.
075 * <p>
076 * TODO: At least tie the cache to the AEP/pipeline, not static.
077 */
078 public abstract class AbstractImgScorer extends AbstractScorer implements ScorerIF
079 {
080 /**Create simple non-parameterised instance. */
081 public AbstractImgScorer() { }
082
083 /**Create parameterised version. */
084 public AbstractImgScorer(final String nameAndParameters)
085 { super(nameAndParameters); }
086
087 /**Create parameterised version. */
088 public AbstractImgScorer(final String baseName, final List<ScorerParam> parameters)
089 { super(baseName, parameters); }
090
091 /**Call the factory to retrieve a still image, handling wrapped IOExceptions and timeouts specially.
092 * @throws IOException if the factory threw IOException
093 * @throws IllegalStateException if the factory throws something unrecognised
094 */
095 protected static RenderedImage callStillImageFactory(final Callable<RenderedImage> stillImageFactory)
096 throws IOException
097 {
098 if(stillImageFactory == null) { throw new IllegalArgumentException(); }
099 try { return(stillImageFactory.call()); }
100 catch(final IOException e) { throw e; }
101 catch(final Exception e) { throw new IllegalStateException("unable to retrieve image", e); }
102 }
103
104 /**Compute score [-1,+1] and confidence[0,+1] for given image; never null.
105 * This routine must NOT alter the image.
106 * <p>
107 * Note that the underlying factory may legitimately throw IOException, especially InterruptedIOException,
108 * in case of transient problems that indicate that a retry should probably be attempted later.
109 * <p>
110 * This is made public mainly so as to facilitate testing.
111 *
112 * @param key unique key identifying image for cacheing intermediate results for example;
113 * if null then this will not be cached
114 * @param stillImageFactory produces on demand a non-null still 2D image or null if not possible;
115 * if greater than 256x256 pixels then will be internally scaled; never null
116 */
117 public abstract ScoreAndConf computeScoreAndConfidenceOnStillImage(Object key, Callable<RenderedImage> stillImageFactory) throws IOException;
118
119 /**Private static 'soft' cache from unique exhibit identifier to expanded (std) thumbnail image.
120 * This exists to avoid having to unpack thumbnail images repeatedly.
121 * <p>
122 * The (big) expanded images are held via SoftReferences to allow them to be ditched
123 * in case of memory shortage, and when individual thumbnails stop being used,
124 * and we don't cache new entries unless there is lots of free memory.
125 * <p>
126 * The cache key is the (full) exhibit name plus other static attributes
127 * on the basis that thumbnails hardly ever change once created.
128 * This also means that we will stop hitting any thumbnail caches
129 * in the data pipeline once we have fetched a thumbnail once.
130 * <p>
131 * This cache is thread-safe and highly concurrent.
132 * <p>
133 * This cache is shared between all derived image Scorer classes.
134 * <p>
135 * TODO: Deal with the case when a thumbnail IS changed/improved.
136 */
137 private static final SoftReferenceMap<Object, BufferedImage> _tnBICache = SoftReferenceMap.<Object, BufferedImage>create(16, true, "_tnBICache");
138
139 /**Interface to accept or reject putative samples in getSamplePoints(). */
140 public static interface ARGBPixelFilter
141 {
142 /**Return true to accept the offered ARGB pixel value and include it amongst the samples. */
143 public boolean accept(int pixelARGB);
144 }
145
146 /**Filter to reject mainly-transparent ARGB samples. */
147 public static final ARGBPixelFilter rejectMainlyTransparentPoints = new ARGBPixelFilter() {
148 /**Returns false for a mainly-transparent point to exclude it from the samples. */
149 public boolean accept(final int pixelARGB)
150 {
151 // Accept mainly-opaque pixels.
152 // alpha == 0xff ==> opaque (we can use it), alpha == 0 ==> transparent/unusable.
153 return((pixelARGB >>> 24) >= 0x80);
154 }
155 };
156
157 /**Collect a set of ARGB samples from the image.
158 * This may return less than the number of samples requested,
159 * eg because the image does not have that many pixels
160 * or because we are rejecting 'transparent' points.
161 * <p>
162 * All samples returned are from unique pixels in the source image.
163 * <p>
164 * If the number of points requested greater than or equal to the number of image pixels
165 * then all pixels are 'sampled' and returned,
166 * else a pseudo-random (but consistent) sample set is returned.
167 * <p>
168 * The order of sample pixels is explicitly undefined
169 * since each sample should be treated independently of its position in the sample set.
170 * <p>
171 * This is made public mainly to assist with testing.
172 *
173 * @param stillImage still image (eg thumbnail) to sample pixels from; never null nor zero-sized
174 * @param maxSamplePoints maximum number of samples to collect; strictly positive
175 * @param rejectMainlyTransparentPoints if true then omit mainly transparent points
176 */
177 public static int[] getSamplePoints(final RenderedImage stillImage,
178 final int maxSamplePoints,
179 final ARGBPixelFilter filter)
180 {
181 if(stillImage == null) { throw new IllegalArgumentException(); }
182 if(maxSamplePoints <= 0) { throw new IllegalArgumentException(); }
183
184 final int h = stillImage.getHeight();
185 final int w = stillImage.getWidth();
186
187 // Don't try to score strange (eg zero-sized) images.
188 if((h < 1) || (w < 1)) { throw new IllegalArgumentException(); }
189
190 final long availablePixels = h * w;
191 assert(availablePixels > 0);
192
193 final boolean smallImage = (availablePixels <= maxSamplePoints);
194 final int maxSampleablePoints = smallImage ? (int) availablePixels : maxSamplePoints;
195
196 // final List<Integer> results = new ArrayList<Integer>(maxSampleablePoints);
197 final int results[] = new int[maxSampleablePoints];
198 int resultCount = 0;
199
200 final int minX = stillImage.getMinX();
201 final int minY = stillImage.getMinY();
202
203 // For sampling we'll need a buffered image;
204 // so we copy the input image if need be.
205 final BufferedImage bi = ImageUtils.convertRenderedImageToBufferedImage(stillImage);
206
207 // For small images sample all pixels.
208 if(smallImage)
209 {
210 for(int x = w; --x >= 0; )
211 {
212 for(int y = h; --y >= 0; )
213 {
214 // Get sample value.
215 final int argb = bi.getRGB(x + minX, y + minY);
216
217 // Reject/filter pixels if requested.
218 if((filter != null) && !filter.accept(argb)) { continue; }
219
220 // Capture sample.
221 results[resultCount++] = argb;
222 }
223 }
224 }
225 // For reasonable-size (not tiny) images,
226 // we sample pixels in a pseudo-random pattern based on image size/shape.
227 else
228 {
229 // Make the sample pattern (random x,y point stream) dependent on image size/shape
230 // to help circumvent some systematic sample pattern problems.
231 // We do not make the sample pattern/seed dependent on sample size
232 // so that as we take more samples all previous values are included.
233 final Random r = new Random(w ^ (((long)h) << 16));
234
235 // To allow in the main for non-square partially-transparent images
236 // but also for images close in pixels to our threshold size
237 // (so that we capture as many of the samplable points as reasonable possible)
238 // we are prepared to attempt the sampling operation
239 // more than the target number of samples
240 // (though we stop as soon as our target sample count is reached).
241 // We also reject would-be duplicate sampling of the same point.
242 assert((new Point(0, 0)).equals(new Point(0, 0)));
243 final Set<Point> alreadySampled = new HashSet<Point>(1 + 2*maxSampleablePoints);
244 for(int i = 2*maxSampleablePoints; (--i >= 0) && (alreadySampled.size() < maxSampleablePoints); )
245 {
246 // Choose a (new) random point and take a sample.
247 final int x = r.nextInt(w);
248 final int y = r.nextInt(h);
249 final Point p = new Point(x, y);
250 if(!alreadySampled.add(p)) { continue; } // Skip if point not new...
251 //System.out.println("[Sample point: "+p+".]");
252
253 // Get sample value.
254 final int argb = bi.getRGB(x + minX, y + minY);
255
256 // Reject/filter pixels if requested.
257 if((filter != null) && !filter.accept(argb)) { continue; }
258
259 // Capture sample.
260 results[resultCount++] = argb;
261 }
262 }
263
264 assert(resultCount <= maxSamplePoints);
265
266 // If all requested samples were generated then return as-is...
267 if(resultCount == results.length) { return(results); }
268
269 // Else trim to size and return.
270 return(Arrays.copyOf(results, resultCount));
271 }
272
273 /**Implement core/generic compute method specially for image types.
274 * For any suitable 2D still image type that has a standard thumbnail,
275 * this uses calls the computeScoreAndConfidence(RenderedImage) method
276 * with that thumbnail.
277 * <p>
278 * We assume the standard thumbnail to be representative of the full image
279 * (and usually good enough for a human to make a judgement from)
280 * without requiring all the system resources of the full image.
281 * <p>
282 * For non-image types (or images with no thumbnails possible)
283 * this returns (0,0) to indicate that it does not have a view.
284 *
285 * @throws IOException in case of I/O difficulties,
286 * or when a thumbnail is not currently available but may be later (upon retry)
287 */
288 public ScoreAndConf computeScoreAndConfidence(final SimpleExhibitPipelineIF dataSource,
289 final Name.ExhibitFull exhibitName)
290 throws IOException
291 {
292 // Compute a new key uniquely identifying this (image) exhibit
293 // from the exhibit static attrs (which should catch most exhibit changes and maybe thumbnail updates).
294 // FIXME: This assumes that the static attrs uniquely identifies the exhibit under all realistic circumstances.
295 final ExhibitStaticAttr esa = dataSource.getAllExhibitImmutableData(-1).getStaticAttr(exhibitName);
296 if(null == esa) { throw new IOException("exhibit not available: "+exhibitName); }
297 final Object key = esa.getKey();
298
299 // We can choose not to cache the (probably large) BufferedImage that we produce
300 // for example if we know that the client is going to cache a version of it
301 // and/or the system does not have loads of memory free.
302 // But we're always happy to use a BufferedImage already cached.
303 final boolean dontCache = (this instanceof AbstractImgSampleScorer) ||
304 !MemoryTools.lotsFree();
305
306 // Create the means to generate the image on demand.
307 final Callable<RenderedImage> stillImageFactory = new Callable<RenderedImage>() {
308 /**Return the image, or null if not possible (eg because the exhibit is not an image). */
309 public RenderedImage call() throws Exception
310 {
311 // Check for a cached image.
312 BufferedImage tnbi = (key == null) ? null : _tnBICache.get(key);
313
314 // If we don't have a cached value
315 // then see if one can be fetched and cached...
316 if(tnbi == null)
317 {
318 final ExhibitTypeParameters exhibitType = ExhibitMIME.getInputFileType(exhibitName);
319 // If this does look like an image type from which we can make thumbnails,
320 // then return null.
321 if((exhibitType == null) ||
322 !exhibitType.canPossiblyCreateThumbnailOfSameMIMEType() ||
323 !(exhibitType.handler instanceof AbstractImageHandler))
324 { return(null); }
325
326 // Try to fetch the (compressed) thumbnail data.
327 final ExhibitThumbnails tns = dataSource.getThumbnails(exhibitName, true);
328
329 // Where thumbnails are not currently available,
330 // but may become so in future,
331 // we throw an exception.
332 if(tns == null) { throw new InterruptedIOException("thumbnail not currently available: may retry: " + exhibitName); }
333
334 // If no standard thumbnail is available (or if it cannot be decoded)
335 // then return null.
336 if(tns.getStandard() == null) { return(null); }
337
338 // Set to the decoded image if successful.
339 final AtomicReference<BufferedImage> decodedImage = new AtomicReference<BufferedImage>();
340
341 // Decode image using memory handler to regulate peak memory use.
342 final Runnable r = new Runnable()
343 {
344 public final void run()
345 {
346 try
347 {
348 decodedImage.set(exhibitType.handler
349 .decodeImage(new ByteArrayInputStream(tns
350 .getStandard().toByteArrray())));
351 }
352 catch(final Exception e) { /* Absorb exception: we can retry later. */ }
353 }
354 };
355 // Estimate memory use based on 4 bytes per pixel for a maximum size std thumbnail.
356 // In this case we won't recover the memory when processing finished (we cache the image)
357 // but it should serve to limit the allowed concurrency of this image decoding.
358 final int estimatedMinBytesRequired = 4 * ExhibitThumbnails.STD_STATIC_IMAGE_TN_LDIM_PX * ExhibitThumbnails.STD_STATIC_IMAGE_TN_LDIM_PX;
359 MemoryTools.runMemoryIntensiveOperation(r, false, estimatedMinBytesRequired);
360 tnbi = decodedImage.get();
361 // If no image could be decoded
362 // then return null.
363 if(tnbi == null) { return(null); }
364
365 // Cache this (non-null) expanded image (usually).
366 if((key != null) && !dontCache)
367 {
368 _tnBICache.put(key, tnbi);
369 assert(_tnBICache.get(key) != null) : "Our value (or possibly a newer one) must now be present";
370 }
371 }
372
373 // Return the image.
374 return(tnbi);
375 }
376 };
377
378 // Score the thumbnail image.
379 return(computeScoreAndConfidenceOnStillImage(key, stillImageFactory));
380 }
381 }