001 /*
002 Copyright (c) 1996-2012, Damon Hart-Davis
003 All rights reserved.
004
005 Redistribution and use in source and binary forms, with or without
006 modification, are permitted provided that the following conditions are
007 met:
008
009 * Redistributions of source code must retain the above copyright
010 notice, this list of conditions and the following disclaimer.
011
012 * Redistributions in binary form must reproduce the above copyright
013 notice, this list of conditions and the following disclaimer in the
014 documentation and/or other materials provided with the
015 distribution.
016
017 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
018 IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
019 TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
020 PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
021 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
022 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
023 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
024 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
025 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
026 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
027 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
028 */
029 package org.hd.d.pg2k.svrCore;
030
031 import java.io.ByteArrayOutputStream;
032 import java.io.File;
033 import java.io.FilterOutputStream;
034 import java.io.IOException;
035 import java.io.InputStream;
036 import java.io.InterruptedIOException;
037 import java.io.ObjectOutputStream;
038 import java.io.OutputStream;
039 import java.io.Serializable;
040 import java.net.HttpURLConnection;
041 import java.net.URI;
042 import java.security.AccessControlException;
043 import java.security.MessageDigest;
044 import java.security.NoSuchAlgorithmException;
045 import java.util.AbstractList;
046 import java.util.ArrayList;
047 import java.util.Arrays;
048 import java.util.Collections;
049 import java.util.Comparator;
050 import java.util.Date;
051 import java.util.Enumeration;
052 import java.util.HashMap;
053 import java.util.HashSet;
054 import java.util.Iterator;
055 import java.util.List;
056 import java.util.Locale;
057 import java.util.Map;
058 import java.util.Queue;
059 import java.util.RandomAccess;
060 import java.util.ResourceBundle;
061 import java.util.Set;
062 import java.util.SortedSet;
063 import java.util.StringTokenizer;
064 import java.util.TreeSet;
065 import java.util.concurrent.ArrayBlockingQueue;
066 import java.util.concurrent.BlockingQueue;
067 import java.util.concurrent.ConcurrentHashMap;
068 import java.util.concurrent.ConcurrentMap;
069 import java.util.concurrent.ExecutionException;
070 import java.util.concurrent.Future;
071 import java.util.concurrent.LinkedBlockingQueue;
072 import java.util.concurrent.atomic.AtomicInteger;
073 import java.util.concurrent.atomic.AtomicReference;
074
075 import net.contrapunctus.lzma.LzmaInputStream;
076 import net.contrapunctus.lzma.LzmaOutputStream;
077
078 import org.apache.tools.bzip2.CBZip2InputStream;
079 import org.apache.tools.bzip2.CBZip2OutputStream;
080 import org.hd.d.pg2k.svrCore.Tuple.Pair;
081 import org.hd.d.pg2k.svrCore.collections.LRUMapAutoSizeForHitRate;
082 import org.hd.d.pg2k.svrCore.props.GenProps;
083 import org.hd.d.pg2k.svrCore.props.GenPropsGenNames;
084 import org.hd.d.pg2k.svrCore.props.LocalProps;
085
086 import ORG.hd.d.IsDebug;
087
088 /**Some general utility methods of use throughout the application.
089 */
090 public final class GenUtils
091 {
092 /**Prevent construction of an instance. */
093 private GenUtils() { }
094
095
096 /**Simple logger instance to log to System.out. */
097 @Deprecated
098 public static final SimpleLoggerIF systemOutLogger = new AbstractSimpleLogger()
099 {
100 public final void log(final String message)
101 { System.out.println(message); }
102 };
103
104 /**Simple logger instance to log to System.err. */
105 @Deprecated
106 public static final SimpleLoggerIF systemErrLogger = new AbstractSimpleLogger()
107 {
108 public final void log(final String message)
109 { System.err.println(message); }
110 };
111
112 /**Simple logger instance to throw away log output. */
113 public static final SimpleLoggerIF nullLogger = new SimpleLoggerIF()
114 {
115 public final void log(final String message) { /* Discard output. */ }
116 public final void log(final String message, final Throwable t) { /* Discard output. */ }
117 };
118
119
120 /**Attach run-time RandomAccess marker to AbstractList base. */
121 public static abstract class AbstractRandomAccessList<T> extends AbstractList<T> implements RandomAccess { }
122
123
124 /**Get instance of our standard good/secure digester (a new instance each time).
125 * This is the SHA1 digest.
126 * <p>
127 * Instances returned by this call must not be shared between threads.
128 */
129 public static MessageDigest getStandardDigest()
130 {
131 try { return(MessageDigest.getInstance(CoreConsts.HASH_SHA1 /*"SHA"*/)); }
132 catch(final NoSuchAlgorithmException e) // Should never happen...
133 { throw new Error("could not find "+CoreConsts.HASH_SHA1+" digester!", e); }
134 }
135
136
137 /**Get application build version in form x.y.z (eg 1.57.15); null if not available. */
138 public static String appVersion()
139 {
140 try
141 {
142 // Fetch and permanently cache the resource bundle under its real locale.
143 // We only provide a properties-based version of the common bundle.
144 final ResourceBundle result = ResourceBundle.getBundle("PG2Kversion");
145 return(result.getString("app.version"));
146 }
147 catch(final Exception e) { e.printStackTrace(); } // Absorb (but log) any error.
148 return(null); // Not available.
149 }
150
151
152 /**Compute the approximate rounded-down log-base-2 of the input.
153 * The input must be non-negative; input zero returns -1.
154 * <p>
155 * In effect this returns a 1-significant-bit answer,
156 * which is useful for "clumping" values that have a large dynamic range.
157 * <p>
158 * @return floor(log<sub>2</sub>(v))
159 */
160 public static int log2Approx(long v)
161 {
162 if(v < 0) { throw new IllegalArgumentException(); }
163
164 int l;
165 for(l = -1; v != 0; ++l)
166 { v >>>= 1; }
167
168 return(l);
169 }
170
171
172 /**Maximum capacity for _gLTD_cache; strictly positive.
173 * The larger this capacity the more memory that we may consume
174 * but the greater the chance we have of making a cache hit.
175 * <p>
176 * If this cache is to help significantly with building the by-word index
177 * then this should be easily large enough to handle the fact
178 * that keys will at best be looked up in sorted order on part of the
179 * exhibit name, and that that may not correspond exactly with the way
180 * that AKA/desc data is organised.
181 * <p>
182 * If this is to help significantly with normal catalogue-page lookup
183 * then this should allow for the typical distribution of page views
184 * and of locales used by visitors.
185 * <p>
186 * This suggests a minimum size of at least several hundred entries
187 * under normal circumstances.
188 * <p>
189 * Aim to allow ~10000 entries per 1GB of heap space (at initialisation)...
190 */
191 private static final int MAX_gLTD_cache_SIZE = Math.max(713, (int) Math.min(22001, Runtime.getRuntime().totalMemory() >> 17));
192
193 /**An LRU cache for lookups in getLocalisedTreeDesc() and private to it; never null.
194 * This is a map from a composite key based on the data hash,
195 * exhibit name, lookup type and locale to the result of the lookup.
196 * <p>
197 * The result text can be extracted with toString().
198 * <p>
199 * (The data hash that we use from the AEP is from just the treedesc
200 * data so that changes in other parts of the AEP do not invalidate
201 * cache entries. This is useful because the treedesc data
202 * usually changes much more slowly than the entire AEP
203 * (and the AEP longHash does not always capture the treedesc state)
204 * and because we can use a shorter hash for quicker key construction.)
205 * <p>
206 * The keys are essentially unique and this not intern()ed,
207 * but the result texts may not be unique,
208 * and thus are intern()ed to try to economise on memory.
209 * <p>
210 * This cache can in principle support multiple AEPs simultaneously
211 * (assuming that their hashes do not collide)
212 * and thus will cope with an update to a new AEP in the Web server.
213 * <p>
214 * We are prepared to empty the cache entirely under memory stress.
215 * <p>
216 * FIXME: convert this from being a static cache, eg to an opaque value passed in by caller
217 */
218 private static final LRUMapAutoSizeForHitRate<CS8Bit,Object> _gLTD_cache =
219 LRUMapAutoSizeForHitRate.<CS8Bit, Object>create(0, MAX_gLTD_cache_SIZE, "_gLTD_cache");
220
221 /**Prefix of descriptive-text keys in tree-text i18n bundles. */
222 public static final String TREE_TEXT_PREFIX_DESCRIPTION = "desc.";
223 /**Prefix of Also-Known-As-keyword-list keys in tree-text i18n bundles. */
224 public static final String TREE_TEXT_PREFIX_AKA = "aka.";
225 /**Default maximum acceptable gap between aka/desc entries for search to continue; non-negative. */
226 public static final int DEFAULT_MAX_DESCAKA_GAP = 4;
227
228 /**Extract (immutable) description tree information as a list; returns "" if none; never null.
229 * Returns an HTML fragment,
230 * zero or more <li>key (AKA keywords) = text</li> values in order,
231 * with localised i18n text extracted about the exhibit.
232 * <p>
233 * This will stop searching after a certain number of failures,
234 * ie nodes where no description/AKA text is available.
235 * This is to reduce search times on long exhibit names
236 * or where there is no suitable text at all.
237 * <p>
238 * Behaviour is undefined if the exhibitName is not syntactically valid.
239 * Note that the exhibitName need not be a current/extant exhibit,
240 * and only the main-words portion is used case-insensitively.
241 * <p>
242 * Note that the returned text is almost always 7-bit or 8-bit,
243 * and thus may be returned as (for example) a Name or Compact7BitString.
244 *
245 * @param aep complete exhibit metadata/properties; never null
246 * @param exhibitName syntactically-valid (not necessarily extant) full exhibit name;
247 * never null
248 * @param getDesc if true, include descriptive text from the tree
249 * @param getAKA if true, include "Also-Known-As" synonym text from the tree
250 * @param akaLinks if true, insert hyperlinks from synonyms
251 * to appropriate places in the catalogue tree where possible
252 * @param mostSpecificOnly if true, show only the most-specific text,
253 */
254 public static final CharSequence getLocalisedTreeDesc(final AllExhibitProperties aep,
255 final CharSequence exhibitName,
256 final LocaleBeanBase localeBean,
257 final boolean getDesc,
258 final boolean getAKA,
259 final boolean akaLinks,
260 final boolean mostSpecificOnly)
261 {
262 assert((exhibitName instanceof Name.ExhibitFull) || ExhibitName.validNameSyntaxBasic(exhibitName)) : "bad exhibit name: "+exhibitName;
263
264 if((aep == null) || (localeBean == null))
265 { throw new IllegalArgumentException(); }
266
267 if(!getDesc && !getAKA)
268 { throw new IllegalArgumentException("no point in asking for neither desc nor AKA"); }
269
270 if(!getAKA && akaLinks)
271 { throw new IllegalArgumentException("no point in asking for AKA links with no AKA"); }
272
273 // All attribute words (to be stripped from names before lookup).
274 final SortedSet<String> attrWordsSortedSet = ExhibitAttrUtils.getAttrWords().getAttrWordsSortedSet();
275 // We lower-case the main component to:
276 // * help reduce the cache key space and increase the cache hit rate,
277 // * reduce work later in converting keys to lower-case.
278 final String mainWordsComponentLC = ExhibitName.getMainWordsComponent(exhibitName, attrWordsSortedSet).toString().toLowerCase();
279 // The category component (expected always to be lower-case).
280 final CharSequence catLC = ExhibitName.getCategoryComponent(exhibitName);
281 if(IsDebug.isDebug) { assert(catLC.toString().equals(catLC.toString().toLowerCase())); }
282
283 //final long kts = System.nanoTime();
284 // Compute non-ambiguous but compact cache lookup key.
285 // Try to do the least work (fewest appends) for the most common cases.
286 final StringBuilder cacheKeySB = new StringBuilder(exhibitName.length());
287 // Organised to try to improve prefix sharing with Name key implementation.
288 cacheKeySB.append(catLC);
289 cacheKeySB.append('/');
290 cacheKeySB.append(mainWordsComponentLC);
291 cacheKeySB.append('|');
292 // Append markers for "uncommon" flag values.
293 if(!getAKA) { cacheKeySB.append('A'); }
294 if(getDesc) { cacheKeySB.append('d'); }
295 if(akaLinks) { cacheKeySB.append('l'); }
296 if(mostSpecificOnly) { cacheKeySB.append('s'); }
297 cacheKeySB.append('|');
298 // Don't include the default locale in the key so as to save time/space,
299 // but do insert a non-standard locale as an extra field.
300 final Locale locale = localeBean.getLocale();
301 if(!locale.equals(I18NTools.DEFAULT_SYSTEM_LOCALE))
302 { cacheKeySB.append(locale.toString()).append('|'); }
303 // We append a fixed-length hash over just our treedesc keys and values,
304 // so we should remain immune to changes elsewhere in the AEP.
305 // Fixed-char-count to avoid ambiguity without needing a separator.
306 // Values limited to 8-bit values for 8-bit key type.
307 final int dataHash = aep.epgi.getTreedescHash();
308 cacheKeySB.append((char) (dataHash & 0xff))
309 .append((char) ((dataHash >>> 8) & 0xff))
310 .append((char) ((dataHash >>> 16) & 0xff))
311 .append((char) ((dataHash >>> 24) & 0xff));
312
313 final CS8Bit cacheKey = new CS8Bit(cacheKeySB);
314
315 // If we have an entry in the cache then return it immediately...
316 final Object fromCache = _gLTD_cache.get(cacheKey);
317 //final long kte = System.nanoTime(); System.out.println("[AKA key gen/use time: "+(kte-kts)+"ns.]");
318 if(fromCache != null)
319 {
320 // We can return a CharSequence directly without expanding it.
321 if(fromCache instanceof CharSequence) { return((CharSequence) fromCache); }
322 // Else convert to a String to return...
323 return(fromCache.toString());
324 }
325
326 // Compute the result...
327
328 //final long rts = System.nanoTime();
329 // Limits maximum (potentially fruitless) depth of search.
330 final int maxGap = DEFAULT_MAX_DESCAKA_GAP;
331 int remainingGap = maxGap - 1; // Allow for section-dir component.
332
333 // Result is collected in this.
334 final StringBuilder result = new StringBuilder();
335
336 // Build up basic common key word by word...
337 final StringBuilder basicKey = new StringBuilder(exhibitName.length());
338 basicKey.append(catLC);
339 final int afterSection = basicKey.length() + 1;
340
341 final ExhibitPropsGlobalImmutable epgi = aep.epgi;
342 final Locale l = localeBean.getLocale();
343
344 final Enumeration<?> wEn = (new StringTokenizer(mainWordsComponentLC, ExhibitName.WORD_SEPS));
345 while(wEn.hasMoreElements() && (--remainingGap >= 0))
346 {
347 // Capture the "old"/previous basic key length.
348 // We do this so that we can easily construct the previous key
349 // or derivatives of it if we want to later.
350 final int oldBkLen = basicKey.length();
351
352 // Extend basic key with new word.
353 final String word = (String) wEn.nextElement();
354 basicKey.append('.').append(word);
355 final String bk = basicKey.toString();
356
357 final String descKeyS = TREE_TEXT_PREFIX_DESCRIPTION + bk;
358 final CharSequence desc = epgi.getLocalisedTreeDescMessage(descKeyS, l);
359 final String akaKeyS = TREE_TEXT_PREFIX_AKA + bk;
360 final CharSequence aka = epgi.getLocalisedTreeDescMessage(akaKeyS, l);
361 // If we found some real description or "AKA", then capture it.
362 final boolean gotDescText = getDesc && !descKeyS.equals(desc);
363 final boolean gotAKAText = getAKA && !akaKeyS.equals(aka);
364 if(gotDescText || gotAKAText)
365 {
366 // Clear any earlier text that we had collected
367 // if we want to keep only the most-specific text available.
368 if(mostSpecificOnly)
369 { result.setLength(0); }
370
371 // We found something, so can reset our counter.
372 remainingGap = maxGap;
373
374 // Make a printable version of the key.
375 // Drop the section category and
376 // show the words of the partial key in order,
377 // space-separated.
378 final String keyP =
379 bk.substring(afterSection).replace('.', ' ');
380 if(!mostSpecificOnly)
381 { result.append("<li><i>").append(keyP); }
382 if(gotAKAText)
383 {
384 if(!mostSpecificOnly)
385 { result.append(" = "); }
386
387 // If not doing links then put in text "as-is".
388 if(!akaLinks)
389 { result.append(aka); }
390 else
391 {
392 // If doing AKA hypertext links
393 // then go through the synonyms one-by-one
394 // looking for related information
395 // and linking to it if any found.
396 final StringTokenizer st = new StringTokenizer(aka.toString(), ",");
397 while(st.hasMoreTokens())
398 {
399 final String s = st.nextToken().trim();
400
401 // Could the current synonym (AKA)
402 // be a single valid "peer" word
403 // that has its own (interesting) AKA/desc text
404 // and related exhibits?
405 final boolean validWord = ExhibitName.validWord(s);
406 boolean doneLink = false; // Hyperlinked if true.
407 if(validWord)
408 {
409 // Look for peer use of AKA term...
410 // (We don't check for peer description here,
411 // as we assume that possible reciprocal
412 // AKA "referencing" may be a better filter.)
413 final String peerBk =
414 basicKey.substring(0, oldBkLen) +
415 '.' + s.toLowerCase();
416 final String peerAKAKey =
417 TREE_TEXT_PREFIX_AKA + peerBk;
418 final CharSequence peerAka = epgi.getLocalisedTreeDescMessage(peerAKAKey, l);
419 if(!TextUtils.contentEquals(peerAKAKey, peerAka))
420 {
421 // OK, have found a peer key, so link.
422 result.append("<a href=\"/").
423 append(peerBk.replace('.', '/')).
424 append("/\">");
425 doneLink = true;
426 if(IsDebug.isDebug) { System.err.println("[Inserted AKA hyperlink from "+bk+" to "+peerBk+".]"); }
427 }
428 }
429
430 result.append(s);
431
432 // Close hyperlink if one was made/opened.
433 if(doneLink) { result.append("</a>"); }
434
435 // If not last on the list
436 // then reinsert the usual separator.
437 if(st.hasMoreTokens())
438 { result.append(", "); }
439 }
440 }
441 }
442 if(!mostSpecificOnly)
443 { result.append("</i>"); }
444 if(gotDescText)
445 { result.append(" = ").append(desc); }
446 if(!mostSpecificOnly)
447 { result.append("</li>"); }
448 }
449 }
450
451 // Cache and return a negative ("") result.
452 if(result.length() == 0)
453 {
454 _gLTD_cache.put(cacheKey, "");
455 return("");
456 }
457
458 // Cache the positive result, compressed and intern()ed if possible.
459 String s = result.toString();
460 //final long rte = System.nanoTime(); System.out.println("[AKA compute time: "+(rte-rts)+"ns.]");
461
462 try { _gLTD_cache.put(cacheKey, MemoryTools.intern(Compact7BitString.convertToCompact7BitString(s, null))); }
463 // Store as full-blown String if we can't compress it for any reason.
464 catch(final IllegalArgumentException e) { _gLTD_cache.put(cacheKey, s = MemoryTools.intern(s)); }
465
466 // Return the (non-null, non-empty) result.
467 return(s);
468 }
469
470 /**If true, fall back to a lookup in the common i18n bundles for section titles and description. */
471 private static final boolean SECTION_LOOKUP_COMMON_FALLBACK = false;
472
473 /**The prefix to look up the section description in the common message bundle, if any. */
474 private static final String SECTION_DESC_PROPNAME_PREFIX = "org.hd.pg2k.section.description.";
475
476 /**The prefix to look up the prettied/i18ned version of a section title in the common message bundle, if any. */
477 private static final String SECTION_TITLE_PROPNAME_PREFIX = "org.hd.pg2k.section.title.";
478
479 /**Find the localised section description for the given category name; returns null if none.
480 * This is sufficiently specialised that we don't follow our normal convention
481 * of returning the request key if localised text is not found.
482 *
483 * @param aep current AEP; never null
484 * @param sectionDir category/section/directory name; never null
485 * @param localeBean current locale; never null
486 * @return section description, or null if none
487 */
488 public static final CharSequence getLocalisedSectionDesc(final AllExhibitProperties aep,
489 final CharSequence sectionDir,
490 final LocaleBeanBase localeBean)
491 {
492 if((aep == null) || (sectionDir == null) || (localeBean == null))
493 { throw new IllegalArgumentException(); }
494
495 // Lookup in treedesc directly as "desc.<category>".
496 final String key = TREE_TEXT_PREFIX_DESCRIPTION + sectionDir;
497 final CharSequence msg = aep.epgi.getLocalisedTreeDescMessage(key, localeBean.getLocale());
498 if(!TextUtils.contentEquals(key, msg))
499 {
500 //if(IsDebug.isDebug) { System.out.println("Found treedesc section description for "+sectionDir+" in locale "+localeBean+ ": " + msg); }
501 return(msg.toString());
502 }
503
504 if(SECTION_LOOKUP_COMMON_FALLBACK)
505 {
506 final String i18nName = GenUtils.SECTION_DESC_PROPNAME_PREFIX + sectionDir;
507 final String desc = localeBean.getLocalisedMessage(i18nName);
508 // We return localised text if found.
509 if(!desc.equals(i18nName))
510 { return(desc); }
511 }
512
513 return(null); // Did not find localised section description.
514 }
515
516 /**Get section title (as HTML); never null.
517 * This attempts to get or compute a section title given the
518 * top-level section directory name from an exhibit (such as "brain_scan").
519 * <p>
520 * The fall-back algorithm is for this to convert all non-alphanumeric
521 * characters in the name to spaces and capitalise the first letter of
522 * the title.
523 * <p>
524 * However, if passed a valid AEP and locale bean then this routine attempts to
525 * look up the directory name (suitably prefixed and suffixed)
526 * in appropriate message catalogue(s) to find a sugared and/or i18ned version.
527 *
528 * @param aep the exhibit properties containing the treedesc i18n text;
529 * may be null
530 * @param sectionDir section/category/directory; never null
531 * @param localeBean locale; can be null
532 */
533 public static String computeSectionTitle(final AllExhibitProperties aep,
534 final CharSequence sectionDir,
535 final LocaleBeanBase localeBean)
536 {
537 // Lookup in treedesc directly as "aka.<category>".
538 if(aep != null)
539 {
540 if(localeBean != null)
541 {
542 final String key = TREE_TEXT_PREFIX_AKA + sectionDir;
543 final CharSequence msg = aep.epgi.getLocalisedTreeDescMessage(key, localeBean.getLocale());
544 if(!TextUtils.contentEquals(key, msg))
545 {
546 //if(IsDebug.isDebug) { System.out.println("Found treedesc section title for "+sectionDir+" in locale "+localeBean+ ": " + msg); }
547 return(msg.toString());
548 }
549 // Failed: fall through to backup algorithm...
550 }
551 }
552
553 if(SECTION_LOOKUP_COMMON_FALLBACK)
554 {
555 if(localeBean != null)
556 {
557 final String key = GenUtils.SECTION_TITLE_PROPNAME_PREFIX + sectionDir;
558 final String i18nedTitle = localeBean.getLocalisedMessage(key);
559 if((i18nedTitle != null) &&
560 (i18nedTitle.length() != 0) &&
561 !i18nedTitle.equals(key)) // Usual graceful failure mode of LocaleBean.
562 { return(i18nedTitle); } // Got i18n version!
563 // Failed: fall through to backup algorithm...
564 }
565 }
566
567 // Fall-back algorithm: compute name from section title...
568 final int length = sectionDir.length();
569 final StringBuilder simpleResult = new StringBuilder(length);
570 for(int i = 0; i < length; ++i)
571 {
572 final char ch = sectionDir.charAt(i);
573 if(!Character.isLetterOrDigit(ch)) // All non-alphanumeric to space.
574 { simpleResult.append(' '); }
575 else if(i == 0) // Capitalise first character.
576 { simpleResult.append(Character.toTitleCase(ch)); }
577 else // Leave as is.
578 { simpleResult.append(ch); }
579 }
580 return(simpleResult.toString());
581 }
582
583
584 /**Returns true iff the full/short exhibit name or URI supplied is "sensitive" in some way.
585 * Can be used to suppress "promotion" of (or advertising against) an exhibit,
586 * or other accidentally-tasteless activity.
587 * <p>
588 * Simply matches (case-sensitively) potentially-problematic substrings,
589 * so has to be used with care to avoid false/silly matches,
590 * the main defence against which is to use sufficiently long and specific substrings.
591 * <p>
592 * TODO: consider switching to regex expression and/or cached lookup
593 */
594 public static boolean isSensitive(final CharSequence nameOrURI,
595 final GenProps gp)
596 {
597 //if(IsDebug.isDebug) { System.err.println("isSensitive("+nameOrURI+")"); }
598 if(nameOrURI == null) { return(false); }
599
600 final String substrsS = gp.getGen().get(GenPropsGenNames.GPGEN_SENSITIVE_NAME_SUBSTRS_KEY);
601 // Return immediately if no sensitive strings flagged.
602 if(substrsS == null) { return(false); }
603
604 final String nameOrURIAsString = nameOrURI.toString(); // FIXME: potentially inefficient conversion to String
605 final String sensitiveWords[] = substrsS.split(" ");
606 for(final String sensitiveWord : sensitiveWords)
607 {
608 // If this substring matches then this name is sensitive.
609 if(nameOrURIAsString.indexOf(sensitiveWord) != -1)
610 { return(true); }
611 }
612
613 // All clear.
614 return(false);
615 }
616
617 /**Number of parents (above child hot spot) to count in in stack trace; strictly positive.
618 * If 1 this counts only the closest interesting parent.
619 * <p>
620 * If significantly greater than 1 then can capture recursion for example,
621 * at the cost of extra work.
622 * <p>
623 * A reasonable value is probably 1 to 32.
624 */
625 private static final int MAX_PARENTS_COUNTED = 10;
626
627 /**Create a performance monitor attached to the given thread, filling in the given map.
628 * This adds data to an existing map,
629 * and multiple performance monitors may concurrently update the same map.
630 * <p>
631 * This thread can be interrupt()ed to stop it collecting,
632 * and it will also stop when the thread being monitored exits.
633 * <p>
634 * This runs as a high-priority thread sampling at a high rate (~1ms).
635 *
636 * @param toObserve the thread to be sampled; never null
637 * @param counts the map from code sample site to sample count; never null
638 * @param prefix if non-null then the lowest method on the stack
639 * whose fully-qualified class name starts with this prefix is logged,
640 * else the lowest method is logged
641 * @param stopBy the time by which we should stop sampling
642 * @param sampleTimeMS approximate minimum milliseconds to sleep between samples;
643 * strictly positive
644 */
645 public static Thread startThreadPerfMonitor(final Thread toObserve,
646 final ConcurrentHashMap<StackTraceElement, AtomicInteger> counts,
647 final String prefix,
648 final long stopBy,
649 final int sampleTimeMS)
650 {
651 return(startThreadPerfMonitor(toObserve, counts, null, prefix, stopBy, sampleTimeMS));
652 }
653
654 /**Create a performance monitor attached to the given thread, filling in the given map.
655 * This adds data to an existing map,
656 * and multiple performance monitors may concurrently update the same map.
657 * <p>
658 * This thread can be interrupt()ed to stop it collecting,
659 * and it will also stop when the thread being monitored exits.
660 * <p>
661 * This runs as a high-priority thread sampling at a high rate (~1ms).
662 *
663 * @param toObserve the thread to be sampled; never null
664 * @param counts the map from (prefix-matching) code sample site to sample count; never null
665 * @param parentCounts map from (prefix-matching) code sample site
666 * to all its (prefix-matching) parent/caller sites;
667 * can be null if not required
668 * @param prefix if non-null then the lowest method on the stack
669 * whose fully-qualified class name starts with this prefix is logged,
670 * else the lowest method is logged
671 * @param stopBy the time by which we should stop sampling
672 * @param sampleTimeMS approximate minimum milliseconds to sleep between samples;
673 * strictly positive
674 */
675 public static Thread startThreadPerfMonitor(final Thread toObserve,
676 final ConcurrentMap<StackTraceElement, AtomicInteger> counts,
677 final ConcurrentMap<StackTraceElement, ConcurrentMap<StackTraceElement, AtomicInteger>> parentCounts,
678 final String prefix,
679 final long stopBy, final int sampleTimeMS)
680 {
681 if((null == toObserve) || (null == counts) || (sampleTimeMS <= 0))
682 { throw new IllegalArgumentException(); }
683
684 final Thread t = new Thread("perf monitor thread for " + toObserve){
685 @Override public final void run()
686 {
687 while(toObserve.isAlive() &&
688 !isInterrupted() &&
689 (System.currentTimeMillis() < stopBy))
690 {
691 final Thread.State state = toObserve.getState();
692 // Omit sample for threads in probably-uninteresting states...
693 if((state == Thread.State.NEW) ||
694 (state == Thread.State.TERMINATED))
695 { continue; }
696 // Get stack trace.
697 final StackTraceElement[] stackTraceElements = toObserve.getStackTrace();
698 // Skip threads with an empty or strange stack.
699 StackTraceElement top;
700 int topIndex = 0;
701 if((stackTraceElements == null) ||
702 (stackTraceElements.length < 1) ||
703 (null == (top = stackTraceElements[0])))
704 { continue; }
705
706 if(null != prefix)
707 {
708 // Look for a better match than top-of-stack.
709 for(int i = 0; i < stackTraceElements.length; ++i)
710 {
711 final StackTraceElement ste = stackTraceElements[i];
712 if(null == ste) { continue; }
713 final String cn = ste.getClassName();
714 if(null == cn) { continue; }
715 final boolean ourCode = cn.startsWith(prefix);
716 if(ourCode)
717 {
718 // We've found a better match,
719 // so use use it and stop searching.
720 top = ste;
721 topIndex = i;
722 break;
723 }
724 }
725 }
726
727 // Get existing counter for this execution site,
728 // or (atomically) create a new zero one.
729 AtomicInteger count;
730 while(null == (count = counts.get(top)))
731 { counts.putIfAbsent(top, new AtomicInteger()); }
732 // Increment counter.
733 count.incrementAndGet();
734
735 if(null != parentCounts)
736 {
737 int parentsCounted = 0;
738 // Increment count of parent(s) of interest to us.
739 for(int i = topIndex + 1; i < stackTraceElements.length; ++i)
740 {
741 final StackTraceElement ste = stackTraceElements[i];
742 if(null == ste) { continue; }
743 final String cn = ste.getClassName();
744 if(null == cn) { continue; }
745 final boolean ourCode = (null == prefix) || cn.startsWith(prefix);
746 if(ourCode)
747 {
748 // Increment a parent count...
749 // atomically creating whatever we need on the way...
750 ConcurrentMap<StackTraceElement, AtomicInteger> submap;
751 while(null == (submap = parentCounts.get(top)))
752 { parentCounts.putIfAbsent(top, new ConcurrentHashMap<StackTraceElement, AtomicInteger>()); }
753 AtomicInteger countP;
754 while(null == (countP = submap.get(ste)))
755 { submap.putIfAbsent(ste, new AtomicInteger()); }
756 // Increment counter.
757 countP.incrementAndGet();
758 // Count limited number of parents...
759 if(++parentsCounted >= MAX_PARENTS_COUNTED) { break; }
760 }
761 }
762 }
763
764 try { Thread.sleep(sampleTimeMS); }
765 catch(final InterruptedException e) { return; /* Terminate on interrupt. */ }
766 }
767 }
768 };
769 t.setDaemon(true);
770 t.setPriority(Thread.MAX_PRIORITY);
771 t.start();
772 return(t);
773 }
774
775 /**Stop a monitor thread and dump its samples.
776 *
777 * @param monitorThread thread being profiled; never null
778 * @param title title to display; can be null
779 * @param counts map from sample site to counts at that site; never null
780 * @param maxToShow max (top) entries to display; strictly positive
781 * @param logger if null no dump is generated, else is log sink
782 *
783 * @throws InterruptedException if this thread is interrupted
784 */
785 public static void stopPerfMonitorandDumpSamples(
786 final Thread monitorThread,
787 final String title,
788 final ConcurrentMap<StackTraceElement, AtomicInteger> counts,
789 final int maxToShow,
790 final SimpleLoggerIF logger)
791 throws InterruptedException
792 { stopPerfMonitorandDumpSamples(monitorThread, title, counts, null, maxToShow, logger); }
793
794 /**Stop a monitor thread and dump its samples.
795 * Note that we only show parent traces for a small top fraction of the hot sites,
796 * and show a fraction of maxToShow parent sites
797 * on the grounds that we otherwise have a huge explosion in output size
798 * for relatively little extra value!
799 *
800 * @param monitorThread thread being profiled; never null
801 * @param title title to display; can be null
802 * @param counts map from sample site to counts at that site; never null
803 * @param parentCounts map from sample site to parents and counts at that site;
804 * can be null if not required
805 * @param maxToShow max (top) entries to display; strictly positive
806 * @param logger if null no dump is generated, else is log sink
807 * @throws InterruptedException if this thread is interrupted
808 */
809 public static <SM extends ConcurrentMap<StackTraceElement, AtomicInteger>> void stopPerfMonitorandDumpSamples(
810 final Thread monitorThread,
811 final String title,
812 final ConcurrentMap<StackTraceElement, AtomicInteger> counts,
813 final ConcurrentMap<StackTraceElement, SM> parentCounts,
814 final int maxToShow, final SimpleLoggerIF logger)
815 throws InterruptedException
816 {
817 // Stop the monitor thread and wait a little while for it to exit.
818 while(monitorThread.isAlive())
819 {
820 monitorThread.interrupt();
821 monitorThread.join(100);
822 }
823
824 // If no dump is required then return now.
825 if(null == logger) { return; }
826
827 // Do the dump...
828 dumpPerfSamples(title, counts, parentCounts, maxToShow, logger);
829 }
830
831 /**Dump a set of performance-monitoring site samples.
832 * We take a snapshot of the current profile stats when we start,
833 * so that it is safe to dump from a map that is still being updated
834 * (though keys/entries must not be <em>removed</em> while we are working)
835 * and an iterator over the counts map and get() must be thread-safe.
836 * <p>
837 * We take a snapshot of the current profile stats when we start,
838 * so that it is safe to dump from a map that is still being updated
839 * (though keys/entries must not be <em>removed</em> while we are working)
840 * and an iterator over the counts map and get() must be thread-safe.
841 *
842 * @param title title to display; can be null
843 * @param counts map from sample site to counts at that site; never null
844 * @param maxToShow max (top) entries to display; strictly positive
845 * @param logger destination of profile results; never null
846 */
847 public static void dumpPerfSamples(final String title,
848 final ConcurrentMap<StackTraceElement, AtomicInteger> counts,
849 final int maxToShow,
850 final SimpleLoggerIF logger)
851 { dumpPerfSamples(title, counts, null, maxToShow, logger); }
852
853 /**Dump a set of performance-monitoring site samples.
854 * We take a snapshot of the current profile stats when we start,
855 * so that it is safe to dump from a map that is still being updated
856 * (though keys/entries must not be <em>removed</em> while we are working)
857 * and an iterator over the counts map and get() must be thread-safe.
858 * <p>
859 * We take a snapshot of the current profile stats when we start,
860 * so that it is safe to dump from a map that is still being updated
861 * (though keys/entries must not be <em>removed</em> while we are working)
862 * and an iterator over the counts map and get() must be thread-safe.
863 *
864 * @param title title to display; can be null
865 * @param counts map from sample site to counts at that site; never null
866 * @param parentCounts map from sample site to parents and counts at that site;
867 * can be null if not required
868 * @param maxToShow max (top) entries to display; strictly positive
869 * @param logger destination of profile results; never null
870 */
871 public static <SM extends ConcurrentMap<StackTraceElement, AtomicInteger>> void dumpPerfSamples(final String title,
872 final ConcurrentMap<StackTraceElement, AtomicInteger> counts,
873 final ConcurrentMap<StackTraceElement, SM> parentCounts,
874 final int maxToShow, final SimpleLoggerIF logger)
875 {
876 // Sort elements by count...
877 final class STEComp implements Comparator<StackTraceElement>
878 {
879 final HashMap<StackTraceElement, Integer> snapshot;
880 final int totalHits;
881 STEComp(final Map<StackTraceElement, AtomicInteger> counts)
882 {
883 // Take a snapshot of the data.
884 final HashMap<StackTraceElement, Integer> snapshot = new HashMap<StackTraceElement, Integer>(counts.size());
885
886 // We have to assume that the iterator will be safe.
887 int totalHits = 0;
888 for(final StackTraceElement ste : counts.keySet())
889 {
890 final Integer count = new Integer(counts.get(ste).get());
891 totalHits += count;
892 snapshot.put(ste, count);
893 }
894 this.snapshot = snapshot;
895 this.totalHits = totalHits;
896 }
897
898 /**Sort highest-first by the count,
899 * then the StackTraceElement content to break ties.
900 */
901 public int compare(final StackTraceElement se0,
902 final StackTraceElement se1)
903 {
904 final int c0 = snapshot.get(se0).intValue();
905 final int c1 = snapshot.get(se1).intValue();
906 if(c0 > c1) { return(-1); /* Correct order. */ }
907 if(c0 < c1) { return(+1); /* Wrong order. */ }
908
909 // Break ties (expensively)
910 // with the full String form of the sample data.
911 return(se0.toString().compareTo(se1.toString()));
912 }
913 }
914
915 // Prepare a set of samples sorted by count (then sample).
916 // We only even insert the best samples.
917 final STEComp comparator = new STEComp(counts);
918 final SortedSet<StackTraceElement> best = new TreeSet<StackTraceElement>(comparator);
919
920 // Crudely add samples.
921 // TODO: make this more efficient.
922 best.addAll(counts.keySet());
923
924 if(title != null) { logger.log(title); }
925 logger.log("Sampled sites: "+(counts.size())+"; max dumped: "+maxToShow+"");
926 logger.log("COUNT\tHit%\tSite");
927
928 // We build each result line in here.
929 final StringBuilder line = new StringBuilder(256);
930
931 // We build the percentage figure in here.
932 final StringBuilder p = new StringBuilder(3);
933
934 // Dump top (highest-count) samples.
935 final Iterator<StackTraceElement> it = best.iterator();
936 for(int i = maxToShow; (--i >= 0) && it.hasNext(); )
937 {
938 final StackTraceElement ste = it.next();
939 line.setLength(0);
940 final int count = comparator.snapshot.get(ste).intValue();
941 line.append(count);
942
943 line.append('\t');
944 p.setLength(0);
945 final int percentage = (100 * count) / comparator.totalHits;
946 p.append(percentage);
947 while(p.length() < 3) { p.insert(0, ' '); }
948 line.append(p).append('%');
949
950 line.append('\t');
951 line.append(ste.toString());
952 logger.log(line.toString());
953
954 // Add in (fewer) parent counts if present,
955 // and if this hotspot is showing a large enough percentage...
956 if(percentage < 1) { continue; }
957 if(parentCounts == null) { continue; }
958 final SM sm = parentCounts.get(ste);
959 if(sm == null) { continue; }
960 final STEComp comparatorP = new STEComp(sm);
961 final SortedSet<StackTraceElement> bestP = new TreeSet<StackTraceElement>(comparatorP);
962 bestP.addAll(sm.keySet());
963 // Dump top (highest-count) samples.
964 final Iterator<StackTraceElement> pit = bestP.iterator();
965 for(int j = Math.max(1, maxToShow/4); (--j >= 0) && pit.hasNext(); )
966 {
967 final StackTraceElement steP = pit.next();
968 line.setLength(0);
969 final int countP = comparatorP.snapshot.get(steP).intValue();
970 line.append(" *").append(countP);
971
972 line.append('\t');
973 // Percentage may be too confusing interleaved with child percentages...
974 // TODO: possibly show as percentages on child scale...
975 // p.setLength(0);
976 // p.append((100 * countP) / comparatorP.totalHits);
977 // while(p.length() < 3) { p.insert(0, ' '); }
978 // line.append(p).append('%');
979
980 line.append('\t');
981 line.append(" (parent) ").append(steP.toString());
982 logger.log(line.toString());
983 }
984 }
985 }
986
987 /**Validate internal Gallery timestamp; returns true if sensible, false otherwise.
988 * Such timestamps should not be before the Gallery was created,
989 * nor much in the future (allowing for some clock skew between consenting participants).
990 */
991 public static final boolean isValidGalleryTimestamp(final long timestamp)
992 {
993 if(timestamp < CoreConsts.GALLERY_EPOC_START) { return(false); }
994 if(timestamp > System.currentTimeMillis() + 2*CoreConsts.MAX_PEER_CLOCK_SKEW_MS) { return(false); }
995 return(true); // Seems OK.
996 }
997
998
999 /**Maximum compression level currently supported by compressData(); never null.
1000 * Until 20070811 this has been BZIP2.
1001 */
1002 public static final CompressionLevel MAX_SUPPORTED_COMPRESSION_LEVEL = CompressionLevel.LZMA;
1003
1004 /**Stream compress/filter; never null.
1005 * Especially valuable for very large amounts of data to large to handle uncompressed.
1006 * <p>
1007 * The output stream is wrapped with the selected compression filter
1008 * that should be byte-for-byte compatible with the output of compressData()
1009 * for the given compression level, generally maximal,
1010 * unless flush() is called mid-stream in which case there may be reduced compression.
1011 * <p>
1012 * Compression may not be finished and all data pushed through
1013 * until close() is called;
1014 * flush() may not be sufficient and may produced less-compressed output.
1015 * <p>
1016 * Errors from the compressor may be transformed into IOException
1017 * for easier handling and recovery by the caller.
1018 * <p>
1019 * This may increase the size of the data in some cases,
1020 * especially where the input array is short.
1021 * <p>
1022 * A compression level of NONE returns the supplied stream unchanged.
1023 *
1024 * @param os output stream of uncompressed data; never null
1025 * @param level compression level; never null and no higher than MAX_SUPPORTED_COMPRESSION_LEVEL
1026 */
1027 public static OutputStream wrapForCompression(final OutputStream os, final CompressionLevel level)
1028 throws IOException
1029 {
1030 if(null == os) { throw new IllegalArgumentException(); }
1031 if(null == level) { throw new IllegalArgumentException(); }
1032 if(level.getLevel() > MAX_SUPPORTED_COMPRESSION_LEVEL.getLevel())
1033 { throw new IllegalArgumentException("compression level too high"); }
1034
1035 switch(level)
1036 {
1037 case NONE:
1038 { return(os); }
1039
1040 case ZIP:
1041 { return(new DefOutputStream(os)); }
1042
1043 case BZIP2:
1044 { return(new CBZip2OutputStream(os, 9)); }
1045
1046 case LZMA:
1047 { return(new LzmaOutputStream(os)); }
1048
1049 default:
1050 { throw new UnsupportedOperationException("compression type unsupported: "+level); }
1051 }
1052 }
1053
1054 /**Compress the supplied binary data as well as possible up to the given compression level; never null.
1055 * Tries all the supported compression methods up to the specified level
1056 * returning whichever results in the smallest result.
1057 * <p>
1058 * (Resource constraints may prevent some of the compression types being applied,
1059 * usually the highest levels.)
1060 * <p>
1061 * This does NOT assume that higher compression levels always compress given data better,
1062 * not does it assume a lower bound on data size worth compressing;
1063 * this tries all the possibilities exhaustively if it can.
1064 * <p>
1065 * This is likely to be very memory- and CPU- intensive; use with care.
1066 * <p>
1067 * This may attempt compressions in parallel on a multi-CPU machine,
1068 * though each compression is likely to thrash the cache
1069 * so this may or may not be good for compression and system performance.
1070 * <p>
1071 * This routine does not alter the content of the input array.
1072 * <p>
1073 * TODO: WRAP AS MEMORY-INTENSIVE OP FOR MORE EXTREME COMPRESSION METHODS
1074 *
1075 * @param in the data to be compressed; never null
1076 * @param maxLevel the maximum level at which to attempt to compress data,
1077 * using a value greater than MAX_SUPPORTED_COMPRESSION_LEVEL is allowed but ineffective;
1078 * never null
1079 *
1080 * @return non-null data compressed as well as possible;
1081 * if compression was not possible the level is NONE
1082 * and the input array is returned in the result as-is
1083 */
1084 public static Tuple.Pair<CompressionLevel,byte[]> compressData(final byte[] in, final CompressionLevel maxLevel)
1085 throws InterruptedException
1086 { return(compressData(in, maxLevel, false)); }
1087
1088 /**Compress the supplied binary data as well as possible up to the given compression level; never null.
1089 * Tries all the supported compression methods up to the specified level
1090 * returning whichever results in the smallest result.
1091 * <p>
1092 * (Resource constraints may prevent some of the compression types being applied,
1093 * usually the highest levels.)
1094 * <p>
1095 * This does NOT assume that higher compression levels always compress given data better,
1096 * not does it assume a lower bound on data size worth compressing;
1097 * this tries all the possibilities exhaustively if it can.
1098 * <p>
1099 * This is likely to be very memory- and CPU- intensive; use with care.
1100 * <p>
1101 * This may attempt compressions in parallel on a multi-CPU machine,
1102 * though each compression is likely to thrash the cache
1103 * so this may or may not be good for compression and system performance.
1104 * <p>
1105 * This routine does not alter the content of the input array.
1106 * <p>
1107 * TODO: WRAP AS MEMORY-INTENSIVE OP FOR MORE EXTREME COMPRESSION METHODS
1108 *
1109 * @param in the data to be compressed; never null
1110 * @param maxLevel the maximum level at which to attempt to compress data,
1111 * using a value greater than MAX_SUPPORTED_COMPRESSION_LEVEL is allowed but ineffective;
1112 * never null
1113 * @param maxOnly if true then only try the maximum compression level, and implicitly 'NONE'
1114 * @return non-null data compressed as well as possible;
1115 * if compression was not possible the level is NONE
1116 * and the input array is returned in the result as-is
1117 */
1118 public static Tuple.Pair<CompressionLevel,byte[]> compressData(final byte[] in, CompressionLevel maxLevel, final boolean maxOnly)
1119 throws InterruptedException
1120 {
1121 if((null == in) || (null == maxLevel))
1122 { throw new IllegalArgumentException(); }
1123
1124 // Coerce maxLevel to reflect what we actually have implemented.
1125 if(maxLevel.getLevel() > MAX_SUPPORTED_COMPRESSION_LEVEL.getLevel())
1126 { maxLevel = MAX_SUPPORTED_COMPRESSION_LEVEL; }
1127
1128 // Default is not to compress at all, returning the input data as-is.
1129 final AtomicReference<Tuple.Pair<CompressionLevel,byte[]>> result =
1130 new AtomicReference<Tuple.Pair<CompressionLevel,byte[]>>(new Tuple.Pair<CompressionLevel,byte[]>(CompressionLevel.NONE, in));
1131
1132 // Handle zero-length (trivial) input specially.
1133 if(in.length == 0) { return(result.get()); }
1134
1135 // Set of tasks/results.
1136 final Queue<Future<?>> tasks = new ArrayBlockingQueue<Future<?>>(CompressionLevel.values().length);
1137
1138 // Iterate over all known compression levels,
1139 // possibly processing them concurrently,
1140 // trying compression at the given level.
1141 for(final CompressionLevel cl : (maxOnly ? Collections.singleton(maxLevel) : Arrays.asList(CompressionLevel.values())))
1142 {
1143 // Skip unusable compression levels.
1144 if((cl == CompressionLevel.NONE) || (cl.getLevel() > maxLevel.getLevel()))
1145 { continue; }
1146
1147 // Submit this compression attempt to run as a CPU-intensive task.
1148 tasks.add(ThreadUtils.computeIntensiveThreadPool.submit(new Runnable(){
1149 public final void run()
1150 {
1151 final byte[] r; // Result for this level.
1152 switch(cl)
1153 {
1154 case ZIP:
1155 {
1156 r = FileTools.compressDeflatableData(in, 0, in.length);
1157 break;
1158 }
1159
1160 case BZIP2:
1161 {
1162 try
1163 {
1164 // Assume that we get *some* compression when sizing the output buffer...
1165 final ByteArrayOutputStream baos = new ByteArrayOutputStream(512+(in.length/2));
1166 // Use maximum BZIP2 compression.
1167 final CBZip2OutputStream bz2os = new CBZip2OutputStream(baos, 9);
1168 bz2os.write(in);
1169 bz2os.flush();
1170 bz2os.close();
1171 r = baos.toByteArray();
1172 break;
1173 }
1174 catch(final IOException e)
1175 {
1176 e.printStackTrace();
1177 throw new RuntimeException(e);
1178 }
1179 }
1180
1181 case LZMA:
1182 {
1183 try
1184 {
1185 // Assume that we get *some* compression when sizing the output buffer...
1186 final ByteArrayOutputStream baos = new ByteArrayOutputStream(512+(in.length/2));
1187 final LzmaOutputStream lzmaos = new LzmaOutputStream(baos);
1188 lzmaos.write(in);
1189 lzmaos.flush();
1190 lzmaos.close();
1191 r = baos.toByteArray();
1192 break;
1193 }
1194 catch(final IOException e)
1195 {
1196 e.printStackTrace();
1197 throw new RuntimeException(e);
1198 }
1199 }
1200
1201 default: return; // Skip unsupported level.
1202 }
1203
1204 assert(r != null); // Compression should not generate null result.
1205 assert(r.length > 0); // Compression should not generate zero-length result.
1206
1207 // If what we have computed looks to be the best result so far
1208 // then atomically replace the result.
1209 // This frees up resources ASAP (eg from sub-optimal results).
1210 final Tuple.Pair<CompressionLevel, byte[]> putativeResult = new Tuple.Pair<CompressionLevel,byte[]>(cl, r);
1211 for( ; ; )
1212 {
1213 final Tuple.Pair<CompressionLevel,byte[]> bestSoFar = result.get();
1214 if(r.length < bestSoFar.second.length)
1215 {
1216 if(!result.compareAndSet(bestSoFar, putativeResult))
1217 { continue; /* Failed to set result yet. */ }
1218 }
1219 break; // This is not the best result, so quit.
1220 }
1221
1222 return;
1223 }
1224 }));
1225
1226 // Because some of the compressors are very resource-intensive,
1227 // we'll wait for earlier tasks to complete
1228 // to ensure that we don't have more tasks running
1229 // than we have CPUs, regardless of the details of the thread pool.
1230 while(!tasks.isEmpty() && (tasks.size() >= Runtime.getRuntime().availableProcessors()))
1231 {
1232 try { tasks.remove().get(); }
1233 catch(final ExecutionException e)
1234 {
1235 // Ignore (though log) error in this particular compression attempt.
1236 e.printStackTrace();
1237 }
1238 }
1239 }
1240
1241 // Wait for all remaining tasks to complete.
1242 for(final Future<?> task : tasks)
1243 {
1244 try { task.get(); }
1245 catch(final ExecutionException e)
1246 {
1247 // Ignore (though log) error in this particular compression attempt.
1248 e.printStackTrace();
1249 }
1250 }
1251
1252 return(result.get()); // Return best result.
1253 }
1254
1255 /**Serialise the given object and compress the result as well as possible up to the given compression level; never null.
1256 * Tries all the supported compression methods up to the specified level
1257 * returning whichever results in the smallest result.
1258 * <p>
1259 * (Resource constraints may prevent some of the compression types being applied,
1260 * usually the highest levels.)
1261 * <p>
1262 * This does NOT assume that higher compression levels always compress given data better,
1263 * not does it assume a lower bound on data size worth compressing;
1264 * this tries all the possibilities exhaustively if it can.
1265 * <p>
1266 * This is likely to be very memory- and CPU- intensive; use with care.
1267 * <p>
1268 * This may attempt compressions in parallel on a multi-CPU machine,
1269 * though each compression is likely to thrash the cache
1270 * so this may or may not be good for compression and system performance.
1271 * <p>
1272 * This routine does not alter the content of the input array.
1273 *
1274 * @param in the object to be compressed; never null
1275 * @param maxLevel the maximum level at which to attempt to compress data,
1276 * using a value greater than MAX_SUPPORTED_COMPRESSION_LEVEL is allowed but ineffective;
1277 * never null
1278 *
1279 * @return non-null data compressed as well as possible;
1280 * if compression was not possible the level is NONE
1281 * and the input array is returned
1282 */
1283 public static Tuple.Pair<CompressionLevel,byte[]> compressObject(final Serializable in, final CompressionLevel maxLevel)
1284 throws IOException, InterruptedException
1285 { return(compressObject(in, maxLevel, false)); }
1286
1287 /**Serialise the given object and compress the result as well as possible up to the given compression level; never null.
1288 * Tries all the supported compression methods up to the specified level
1289 * returning whichever results in the smallest result.
1290 * <p>
1291 * (Resource constraints may prevent some of the compression types being applied,
1292 * usually the highest levels.)
1293 * <p>
1294 * This does NOT assume that higher compression levels always compress given data better,
1295 * not does it assume a lower bound on data size worth compressing;
1296 * this tries all the possibilities exhaustively if it can.
1297 * <p>
1298 * This is likely to be very memory- and CPU- intensive; use with care.
1299 * <p>
1300 * This may attempt compressions in parallel on a multi-CPU machine,
1301 * though each compression is likely to thrash the cache
1302 * so this may or may not be good for compression and system performance.
1303 * <p>
1304 * This routine does not alter the content of the input array.
1305 *
1306 * @param in the object to be compressed; never null
1307 * @param maxLevel the maximum level at which to attempt to compress data,
1308 * using a value greater than MAX_SUPPORTED_COMPRESSION_LEVEL is allowed but ineffective;
1309 * never null
1310 * @param maxOnly if true then only try the maximum compression level, and implicitly 'NONE'
1311 * @return non-null data compressed as well as possible;
1312 * if compression was not possible the level is NONE
1313 * and the input array is returned
1314 */
1315 public static Tuple.Pair<CompressionLevel,byte[]> compressObject(final Serializable in, final CompressionLevel maxLevel, final boolean maxOnly)
1316 throws IOException, InterruptedException
1317 {
1318 if((null == in) || (null == maxLevel))
1319 { throw new IllegalArgumentException(); }
1320
1321 final ByteArrayOutputStream baos = new ByteArrayOutputStream();
1322 final ObjectOutputStream oos = new ObjectOutputStream(baos);
1323 oos.writeObject(in);
1324 oos.close();
1325 return(compressData(baos.toByteArray(), maxLevel, maxOnly));
1326 }
1327
1328 /**Wrap input stream for decompression according to the compression level; never null.
1329 * If an unsupported compression level is selected then an exception is thrown.
1330 * <p>
1331 * If NONE is chosen as the compression level then the input stream is returned as-is.
1332 * <p>
1333 * Can decompress data compressed with compressData()/compressObject().
1334 */
1335 public static final InputStream wrapForDecompression(final InputStream in,
1336 final CompressionLevel cl)
1337 throws IOException
1338 {
1339 if((null == in) || (null == cl))
1340 { throw new IllegalArgumentException(); }
1341
1342 switch(cl)
1343 {
1344 // NONE means no compression, so return the stream unwrapped.
1345 case NONE: { return(in); }
1346
1347 // ZIP is so CPU-efficient that a data pump is probably not very useful
1348 // on single-CPU machines, but may be helpful beyond that.
1349 // When we do use a data-pump, we allow for good (10x) compression
1350 // given the ZIP input buffer of 32kB,
1351 // implying ~320kB+ of buffering is probably good.
1352 // We assume that ZIP does a reasonable job of buffering its input.
1353 case ZIP:
1354 {
1355 // Only start a new thread if upstream data is not flowing well.
1356 return(dataPump(new DefInputStream(in), 512 * 1024, false));
1357 }
1358
1359 // BZIP2 needs some wrapping for robustness and performance.
1360 // BZIP2 reads one byte at a time, so some sort of input buffering is vital.
1361 // BZIP2 can probably benefit from a data pump to multi-thread work and to keep data flow smooth.
1362 case BZIP2:
1363 {
1364 // We wrap this decompressor in two data pumps with output buffering
1365 // much larger than the maximum input/uncompressed 900kB block size
1366 // (to optimistically allow for good (~4x) compression!)
1367 // and input buffering much larger than the 900kB block size
1368 // which should help to keep the inbound compressed bytes flowing smoothly
1369 // in the face of BZIP2's very lumpy progress.
1370 // We assume that the dataPump() will also support BZIP2's
1371 // inefficient byte-at-a-time read()s reasonably efficiently.
1372 final InputStream cis;
1373 try { cis = new CBZip2InputStream(dataPump(in, 2*1024*1024, true)); }
1374 catch(final NullPointerException e)
1375 {
1376 // NPE is actually caused by a corrupt input stream.
1377 throw new IOException("corrupt/invalid input to BZIP2 decompressor");
1378 }
1379 if(Runtime.getRuntime().availableProcessors() < 2)
1380 { return(cis); }
1381 // Only put a data pump on the output of the compressor
1382 // if we have multiple CPUs and thus may improve throughput.
1383 return(dataPump(cis, 4*1024*1024, false));
1384 }
1385
1386 // LZMA uses lots of memory but may not be desperately heavy on CPU.
1387 case LZMA:
1388 {
1389 // FIXME: DHD20070913: LIS ~0.9 does some bad things that might throw an Error... (tries to access a system property)
1390 try { return(new LzmaInputStream(dataPump(in, 1024*1024, true))); }
1391 catch(final Throwable e) { throw new IOException("threw a SecurityException", e); }
1392 }
1393 }
1394
1395 throw new IOException("unsupported compression level");
1396 }
1397
1398 /**Basic dataPump() read size (bytes) for efficient operations; strictly positive. */
1399 private static final int DATA_PUMP_BLOCK_SIZE = 8192;
1400
1401 /**Wraps InputStream with a data pump (if possible); never null.
1402 * This can be used to keep data flowing in a pipeline with "lumpy" CPU use,
1403 * such as a stream decompressor with large input blocks.
1404 * <p>
1405 * This also allows stages on either side of the pump to run in separate threads
1406 * which can improve throughput further on a multi-CPU JVM.
1407 * <p>
1408 * In general this should be transparent to the user,
1409 * with close()s, exceptions, etc, being propagated much as usual.
1410 * <p>
1411 * In particular, a close() on the returned stream will cause a close()
1412 * on the argument input stream.
1413 * <p>
1414 * This routine may return the input stream if it cannot create a pump.
1415 * <p>
1416 * Instances of the returned class may not be thread-safe.
1417 * This is so that we can maximise performance, especially for small reads.
1418 * <p>
1419 * If flagged as aggressive then this routine always forces
1420 * the creation of an extra thread to do read-ahead,
1421 * else this only does so when an attempted read from down-stream
1422 * cannot be immediately (partly or fully) satisfied.
1423 * Thus a "non-aggressive" data pump may save some overhead
1424 * where it is not needed, eg on short/fast streams.
1425 *
1426 * @param is input data stream; never null
1427 * @param bufferSize approximate maximum amount of data to read-ahead;
1428 * strictly positive
1429 * @param aggressive if true then always start another thread,
1430 * else possibly do so only to avoid blocking
1431 */
1432 public static final InputStream dataPump(final InputStream is,
1433 final int bufferSize,
1434 final boolean aggressive)
1435 {
1436 if((is == null) || (bufferSize <= 0))
1437 { throw new IllegalArgumentException(); }
1438
1439 //if(IsDebug.isDebug) { System.out.println("[dataPump("+is+","+bufferSize+") starting...]"); }
1440
1441 // We must override at least the read() and close() methods.
1442 // Instances of this class are not thread-safe.
1443 return(new InputStream(){
1444 /**If true then close() has already been called. */
1445 private volatile boolean closed;
1446
1447 /**Close the stream(s), release resources.
1448 * Is idempotent, ie can be called more than once without ill effect.
1449 * <p>
1450 * This is not synchronized so as to allow an asynchronous call.
1451 */
1452 @Override public void close() throws IOException
1453 {
1454 if(!closed)
1455 {
1456 // Mark stream as closed immediately.
1457 closed = true;
1458
1459 // Ensure that the underlying input is closed directly,
1460 // possibly asynchronously...
1461 try { is.close(); }
1462 catch(final Exception e) { /* Ignore close() errors. */ }
1463
1464 // Explicitly interrupt the readerThread, if any (and alive),
1465 // until it goes away.
1466 final Thread t = readerThread;
1467 if(t != null)
1468 {
1469 while(t.isAlive())
1470 {
1471 t.interrupt();
1472 try { Thread.sleep(100); }
1473 catch(final InterruptedException e)
1474 { throw new InterruptedIOException("could not kill reader thread"); }
1475 }
1476 }
1477 }
1478
1479 // Release any buffers held.
1480 readAhead.clear();
1481 }
1482
1483 /**Private buffer used by read(void) to avoid lots of heap churn; never null. */
1484 private final byte[] readBuf = new byte[1];
1485
1486 /**This is potentially very inefficient but must be provided.
1487 * Unfortunately it is also the only read method called by
1488 * (for example) CBZip2InputStream,
1489 * so we cannot have it be massively slow.
1490 */
1491 @Override public int read() throws IOException
1492 {
1493 //if(IsDebug.isDebug) { System.out.println("[dataPump() read(1).]"); }
1494 // Fast-path for most reads...
1495 // ie when we have data immediately available on the head.
1496 final Pair<Integer, byte[]> head = readAhead.peek();
1497 if(head != null)
1498 {
1499 final int available = head.first - readAheadHeadOffset;
1500 assert(available > 0) : "we must never have an empty head of queue";
1501 final int result = head.second[readAheadHeadOffset++] & 0xff;
1502 assert(readAheadHeadOffset <= head.first.intValue());
1503 // Remove any now-empty head item.
1504 if(readAheadHeadOffset == head.first.intValue())
1505 {
1506 readAhead.remove();
1507 readAheadHeadOffset = 0;
1508 }
1509 return(result);
1510 }
1511
1512 // Fall back to main generic routine for tricky cases,
1513 // eg starting the reader thread, blocking for data, and detecting EOF.
1514 final int n = read(readBuf, 0, 1);
1515 if(n != 1) { return(-1); /* EOF */ }
1516 return(readBuf[0] & 0xff);
1517 }
1518
1519 /**Reader thread to suck data up the pipe.
1520 * The thread is started only once we have determined that is needed
1521 * (eg to avoid unnecessary overhead on small/fast transfers)
1522 * usually at the first read that would have blocked.
1523 * <p>
1524 * Once non-null this is not set null again.
1525 */
1526 private Thread readerThread;
1527
1528 /**The maximum size of the read-ahead queue; strictly positive.
1529 * Determined by a block size and the requested read-ahead buffering.
1530 * <p>
1531 * If we have lots of mostly-empty buffers
1532 * then much less actual data will be queued then the user requested.
1533 */
1534 private final int maxQueueLength = 1 + (bufferSize/DATA_PUMP_BLOCK_SIZE);
1535
1536 /**Queue of data read by the readerThread; never null.
1537 * This is a queue of (block-size,data) tuples.
1538 */
1539 private final BlockingQueue<Tuple.Pair<Integer,byte[]>> readAhead =
1540 new LinkedBlockingQueue<Tuple.Pair<Integer,byte[]>>(maxQueueLength);
1541
1542 /**Any (IO) Exception caught by the reader thread, or null if none. */
1543 private final AtomicReference<IOException> readerEx = new AtomicReference<IOException>();
1544
1545 /**Offset into current head item of readAhead queue; usually zero. */
1546 private int readAheadHeadOffset;
1547
1548 /**Object notified when new data is queued. */
1549 private final Object signalObject = new Object();
1550
1551 /**This is the generic efficient bulk-data read method.
1552 * If asked for one or more bytes of input
1553 * will block until at least one byte is ready to return or EOF is reached,
1554 * but will otherwise avoid blocking by returning what is immediate to hand.
1555 */
1556 @Override public int read(final byte[] b, int off, int len) throws IOException
1557 {
1558 //if(IsDebug.isDebug) { System.out.println("[dataPump() read(len="+len+").]"); }
1559 // If already closed then quit immediately.
1560 if(closed) { return(-1); }
1561
1562 // Handle trivial zero-length read immediately.
1563 if(len == 0) { return(0); }
1564
1565 // If we are not aggressive and have not yet started read-ahead
1566 // then attempt to read without blocking from the underlying stream.
1567 if(!aggressive && (readerThread == null))
1568 {
1569 final int avail = is.available();
1570 // We can continue to eschew a read-ahead thread for now
1571 // if there is some data ready.
1572 if(avail > 0)
1573 {
1574 // Read what we can without blocking.
1575 final int n = is.read(b, off, Math.min(avail, len));
1576 if(n < 1) { close(); return(-1); /* EOF */ }
1577 return(n);
1578 }
1579 // Fall through to start read-ahead thread.
1580 }
1581
1582 try
1583 {
1584 // Create the readerThread on the first call.
1585 if(readerThread == null)
1586 {
1587 readerThread = new Thread("GenUtils.dataPump() reader thread"){
1588 @Override public final void run()
1589 {
1590 try
1591 {
1592 //if(IsDebug.isDebug) { System.out.println("[dataPump() started reader thread...]"); }
1593 for( ; ; )
1594 {
1595 final int available = is.available();
1596 // Read what's immediately available,
1597 // but if none then block for a full chunk.
1598 final int bufSize = (available < 1) ? DATA_PUMP_BLOCK_SIZE :
1599 Math.min(available, DATA_PUMP_BLOCK_SIZE);
1600 // TODO: get buffer from a pool if available...
1601 byte buf[] = new byte[bufSize];
1602 final int n = is.read(buf, 0, bufSize);
1603 //if(IsDebug.isDebug) { System.out.println("[dataPump() read "+n+" bytes in buffer of size "+bufSize+".]"); }
1604 // Quit gracefully at EOF.
1605 if(n <= 0) { return; }
1606 // To avoid wasting memory in the queue,
1607 // if we only did a tiny read
1608 // and the buffer is unlikely to be consumed/freed quickly
1609 // then copy the data into a smaller buffer.
1610 if((n != bufSize) && (n < (bufSize>>1)) && !readAhead.isEmpty())
1611 {
1612 final byte newBuf[] = new byte[n];
1613 System.arraycopy(buf, 0, newBuf, 0, n);
1614 buf = newBuf;
1615 //if(IsDebug.isDebug) { System.out.println("[dataPump() reallocated buffer.]"); }
1616 }
1617 // Else if we got some data, try to queue it.
1618 readAhead.put(new Tuple.Pair<Integer,byte[]>(Integer.valueOf(n), buf));
1619 // Wake/notify the consumer thread.
1620 synchronized(signalObject) { signalObject.notifyAll(); }
1621 }
1622 }
1623 catch(final IOException e)
1624 {
1625 // Capture the error to re-throw for the caller
1626 // on its next read() operation.
1627 readerEx.set(e);
1628 // And then exit immediately.
1629 return;
1630 }
1631 catch(final InterruptedException e)
1632 {
1633 final IOException err = new InterruptedIOException("interrupted");
1634 err.initCause(e);
1635 // Capture the error to re-throw for the caller
1636 // on its next read() operation.
1637 readerEx.set(err);
1638 // And then exit immediately.
1639 return;
1640 }
1641 }
1642 };
1643 readerThread.setDaemon(true); // Don't stop the JVM exiting.
1644 readerThread.start();
1645 }
1646
1647 // Run until there seems to be nothing left to do...
1648 for( ; ; )
1649 {
1650 // While there is queued input,
1651 // try to return it to the caller.
1652 // We do not block.
1653 if(!readAhead.isEmpty())
1654 {
1655 int bytesRead = 0;
1656 do
1657 {
1658 // Compute what is immediately available on the head of the queue
1659 // minus any already returned from this value already.
1660 final Pair<Integer, byte[]> head = readAhead.peek();
1661 final int available = head.first - readAheadHeadOffset;
1662 assert (available > 0) : "we must never have an empty head of queue";
1663 final int toCopy = Math.min(len, available);
1664 // Copy data to the caller's buffer.
1665 System.arraycopy(head.second, readAheadHeadOffset, b, off, toCopy);
1666 // Adjust our notion of the amount read from this item.
1667 readAheadHeadOffset += toCopy;
1668 off += toCopy;
1669 len -= toCopy;
1670 bytesRead += toCopy;
1671 assert(readAheadHeadOffset <= head.first.intValue());
1672 // If we've exhausted data from the HEAD then remove() it.
1673 if(readAheadHeadOffset == head.first.intValue())
1674 {
1675 readAhead.remove();
1676 readAheadHeadOffset = 0;
1677 }
1678 } while(!readAhead.isEmpty() && (len > 0));
1679
1680 // Return the data to the caller.
1681 //if(IsDebug.isDebug) { System.out.println("[dataPump() read()="+bytesRead+".]"); }
1682 assert(bytesRead > 0);
1683 return(bytesRead);
1684 }
1685 // Quit if the reader thread has died
1686 // (with assumed memory-release semantics)
1687 // and the read-ahead queue is still empty.
1688 else if(!readerThread.isAlive() && readAhead.isEmpty())
1689 { break; }
1690
1691 final IOException err = readerEx.get();
1692 // Quit (and close()) if the reader thread encountered an error.
1693 // (Once we've dealt with any pending/queued input...)
1694 if(err != null)
1695 {
1696 close();
1697 throw err;
1698 }
1699
1700 // Wait a little while for more data and then try reading again.
1701 // We should wake immediately when there is something to do.
1702 // We don't block for ever so that a missed notification is not fatal.
1703 synchronized(signalObject)
1704 {
1705 while(readerThread.isAlive() && readAhead.isEmpty())
1706 { signalObject.wait(101); }
1707 }
1708 }
1709
1710 // Assume that we have now reached EOF...
1711 //if(IsDebug.isDebug) { System.out.println("[dataPump().read() hit EOF...]"); /* Most expected uses will know they are at EOF without getting this explicit return. */ }
1712 close(); // Release resources...
1713 return(-1); // Indicate EOF to the caller.
1714 }
1715 catch(final IOException e)
1716 {
1717 // In case of error try to ensure that resources are released.
1718 close(); // Try to free upstream resources ASAP.
1719 throw e; // Rethrow the original exception.
1720 }
1721 catch(final InterruptedException e)
1722 {
1723 // In case of interruption try to ensure that resources are released.
1724 close(); // Try to free upstream resources ASAP.
1725 final IOException err = new InterruptedIOException("interrupted");
1726 err.initCause(e);
1727 throw err;
1728 }
1729 }
1730
1731 /**Indicate how many bytes should be immediately available without blocking; non-negative.
1732 */
1733 @Override public int available() throws IOException
1734 {
1735 // If the reader thread is not yet started
1736 // then report what upstream says is available.
1737 if(readerThread == null)
1738 { return(is.available()); }
1739
1740 // Else conservatively report what we have available
1741 // on the head of the queue right now
1742 // (always at least 1 if the queue not empty)
1743 // plus at least one byte for each queued block behind!
1744 if(!readAhead.isEmpty())
1745 {
1746 final int onHead = readAhead.peek().first - readAheadHeadOffset;
1747 assert(onHead > 0);
1748 return(onHead + (readAhead.size()-1));
1749 }
1750
1751 // Nothing obviously immediately available.
1752 return(0);
1753 }
1754 });
1755 }
1756
1757 /**Avoid compile-time complaints about casts believed to be correct.
1758 * C/o http://weblogs.java.net/blog/emcmanus/archive/2007/03/getting_rid_of.html.
1759 */
1760 @SuppressWarnings("unchecked")
1761 public static <T> T cast(final Object x) { return((T) x); }
1762
1763 /**Presence of a file in this location is an indication of power shortage by default.
1764 * The file is relative to the JVM's working directory.
1765 * <p>
1766 * If any local-props value is supplied then this is ignored.
1767 */
1768 public static final File DEFAULT_LOWPOWER_FLAG_FILENAME = new File("LOW_POWER.flag");
1769
1770 /**Time and result of last check of system power state; the least significant bit indicates the state detected.
1771 * When the last bit is 1 it indicates that power is low, else none was.
1772 * <p>
1773 * Marked volatile for thread-safe lock-free access.
1774 * <p>
1775 * The initial state indicates that the system should start in power-conserving mode
1776 * <em>unless</em> this instance is in fast-start mode.
1777 */
1778 private static volatile long _lp_lastCheck = (System.currentTimeMillis() & ~1) |
1779 (LocalProps.fastStartMode() ? 0 : 1);
1780
1781 /**Minimum interval for check/update of file flags in ms; strictly positive.
1782 * Set to avoid expensive and redundant tests.
1783 * <p>
1784 * Should probably be of the order of a few seconds.
1785 */
1786 private static final int _LP_MIN_RECHECK_MS = 5131;
1787
1788 /**Minimum interval for check/update of URI flags in ms; strictly positive.
1789 * Chosen to avoid expensive and redundant tests.
1790 * <p>
1791 * Should probably be of the order of a few tens of seconds
1792 * as network connections may have all sorts of costs and be uncacheable,
1793 * though checking some factors such as local grid overload (via mains frequency)
1794 * could profitably be done more often.
1795 */
1796 private static final int _LP_MIN_RECHECK_URI_MS = 31311;
1797
1798 /**Maximum interval for check/update of URI flags in ms; strictly positive.
1799 * Much larger than _LP_MIN_RECHECK_URI_MS to allow a large dynamic range
1800 * and thus efficient adaptation to the actual underlying information rate.
1801 * <p>
1802 * Should probably be of the order of a an hour or more.
1803 */
1804 private static final int _LP_MAX_RECHECK_URI_MS = _LP_MIN_RECHECK_URI_MS << 8;
1805
1806 /**Maximum time that a check/update of URI flags may take in ms; strictly positive.
1807 * Long enough to only allow a new check to be launched
1808 * if a previous one has failed in some way not otherwise caught.
1809 * <p>
1810 * Should probably be several minutes at least.
1811 */
1812 private static final int _LP_MAX_CHECK_TIME_URI_MS = 10 * 60 * 1001;
1813
1814 /**Private lock for mustConservePower() while working. */
1815 private static final Object _mcp_lock = new Object();
1816
1817 /**Immutable information about each remote URI flag.
1818 */
1819 private static final class RemoteFlagInfo
1820 {
1821 /**Flag URI; not null. */
1822 final URI flagURI;
1823 /**Status of the remote flag; null if not known.
1824 * TRUE indicates that the remote flag is definitely present
1825 * and thus we should be in low-power mode.
1826 * <p>
1827 * FALSE indicates that the remote flag is definitely absent
1828 * and thus we need not be in low-power mode
1829 * (if no other flags are present).
1830 * <p>
1831 * A null value indicates that the remote value is unknown.
1832 */
1833 final Boolean flagIsPresent;
1834 /**Time of the last completed poll, or zero if none. */
1835 final long lastPollCompleted;
1836 /**Time we started (re)checking the status of this flag, or zero if no check is underway. */
1837 final long checkStarted;
1838 /**Time of last state change, or zero if none. */
1839 final long lastStateChange;
1840
1841 /**Returns true iff a check is currently being run.
1842 * We consider a check to be running if started at all(non-zero)
1843 * and if started recently enough.
1844 */
1845 public boolean checkIsRunning()
1846 {
1847 return((0 != checkStarted) &&
1848 (System.currentTimeMillis() - checkStarted < _LP_MAX_CHECK_TIME_URI_MS));
1849 }
1850
1851 /**Returns true if a new check should be started.
1852 * This is taken never to be true if a check is running,
1853 * nor if the last poll very recently completed.
1854 */
1855 public boolean checkIsNeeded()
1856 {
1857 // If another poll for this URI is still running then don't check again yet.
1858 if(checkIsRunning()) { return(false); }
1859 // Don't recheck too recently after the last poll.
1860 if(lastPollCompleted != 0)
1861 {
1862 final long now = System.currentTimeMillis();
1863 final long timeSinceLastPollCompleted = now - lastPollCompleted;
1864 // If the last poll finished recently then don't check again yet.
1865 // Unless the current status is FALSE (low-power not required)
1866 // then expand the time before the next check well beyond the minimum.
1867 final int basicRecheckTime = ((Boolean.FALSE == flagIsPresent) ? _LP_MIN_RECHECK_URI_MS : (_LP_MIN_RECHECK_URI_MS<<2));
1868 assert(basicRecheckTime <= _LP_MAX_RECHECK_URI_MS);
1869
1870 // If we know the time since the last state change
1871 // then we may be able to adaptively delay the next check to the minimum of
1872 // a small fraction of the time that the state has been unchanged
1873 // and a small multiple of the basic proposed time.
1874 // (This is similar to how some simple HTTP caches work in the absence of cache headers.)
1875 final long modifiedRecheckTime;
1876 if(lastStateChange != 0)
1877 {
1878 final long timeSinceLastStateChange = now - lastStateChange;
1879 modifiedRecheckTime = Math.max(basicRecheckTime, // At least the basic time specified...
1880 Math.min(Math.min(timeSinceLastStateChange >> 4, basicRecheckTime << 3), _LP_MAX_RECHECK_URI_MS));
1881 }
1882 else
1883 {
1884 modifiedRecheckTime = basicRecheckTime; // Don't modify the basic recheck time...
1885 }
1886 assert(modifiedRecheckTime <= _LP_MAX_RECHECK_URI_MS);
1887
1888 // A check is needed if the suggested recheck time (capped by the maximum allowed)
1889 // has been exceeded since the last completed poll.
1890 if(timeSinceLastPollCompleted < modifiedRecheckTime)
1891 { return(false); }
1892 }
1893 return(true); // Time to check again.
1894 }
1895
1896 /**Returns true if the last check was so long ago that any data held is probably stale.
1897 */
1898 public boolean isStale()
1899 {
1900 // Definitely stale if no previous poll!
1901 if(lastPollCompleted == 0) { return(true); }
1902 // If the last poll was completed a very long time ago
1903 // (much longer than the maximum poll interval)
1904 // then regard the data as stale.
1905 final long timeSinceLastPollCompleted = System.currentTimeMillis() - lastPollCompleted;
1906 final boolean tooLong = timeSinceLastPollCompleted > (_LP_MAX_RECHECK_URI_MS<<1);
1907 if(tooLong) { assert checkIsNeeded() : "Check must be needed if this is stale"; }
1908 return(tooLong);
1909 }
1910
1911 /**Create a new instance from scratch.
1912 * Private so that only instance methods can call it
1913 * to provide modified versions of existing instances for example. */
1914 private RemoteFlagInfo(final URI flagURI,
1915 final Boolean flagIsPresent,
1916 final long lastPollCompleted, final long checkStarted, final long lastStateChange)
1917 {
1918 if(null == flagURI) { throw new IllegalArgumentException(); }
1919 this.flagURI = flagURI;
1920 this.flagIsPresent = flagIsPresent;
1921 this.lastPollCompleted = lastPollCompleted;
1922 this.checkStarted = checkStarted;
1923 this.lastStateChange = lastStateChange;
1924 }
1925
1926 /**Construct an (initial) instance with unknown status and no check underway. */
1927 RemoteFlagInfo(final URI flagURI)
1928 { this(flagURI, null, 0, 0, 0); }
1929
1930 /**Note the start of a status check (called just before starting a check).
1931 * Returns new instance with all values unchanged except 'checkStarted'
1932 * which is set to 'now'.
1933 */
1934 RemoteFlagInfo startCheckNow()
1935 {
1936 final long now = System.currentTimeMillis();
1937 return(new RemoteFlagInfo(flagURI, flagIsPresent, lastPollCompleted, now, lastStateChange));
1938 }
1939
1940 /**Mark remote flag check complete with the given status.
1941 * Marks the time of the poll completion,
1942 * and clears the 'being checked' flag.
1943 * <p>
1944 * If the status has changed, this is noted.
1945 */
1946 public RemoteFlagInfo checkComplete(final Boolean remoteFlagIsNowPresent)
1947 {
1948 final long now = System.currentTimeMillis();
1949 final boolean statusChanged = (null != remoteFlagIsNowPresent) ? !remoteFlagIsNowPresent.equals(flagIsPresent) : (flagIsPresent != null);
1950 return(new RemoteFlagInfo(flagURI, remoteFlagIsNowPresent, now, 0, statusChanged ? now : lastStateChange));
1951 }
1952 }
1953
1954 /**Low-power URI flags and their status; never null.
1955 * Is thread-safe to allow async updates.
1956 * <p>
1957 * Is periodically purged of stale URI keys to avoid 'leaking' memory.
1958 */
1959 private static final ConcurrentMap<URI,RemoteFlagInfo> lowerPowerFileURIStatus = new ConcurrentHashMap<URI,RemoteFlagInfo>();
1960
1961 /**Extreme-low-power flag status, true if extreme conservation is required; initially false.
1962 * Marked volatile for lock-free thread-safe access.
1963 */
1964 private static volatile boolean extremeLowPower;
1965
1966 /**If this returns true then the system must temporarily conserve power as much as possible.
1967 * This may be because, for example, we are running on battery power,
1968 * and/or the (remaining) battery capacity may be very limited.
1969 * <p>
1970 * This may also be because the system is running too hot and/or too noisy for example.
1971 * <p>
1972 * This cap the rate at which underlying data source(s) are polled.
1973 * <p>
1974 * This may block (very briefly) to check filesystem flags,
1975 * but URI (ie probably-remote) flags are checked asynchronously and much less frequently
1976 * and only if the local filesystem flags do not request a low-power state,
1977 * so as to save bandwidth and time and effort.
1978 * <p>
1979 * This condition is usually assumed to be transient, not permanent.
1980 * <p>
1981 * Note that if more than one filesystem flag is listed,
1982 * the last one is taken to indicate 'extreme' energy shortage/conservation.
1983 */
1984 public static final boolean mustConservePower()
1985 {
1986 // If it is too soon since we last polled our power-status to poll it again
1987 // then return whatever status we last observed.
1988 // We do this without taking any lock to make the usual path as fast as possible.
1989 final long now = System.currentTimeMillis();
1990 final long lastCheck = _lp_lastCheck;
1991 // True if old power state was low.
1992 final boolean wasLow = ((lastCheck & 1) == 1);
1993 // Recheck (much) less often when already in the power-conserving state.
1994 // This helps conserve by being reluctant to leave power-conserving mode.
1995 if(now - lastCheck <= (wasLow ? (_LP_MIN_RECHECK_MS<<3) : _LP_MIN_RECHECK_MS))
1996 { return(wasLow); }
1997
1998 // Only use the lock if actually about to check the filesystem flags.
1999 // We avoid doing this work wastefully in concurrent threads...
2000 synchronized(_mcp_lock)
2001 {
2002 // Double-check that no one got in ahead of us,
2003 // returning the updated state if they did so
2004 // rather than doing the work again...
2005 if(System.currentTimeMillis() - _lp_lastCheck <= _LP_MIN_RECHECK_MS) { return(((_lp_lastCheck & 1) == 1)); }
2006
2007 // Only look for local file-based flags if general filesystem access is permitted.
2008 final boolean nowLowLocal;
2009 final List<File> localFlagsPresent;
2010 final List<File> effectiveLocalFlags;
2011 if(!LocalProps.getNoGeneralFileAccess())
2012 {
2013 // Get local properties list of low-power warning filesystem flags.
2014 final List<File> lowerPowerFileFlags = LocalProps.getLowerPowerFileFlags();
2015 // Iff empty, use default single flag in PWD.
2016 effectiveLocalFlags = (lowerPowerFileFlags.isEmpty()) ? Collections.singletonList(DEFAULT_LOWPOWER_FLAG_FILENAME) :
2017 lowerPowerFileFlags;
2018 assert(!effectiveLocalFlags.isEmpty());
2019
2020 // Find out if the system is short of power now from local flags only.
2021 localFlagsPresent = new ArrayList<File>(effectiveLocalFlags.size());
2022 for(final File f : effectiveLocalFlags)
2023 {
2024 // Check if this 'low-power' flag file is present.
2025 try { if(f.exists() && f.isFile()) { localFlagsPresent.add(f); } }
2026 // Do not assume that inability to access a flag means that it is set.
2027 // Don't complain in the case that it's just our default flag...
2028 catch(final AccessControlException e)
2029 {
2030 if(!f.equals(DEFAULT_LOWPOWER_FLAG_FILENAME) && !lowerPowerFileFlags.isEmpty())
2031 { System.err.println("WARNING: unable to check for power-flag file: "+f+" ... "+e.getMessage()); }
2032 }
2033 }
2034 nowLowLocal = !localFlagsPresent.isEmpty();
2035
2036 // Set 'extreme' flag if last flag is set...
2037 if(lowerPowerFileFlags.size() > 1)
2038 { extremeLowPower = localFlagsPresent.contains(lowerPowerFileFlags.get(lowerPowerFileFlags.size()-1)); }
2039 else
2040 { extremeLowPower = false; }
2041 }
2042 else
2043 {
2044 nowLowLocal = false; // No local file flags to test...
2045 effectiveLocalFlags = localFlagsPresent = Collections.emptyList();
2046 extremeLowPower = false;
2047 }
2048
2049 // If there are no local filesystem flags set
2050 // then any URI-based flags will need to be checked
2051 // (ie we don't waste network bandwidth when we already locally know to conserve).
2052 // We may wish to prevent exit from a low-power state until they've been checked,
2053 // so for example we regard the (initial) unknown remote status as requiring conservation.
2054 final Set<URI> lowerPowerFileURIs = LocalProps.getLowerPowerFileURIs();
2055 final Set<URI> remoteFlagsNotAbsent = new HashSet<URI>();
2056 if(!nowLowLocal)
2057 {
2058 // Purge any previous URIs now now longer to be monitored to avoid memory retention/'leaks'.
2059 lowerPowerFileURIStatus.keySet().retainAll(lowerPowerFileURIs);
2060 // Now if we are actually monitoring some URIs, do it..
2061 if(!lowerPowerFileURIs.isEmpty())
2062 {
2063 for(final URI u : lowerPowerFileURIs)
2064 {
2065 // Get current remote flag status.
2066 RemoteFlagInfo rfi; // Will be non-null when we're done.
2067 // Create initial mapping if absent...
2068 while(null == (rfi = lowerPowerFileURIStatus.get(u)))
2069 { lowerPowerFileURIStatus.putIfAbsent(u, new RemoteFlagInfo(u)); }
2070
2071 // Note the remote status as indicating that low power is required
2072 // unless the remote flag was definitely absent when last checked
2073 // and was checked recently enough ago.
2074 if(!Boolean.FALSE.equals(rfi.flagIsPresent) || rfi.isStale())
2075 { remoteFlagsNotAbsent.add(u); }
2076
2077 // If we need to update/replace a stale result then do so now.
2078 if(rfi.checkIsNeeded())
2079 {
2080 // Attempt to start a polling thread without blocking.
2081 // If the thread pool was full then we do nothing...
2082 final RemoteFlagInfo rfiBeforeStart = rfi;
2083 ThreadUtils.nonCPUThreadPoolDiscardable.submit(new Runnable() {
2084 public void run()
2085 {
2086 final long checkStart = System.currentTimeMillis();
2087 // If the status has changed since we were launched
2088 // then we abort, else we mark the URI check as started...
2089 Boolean remoteFlagIsPresent = null; // Status not known (yet).
2090 final RemoteFlagInfo rfiStartingCheck = rfiBeforeStart.startCheckNow();
2091 try
2092 {
2093 if(!lowerPowerFileURIStatus.replace(u, rfiBeforeStart, rfiStartingCheck))
2094 {
2095 System.err.println("Failed to start low-power check for "+u); // Shouldn't really happen...
2096 return; // Abort!
2097 }
2098 System.out.println("Low-power check started for "+u+"; previously present="+rfiBeforeStart.flagIsPresent);
2099 final HttpURLConnection connection = (HttpURLConnection) u.toURL().openConnection();
2100 connection.setAllowUserInteraction(false);
2101 connection.setUseCaches(false);
2102 // Set timeouts well within the maximum time allowed for a check.
2103 connection.setConnectTimeout(_LP_MAX_CHECK_TIME_URI_MS / 2);
2104 connection.setReadTimeout(_LP_MAX_CHECK_TIME_URI_MS / 2);
2105 // For an HTTP connection we only need HEAD, not body content...
2106 connection.setRequestMethod("HEAD");
2107 // Now get the status...
2108 final int statusHTTP = connection.getResponseCode();
2109 // Convert to our final status form iff we recognise the HTTP code.
2110 switch(statusHTTP)
2111 {
2112 case 200: { remoteFlagIsPresent = Boolean.TRUE; break; }
2113 case 404: { remoteFlagIsPresent = Boolean.FALSE; break; }
2114 default:
2115 {
2116 System.err.println("Unrecognised HTTP status "+statusHTTP+" for low-power flag "+u);
2117 break;
2118 }
2119 }
2120 }
2121 catch(final IOException e)
2122 {
2123 // Absorb I/O problem, resulting in status unknown, though log short summary.
2124 System.err.println("Failed to get status of low-power flag "+u+": "+e.getMessage());
2125 }
2126 finally
2127 {
2128 // Mark check as complete (providing ours was the last check to start).
2129 final RemoteFlagInfo rfiFinishedCheck = rfiStartingCheck.checkComplete(remoteFlagIsPresent);
2130 if(!lowerPowerFileURIStatus.replace(u, rfiStartingCheck, rfiFinishedCheck))
2131 {
2132 System.err.println("Failed to mark low-power check complete for "+u); // Shouldn't really happen...
2133 }
2134 }
2135 final long checkEND = System.currentTimeMillis();
2136 System.out.println("Low-power check complete for "+u+" after "+(checkEND-checkStart)+"ms; now present="+remoteFlagIsPresent);
2137 }
2138 });
2139 // Attempt at most one (new) status poll on each time here...
2140 break;
2141 }
2142 }
2143 }
2144 }
2145 // Status from remote flags.
2146 final boolean nowLowRemote = !remoteFlagsNotAbsent.isEmpty();
2147
2148 // Overall status...
2149 final boolean nowLow = nowLowLocal || nowLowRemote;
2150
2151 // Log any state change/transition.
2152 if(wasLow != nowLow)
2153 {
2154 // Only a transition *to* low power needs a warning.
2155 (wasLow ? System.out : System.err).println((wasLow?"INFO":"WARNING") + ": SYSTEM POWER STATE CHANGED: now "+(nowLow?(extremeLowPower?"EXTREMELY LOW":"LOW"):"OK")+" at "+(new Date()));
2156 // Report set of flags active at the transition.
2157 if(nowLowLocal)
2158 { System.err.println("INFO: SYSTEM POWER STATE FLAGS ACTIVE: " + new ArrayList<File>(localFlagsPresent)); }
2159 if(nowLowRemote)
2160 { System.err.println("INFO: REMOTE POWER STATE FLAGS ACTIVE (or not testable): " + new ArrayList<URI>(remoteFlagsNotAbsent)); }
2161 System.out.println("INFO: SYSTEM POWER STATE FLAGS AVAILABLE: " + new ArrayList<File>(effectiveLocalFlags) + " REMOTE: "+lowerPowerFileURIs);
2162 }
2163
2164 // Cache the current status.
2165 final long rightNow = System.currentTimeMillis();
2166 _lp_lastCheck = nowLow ? (rightNow | 1) : (rightNow & ~1);
2167
2168 // Note when not in low-power mode.
2169 if(!nowLow)
2170 { lastInNormalPowerMode = rightNow; }
2171
2172 // Return the current status.
2173 return(nowLow);
2174 }
2175 }
2176
2177 /**Minimum time that system has to be continuously in low-power mode to trigger 'long term' measures (ms); strictly positive.
2178 * Somewhat more than one or more 24h periods
2179 * to allow for daily energy cycles for systems powered by renewables
2180 * and for clock-driven notions of low-power mode (eg driven by ToD/HH grid load and carbon-intensity).
2181 * <p>
2182 * Not so long as to remove responsiveness to actual energy issues,
2183 * especially after system (re)start.
2184 * <p>
2185 * Small random factor to help avoid collisions between different servers.
2186 */
2187 public static final long MIN_LONG_TERM_LOW_POWER_MS = 27 * 3600 * 1000L + Rnd.fastRnd.nextInt(3600*1000);
2188
2189 /**Last time system was not in low-power mode.
2190 * Initially 'now' so that system does not start in long-term low-power measures.
2191 * <p>
2192 * Marked volatile to allow lock-free thread-safe access.
2193 */
2194 private static volatile long lastInNormalPowerMode = System.currentTimeMillis();
2195
2196 /**If true then the system has been in a low-power state for a long time.
2197 * Typically implies power has been low for (much) more than 24 hours,
2198 * and thus longer-term changes in behaviour to save energy are in order.
2199 */
2200 public static boolean mustConservePowerLongTerm()
2201 {
2202 return((System.currentTimeMillis() - lastInNormalPowerMode) > MIN_LONG_TERM_LOW_POWER_MS);
2203 }
2204
2205 /**Approximate minimum time to long-term low-power mode (ms).
2206 * Can be negative to indicate how long the system has been in long-term low-power mode already.
2207 * <p>
2208 * Will stay at approximately maximum time if not currently in low-power mode.
2209 */
2210 public static long approxMinMSToStartLongTermLowPowerMode()
2211 {
2212 return(MIN_LONG_TERM_LOW_POWER_MS - (System.currentTimeMillis() - lastInNormalPowerMode));
2213 }
2214
2215 /**If true then the system is in an extreme low-power state and may sacrifice some functionality to conserve.
2216 * This is true if either the system has been in a low-power state
2217 * for much longer than the MIN_LONG_TERM_LOW_POWER_MS limit, eg at least twice that,
2218 * OR if the 'extreme low power' (last local filesystem) flag is set.
2219 */
2220 public static boolean mustConservePowerExtreme()
2221 {
2222 return(extremeLowPower ||
2223 ((System.currentTimeMillis() - lastInNormalPowerMode) > (MIN_LONG_TERM_LOW_POWER_MS<<1)));
2224 }
2225
2226 /**If this returns true then the system must permanently conserve CPU or is has extreme energy shortage.
2227 * This may be because, for example, we are running on extremely limited battery reserves,
2228 * or because we are running in a CPU-metered (eg "cloud"/VPS) environment,
2229 * or possibly because of issues such as system over-heating.
2230 * <p>
2231 * This may block (very briefly).
2232 */
2233 public static boolean mustConserveCPU()
2234 {
2235 return(LocalProps.isCloudMirrorInstance() || mustConservePowerExtreme());
2236 }
2237
2238
2239 /**Returns immutable SortedSet of the n best items retrieved via the supplied iterator; never null but may be empty.
2240 * Only retains at most n items at one time.
2241 * <p>
2242 * If less then n items are available then all are returned
2243 * else the least n by the supplied Comparator are returned.
2244 * <p>
2245 * Minimises memory overhead by only retaining at most n items while working.
2246 *
2247 * @param n maximum size of head (least-value-items) set to return; non-negative
2248 * @param it iterator over items to consider for inclusion in the result; non-null
2249 * @param comparator Comparator over items to include; non-null
2250 */
2251 public static <T extends Comparable<T>> SortedSet<T> leastN(final int n, final Iterator<T> it, final Comparator<T> comparator)
2252 {
2253 if(n < 0) { throw new IllegalArgumentException(); }
2254 if(it == null) { throw new IllegalArgumentException(); }
2255 if(comparator == null) { throw new IllegalArgumentException(); }
2256
2257 final SortedSet<T> result = new TreeSet<T>(comparator);
2258
2259 // Deal with this special case here primarily to simplify the logic below.
2260 if(n == 0) { return(Collections.unmodifiableSortedSet(result)); }
2261
2262 // Until we reach the size cap, copy stuff in blindly.
2263 while(it.hasNext() && (result.size() < n))
2264 { result.add(it.next()); }
2265
2266 // Now only add items better than the last() by replacing the last().
2267 while(it.hasNext())
2268 {
2269 final T next = it.next();
2270 // If next item no earlier than last item then discard next item.
2271 final T last = result.last();
2272 if(comparator.compare(last, next) <= 0)
2273 { continue; }
2274 // The next item is earlier than the last item,
2275 // so zap last and insert next
2276 // if next is new (ie not a duplicate of an already-held value)...
2277 if(result.add(next))
2278 { result.remove(last); }
2279 assert(result.size() == n) : "size should remain exactly at maximum";
2280 }
2281
2282 return(Collections.unmodifiableSortedSet(result));
2283 }
2284
2285
2286 /**Prevents more than a specified number of bytes being written to the output stream.
2287 * Attempting to exceed the limit will result in an IOException being thrown.
2288 * <p>
2289 * Specifically, this will veto the first write that would exceed the limit
2290 * and all subsequent (non-zero) writes.
2291 * <p>
2292 * Data up until the limit will be written.
2293 * <p>
2294 * Thread-safe.
2295 */
2296 public final static class LengthLimitedOutputStream extends FilterOutputStream
2297 {
2298 /**The maximum number of bytes to be written; non-negative. */
2299 private final int limit;
2300
2301 /**The number of bytes written so far; non-negative.
2302 * Advances monotonically.
2303 * <p>
2304 * Accessed under the instance lock only.
2305 */
2306 private int bytesWritten;
2307
2308 /**Retrieve the number of bytes written so far; non-negative. */
2309 public synchronized int getBytesWritten() { return(bytesWritten); }
2310
2311 /**Creates an output stream with the specified write limit.
2312 * @param out the output stream
2313 * @param limit maximum number of bytes to be allowed; non-negative
2314 */
2315 public LengthLimitedOutputStream(final OutputStream out, final int limit)
2316 {
2317 super(out);
2318 if(limit < 0) { throw new IllegalArgumentException(); }
2319 this.limit = limit;
2320 }
2321
2322 @Override public synchronized void write(final int b)
2323 throws IOException
2324 {
2325 if(bytesWritten == limit)
2326 { throw new IOException("limit reached"); }
2327 out.write(b);
2328 ++bytesWritten;
2329 }
2330
2331 @Override public synchronized void write(final byte[] b, final int off, final int len)
2332 throws IOException
2333 {
2334 if(len < 0) { throw new IllegalArgumentException(); }
2335 if(len == 0) { return; }
2336 // The simple case where the entire write can be done.
2337 if(bytesWritten == limit)
2338 { throw new IOException("limit reached"); }
2339 final int spaceLeft = limit - bytesWritten;
2340 if(spaceLeft >= len)
2341 {
2342 out.write(b, off, len);
2343 bytesWritten += len;
2344 return;
2345 }
2346 // Write as much as we can...
2347 if(spaceLeft > 0) { out.write(b, off, spaceLeft); }
2348 bytesWritten = limit;
2349 // Could not write everything so throw an exception.
2350 throw new IOException("limit reached after partial write");
2351 }
2352 }
2353
2354 /**Generates a number in the range 0 to n-1 based mainly on the low-order bits of seed; non-negative.
2355 * This is a simple and cheap alternative to (say)
2356 * <code>(new Random(seed)).nextInt(n))</code>
2357 * to get a reasonably-evenly-distributed value based on a much larger one,
2358 * such as currentTimeMillis().
2359 * <p>
2360 * The seed value should have much more than 2^n different values in its low-order bits.
2361 * <p>
2362 * A basic modulo function is used in this implementation.
2363 *
2364 * @param seed input whose value will be used to generate n.
2365 * @param n small strictly positive exclusive upper bound on returned value
2366 * @return non-negative value less than n
2367 */
2368 public static final int getIntHashedSimple(final long seed, final int n)
2369 {
2370 if(n <= 0) { throw new IllegalArgumentException(); }
2371 if(seed < 0) { return((int) ((seed >>> 1) % n)); }
2372 return((int) (seed % n));
2373 }
2374 }