001 /*
002 Copyright (c) 1996-2011, Damon Hart-Davis
003 All rights reserved.
004
005 Redistribution and use in source and binary forms, with or without
006 modification, are permitted provided that the following conditions are
007 met:
008
009 * Redistributions of source code must retain the above copyright
010 notice, this list of conditions and the following disclaimer.
011
012 * Redistributions in binary form must reproduce the above copyright
013 notice, this list of conditions and the following disclaimer in the
014 documentation and/or other materials provided with the
015 distribution.
016
017 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
018 IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
019 TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
020 PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
021 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
022 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
023 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
024 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
025 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
026 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
027 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
028 */
029 package org.hd.d.pg2k.webSvr.exhibit;
030
031 import java.io.ByteArrayInputStream;
032 import java.io.File;
033 import java.io.IOException;
034 import java.io.InvalidObjectException;
035 import java.io.ObjectInputStream;
036 import java.io.Serializable;
037 import java.io.UnsupportedEncodingException;
038 import java.lang.ref.WeakReference;
039 import java.net.InetAddress;
040 import java.net.UnknownHostException;
041 import java.nio.ByteBuffer;
042 import java.util.ArrayList;
043 import java.util.Arrays;
044 import java.util.BitSet;
045 import java.util.Collections;
046 import java.util.Comparator;
047 import java.util.HashSet;
048 import java.util.Iterator;
049 import java.util.List;
050 import java.util.Map;
051 import java.util.NoSuchElementException;
052 import java.util.Observable;
053 import java.util.Observer;
054 import java.util.Set;
055 import java.util.concurrent.ArrayBlockingQueue;
056 import java.util.concurrent.BlockingQueue;
057 import java.util.concurrent.ConcurrentHashMap;
058 import java.util.concurrent.TimeUnit;
059 import java.util.concurrent.atomic.AtomicReference;
060 import java.util.concurrent.locks.ReentrantLock;
061
062 import javax.servlet.ServletContext;
063
064 import org.hd.d.pg2k.ai.scorer.ScorerCacheIF;
065 import org.hd.d.pg2k.ai.scorer.ScorerCacheImpl;
066 import org.hd.d.pg2k.svrCore.AddrTools;
067 import org.hd.d.pg2k.svrCore.AllExhibitImmutableData;
068 import org.hd.d.pg2k.svrCore.AllExhibitProperties;
069 import org.hd.d.pg2k.svrCore.CoreConsts;
070 import org.hd.d.pg2k.svrCore.ExhibitName;
071 import org.hd.d.pg2k.svrCore.ExhibitPropsComputableMutable;
072 import org.hd.d.pg2k.svrCore.ExhibitPropsLoadable;
073 import org.hd.d.pg2k.svrCore.ExhibitStaticAttr;
074 import org.hd.d.pg2k.svrCore.ExhibitThumbnails;
075 import org.hd.d.pg2k.svrCore.FileTools;
076 import org.hd.d.pg2k.svrCore.GenUtils;
077 import org.hd.d.pg2k.svrCore.LocaleBeanBase;
078 import org.hd.d.pg2k.svrCore.MemoryTools;
079 import org.hd.d.pg2k.svrCore.Name;
080 import org.hd.d.pg2k.svrCore.Name.ExhibitFull;
081 import org.hd.d.pg2k.svrCore.PGMasterNotInServiceException;
082 import org.hd.d.pg2k.svrCore.Rnd;
083 import org.hd.d.pg2k.svrCore.SimpleLoggerIF;
084 import org.hd.d.pg2k.svrCore.Stratum;
085 import org.hd.d.pg2k.svrCore.TextUtils;
086 import org.hd.d.pg2k.svrCore.ThreadUtils;
087 import org.hd.d.pg2k.svrCore.Tuple;
088 import org.hd.d.pg2k.svrCore.MIME.ExhibitMIME;
089 import org.hd.d.pg2k.svrCore.datasource.ExhibitDataFileSource;
090 import org.hd.d.pg2k.svrCore.datasource.ExhibitDataSimpleCache;
091 import org.hd.d.pg2k.svrCore.datasource.SimpleExhibitPipelineIF;
092 import org.hd.d.pg2k.svrCore.location.GeoUtils;
093 import org.hd.d.pg2k.svrCore.props.GenProps;
094 import org.hd.d.pg2k.svrCore.props.LocalProps;
095 import org.hd.d.pg2k.svrCore.vars.EventPeriod;
096 import org.hd.d.pg2k.svrCore.vars.EventVariableValue;
097 import org.hd.d.pg2k.svrCore.vars.SimpleVariableDefinition;
098 import org.hd.d.pg2k.svrCore.vars.SimpleVariableValue;
099 import org.hd.d.pg2k.svrCore.vars.SystemVariables;
100 import org.hd.d.pg2k.webSvr.util.HTMLThumbnailInsertGenerators;
101 import org.hd.d.pg2k.webSvr.util.LocaleBean;
102 import org.hd.d.pg2k.webSvr.util.SearchResultSimpleCache;
103 import org.hd.d.pg2k.webSvr.util.WebConsts;
104 import org.hd.d.pg2k.webSvr.util.WebUtils;
105 import org.hd.d.pg2k.webSvr.virtualHosts.AlohaEarth.AEParams;
106 import org.hd.d.pg2k.webSvr.virtualHosts.AlohaEarth.AEUtils;
107 import org.hd.d.pg2k.webSvr.virtualHosts.AlohaEarth.AlohaEarthMapCache;
108
109 import ORG.hd.d.jIndexer.server.InvertedIndexException;
110 import ORG.hd.d.jIndexer.server.JIndexBean;
111
112 /**JavaBean encapsulating access to exhibit data and meta-data.
113 * This is expected to exist in one copy for the exhibit servlet,
114 * enabling (cached) access exhibit data and meta data.
115 * <p>
116 * This also provides access to shared data-pipeline and cache
117 * facilities, and is expected to be created on demand at application
118 * level. This, therefore, is designed to be used as a bean,
119 * and is suitable for direct instantiation from a JSP.
120 * <p>
121 * We have a public no-arg constructor to allow use from JSPs.
122 * <p>
123 * This object is thread-safe, and maximises concurrency while
124 * synchronising where necessary. (Quick calls are synchronized
125 * on this instance; longer-running calls may be synchronized on a
126 * private lock.)
127 * <p>
128 * This creates an internal thread to maintain the cache, and the
129 * thread is destroyed when the object is GCed. We only hold a
130 * SoftReference back to this instance from the background thread.
131 * <p>
132 * External callers can call the poll() method, though it is not
133 * recommended. They can also call the destroy method to close
134 * down the cache and stop the background thread, though this
135 * object must not be used thereafter.
136 * <p>
137 * When initialised for first use a number of parameters need to be set.
138 * (For JSP, these should only be done upon creation, eg between the
139 * useBean ... /useBean tags, to save cycles.)
140 * Many of the actual values passed are not stored but just used
141 * to extract other necessary parameters.
142 * Some parameters, notably contextPath and servletContext, should be
143 * set just before each use after retrieving the data from
144 * attributes, in case this instance has been passivated and restored.
145 * <p>
146 * (The servlet context parameter should be set every time the bean
147 * is used so that we always have a valid value to hand; we don't
148 * persist this if the bean is serialised.)
149 * <p>
150 * If an attempt to made to use the bean without initialising it, or
151 * to reinitialise it with conflicting values, an exception is thrown.
152 * <p>
153 * If an attempt is made to serialise/deserialise is made,
154 * the internal cache is reconstructed; careless use may result in
155 * multiple cache copies per servlet context which is wasteful,
156 * and may be unsafe.
157 * <p>
158 * One item created here is the by-word inverted index of the exhibits,
159 * created on first use.
160 * <p>
161 * This supports simple logging; by preference to the servlet context log,
162 * but failing that to System.out.
163 * <p>
164 * This is Serializable so as to be able to be stored in a servlet session;
165 * nothing especially long-lived or sensitive.
166 */
167 public final class DataSourceBean implements SimpleExhibitPipelineIF,
168 SimpleLoggerIF,
169 Serializable
170 {
171 /**Name that should be used for unique application-level instance. */
172 public static final String BEAN_NAME = "dataSource";
173
174 /**Factory method to create/return the unique application-level instance; never returns null.
175 * Makes an instance at application level, or returns an extant one
176 * if present.
177 * <p>
178 * Grabs appropriate locks to ensure that there are no data races
179 * and that there is at most one instance per servlet context.
180 * <p>
181 * The instance has the name ``dataSource'' and can be accessed
182 * from JSPs by that name.
183 *
184 * @param ctxt current servlet context; must not be null
185 */
186 public static DataSourceBean getApplicationInstance(final ServletContext ctxt)
187 {
188 DataSourceBean ds;
189
190 boolean created = false;
191 synchronized(ctxt)
192 {
193 ds = (DataSourceBean) ctxt.getAttribute(BEAN_NAME);
194 if(ds == null)
195 {
196 // Create and store a new instance if necessary.
197 ds = new DataSourceBean();
198 ds.setSlave(ctxt.getInitParameter(CoreConsts.WAR_CTXTPARAM_DISPOSITION));
199 ctxt.setAttribute(BEAN_NAME, ds);
200 created = true;
201 }
202 }
203
204 // We take this opportunity to set the servlet context,
205 // possibly starting a background thread.
206 //
207 // We avoid doing this inside the lock on the context itself
208 // to avoid any possibility of deadlock.
209 ds.setServletContext(ctxt);
210
211 // When we have newly created the DSB instance
212 // do some warm-up asynchronously if we have the resources (we often won't)
213 // to try to minimise the delay seen by the first visitor(s).
214 if(created && !GenUtils.mustConservePower())
215 {
216 System.out.println("[DataSourceBean created.]");
217
218 // We also ensure that a couple of other things are warmed up
219 // on a separate thread, partially because they may be I/O bound.
220 // We also help take advantage of any extra CPUs available.
221 // This may not happen if the system is short of spare resources.
222 ThreadUtils.nonCPUThreadPoolDiscardable.submit(new Runnable(){
223 public final void run()
224 {
225 // Get the IP-->location data loaded.
226 try { GeoUtils.getRegionByAddress(InetAddress.getLocalHost(), false); }
227 catch(final UnknownHostException e) { /* Ignore... */ }
228
229 // Prime the DNS cache/lookup.
230 AddrTools.hasARecordQuick(CoreConsts.MAIN_DATA_HOST);
231
232 // Warm up and (re)seed the random number generators.
233 Rnd.fastRnd.setSeed(System.nanoTime() + Rnd.goodRnd.nextInt());
234 }
235 });
236 }
237
238 return(ds);
239 }
240
241
242 /**Public no-arg constructor for ease of use as a JavaBean.
243 * This defers as much work as it reasonably can.
244 * <p>
245 * However, if the system property
246 * CoreConsts.WAR_SYSPROPNAME_DISPOSITION_PRESET
247 * is set to a usable value, this may allow us to start the
248 * cache sooner.
249 */
250 public DataSourceBean()
251 {
252 if(ORG.hd.d.IsDebug.isDebug) { logger.log("[DataSourceBean: instance created: " + this + ".]"); }
253
254 // If this is marked as a cloud mirror instance,
255 // then this is a slave.
256 if(LocalProps.isCloudMirrorInstance())
257 { setSlave(true); }
258
259 // Set the master/slave disposition
260 // explicitly if the use has tweaked the system properties
261 // with an explicit ``master'' or ``slave'' or ``waronly'' value.
262 else
263 { setSlave(System.getProperty(CoreConsts.WAR_SYSPROPNAME_DISPOSITION_PRESET)); }
264
265 if(ORG.hd.d.IsDebug.isDebug && (isSlave() != null)) { logger.log("[DataSourceBean: preset disposition ``"+ isSlave() +"'': " + this + ".]"); }
266
267 // Register the memory-release call-backs.
268 MemoryTools.registerRecurrentEmergencyFreeHandle(_efh);
269
270 // Verify what we've been given.
271 try { validateObject(); }
272 catch(final InvalidObjectException e)
273 { throw new IllegalArgumentException(e.getMessage()); }
274 }
275
276 /**If true, ensure that index is up-to-date on each poll() call if possible.
277 * If true, this minimises delay for users,
278 * but can seriously impact start-up time and may perform significant
279 * unnecessary work when new items are being loaded into the database.
280 * <p>
281 * We may try to avoid a forced rebuild if the system is busy
282 * or else do each (re)build in a low-priority thread.
283 */
284 private static final boolean FORCE_INDEX_REBUILD_IN_POLL = true;
285
286
287 /**Our logger which falls back to System.out if servlet log not available; never null. */
288 private final WebUtils.ServletLoggerWithFallback logger = new WebUtils.ServletLoggerWithFallback();
289
290 /**Get logger; never null.
291 * Allow external users to use our managed logger.
292 */
293 public SimpleLoggerIF getLogger()
294 { return(logger); }
295
296 /**Log the given message.
297 * Allow external users to use our managed logger.
298 */
299 public void log(final String message)
300 { logger.log(message); }
301
302
303 /**Key for systematic EPCM recomputation (linked to AEP), Iterator<Name.ExhibitFull>; never null. */
304 private final AEPLinkedKey iterEPCM = new AEPLinkedKey("iterEPCM");
305
306 /**'Dead' iterator with hasNext() always false; never null. */
307 private static final Iterator<Name.ExhibitFull> deadIt = new Iterator<Name.ExhibitFull>() {
308 /**Always false. */
309 public boolean hasNext() { return(false); }
310 /**Always throws exception. */
311 public Name.ExhibitFull next() { throw new NoSuchElementException(); }
312 /**Remove not supported. */
313 public void remove() { throw new UnsupportedOperationException(); }
314 };
315
316 /**Poll periodically (of the order of a second) to do background tasks.
317 * One job we do is post some properties where they can be found
318 * by other system components...
319 * <p>
320 * Not expected to be called by more than one thread at a time.
321 */
322 public final void poll(final GenProps gp)
323 throws IOException
324 {
325 // Last error encountered...
326 IOException lastErr = null;
327
328 final long startTime = System.currentTimeMillis();
329
330 // Get the pipeline.
331 final SimpleExhibitPipelineIF pipeline = _getPipeline();
332
333 // Set the cache aggressiveness depending on how busy we are.
334 if(pipeline instanceof ExhibitDataSimpleCache)
335 {
336 // Make the cache aggressive iff we are not too busy...
337 final boolean overloaded = WebUtils.isOverloaded(_servletContext);
338 ((ExhibitDataSimpleCache) pipeline).setAggressive(
339 isAggressive() && !overloaded);
340 }
341
342 // Do any upstream work...
343 try { pipeline.poll(gp); }
344 catch(final IOException e) { lastErr = e; }
345
346
347 // Now do our background work, essential stuff first...
348
349 // Keep any posted properties up to date.
350 try { _postProps(); }
351 catch(final IOException e) { lastErr = e; }
352
353 try
354 {
355 // Notify any Observer(s) registered with us
356 // and clear the AEP-linked store
357 // when the AEP content hash has changed.
358 // This is typically used to help clear caches quickly
359 // and we do not promise to update them every time or immediately,
360 // ie this mechanism is lossy and only "best efforts".
361 //
362 // TO AVOID DEADLOCKS, DON'T DO THIS IN THE SCOPE OF OUR LOCKS.
363 final EDVHObservable obs = _observable;
364
365 final int evdh = exhibitDataVersionHash();
366 obs.setCurrentEDVH(evdh);
367
368 // Iff there has been a change, spin off a thread to deal with it
369 // to try to avoid deadlocks.
370 //
371 // Pass the current hash value to help Observers avoid a redundant
372 // cache clear if they have already discarded/rebuilt their cache.
373 if(obs.hasChanged())
374 {
375 final Map<AEPLinkedKey, Object> sl = storeLinked.get();
376 logger.log("[DataSourceBean notifying observers of change in exhibit data set, count: " + obs.countObservers() + ", map entries: "+((sl==null)?-1:sl.size())+".]");
377
378 // First clear any AEP-linked store.
379 storeLinked.set(null); // Fast and helps GC!
380
381 // This will be spun off in a separate thread,
382 // if none already running.
383 obs.notifyObserversInNewThread(new Integer(evdh));
384 }
385 }
386 catch(final IOException e) { lastErr = e; }
387
388
389 // NON-ESSENTIAL (DISCRETIONARY) WORK BELOW THIS POINT...
390
391 // If (temporarily) conserving power then some of the time return immediately
392 // to short-cut a lot of the logic and potential effort
393 // on non-essential tasks.
394 final boolean mustConservePower = GenUtils.mustConservePower();
395 if(mustConservePower && Rnd.fastRnd.nextBoolean())
396 {
397 if(lastErr != null) { throw lastErr; }
398 return;
399 }
400
401 // Chose to do some of the work only if positively twiddling thumbs...
402 final ServletContext servletContext = _servletContext;
403 final boolean lightlyLoaded = (!mustConservePower) && (_servletContext != null) &&
404 WebUtils.isLightlyLoaded(_servletContext);
405 final boolean overloaded = WebUtils.isOverloaded(_servletContext);
406 // If overloaded then some of the time return immediately
407 // to short-cut a lot of the logic and potential effort
408 // on non-essential tasks.
409 if(overloaded && Rnd.fastRnd.nextBoolean())
410 {
411 if(lastErr != null) { throw lastErr; }
412 return;
413 }
414
415 // IMPORTANT: keep the by-word index fresh.
416 // Force pro-active rebuild of out-of-date index
417 // if this is a fast-start system or it is in any case not struggling.
418 if(FORCE_INDEX_REBUILD_IN_POLL && !byWordIndexIsAvailableAndUpToDate() &&
419 (LocalProps.fastStartMode() ||
420 (!mustConservePower && !overloaded)))
421 {
422 // Make sure that the by-word index is always up to date
423 // since it is heavily used (eg in the catalogue pages).
424 try { _rebuildJIB(); }
425 catch(final IOException e) { lastErr = e; }
426 }
427
428 // IMPORTANT: keep the newest/best exhibits and thumbnails fresh.
429 // Keep the "best" and newest exhibits/thumbnails up to date and 'warm' in memory
430 // as long as we are not very busy/overloaded/stressed and have not taken too long so far.
431 // We don't do this on every round but it is fairly high priority.
432 try
433 {
434 if(!overloaded && !mustConservePower && !MemoryTools.isMemoryStressed() &&
435 ((System.currentTimeMillis() - startTime) < MEAN_POLL_INTERVAL_MS/2) &&
436 Rnd.fastRnd.nextBoolean())
437 { _preloadBestAndNewest(gp, startTime, pipeline, servletContext); }
438 }
439 catch(final IOException e) { /* IGNORE BENIGN ERRORS */ /* lastErr = e; */ }
440
441 // Incrementally pre-compute parts of very popular catalogue pages.
442 try
443 {
444 if(lightlyLoaded && Rnd.fastRnd.nextBoolean())
445 { _precomputePopularPages(gp, startTime); }
446 }
447 catch(final IOException e) { /* IGNORE BENIGN ERRORS */ /* lastErr = e; */ }
448 catch(final Exception e) { e.printStackTrace(); } // Not expected...
449
450 // Keep the AI exhibit Scorers stats up to date
451 // (when we are not overloaded/stressed nor permanently conserving CPU cycles).
452 // Because each call may take a very long time to complete,
453 // this call uses a low-priority (discardable) background thread to do the work.
454 // We try to make use of anything that AH clients have been kind enough to do for us,
455 // but even in this case we don't do work on most poll() cycles so as to keep CPU load reasonable.
456 if(!SCORER_BG_EMERGENCY_CUTOFF && !GenUtils.mustConserveCPU() && !overloaded && !MemoryTools.isMemoryStressed() &&
457 (lightlyLoaded || (getScorerCache().hasQueuedExternalScorer() && (Rnd.fastRnd.nextInt(7) == 0))))
458 {
459 try { getScorerCache().poll(); }
460 catch(final IOException e) { /* IGNORE BENIGN ERRORS */ /* lastErr = e; */ }
461 }
462
463 // Try to keep EPCM values current unless overloaded/stressed
464 // and when not conserving power temporarily or permanently.
465 // since this will keep page-display times shorter for example.
466 // (Run with a lower duty cycle but don't stop when not lightly loaded.)
467 // As usual, avoid doing this work on every tick to avoid starvation elsewhere.
468 if(!!overloaded && !GenUtils.mustConserveCPU() && !MemoryTools.isMemoryStressed() && Rnd.fastRnd.nextBoolean())
469 { _keepEPCMUpToDate(gp, startTime, lightlyLoaded, overloaded); }
470
471 // Ensure that the basic AlohaEarth data is prepared
472 // as long as we are not too busy and have not taken too long so far.
473 // We don't do this on every round.
474 try
475 {
476 if(lightlyLoaded && ((System.currentTimeMillis() - startTime) < MEAN_POLL_INTERVAL_MS/2) && Rnd.fastRnd.nextBoolean())
477 {
478 final AlohaEarthMapCache aemfb = AEUtils.getAemfb(this);
479 if((servletContext != null) && Rnd.fastRnd.nextBoolean())
480 {
481 // Compute most data for basic/default view.
482 final AEParams aeps = new AEParams();
483 aemfb.selectViewLocationLabels(this, aeps);
484 // Don't usually load/compute image(s) until actually needed
485 // as they can take up quite a bit of memory
486 // and load time should not be that terrible,
487 // and we won't let the most expensive go again once loaded.
488 // if(MemoryTools.lotsFree()) { aemfb.selectMapEncodedImage(aeps, servletContext); }
489 }
490 }
491 }
492 catch(final IOException e) { lastErr = e; }
493
494 // Prime the DNS cache/lookup and/or keep it live and primed.
495 // This may block for a little while, so do it last.
496 if(lightlyLoaded) { AddrTools.hasARecordQuick(CoreConsts.MAIN_DATA_HOST); }
497
498 // Rethrow any subordinate error fielded until now.
499 if(lastErr != null) { throw lastErr; }
500 }
501
502 /**Attempt to to precompute or keep like some pages likely to be popular/busy. */
503 private void _precomputePopularPages(final GenProps gp, final long startTime)
504 throws IOException
505 {
506 final AllExhibitProperties aep = getAllExhibitProperties(-1);
507
508 // Precompute parts of popular pages viewed today and yesterday,
509 // with more recent and more popular ones done sooner...
510 byDay: for(final boolean today : new boolean[]{true, false})
511 {
512 // Get today's/yesterday's most visited pages...
513 final EventVariableValue evv = getEventValue(SystemVariables.ACCESSPATTERN_CAT_PAGE_VIEW, EventPeriod.VLONG, today);
514 final int totalDistinctValues = evv.getTotalDistinctValues();
515
516 // Any IOException will stop this round of precomputation.
517 if(totalDistinctValues > 0)
518 {
519 // Max pages to systematically precompute parts of.
520 // Cap this to the few most popular for maximum return on effort.
521 final int pp = Math.min(totalDistinctValues, 7 + WebConsts.SINGLE_PAGE_CONTACT_SHEET_TN_COUNT);
522
523 // Precompute the search results, most-popular-page first.
524 for(int i = 0; (i < pp) ; ++i)
525 {
526 // Give up for pages with low-ish (just noise?) visit counts...
527 if(evv.getCountByRank(i) < 3) { break; }
528 final String exhibitShortName = (String) evv.getValueByRank(i);
529 final Name.ExhibitFull fullName = aep.aeid.getFullName(exhibitShortName);
530 if(fullName != null)
531 {
532 // Do the search so as to cache the results...
533 final Set<Name.ExhibitFull> r = new HashSet<Name.ExhibitFull>(SearchResultSimpleCache.doCachedCatPageSimilarItems(this,
534 fullName,
535 SearchResultSimpleCache.ABS_MAX_RESULTS));
536 // Compute and cache any metadata...
537 WebUtils.getCatPageExhibitMetaDataHTML(this, fullName);
538 // Compute and cache thumbnail availability...
539 // Try to force thumbnail creation for top-visited pages...
540 if(WebUtils.TN_AVAIL_CACHE)
541 { WebUtils.exhibitHasThumbnail(this, fullName, true, true); }
542 // Now ensure that the EPCM is up-to-date for this exhibit
543 // and many/most related exhibits for which we will show ratings.
544 r.add(fullName);
545 for(final Name.ExhibitFull fn : r)
546 { aep.getExhibitPropsComputableMutable(fn, false, gp, this, getScorerCache()); }
547 }
548
549 // Stop if out of time... (Having done at least some work...)
550 if((System.currentTimeMillis() - startTime) > MEAN_POLL_INTERVAL_MS/2) { break byDay; }
551 }
552 }
553 }
554 }
555
556 /**Attempt to keep EPCM values up to date from within poll. */
557 private void _keepEPCMUpToDate(final GenProps gp, final long startTime,
558 final boolean lightlyLoaded, final boolean overloaded)
559 throws IOException
560 {
561 final AllExhibitProperties aep = getAllExhibitProperties(-1);
562 // Limit EPCM updates to a small duty cycle, much larger if lightly loaded.
563 final long endTime = startTime + (lightlyLoaded ? (MEAN_POLL_INTERVAL_MS/2) : (MEAN_POLL_INTERVAL_MS/32));
564 // Always perform at least one update attempt if we get this far.
565 do {
566 // Systematically, in most-needed-first order,
567 // force-update the EPCMs of one of the entire set of live exhibits
568 // Create a new iterator if the extant one is null or expired.
569 final Iterator<Name.ExhibitFull> nextEPCM = (Iterator<Name.ExhibitFull>) getAEPLinkedValue(iterEPCM);
570 if((null == nextEPCM) || !nextEPCM.hasNext())
571 {
572 // Replace dead iterator.
573 // Don't bother attempting any EPCM computation this time
574 // (since setting up the iterator take some effort or the list may in fact be empty).
575 final List<Name.ExhibitFull> allExhibits = aep.aeid.getAllExhibitNamesSorted();
576
577 // Collecting and filtering a new recompute set is likely expensive.
578 //
579 // We take a null iterator to indicate that the AEP has changed
580 // and so we should unconditionally compile a list of EPCM recomputations
581 // (mainly for any new exhibits just introduced),
582 // else we take a non-null (but dead) iterator as being the 'normal' state
583 // and thus postpone recomputation of the set of EPCMs needing work
584 // in inverse proportion to the amount of likely work needed.
585 // We do this by choosing an exhibit completely at random
586 // and only continuing if that exhibit's EPCM value actually needs work
587 // (ie put off expensively creating a new iterator for another tick if it does not).
588 // That should keep the work rate ~O(1) regardless of exhibit count.
589 // We free up any previous possibly-large backing store ASAP.
590 if(allExhibits.isEmpty()) { putAEPLinkedValue(iterEPCM, deadIt); /* Help GC. */ break; }
591 if(null != nextEPCM)
592 {
593 final ExhibitPropsComputableMutable sample = aep.getExhibitPropsComputableMutable(allExhibits.get(Rnd.fastRnd.nextInt(allExhibits.size())));
594 if((null != sample) && !sample.isStale()) { putAEPLinkedValue(iterEPCM, deadIt); /* Help GC. */ break; }
595 }
596
597 // Collect a list of just those exhibits that do need EPCM (re)calculation...
598 // Usually only a small fraction of the EPCMs will need (re)calculating on each round.
599 final ArrayList<Name.ExhibitFull> exhibits = new ArrayList<Name.ExhibitFull>((allExhibits.size()/8)+1);
600 for(final Name.ExhibitFull ex : allExhibits)
601 {
602 final ExhibitPropsComputableMutable epcm = aep.getExhibitPropsComputableMutable(ex);
603 if((null == epcm) || epcm.isStale()) { exhibits.add(ex); }
604 }
605 // (trim size to avoid wasting memory...)
606 exhibits.trimToSize();
607 // ... and sort the exhibits most-urgent first.
608 Collections.sort(exhibits, new SortByEPCMRecalcUrgency(aep));
609 // Don't worry about races to save this new value (shouldn't be possible).
610 putAEPLinkedValue(iterEPCM, exhibits.iterator());
611 // That is enough work done in this tick...
612 // (And avoids getting stuck looping when there are zero exhibits.)
613 break;
614 }
615 else
616 {
617 final Name.ExhibitFull next = nextEPCM.next();
618 // Stop if the next selected exhibit is already completely up-to-date.
619 // The first pass after start-up should relatively quickly ensure that
620 // nothing is left totally uncomputed nor trivially stale.
621 // Stochastically this gradually reduces work rate to a trickle in equilibrium near 100%.
622 final ExhibitPropsComputableMutable epcm = aep.getExhibitPropsComputableMutable(next);
623 if((null != epcm) && !epcm.isStale()) { break; }
624 // Force-compute and cache this exhibit's stale/absent EPCM value.
625 final ScorerCacheIF scorerCache = getScorerCache();
626 if(!overloaded)
627 {
628 // Unless overloaded, spawn work in the background where possible,
629 // running in this poll() thread only when the thread pool is full.
630 // This should make good use of multiple CPUs.
631 ThreadUtils.lowPriorityThreadPool.submit(new Runnable() {
632 public void run() { aep.getExhibitPropsComputableMutable(next, false, gp, DataSourceBean.this, scorerCache); }
633 });
634 }
635 else { aep.getExhibitPropsComputableMutable(next, false, gp, this, scorerCache); }
636 }
637 } while(System.currentTimeMillis() < endTime);
638 }
639
640 /**Attempt to preload the best and newest exhibits and/or keep them fresh/live. */
641 private void _preloadBestAndNewest(
642 final GenProps gp,
643 final long startTime,
644 final SimpleExhibitPipelineIF pipeline,
645 final ServletContext servletContext) throws IOException
646 {
647 // Eager precacheing/preloading of key exhibit thumbnails if fast-starting
648 // or have a reasonable amount of free memory...
649 final boolean fastPreloadKeyExhibitTns = LocalProps.fastStartMode() || MemoryTools.lotsFree();
650 // Now keep up-to-date enough best/new for the front page at least
651 // and not enough to overflow any of the underlying caches even at minimum size.
652 final int toGet = Rnd.fastRnd.nextInt(7) + (fastPreloadKeyExhibitTns ? 1+WebConsts.SINGLE_PAGE_CONTACT_SHEET_TN_COUNT : 11); // Number to force loading of.
653 final Name.ExhibitFull exhibits[];
654 if(Rnd.fastRnd.nextBoolean())
655 {
656 // Keep "best" exhibit set and thumbnails up to date.
657 // Get the "creme de la creme" as shown on the "best" page,
658 // and sometimes a selection from further down the "best" set,
659 // to represent the sampling done on on the front page.
660 // Also fetch a little more than either of those would
661 // to try to do any just-out-of-sight updates preemptively.
662 exhibits = HTMLThumbnailInsertGenerators.getBestExhibitSelection(servletContext,
663 toGet,
664 Rnd.fastRnd.nextBoolean() ? null : Rnd.goodRnd, false); // Help churn good rnd generator a little!
665 }
666 else
667 {
668 // Make sure that we have fetched and have fresh
669 // a selection of the very newest exhibit thumbnails,
670 // but not so many as to displace other things from caches.
671 // This should help right after new exhibits are added.
672 exhibits = HTMLThumbnailInsertGenerators.getNewExhibitSelection(servletContext,
673 toGet,
674 null, false);
675 }
676
677 // Try to keep the selected exhibits' thumbnails, etc, "live".
678 final AllExhibitProperties aep = getAllExhibitProperties(-1);
679 for(int i = 0; i < exhibits.length; ++i) // Do best/newest first.
680 {
681 final Name.ExhibitFull exhibitName = exhibits[i];
682 // Get thumbnail fetched/cached
683 // and get status into UI's cache if possible.
684 if((pipeline.getThumbnails(exhibitName, true) != null) && WebUtils.TN_AVAIL_CACHE)
685 {
686 // Update cache status for UI's benefit.
687 WebUtils.exhibitHasThumbnail(this, exhibitName, false, false);
688 }
689 // Get the 'similar exhibits' search computed.
690 SearchResultSimpleCache.doCachedCatPageSimilarItems(this, exhibitName, 1);
691 // Keep EPCM up to date unless permanently conserving CPU cycles...
692 if(!GenUtils.mustConserveCPU())
693 { aep.getExhibitPropsComputableMutable(exhibitName, false, gp, this, getScorerCache()); }
694
695 // Stop if we've run out of time...
696 if((System.currentTimeMillis() - startTime) > MEAN_POLL_INTERVAL_MS/2)
697 { break; }
698 }
699 }
700
701 /**If true then we stop all poll()-driven background/speculative Scorer processing.
702 * This will reduce system functionality (and predictive ability),
703 * but may be necessary where the strain of the extra processing is too high
704 * for a given machine, or where resources are shared with other heavy load.
705 * <p>
706 * A "true" value will have to be supplied on the the command line to effect this stop.
707 */
708 private static final boolean SCORER_BG_EMERGENCY_CUTOFF = Boolean.getBoolean("org.hd.d.pg2k.ai.scorer.EMERGENCYSTOP");
709
710 /**How often do we repost properties (ms, approx); strictly positive.
711 * This should be at least once per minute.
712 */
713 private final int PROPS_REPOST_INTERVAL_MS = 15001 + Rnd.fastRnd.nextInt(5001);
714
715 /**Note of the last time that we successfully ran raw properties repost without error.
716 * Private to _postProps().
717 */
718 private long _lastGoodPostProps;
719
720 /**Update and post properties to application level as required.
721 * Operates under the instance lock for thread safety.
722 */
723 private synchronized void _postProps()
724 throws IOException
725 {
726 final long now = System.currentTimeMillis();
727
728 // Exit quickly if nothing to do yet.
729 if((now - _lastGoodPostProps) < PROPS_REPOST_INTERVAL_MS)
730 { return; }
731
732 // Get the servlet context to be able to set app-level attributes.
733 final ServletContext ctx = _servletContext;
734 if(ctx == null) { return; } // No context set yet; give up.
735
736 // Post the GenSec general security props.
737 // Maybe in future it would be worth checking the
738 // timestamp for an updated value before reposting.
739 ctx.setAttribute("org.hd.d.pg2k.props.gensec", getGenSecProps(-1));
740
741 // Done! Mark successful completion...
742 _lastGoodPostProps = now;
743 }
744
745 /* Non-javadoc: delegates to pipeline. */
746 public Stratum getStratum() throws IOException
747 {
748 return(_getPipeline().getStratum());
749 }
750
751
752 /**Set true when the servlet is destroyed. */
753 private volatile boolean destroyed;
754
755 /**Free up some system resources and make our poller thread go away.
756 * This instance is unusable once this methid has been called.
757 */
758 public final void destroy()
759 {
760 // Ask our threads to die ASAP.
761 destroyed = true;
762
763 try
764 {
765 // Save Scorer work, if any.
766 final ScorerCacheIF scorers = getScorerCache();
767 if(scorers != null) { scorers.destroy(); }
768 }
769 finally
770 {
771 // Shut down pipeline.
772 final SimpleExhibitPipelineIF p = _dataPipeline;
773 try { if(p != null) { p.destroy(); } }
774 finally
775 {
776 // Free up resources.
777 _slave = null; // Prevent (re)construction of data pipeline.
778
779 _dataPipeline = null; // Let any cache (etc) go...
780
781 // Discard any data we manage.
782 storeLinked.set(null);
783 storeUnlinked.set(null);
784
785 // Discard any observers; don't do this in the scope of any lock.
786 _observable.deleteObservers();
787
788 logger.setContext(null); // Drop reference to the servlet logger.
789
790 if(ORG.hd.d.IsDebug.isDebug) { logger.log("[DataSourceBean: instance destroyed: " + this + ".]"); }
791 }
792 }
793 }
794
795 /**Check if destroyed... */
796 private boolean isDestroyed()
797 { return(destroyed); }
798
799
800 /**Operates the poll()ing/background Thread.
801 * Retains only a WeakReference to the bean instance to as to allow GC.
802 */
803 private static final class BackGroundThread extends Thread
804 {
805 /**Non-strong reference back to the bean instance; never null but the referent may be. */
806 private final WeakReference<DataSourceBean> beanInstance;
807
808 /**Approx average interval between calls to doBG() in ms; maximum interval usually no more than twice this.
809 * Of the order of a second to allow processing of resource-usage monitoring,
810 * which requires the fastest processing.
811 * <p>
812 * Different per instance to make collisions, etc, less likely.
813 */
814 private final int basicBgInterval = 1 + ((3*MEAN_POLL_INTERVAL_MS)/4) + Rnd.fastRnd.nextInt(1+(MEAN_POLL_INTERVAL_MS/2));
815
816 /**We'll back off to an upper limit if we encounter errors in polling.
817 * We do this mainly to avoid flooding logs if things are badly broken,
818 * but we usually expect to stay at or close to the minimum bg interval.
819 * <p>
820 * This is short enough not to completely wreck
821 * performance/interaction, or (eg) break the variable pipeline.
822 */
823 private final int maxBgInterval = 15017 + Rnd.fastRnd.nextInt(1001 + basicBgInterval);
824
825 private BackGroundThread(final DataSourceBean dsb)
826 {
827 super("DataSourceBean: poll() thread");
828 beanInstance = new WeakReference<DataSourceBean>(dsb);
829 }
830
831 /**Call poll(). */
832 @Override public final void run()
833 {
834 try {
835 // Private copy of generic properties.
836 // Start with default/empty set of properties...
837 GenProps genProps = new GenProps();
838
839 // Last GenProps poll time.
840 long _lGpPoll = 0;
841
842 // Start polling slowly since beans (etc) may not even
843 // be deployed yet and the system in general may be `warming up'.
844 // This will probably cause some ugliness in the data pipeline
845 // to start with, but a single successful poll cycle will
846 // restore us to maximum clip anyway...
847 int basicSleepInterval = (3 * basicBgInterval) + Rnd.fastRnd.nextInt(5011 + 2*basicBgInterval);
848
849 for( ; ; )
850 {
851 try {
852 // Interval in ms to sleep before next doBg(); different each time.
853 final int bgInterval = 1 + (basicSleepInterval/2 + Rnd.fastRnd.nextInt(1 | basicSleepInterval)) +
854 // Slow down polling significantly when (temporarily) conserving power...
855 // This may let the CPU sleep longer and deeper for example.
856 (GenUtils.mustConservePower() ? (4*MEAN_POLL_INTERVAL_MS) : 0);
857
858 // Default to lengthening the sleep interval
859 // (in case an error is going to be encountered)
860 // but reset it if all is well post hoc.
861 basicSleepInterval = Math.min(basicSleepInterval * 2, maxBgInterval);
862
863 // Now sleep...
864 Thread.sleep(bgInterval);
865
866 final DataSourceBean ds = beanInstance.get();
867
868 // If the bean has been GCed then quit this thread/polling.
869 if(ds == null) { return; }
870
871 // If the bean has had destroy() called then quit the thread.
872 if(ds.isDestroyed()) { return; }
873
874 // If not yet initialised then just sleep until ready...
875 if(ds.isSlave() == null)
876 { continue; }
877
878 final long start = System.currentTimeMillis();
879
880 try {
881 // We get the new system properties, using the
882 // specified recheck interval (the default being quite short).
883 if((start - _lGpPoll > genProps.getWEBSVR_SYSPROPS_RECHECK_MS()) ||
884 (start < _lGpPoll)) // Clock wobble?
885 {
886 final GenProps newGp = ds.getGenProps(genProps.timestamp);
887 if(newGp != null) { genProps = newGp; }
888 _lGpPoll = start;
889 }
890
891 // Now do normal background processing...
892 ds.poll(genProps);
893 }
894
895 // Silently discard `master not in service' errors.
896 // We don't want these to lengthen our sleep time.
897 catch(final PGMasterNotInServiceException e) { }
898
899 // Compute time taken to do this round of polling.
900 final long timeTaken = (System.currentTimeMillis() - start);
901
902 // If we get here without error,
903 // we can reset polling to the maximum rate
904 // (though we try to avoid spending more than 20% of time in poll(), even of one CPU).
905 basicSleepInterval = Math.max(basicBgInterval, Math.min(basicSleepInterval, 4 * (int) timeTaken));
906 }
907 catch(final IOException e)
908 {
909 // Quietly ignore IOExceptions as we expect some in the
910 // normal course of business. They may be very regular and
911 // we don't want to bloat any log files...
912 //System.err.println("In bg thread run()...");
913 //e.printStackTrace();
914 continue; // Allow sleep time to increase...
915 }
916 catch(final Throwable t)
917 {
918 // Report unusual errors...
919 t.printStackTrace();
920 continue; // Allow sleep time to increase...
921 }
922 }
923 }
924 finally
925 {
926 System.out.println("[Quit DataSourceBean cache/poll()/background thread...]");
927 }
928 }
929 }
930
931 /**Sorts first those exhibits whose EPCM recalculation/check is most urgent.
932 * Null (uncomputed) values first, then trivially stale, then stale,
933 * then ties are broken in a stable way to ensure a total ordering
934 * (other than in the case of completely-uncalculated values,
935 * where the original ordering is left intact, in part for speed).
936 * <p>
937 * This sort may misbehave if underlying EPCM values are recomputed while this works.
938 * <p>
939 * Public visibility primarily to allow testing.
940 */
941 public static final class SortByEPCMRecalcUrgency implements Comparator<Name.ExhibitFull>
942 {
943 /**AEP that we work with; never null. */
944 private final AllExhibitProperties aep;
945
946 /**Construct with (non-null) AEP whose EPCM values we will be sorting on the basis of. */
947 private SortByEPCMRecalcUrgency(final AllExhibitProperties aep)
948 {
949 if(aep == null) { throw new IllegalArgumentException(); }
950 this.aep = aep;
951 }
952
953 /**Sort in most-urgent-first order.
954 * Null (uncomputed) values first, then trivially stale, then stale,
955 * then ties broken in some reasonably stable way.
956 */
957 public int compare(final Name.ExhibitFull ex1, final Name.ExhibitFull ex2)
958 {
959 final ExhibitPropsComputableMutable epcm1 = aep.getExhibitPropsComputableMutable(ex1);
960 final ExhibitPropsComputableMutable epcm2 = aep.getExhibitPropsComputableMutable(ex2);
961
962 // Sort null (completely uncomputed) entries first (and quickly),
963 // else chose order that avoids starving whole Gallery sections alphabetically!
964 if(null == epcm1) { return((null != epcm2) ? -1 : ((ex1.hashCode()>>>1) - (ex2.hashCode()>>>1))); }
965 if(null == epcm2) { return(+1); }
966
967 // Sort soonest-expiring-first
968 // (with trivially-stale ahead of all non-trivial instances),
969 // ie those items first that are most stale or soonest to become stale
970 // as a fairly good measure of urgency.
971 final long bb1 = epcm1.bestBefore();
972 final long bb2 = epcm2.bestBefore();
973 if(bb1 < bb2) { return(-1); }
974 if(bb1 > bb2) { return(+1); }
975
976 // Now break ties by hash (differently from null instances)
977 // to avoid starving Gallery sections alphabetically when short of CPU.
978 final long eh1 = ex1.hashCode();
979 final long eh2 = ex2.hashCode();
980 if(eh1 > eh2) { return(-1); }
981 if(eh1 < eh2) { return(+1); }
982
983 // Break any remaining tie by full name to ensure a total ordering.
984 return(TextUtils.compare(ex1, ex2));
985 }
986 }
987
988 /**An Observable that changes when its "currentEDVH" value changes.
989 * This currentEDVH is the hash over the exhibit data.
990 * <p>
991 * When we call notifyObserversInNewThread() we do so where possible with a single
992 * Integer argument carrying the current hash value.
993 * An Observer may ignore a call when the value
994 * matches its current hash in case it has already rebuilt its cache.
995 * <p>
996 * Note that we build this on top of the default Observable implementation
997 * which holds no locks while delivering notifications,
998 * which should help us avoid deadlocks.
999 */
1000 private static final class EDVHObservable extends Observable
1001 {
1002 /**The current hash of the exhibits: when this changes then the whole Observable hasChanged(). */
1003 private int currentEVDH;
1004 /**Set the currentEVDH; when changed hasChanged() returns true. */
1005 synchronized final void setCurrentEDVH(final int currentEVDH)
1006 {
1007 if(this.currentEVDH != currentEVDH)
1008 {
1009 setChanged();
1010 this.currentEVDH = currentEVDH;
1011 }
1012 }
1013
1014 /**Thread, if any, being used to deliver notifications.
1015 * Private to notifyObserversInNewThread() and accessed under the instance lock.
1016 */
1017 private Thread deliveryThread;
1018
1019 /**Spins off background thread to call super.notifyObserversInNewThread().
1020 * Will not attempt to start a new thread to do this
1021 * if the last one appears not to have died,
1022 * so will in that case just not send any notifications.
1023 * <p>
1024 * No lock is held while notifications are delivered;
1025 * the instance lock may be briefly held while the thread
1026 * field is being examined and a thread launched.
1027 *
1028 * @param arg any object
1029 */
1030 final synchronized void notifyObserversInNewThread(final Object arg)
1031 {
1032 // If last thread still running, do not attempt to start another.
1033 if((deliveryThread != null) &&
1034 deliveryThread.isAlive())
1035 {
1036 System.err.println("DataSourceBean: previous notification thread still running, new one aborted.");
1037 return;
1038 }
1039
1040 final Observable obs = this;
1041 final Thread notifierThread =
1042 (new Thread("DataSourceBean: background Observer notifier thread")
1043 { @Override
1044 public final void run() { obs.notifyObservers(arg); } }
1045 );
1046 deliveryThread = notifierThread;
1047 notifierThread.setDaemon(true);
1048 notifierThread.start(); // Ensures thread isAlive().
1049 }
1050 }
1051
1052 /**Can be observed by an Observer that wants to know when the underlying data has changed; never null.
1053 * Observers are notified when our hash changes,
1054 * for example so that they can clear any now-stale cache and help GC.
1055 * <p>
1056 * Volatile so that access to the reference itself does not need a lock.
1057 * <p>
1058 * May be cleared once a notification has been delivered during a poll(),
1059 * ie may be at most one-shot.
1060 * <p>
1061 * Missed notifications are not considered very important.
1062 */
1063 private volatile transient EDVHObservable _observable = new EDVHObservable();
1064
1065 /**Register Observer to be told when exhibit data changes.
1066 * This will probably be poll driven.
1067 * <p>
1068 * We may discard Observers at will, and will certainly do so
1069 * if serialised and deserialised.
1070 */
1071 public void addObserver(final Observer obs)
1072 // throws IOException
1073 {
1074 _observable.addObserver(obs);
1075
1076 if(ORG.hd.d.IsDebug.isDebug) { logger.log("[DataSourceBean observers: " + _observable.countObservers() + ".]"); }
1077 }
1078
1079 /**Remove an Observer.
1080 * An Observer may have no further need to keep observing,
1081 * eg because its job is done and it wants to be GCed.
1082 */
1083 public void deleteObserver(final Observer o)
1084 {
1085 _observable.deleteObserver(o);
1086
1087 if(ORG.hd.d.IsDebug.isDebug) { logger.log("[DataSourceBean observers: " + _observable.countObservers() + ".]"); }
1088 }
1089
1090
1091
1092 /**Indicates if this is a master or a slave Web server, or not known.
1093 * "Not known" is indicated by a null value.
1094 * <p>
1095 * This value may be persisted.
1096 * <p>
1097 * Volatile to be accessed without a lock by isSlave() and setSlave().
1098 * <p>
1099 * Must be set non-null to initialise the bean for use.
1100 * <p>
1101 * May be able to be set immediately from the context.
1102 */
1103 private volatile Boolean _slave;
1104
1105 /**Get status of this server; master, slave or unknown.
1106 * Once initialised for use the status is known and
1107 * is either TRUE or FALSE; until then it is null indicating unknown.
1108 * Once set non-null its value cannot be changed.
1109 */
1110 public Boolean isSlave()
1111 { return(_slave); }
1112
1113 /**Set disposition.
1114 * If the parameter is true, the disposition is set to slave mode.
1115 * If the parameter is false, the disposition is set to master mode.
1116 * <p>
1117 * If value is passed that conflicts with a non-null value
1118 * already set, an IllegalStateException is thrown, ie the disposition
1119 * cannot be changed once set.
1120 */
1121 private void setSlave(final boolean isASlave)
1122 throws IllegalStateException
1123 {
1124 final Boolean currentState = _slave;
1125
1126 // Do not allow changes in disposition.
1127 if((currentState != null) && (currentState.booleanValue() != isASlave))
1128 { throw new IllegalStateException("master/slave disposition conflict"); }
1129
1130 // Set disposition.
1131 _slave = (isASlave ? Boolean.TRUE : Boolean.FALSE);
1132 }
1133
1134
1135
1136 /**Set slave from context or property value.
1137 * The value passed is ignored if null,
1138 * and is not case-insensitively-equal to ``master'' or ``slave'' or ``waronly'',
1139 * else it is set to slave or master mode as appropriate.
1140 */
1141 private synchronized void setSlave(String masterSlave)
1142 throws IllegalStateException
1143 {
1144 if(masterSlave == null) { return; }
1145
1146 masterSlave = masterSlave.toLowerCase();
1147
1148 // Only look for the exact words, albeit case-insensitive.
1149 if(masterSlave.equals(CoreConsts.WAR_CTXTPARAM_DISPOSITION_MASTER))
1150 { setSlave(false); /* _dispositionToken = masterSlave; */ }
1151 else if(masterSlave.equals(CoreConsts.WAR_CTXTPARAM_DISPOSITION_SLAVE))
1152 { setSlave(true); /* _dispositionToken = masterSlave; */ }
1153 else if(masterSlave.equals(CoreConsts.WAR_CTXTPARAM_DISPOSITION_WARONLY))
1154 { setSlave(false); /* _dispositionToken = masterSlave; */ }
1155 }
1156
1157 /**We examine the context path to see if we can tell if we are slave or master.
1158 * We are definitely a master server if our context path is not the empty string.
1159 * All slaves definitely have an empty-string context path, though a
1160 * ``waronly'' master does too.
1161 * <p>
1162 * This must be set before calling any of the get...() methods of
1163 * the bean (since we may need this to set up our cache, for example)
1164 * and is idempotent so long as we call it with the same value.
1165 * If we call it with a different value the call may be aborted.
1166 */
1167 public void setContextPath(final String cPath)
1168 throws IllegalStateException
1169 {
1170 if(cPath.length() != 0) { setSlave(false); }
1171 }
1172
1173
1174 /**A valid servlet context, or null.
1175 * We don't persist the value if the bean is serialised.
1176 * <p>
1177 * Declared volatile to allow safe lock-free access.
1178 * <p>
1179 * Should only be accessed by setServletContext() for write and
1180 * generally by _getPipeline() and _postProps() and getServletContext() for read.
1181 */
1182 private transient volatile ServletContext _servletContext;
1183
1184 /**The aggressiveness flag; defaulting to false.
1185 * Is set when we set the context; we do not bother to persist it.
1186 * <p>
1187 * Is volatile so that no lock is needed to access it.
1188 * <p>
1189 * Private to setServletContext() and isAggressive().
1190 */
1191 private transient volatile boolean aggressive;
1192
1193 /**Check if we are going to be aggressive in cacheing.
1194 * Defaults to false; is only valid once setServletContext() has been called.
1195 */
1196 public boolean isAggressive()
1197 { return(aggressive); }
1198
1199 /**Set a valid servlet context.
1200 * We don't really care which it is so long as is valid
1201 * (eg not null).
1202 * <p>
1203 * We don't persist the value if the bean is serialised.
1204 * <p>
1205 * Note that if we are going to be aggressive <em>and</em>
1206 * we already know if we are master or slave then we automatically
1207 * start the data pipeline and thread if necessary, but not inside
1208 * the instance lock.
1209 * <p>
1210 * If the context path is set before the servlet context
1211 * and we are in aggressive mode then that will force an
1212 * immediate pipeline start.
1213 */
1214 public void setServletContext(final ServletContext context)
1215 {
1216 if(context == null)
1217 { throw new IllegalArgumentException("invalid null servlet context"); }
1218
1219 logger.setContext(context);
1220
1221 synchronized(this)
1222 {
1223 // Only have the side-effects purely to do with setting the
1224 // context if it has changed.
1225 // We save a bit of redundant parsing, etc, this way.
1226 if(_servletContext != context)
1227 {
1228 _servletContext = context;
1229
1230 // Find out how aggressive we should be...
1231 final String aggressiveS =
1232 context.getInitParameter(CoreConsts.WAR_CTXTPARAM_AGGRESSIVE_CACHE);
1233 if(aggressiveS != null)
1234 {
1235 aggressive = Boolean.valueOf(aggressiveS).booleanValue();
1236 if(ORG.hd.d.IsDebug.isDebug) { logger.log("[DataSourceBean: isAggressive()=="+aggressive+" ("+aggressiveS+") for: " + this + ".]"); }
1237 }
1238 else // Default to not aggressive unless in fast-start mode.
1239 { aggressive = LocalProps.fastStartMode(); }
1240 }
1241 }
1242
1243 if(isAggressive() && (isSlave() != null))
1244 {
1245 // Force pipeline start.
1246 _getPipeline();
1247 }
1248 }
1249
1250 /**Get the servlet context associated with this bean; may be null.
1251 * Thread-safe access.
1252 */
1253 public ServletContext getServletContext()
1254 { return(_servletContext); }
1255
1256
1257 /**Private lock to ensure only one thread tries to build _dataPipeline. */
1258 private static final Object _dataPipeline_build_lock = new Object();
1259
1260 /**Non-persistable data pipeline.
1261 * This is reconstructed under the pipeline lock
1262 * if found to be null and if slave is non-null.
1263 * <p>
1264 * Should be accessed only by _getPipeline(),
1265 * and (without a lock) nulled out in destroy().
1266 */
1267 private transient volatile SimpleExhibitPipelineIF _dataPipeline;
1268
1269
1270 /**Approximate target mean interval between poll() calls (ms); strictly positive.
1271 * Polling may run (much) slower than this in energy-conserving mode.
1272 * <p>
1273 * Normally polling is just about once per second.
1274 */
1275 private static final int MEAN_POLL_INTERVAL_MS = 1017;
1276
1277 /**Start background thread... */
1278 private static void _startBackgroundThread(final DataSourceBean dsb)
1279 {
1280 // Now that we have just created a cache,
1281 // we'll create a thread to manage it,
1282 // but not under our instance lock.
1283 // We do this to avoid any deadlock on the object,
1284 // and we assume that this cannot cause any significant race hazard.
1285
1286 // Start background thread after pipeline is set up...
1287 final Thread th = new BackGroundThread(dsb);
1288 th.setDaemon(true);
1289 th.start();
1290 dsb.logger.log("[Started DataSourceBean poll()/background thread...]");
1291 }
1292
1293
1294 /**Get data pipeline; never null.
1295 * Will return any extant pipeline, or try to create one
1296 * providing that the bean has been initialised
1297 * and knows if this is a master (getting its data directly from EJB)
1298 * or a slave (getting data from the master server).
1299 * <p>
1300 * Operates under the instance lock except to start any background thread.
1301 * <p>
1302 * Should be small enough to inline, statically or with JIT compiler.
1303 *
1304 * @throws IllegalStateException if the cache cannot be created
1305 * due to incorrect initialisation
1306 */
1307 private SimpleExhibitPipelineIF _getPipeline()
1308 throws IllegalStateException
1309 {
1310 // If the pipeline is not null then we can use it as-is.
1311 final SimpleExhibitPipelineIF dp = _dataPipeline;
1312 if(dp != null) { return(dp); }
1313
1314 // Pipeline needs to be created.
1315 // Should be called (about) once during the life of any DSB instance.
1316 return(_createPipeline());
1317 }
1318
1319 /**Do the heavy-lifting of creating the data pipeline; never null.
1320 * Should be called (about) once during the life of any DSB instance.
1321 * @return the non-null pipeline
1322 * @throws IllegalStateException if the cache cannot be created
1323 * due to incorrect initialisation
1324 */
1325 private SimpleExhibitPipelineIF _createPipeline()
1326 throws IllegalStateException
1327 {
1328 boolean createdCache = false;
1329 try {
1330 // Build the cache under the pipeline build lock
1331 // to eliminate any possibility of doing it multiple times.
1332 // It is vital that we don't build multiple cache instances.
1333 synchronized(_dataPipeline_build_lock)
1334 {
1335 // Return any extant pipeline immediately.
1336 if(_dataPipeline != null) { return(_dataPipeline); }
1337
1338 // If destroyed, do not create a new pipeline!
1339 if(destroyed) { throw new IllegalStateException("Cannot create pipeline: shutting down"); }
1340
1341 // Pipeline needs creating but we don't know where
1342 // to find its data: abort the call.
1343 final Boolean iS = isSlave();
1344 if(iS == null)
1345 { throw new IllegalStateException("bean not initialised for master/slave"); }
1346
1347 // We need a valid servlet context; abort if we don't have one.
1348 final ServletContext sCtxt = _servletContext;
1349 if(sCtxt == null)
1350 { throw new IllegalStateException("DataSourceBean needs servlet context"); }
1351
1352 // Find the temporary dir we can use for a file-based cache.
1353 final File cacheDir =
1354 (File) (sCtxt.getAttribute("javax.servlet.context.tempdir"));
1355
1356 // Select our data source (HTTP tunnel or EJB).
1357 final SimpleExhibitPipelineIF dataSource = iS.booleanValue() ?
1358 // This is a slave, so get our data over HTTP...
1359 TunnelServlet.createFromContext(sCtxt) :
1360 // This is a `master'...
1361 ((SimpleExhibitPipelineIF) new ExhibitDataFileSource());
1362
1363 // Wrap a cache round the data source.
1364 final ExhibitDataSimpleCache cache =
1365 ExhibitDataSimpleCache.cacheFactory(dataSource, cacheDir, logger);
1366
1367 // Set the cache aggressiveness flag.
1368 cache.setAggressive(isAggressive());
1369
1370 // Now save the new pipeline.
1371 _dataPipeline = cache;
1372
1373 // We created a new cache
1374 // so start a new poll() thread in a moment...
1375 createdCache = true;
1376 return(cache);
1377 }
1378 }
1379 catch(final IOException e)
1380 {
1381 throw new IllegalStateException("could not create pipeline due to IOException: " + e.getMessage());
1382 }
1383 finally
1384 {
1385 if(createdCache) { _startBackgroundThread(this); }
1386 }
1387 }
1388
1389
1390 /**Get the static attributes for a given exhibit.
1391 * Returns null if the named exhibit does not exist.
1392 * <p>
1393 * Convenience method supporting wider variety of types
1394 * including older String-valued argument.
1395 *
1396 * @deprecated use ExhibitFull version if possible for efficiency
1397 */
1398 @Deprecated
1399 public ExhibitStaticAttr getStaticAttr(final CharSequence name)
1400 throws IOException
1401 {
1402 return(getStaticAttr(ExhibitFull.create(name)));
1403 }
1404
1405
1406 /**Get the static attributes for a given exhibit.
1407 * Returns null if the named exhibit does not exist.
1408 */
1409 public ExhibitStaticAttr getStaticAttr(final ExhibitFull name)
1410 throws IOException
1411 {
1412 return(_getPipeline().getStaticAttr(name));
1413 }
1414
1415 /**Get a chunk of the raw exhibit binary.
1416 * The start index and the index after the last required
1417 * byte is supplied. The start value must be non-negative
1418 * and the afterEnd value no smaller than start and no larger
1419 * than the exhibit data length.
1420 */
1421 public void getRawFile(final ByteBuffer buf,
1422 final Name.ExhibitFull exhibitName, final int position, final boolean dontCache)
1423 throws IOException
1424 {
1425 _getPipeline().getRawFile(buf, exhibitName, position, dontCache);
1426 }
1427
1428 /**Gets all static exhibit data if its timestamp is not that specified.
1429 * If the time specified is negative the object will be returned unconditionally.
1430 * <p>
1431 * If no exhibits are currently installed a default set with a zero
1432 * timestamp is returned.
1433 * <p>
1434 * If the caller's copy appears to be up-to-date (eg the oldStamp
1435 * matches that that would have been returned) null is returned.
1436 */
1437 public AllExhibitImmutableData getAllExhibitImmutableData(final long oldStamp)
1438 throws IOException
1439 {
1440 return(_getPipeline().getAllExhibitImmutableData(oldStamp));
1441 }
1442
1443 /**Gets set of all exhibit properties if its hash is not that specified.
1444 * If the hash specified is negative the object will be returned unconditionally.
1445 * <p>
1446 * If no exhibits are currently installed a default set with a zero
1447 * timestamp is returned.
1448 * <p>
1449 * If the caller's copy appears to be up-to-date
1450 * (eg the oldHash matches that that would have been returned
1451 * then null is returned.
1452 */
1453 public AllExhibitProperties getAllExhibitProperties(final long oldHash)
1454 throws IOException
1455 {
1456 return(_getPipeline().getAllExhibitProperties(oldHash));
1457 }
1458
1459
1460 /**Computes a hash on the versions of immutable and mutable exhibit data currently held.
1461 * This value is computed on the timestamps/hashes of the exhibit data
1462 * and properties available through this bean. If this hash changes
1463 * then some of the underlying exhibit data has changed
1464 * and users of the bean should recompute values based on this bean.
1465 */
1466 public final int exhibitDataVersionHash()
1467 throws IOException
1468 {
1469 // This is derived from all the exhibit properties.
1470 final AllExhibitProperties aep = getAllExhibitProperties(-1);
1471 return(((int) (aep.longHash >>> 32)) + ((int) aep.longHash));
1472 }
1473
1474 /**Encoding we use to convert name or name+description to byte stream for indexing.
1475 * We can assume 7-bit ASCII or 8-bit ISO-8859-1.
1476 */
1477 private static final String NAMEDESC_ENCODING = CoreConsts.FILE_ENCODING_8859_1;
1478
1479 /**Get the by-word-index, possibly recreating it (without blocking if possible) if not cached or if out of date; null if not available.
1480 * This is created from the full exhibit names and any extant description.
1481 * <p>
1482 * We know all names to be 7-bit (printable) ASCII,
1483 * and assume all descriptions to be either 7-bit ASCII or
1484 * at most 8-bit ISO-8859-1 (Latin-1) text,
1485 * and we use that when converting to a byte stream for indexing.
1486 * <p>
1487 * This may not force a synchronous rebuild since this may take too long,
1488 * and therefore may return a stale or null index
1489 * while a new one is being computed.
1490 * This may also defer a build temporarily if the system is heavily loaded.
1491 * <p>
1492 * THUS: keys/docnames held in the index must be relatively stable and sane
1493 * even across a (minor) change of AEP, eg exhibit short names or equivalent,
1494 * so that a lookup result is still very likely to make sense.
1495 */
1496 private JIndexBean _getJIB()
1497 throws IOException
1498 {
1499 // Attempt async rebuild if not available/current, but may block...
1500 if(!byWordIndexIsAvailableAndUpToDate())
1501 { _rebuildJIB(); }
1502
1503 // Return whatever index we have to hand, or null if none.
1504 return(_getJIB_cache);
1505 }
1506
1507 /**Returns true if there is a by-word index available.
1508 * Returns false until one is built or loaded.
1509 * <p>
1510 * If this returns true it does not guarantee that the index is up-to-date,
1511 * simply that one exists and can be used, probably without blocking.
1512 */
1513 public boolean byWordIndexIsAvailable()
1514 { return(_getJIB_cache != null); }
1515
1516 /**Returns true if there is a by-word index available and it is up-to-date.
1517 * Returns false until one is (re)built or loaded.
1518 */
1519 public boolean byWordIndexIsAvailableAndUpToDate()
1520 {
1521 final JIndexBean jib = _getJIB_cache;
1522 if(jib == null) { return(false); /* Fast path at (busy?) startup! */ }
1523 try
1524 {
1525 final int eDVH = exhibitDataVersionHash();
1526 return(jib.getRefValue() == eDVH);
1527 }
1528 catch(final IOException e)
1529 {
1530 return(false); /* Treat failure as implying busy... */
1531 }
1532 }
1533
1534 /**Private lock for _rebuildJIB.
1535 * Attempts by another thread to enter the lock are vetoed immediately,
1536 * preventing other threads from blocking pointlessly.
1537 */
1538 private final ReentrantLock _rebuildJIB_lock = new ReentrantLock();
1539
1540 /**If true then attempt to cache index in a file for potentially-faster system restart. */
1541 private static final boolean CACHE_INDEX_AS_FILE = false;
1542
1543 /**Name under which we attempt to cache the by-word index on disc (relative to servlet temp directory) if we do; not null.
1544 * The format is implementation-dependent,
1545 * and could be a serialised object or a saved index file.
1546 */
1547 private static final String CACHED_BYWORD_INDEX_FNAME = "_cached_byWord_Index.ser.gz";
1548
1549 /**Rebuild the JIB (index bean) if necessary and possible, avoiding blocking if possible.
1550 * Thread safe, and synchronized on a private lock
1551 * in order to avoid wasting CPU cycles with multiple update attempts.
1552 * <p>
1553 * Directly updates the cache used by _getJIB()
1554 * if the index needs recomputing,
1555 * ie if it is null or out of date wrt the AEP.
1556 * <p>
1557 * If a second thread tries to enter while one is already rebuilding
1558 * the index, then the second thread returns immediately (does not block),
1559 * but meaning that the cache will not be guaranteed rebuilt
1560 * by the time that the second thread returns,
1561 * so any caller has to be prepared to deal with the cache still null
1562 * when this routine returns.
1563 * <p>
1564 * If the system is too busy then we will not attempt to (re)build the index.
1565 * <p>
1566 * Note that normally a rebuild is attempted in the background
1567 * and at low priority and may be vetoed if the system is busy.
1568 * However, were there is no index present at all this will
1569 * force a higher-priority build and may possibly block until done.
1570 *
1571 * @throws IOException
1572 */
1573 private void _rebuildJIB()
1574 throws IOException
1575 {
1576 // Return immediately if index exists and is up-to-date.
1577 final JIndexBean jib = _getJIB_cache;
1578 final int eDVH = exhibitDataVersionHash();
1579 if((jib != null) && (jib.getRefValue() == eDVH))
1580 { return; }
1581
1582 // Return immediately (thus avoiding starting a(nother) useless thread)
1583 // if a rebuild appears already to be in progress.
1584 if(_rebuildJIB_lock.isLocked())
1585 { return; }
1586
1587 //if(IsDebug.isDebug) { (new Throwable("Requesting build of by-word index...")).printStackTrace(System.out); }
1588
1589 // Measure start time so as to capture thread start-up overhead.
1590 final long startTime = System.currentTimeMillis();
1591
1592 // The index needs (re)building,
1593 // so launch a (usually low-priority background thread) to do it if possible,
1594 // but have any such thread abort quickly
1595 // if one is already working on the task.
1596 // This may waste a small amount of effort
1597 // spinning off and killing some redundant background tasks.
1598 // Note that normally if the system is busy this build request will be silently discarded
1599 // and the invoking thread/routine be blocked.
1600 // This may complete the build in the caller's thread (synchronously)
1601 // iff there is currently no index at all and the chosen thread pool is already full.
1602 final AllExhibitProperties aep = getAllExhibitProperties(-1);
1603 // We refuse to construct an index from an empty AEP.
1604 if(aep.aeid.length == 0) { return; }
1605 final boolean noIndexPresentYet = (null == _getJIB_cache);
1606 (noIndexPresentYet ? ThreadUtils.computeIntensiveThreadPool : ThreadUtils.lowPriorityThreadPoolDiscardable).execute(new Runnable(){
1607 /**Rebuild the index if still necessary. */
1608 public void run()
1609 {
1610 // Return immediately if index already being (re)built.
1611 if(!_rebuildJIB_lock.tryLock()) { return; }
1612 try
1613 {
1614 final JIndexBean jib2 = _getJIB_cache;
1615 // If index now up-to-date then exit quickly...
1616 if((jib2 != null) && (jib2.getRefValue() == eDVH))
1617 { return; }
1618
1619 logger.log("Building by-word index..." + (noIndexPresentYet ? " (was absent)" : ""));
1620
1621 // If we have a servlet context
1622 // and we have no index in memory at all
1623 // for example because we have just started up,
1624 // then try to reload a cached index first
1625 // even though it may be somewhat stale,
1626 // and then (re)compute an up-to-date index.
1627 final ServletContext sCtxt = _servletContext;
1628 final File cacheDir = (sCtxt == null) ? null : (File) (sCtxt.getAttribute("javax.servlet.context.tempdir"));
1629 final File cacheFile = (cacheDir == null) ? null : new File(cacheDir, CACHED_BYWORD_INDEX_FNAME);
1630 // TODO
1631 // if((jib2 == null) && (cacheFile != null) && cacheFile.canRead())
1632 // {
1633 //logger.log("[Reloading cached by-word index...]");
1634 //
1635 // // Don't let an error during reload de-rail us...
1636 // try
1637 // {
1638 // final JIndexBean jib4 = _getJIB_cache =
1639 // new JIndexBean();
1640 //
1641 //logger.log("[Reload of by-word index complete: " + ((System.currentTimeMillis() - startTime + 500) / 1000) + "s.]");
1642 //
1643 // // If index in fact up-to-date then exit quickly...
1644 // if((jib4 != null) && (jib4.getRefValue() == eDVH))
1645 // { return; }
1646 // }
1647 // catch(final Exception e)
1648 // {
1649 // e.printStackTrace(); /* Absorb any error. */
1650 // }
1651 // }
1652
1653 // Either could not load cached index or it is stale...
1654
1655 logger.log("[Building by-word index...]");
1656
1657 final JIndexBean jib3 = computeByWordIndex(aep);
1658 jib3.setRefValue(eDVH); // Store up-to-date hash as the ref value of the index.
1659 _getJIB_cache = jib3; // Store ref to the the new index atomically.
1660
1661 final long endTime = System.currentTimeMillis();
1662 logger.log("[Build of by-word index complete: " + ((endTime - startTime + 500) / 1000) + "s.]");
1663
1664 // Attempt to cache our newly-computed index if possible.
1665 // Don't attempt to save an empty index...
1666 if(CACHE_INDEX_AS_FILE && (cacheFile != null) && (aep.aeid.length > 0))
1667 {
1668 // try
1669 {
1670 FileTools.serialiseToFile(jib3, cacheFile, true, true);
1671 logger.log("[Cached by-word index: " + ((System.currentTimeMillis() - endTime + 500) / 1000) + "s.]");
1672 }
1673 // catch(final InvertedIndexException e)
1674 // { e.printStackTrace(); /* Absorb any error. */ }
1675 }
1676 }
1677 catch(final InterruptedException e) { e.printStackTrace(); /* Retry later. */ }
1678 catch(final IOException e) { e.printStackTrace(); /* Retry later. */ }
1679 finally { _rebuildJIB_lock.unlock(); }
1680 }
1681 });
1682 }
1683
1684 /**Computes a (compact) by-word index of the current exhibits; never null.
1685 * This is based on the exhibit names, descriptions,
1686 * and any "AKA" (Also-Known-As) keyword data.
1687 *
1688 * @return compacted by-word index
1689 *
1690 * @throws IOException
1691 */
1692 public static JIndexBean computeByWordIndex(final AllExhibitProperties aep)
1693 throws IOException, InterruptedException
1694 {
1695 if(aep == null) { throw new IllegalArgumentException(); }
1696
1697 // In the trival case of no exhibits, return a new empty index.
1698 if(aep.aeid.length == 0) { return(new JIndexBean()); }
1699
1700 // Queue of (name,indexable-8-bit-text) values.
1701 // We can pick an implementation to allow a reasonable queue of work,
1702 // but without excessive memory consumption for queued data.
1703 // This is the main communication between our worker threads.
1704 final BlockingQueue<Tuple.Pair<Name.ExhibitFull, byte[]>> workQueue =
1705 new ArrayBlockingQueue<Tuple.Pair<Name.ExhibitFull, byte[]>>(Math.min(256, aep.aeid.length));
1706
1707 // Exception to be rethrown from the text-generation thread, if any.
1708 final AtomicReference<InterruptedException> bgException = new AtomicReference<InterruptedException>();
1709
1710 final long start = System.currentTimeMillis();
1711
1712 // In one thread generate the text to be indexed.
1713 // We don't launch this in one of the shared thread pools
1714 // because we need to guarantee that it always gets launched immediately
1715 // and in a separate thread so as to avoid deadlock.
1716 final Thread t1 = new Thread("by-word-index text-generation thread"){
1717 @Override public final void run()
1718 {
1719 // Get a default-locale LocaleBean
1720 // to extract the descriptive/AKA text via.
1721 final LocaleBeanBase lb = new LocaleBean();
1722
1723 // Push the data in, indexed by short name.
1724 // The text data always contains the virtual or short name,
1725 // with some additional descriptive text where available.
1726 // We use the virtual or short name since it omits the "noise"
1727 // of any intermediate directory names.
1728 //
1729 // We work through the names in sorted (short name) order
1730 // to improve performance and memory footprint through locality,
1731 // and also makes the discarding of duplicate index entries more consistent.
1732 final List<Name.ExhibitShort> shortNames = Arrays.asList(aep.aeid.getAllExhibitShortNamesArraySorted());
1733 final StringBuilder scratch = new StringBuilder(2048); // Used (and reused!) to build text to be indexed.
1734 // We avoid indexing two items with exactly the same index text.
1735 // We use the texts' hashes as a proxy to avoid holding on to the full texts...
1736 final Set<Integer> indexTexts = new HashSet<Integer>(shortNames.size() * 2);
1737 try
1738 {
1739 for(final Name.ExhibitShort shortName : shortNames)
1740 {
1741 final Name.ExhibitFull exhibitFullName = shortName.getFullName();
1742 String mType = ""; // MIME type/name, "eg video/mpeg".
1743 try { mType = ExhibitMIME.getMIMEType(exhibitFullName); }
1744 catch(final IOException e) { e.printStackTrace(); /* Shouldn't happen; whinge but ignore if it does. */ }
1745 final ExhibitPropsLoadable epl = aep.getExhibitPropsLoadable(exhibitFullName);
1746 final String description = epl.getDescription();
1747 final String akaText = GenUtils.getLocalisedTreeDesc(
1748 aep, exhibitFullName, lb, false, true, false, false).toString(); // Bare AKA text only.
1749 // Index most of the short name (excluding the number-in sequence in particular)
1750 // the MIME type, and whichever of the AKA and descriptions are present.
1751 // We'll auto-detect before indexing if these might contain HTML.
1752 scratch.setLength(0); // Clear the buffer...
1753 // Get main canonical search term based on main (and attribute) words...
1754 scratch.append(SearchResultSimpleCache.computeRawSearchTermFromExhibitName(exhibitFullName));
1755 // Add in auth name and extension for manual lookup on these terms...
1756 scratch.append(' ').append(ExhibitName.getAuthorComponent(exhibitFullName));
1757 scratch.append(' ').append(ExhibitName.getExtensionComponent(exhibitFullName));
1758 // Now add MIME type.
1759 scratch.append(' ').append(mType);
1760 // Now add aka/description text.
1761 if(akaText != null)
1762 { scratch.append(' ').append(akaText); }
1763 if(description != null)
1764 { scratch.append(' ').append(description); }
1765 // Abort if index text is not new.
1766 final String indexText = scratch.toString();
1767 if(!indexTexts.add(indexText.hashCode()))
1768 {
1769 //System.out.println("Aborting indexing (due to duplicate index text) of "+exhibitFullName);
1770 continue;
1771 }
1772 // TODO: replace() getBytes with more efficient conversion?
1773 workQueue.put(new Tuple.Pair<Name.ExhibitFull, byte[]>(exhibitFullName, indexText.getBytes(NAMEDESC_ENCODING)));
1774 }
1775
1776 // Insert 'poison' item all-null to terminate.
1777 workQueue.put(new Tuple.Pair<Name.ExhibitFull, byte[]>(null, null));
1778
1779 System.out.println("INFO: indexed "+indexTexts.size()+" out of "+shortNames.size()+" exhibit(s) in "+(System.currentTimeMillis()-start)+"ms."); // TODO: convert to use logger...
1780 }
1781 catch(final InterruptedException e)
1782 {
1783 // Note any explicit interruption and quit immediately...
1784 bgException.set(e);
1785 return;
1786 }
1787 catch(final UnsupportedEncodingException e)
1788 {
1789 throw new Error("Should not happen", e);
1790 }
1791 }
1792 };
1793 t1.setDaemon(true);
1794 t1.start(); // Start generating the text to be indexed.
1795
1796 // This will be our new index.
1797 final JIndexBean jib = new JIndexBean();
1798 jib.setIndexFragments(false); // No fragments.
1799
1800 try
1801 {
1802 // Keep running until the text-generation thread is dead
1803 // and the work queue is empty.
1804 while(!workQueue.isEmpty() || t1.isAlive())
1805 {
1806 // Get the next doc, waiting a short while for the next entry.
1807 final Tuple.Pair<Name.ExhibitFull,byte[]> doc = workQueue.poll(100, TimeUnit.MILLISECONDS);
1808 if(doc == null) { continue; } // Try again...
1809 // Stop quickly when we get the poison all-null terminating entry.
1810 if(doc.first == null) { break; }
1811
1812 final byte[] text = doc.second;
1813
1814 // Index as HTML if we find a '<' or '&' HTML meta character.
1815 boolean asHTML = false;
1816 for(int i = text.length; --i >= 0; )
1817 {
1818 final byte b = text[i];
1819 if((b == '<') || (b == '&'))
1820 {
1821 asHTML = true;
1822 break;
1823 }
1824 }
1825
1826 jib.setDocumentSimple(asHTML,
1827 doc.first.getShortName().persistableKey().toString(), // Use short tokens for JIB document names to save space.
1828 new ByteArrayInputStream(text));
1829 }
1830 }
1831 catch(final InvertedIndexException e)
1832 {
1833 final IOException err = new IOException("index-generation error: " + e.getMessage());
1834 err.initCause(e);
1835 throw err;
1836 }
1837
1838 // If the text-generation thread was interrupted
1839 // then (re)throw the exception now
1840 // rather than returning a partial index.
1841 final InterruptedException ie = bgException.get();
1842 if(ie != null) { throw ie; }
1843
1844 // Now tidy up and save some memory in a background thread,
1845 // since compact() allows other operations to be interleaved with its own.
1846 // This is CPU-intensive, but not very memory-intensive.
1847 // If the system is potentially short of CPU, this will run synchronously,
1848 // ie this thread will block until compaction is complete.
1849 ThreadUtils.lowPriorityThreadPool.submit(new Runnable(){
1850 public final void run() { jib.compact(); }
1851 });
1852
1853 // We return the new index for use,
1854 // though compaction may be continuing safely in the background.
1855 return(jib);
1856 }
1857
1858
1859 /**Cache for, and private to, _getJIB(); initially null.
1860 * We do not attempt to persist this data, so it is transient.
1861 * <p>
1862 * Is volatile so that access without a lock is safe.
1863 */
1864 private volatile transient JIndexBean _getJIB_cache;
1865
1866 /**Canonicalises a simple by-word query string and sanitises it; useful for potentially-unsafe user input.
1867 * This not only potentially makes the query string small(er),
1868 * it also shows the query that is actually being performed,
1869 * whatever the user thinks that they are doing!
1870 * <p>
1871 * This also converts the query string to pure ASCII,
1872 * having first converted any non-ISO-Latin-1 characters
1873 * to spaces, trimmed any obvious whitespace, and truncated
1874 * to the given length (if non-negative).
1875 * <p>
1876 * This can be directly handed the result of an HTTP request
1877 * parameter, for example. The output is suitable to hand to
1878 * findExhibitsByWord().
1879 * <p>
1880 * This keeps words in the order supplied especially the first.
1881 * <p>
1882 * This returns an empty sequence (eg "") if the input is null.
1883 * <p>
1884 * If the input string does not need transforming
1885 * then it is returned as-is.
1886 * <p>
1887 * Any new CharSequence returned is immutable.
1888 */
1889 public static CharSequence canonicaliseSimpleByWordQuery(final CharSequence s, final int maxLength)
1890 {
1891 if(s == null) { return(Name.EMPTY); }
1892 final String canonicalised = JIndexBean.canonicaliseSimpleByWordQuery(s.toString(), maxLength);
1893 if(TextUtils.contentEquals(s, canonicalised)) { return(s); } // Avoid unnecessary copies.
1894 // Try it as a Name instead to save space in case it hangs around for a while...
1895 try { return(Name.create(canonicalised)); }
1896 // ...but return an ordinary String in case of any difficulties.
1897 catch(final IllegalArgumentException e ) { return(canonicalised); }
1898 }
1899
1900 /**Query type or findExhibitsByWord() to return entries matching any word. */
1901 public static final int FEBY_MATCH_TYPE_ANY = -1;
1902 /**Query type or findExhibitsByWord() to return entries matching most words. */
1903 public static final int FEBY_MATCH_TYPE_MOST = 0;
1904 /**Query type or findExhibitsByWord() to return entries matching all words. */
1905 public static final int FEBY_MATCH_TYPE_ALL = 1;
1906
1907 /**Perform simple query on the exhibit data without filtering.
1908 * The query should be a simple whitespace-separated string
1909 * of query words.
1910 * <p>
1911 * This returns an immutable ranked list of (full) exhibit names;
1912 * possibly zero-length if no matches (but never null).
1913 * <p>
1914 * The underlying index is (re)built on first use or
1915 * when the underlying data has changed.
1916 * <p>
1917 * This may take longer than we'd like and use more memory than we'd like.
1918 *
1919 * @param query a plain-text query of search words, space-separated
1920 * @param matchType the number of input words to match;
1921 * -1 means any word,
1922 * 0 means most words,
1923 * 1 means all words.
1924 * @param maxResults is the maximum number of results returned; should
1925 * be at least twice the maximum shown to reduce the risk
1926 * of having abandoned good results prematurely
1927 */
1928 public List<Name.ExhibitFull> findExhibitsByWord(final String query,
1929 final int matchType,
1930 final int maxResults)
1931 throws IOException
1932 {
1933 return(findExhibitsByWord(query, matchType, maxResults, null));
1934 }
1935
1936 /**Perform simple query on the exhibit data with candidate filtering; never null.
1937 * The query should be a simple whitespace-separated string
1938 * of query words.
1939 * <p>
1940 * This returns an immutable ranked list of (full) exhibit names;
1941 * possibly zero-length if no matches (but never null).
1942 * <p>
1943 * The underlying index is (re)built on first use or
1944 * when the underlying data has changed.
1945 * <p>
1946 * This may take longer than we'd like and use more memory
1947 * than we'd like.
1948 *
1949 * @param query a plain-text query of search words, space-separated
1950 * @param matchType the number of input words to match;
1951 * -1 means any word,
1952 * 0 means most words,
1953 * 1 means all words.
1954 * @param maxResults is the maximum number of results returned; should
1955 * be at least twice the maximum shown to reduce the risk
1956 * of having abandoned good results prematurely
1957 * @param docFilter if non-null, any documents for which the
1958 * accept method returns false are excluded from the search
1959 */
1960 public List<Name.ExhibitFull> findExhibitsByWord(final CharSequence query,
1961 final int matchType,
1962 final int maxResults,
1963 final JIndexBean.SearchFilterByName docFilter)
1964 throws IOException
1965 {
1966 if(LOG_QUERIES)
1967 { logger.log("QUERY" + ((docFilter == null) ? "" : " (filtered)") + ": " + query); }
1968
1969 final JIndexBean jIndexBean = _getJIB();
1970
1971 // If there is no index available yet
1972 // then return an empty answer (not null).
1973 if(jIndexBean == null) { return(Collections.emptyList()); }
1974
1975 final AllExhibitImmutableData aeid = getAllExhibitImmutableData(-1);
1976
1977 // Do the lookup/search in the index.
1978 try
1979 {
1980 // The search returns short exhibit names.
1981 final String[] simpleSearchShortDocNames = jIndexBean.simpleSearchToDocNames(query.toString(), matchType, maxResults, docFilter);
1982 final List<Name.ExhibitFull> result = new ArrayList<ExhibitFull>(simpleSearchShortDocNames.length);
1983 for(final String sn : simpleSearchShortDocNames)
1984 {
1985 final ExhibitFull fullName = aeid.getFullNameFromPeristableKey(sn);
1986 if(null == fullName) { continue; } // Shouldn't really happen...
1987 result.add(fullName);
1988 }
1989
1990 return(Collections.unmodifiableList(result));
1991 }
1992 // Handle unexpected errors/exceptions gracefully, and log them.
1993 catch(final Exception e)
1994 {
1995 e.printStackTrace();
1996 return(Collections.emptyList());
1997 }
1998 }
1999
2000 /**If true, log queries made on the system for analysis. */
2001 public static final boolean LOG_QUERIES = false;
2002
2003 /**Gets the general properties as a GenProps object if its timestamp is not that specified.
2004 * If the time specified is negative the object will be returned unconditionally.
2005 * <p>
2006 * If no props are currently installed/available
2007 * then a default set with a zero timestamp is returned.
2008 * <p>
2009 * If the caller's copy appears to be up-to-date (eg the oldStamp
2010 * matches that that would have been returned) then null is returned.
2011 */
2012 public org.hd.d.pg2k.svrCore.props.GenProps getGenProps(final long oldStamp)
2013 throws IOException
2014 {
2015 return(_getPipeline().getGenProps(oldStamp));
2016 }
2017
2018 /**Gets the security properties as a Properties object if its timestamp is not that specified.
2019 * If the time specified is negative the object will be returned unconditionally.
2020 * <p>
2021 * If no props are currently installed/available a default set with a zero
2022 * timestamp is returned.
2023 * <p>
2024 * If the caller's copy appears to be up-to-date (eg the oldStamp
2025 * matches that that would have been returned) then null is returned.
2026 */
2027 public java.util.Properties getGenSecProps(final long oldStamp)
2028 throws IOException
2029 {
2030 return(_getPipeline().getGenSecProps(oldStamp));
2031 }
2032
2033 /**Gets the thumbnails for an exhibit.
2034 * A data source is at liberty to refuse to compute thumbnails
2035 * in which case it may return null, else it returns a
2036 * non-null value which may include the `could-not-compute'
2037 * value to indicate that a thumbnail/sample cannot be made
2038 * for this exhibit and no attempt need be made in future.
2039 * <p>
2040 * As a kindness to callers, we convert any IOException thrown
2041 * to returning a null instead, to fold the two cases together.
2042 *
2043 * @param create if true, and no thumbnail yet exists, try to
2044 * create one if possible, else only return an existing one
2045 */
2046 public ExhibitThumbnails getThumbnails(final Name.ExhibitFull name, final boolean create)
2047 {
2048 try { return(_getPipeline().getThumbnails(name, create)); }
2049 catch(final IOException e)
2050 {
2051 return(null); // Just one failure case to deal with.
2052 }
2053 }
2054
2055 public void setVariable(final SimpleVariableValue newValue)
2056 throws IOException
2057 {
2058 // Delegate.
2059 _getPipeline().setVariable(newValue);
2060 }
2061
2062 public int setVariables(final SimpleVariableValue[] newValues)
2063 throws IOException
2064 {
2065 // Delegate.
2066 return(_getPipeline().setVariables(newValues));
2067 }
2068
2069 public SimpleVariableValue getVariable(final SimpleVariableDefinition var)
2070 throws IOException
2071 {
2072 // Delegate.
2073 return(_getPipeline().getVariable(var));
2074 }
2075
2076 public SimpleVariableValue[] getVariables(final long changedSince)
2077 throws IOException
2078 {
2079 // Delegate.
2080 return(_getPipeline().getVariables(-1));
2081 }
2082
2083 /**Get the current partial, or previous full, event set at the specified interval; never null.
2084 * This is a simplified interface to return either the current event set
2085 * that is being collected, or the previous completed set.
2086 * <p>
2087 * The current set is the most timely, but may not contain enough data
2088 * to be meaningful if the new interval has just started.
2089 * <p>
2090 * The previous set is complete and thus most likely to have enough samples
2091 * to be useful, but is not completely current.
2092 *
2093 * @param def event definition (must be for an event); never null
2094 * @param intervalSelector one of EVENT_INTERVAL_SELECTOR_xxx values
2095 * @param current if true the current event set is returned,
2096 * else the previous complete set is returned
2097 *
2098 * @return requested event set; may be empty but not null if requested set not available
2099 */
2100 public EventVariableValue getEventValue(final SimpleVariableDefinition def,
2101 final EventPeriod intervalSelector,
2102 final boolean current)
2103 {
2104 return(_getPipeline().getEventValue(def, intervalSelector, current));
2105 }
2106
2107 /**Get the specified event sets for the specified intervals; never null.
2108 * This allows retrieval of zero or more event sets for the specified
2109 * interval size.
2110 * <p>
2111 * Requests for more than SystemVariables.EVENT_SAMPLES_RETAINED in the
2112 * past (or for the future!) cannot be satisfied and data will not be
2113 * returned for them.
2114 * <p>
2115 * Usually not more than SystemVariables.EVENT_SAMPLES_RETAINED samples
2116 * will be returned in response to any one request as a safety measure.
2117 * <p>
2118 * (An implementation that is not an end-point may go upstream to fetch
2119 * missing values and cache them to satisfy future requests.)
2120 *
2121 * @param def event definition (must be for an event); never null
2122 * @param intervalSelector one of EVENT_INTERVAL_SELECTOR_xxx values
2123 * @param intervalNumber a time (as from System.currentTimeMillis())
2124 * which identifies the first interval for which data is potentially
2125 * required; if too far in the past or future then possibly no data
2126 * will be available,
2127 * zero is used to access the "all" bucket
2128 * @param whichValues each true bit represents a slot for which data is
2129 * required, bit 0 indicating data from the slot within which
2130 * firstIntervalTime is located, bit 1 the previous slot, etc
2131 *
2132 * @return as many of the requested values as available,
2133 * at least long enough to return all the available values,
2134 * with [0] corresponding to bit 0 in the BitSet;
2135 * may contain nulls or be zero-length but is never null
2136 */
2137 public EventVariableValue[] getEventValues(final SimpleVariableDefinition def,
2138 final EventPeriod intervalSelector,
2139 final long intervalNumber,
2140 final BitSet whichValues)
2141 {
2142 return(_getPipeline().getEventValues(def, intervalSelector, intervalNumber, whichValues));
2143 }
2144
2145 /**Synchronise variables with upstream values.
2146 * Pushes updated values upstream to the source,
2147 * calls sync on the source if called with the "force" argument true,
2148 * and then retrieves changed values from upstream.
2149 * <p>
2150 * When called with force==true, this acts like a full "memory barrier",
2151 * flushing all write-cached items downstream immediately and afterwards
2152 * getting the value of all upstream values with getVariables(-1),
2153 * but may be expensive in terms of CPU or bandwidth, so use sparingly.
2154 * <p>
2155 * When called with force=false, this incrementally flushes outstanding
2156 * writes and will then fetch all, or only new, values from upstream,
2157 * so is potentally much less resource-intensive.
2158 * In particular, this does not propagate the sync() upstream.
2159 * <p>
2160 * In any case, it is rarely the right thing for a casual user
2161 * to vall this as it may be very expensive.
2162 *
2163 * @param force if true, this will force a full write flush,
2164 * a full sync upstream,
2165 * then full read with getVariables(-1),
2166 * to get the effect of a full "barrier";
2167 * otherwise, in general, a more incremental and non-propagating
2168 * mode is used which still does a write flush but may chose
2169 * to do a partial read of "new" upstream values
2170 *
2171 * @throws IOException if one is received from upstream
2172 */
2173 public void syncVariables(final boolean force)
2174 throws IOException
2175 {
2176 // Delegate.
2177 _getPipeline().syncVariables(force);
2178 }
2179
2180 /**Get requested Properties selected by key and versionID.
2181 * Fetches a Properties set unconditionally (versionID == -1)
2182 * else if the versionID presented is not current.
2183 *
2184 * @param key selector (with possible embedded sub-key)
2185 * for desired properties set; never null
2186 * @param versionID if -1 then map is always returned if available,
2187 * else must be non-negative and null is returned if the versionID
2188 * presented matches that of the current version
2189 * (ie if the caller has presumably got the up-to-date version);
2190 * may be a timestamp or a hash or other value,
2191 * and by convention is zero only for an empty properties set
2192 *
2193 * @return null, or Properties map guaranteed to contain only
2194 * String keys and values
2195 */
2196 public java.util.Properties getProperties(final PropsKey key,
2197 final long versionID)
2198 throws IOException
2199 {
2200 return(_getPipeline().getProperties(key, versionID));
2201 }
2202
2203
2204 /**Base of (immutable) lookup key for AEP-linked and non-AEP-linked tables.
2205 * Each object instance is unique,
2206 * and thus also can be statically created and kept private by its user.
2207 * <p>
2208 * Contains optional key useful for debugging/tracing.
2209 * <p>
2210 * This is not meaningfully Serializable or Cloneable.
2211 */
2212 private static class KeyBase
2213 {
2214 /**Optional comment key (may be null). */
2215 public KeyBase(final String optComment) { comment = optComment; }
2216
2217 /**Hashcode depends only on object instance; not overridable. */
2218 @Override
2219 public final int hashCode()
2220 { return(System.identityHashCode(this)); }
2221
2222 /**Depends only on == and is not overridable. */
2223 @Override
2224 public final boolean equals(final Object obj)
2225 { return(this == obj); }
2226
2227 /**Comment; may be null. */
2228 public final String comment;
2229
2230 /**Render as a String; never null. */
2231 @Override
2232 public final String toString()
2233 { return((comment != null) ? comment : "(null-comment)"); }
2234 }
2235
2236 /**Key for AEP-linked store. */
2237 public static final class AEPLinkedKey extends KeyBase
2238 { public AEPLinkedKey(final String comment) { super(comment); } }
2239
2240 /**Key for non-AEP-linked store. */
2241 public static final class UnlinkedKey extends KeyBase
2242 { public UnlinkedKey(final String comment) { super(comment); } }
2243
2244 /**Thread-safe (highly-concurrent) AEP-linked store; initialised on demand; never null after construction/deserialisation.
2245 * May be cleared if the system becomes very short of memory.
2246 */
2247 private final AtomicReference<ConcurrentHashMap<AEPLinkedKey, Object>> storeLinked = new AtomicReference<ConcurrentHashMap<AEPLinkedKey,Object>>(null);
2248
2249 /**Thread-safe (highly-concurrent) non-AEP-linked store; initialised on demand; never null after construction/deserialisation.
2250 * May be cleared if the system becomes very short of memory.
2251 */
2252 private final AtomicReference<ConcurrentHashMap<UnlinkedKey, Object>> storeUnlinked = new AtomicReference<ConcurrentHashMap<UnlinkedKey,Object>>(null);
2253
2254 /**Emergency-free hook for when very low on memory.
2255 * Holds no reference back to the DataSourceBean,
2256 * only to the internal clearable caches.
2257 * <p>
2258 * Must be set up at construction and at deserialisation.
2259 */
2260 private static final class EFH implements MemoryTools.RecurrentEmergencyFreeHandle, Serializable
2261 {
2262 /**Unique serial ID.*/
2263 private static final long serialVersionUID = -2219560746599156471L;
2264 private final AtomicReference<?>[] caches;
2265 EFH(final AtomicReference<?>[] caches) { this.caches = caches; }
2266 public void run()
2267 {
2268 for(final AtomicReference<?> ar : caches)
2269 { ar.set(null); }
2270 System.err.println("DataSourceBean: emergency free: cleared linked and unlinked stores");
2271 }
2272 }
2273
2274 /**Set up emergency to clear these stores under extreme memory stress. */
2275 private final EFH _efh = new EFH(new AtomicReference<?>[] { storeLinked, storeUnlinked } );
2276
2277 /**Get AEP-linked store, creating if necessary; never null.
2278 */
2279 private ConcurrentHashMap<AEPLinkedKey, Object> getStoreLinked()
2280 {
2281 ConcurrentHashMap<AEPLinkedKey, Object> sl;
2282 // Create new store if needed.
2283 while((sl = storeLinked.get()) == null)
2284 { storeLinked.compareAndSet(null, new ConcurrentHashMap<AEPLinkedKey, Object>()); }
2285 return(sl);
2286 }
2287
2288 /**Get value against supplied key in AEP-linked store; result may be null.
2289 * May be cleared at any time, and will be cleared when the AEP changes.
2290 * <p>
2291 * This is designed for good concurrency and as little locking as possible.
2292 */
2293 public Object getAEPLinkedValue(final AEPLinkedKey key)
2294 { return(getStoreLinked().get(key)); }
2295
2296 /**Store value against supplied key in AEP-linked store.
2297 * A null value is used to clear any extant entry.
2298 * <p>
2299 * This is designed for good concurrency and as little locking as possible.
2300 * <p>
2301 * This may be cleared automatically if we become very short of memory.
2302 *
2303 * @return any previous value, or null if none
2304 */
2305 public Object putAEPLinkedValue(final AEPLinkedKey key, final Object value)
2306 {
2307 final ConcurrentHashMap<AEPLinkedKey, Object> sl = getStoreLinked();
2308 if(value == null)
2309 { return(sl.remove(key)); }
2310 else
2311 { return(sl.put(key, value)); }
2312 }
2313
2314 /**Replace value for supplied key in AEP-linked store.
2315 * This (atomically) replaces the value for the given key
2316 * only if the extant value is that supplied.
2317 * <p>
2318 * Null values are not allowed.
2319 * <p>
2320 * This is designed for good concurrency and as little locking as possible.
2321 * <p>
2322 * This may be cleared automatically if we become very short of memory.
2323 *
2324 * @return true if the value was replaced
2325 */
2326 public boolean replaceAEPLinkedValue(final AEPLinkedKey key, final Object oldValue, final Object newValue)
2327 {
2328 final ConcurrentHashMap<AEPLinkedKey, Object> sl = getStoreLinked();
2329 return(sl.replace(key, oldValue, newValue));
2330 }
2331
2332 /**Remove any value associated with the supplied key in AEP-linked store.
2333 */
2334 public Object removeAEPLinkedValue(final AEPLinkedKey key)
2335 { return(getStoreLinked().remove(key)); }
2336
2337 /**Store value against supplied key in AEP-linked store only if no value already mapped for that key.
2338 * A null value is used to clear any extant entry.
2339 * <p>
2340 * This is designed for good concurrency and as little locking as possible.
2341 * <p>
2342 * This may be cleared automatically if we become very short of memory.
2343 *
2344 * @return any previous value, or null if none
2345 */
2346 public Object putIfAbsentAEPLinkedValue(final AEPLinkedKey key, final Object value)
2347 {
2348 final ConcurrentHashMap<AEPLinkedKey, Object> sl = getStoreLinked();
2349 if(value == null)
2350 { return(sl.remove(key)); }
2351 else
2352 { return(sl.putIfAbsent(key, value)); }
2353 }
2354
2355 /**Get non-AEP-linked store, creating if necessary; never null.
2356 */
2357 private ConcurrentHashMap<UnlinkedKey, Object> getStoreUnlinked()
2358 {
2359 ConcurrentHashMap<UnlinkedKey, Object> su;
2360 // Create new store if needed.
2361 while((su = storeUnlinked.get()) == null)
2362 { storeUnlinked.compareAndSet(null, new ConcurrentHashMap<UnlinkedKey, Object>()); }
2363 return(su);
2364 }
2365
2366 /**Get value against supplied key in non-AEP-linked store; result may be null.
2367 * This is designed for good concurrency and as little locking as possible.
2368 */
2369 public Object getUnlinkedValue(final UnlinkedKey key)
2370 { return(getStoreUnlinked().get(key)); }
2371
2372 /**Store value against supplied key in non-AEP-linked store.
2373 * A null value is used to clear any extant entry.
2374 * <p>
2375 * This is designed for good concurrency and as little locking as possible.
2376 * <p>
2377 * This may be cleared automatically if we become very short of memory.
2378 *
2379 * @return any previous value, or null if none
2380 */
2381 public Object putUnlinkedValue(final UnlinkedKey key, final Object value)
2382 {
2383 final ConcurrentHashMap<UnlinkedKey, Object> su = getStoreUnlinked();
2384 if(value == null)
2385 { return(su.remove(key)); }
2386 else
2387 { return(su.put(key, value)); }
2388 }
2389
2390 /**Replace value for supplied key in non-AEP-linked store.
2391 * This (atomically) replaces the value for the given key
2392 * only if the extant value is that supplied.
2393 * <p>
2394 * Null values are not allowed.
2395 * <p>
2396 * This is designed for good concurrency and as little locking as possible.
2397 * <p>
2398 * This may be cleared automatically if we become very short of memory.
2399 *
2400 * @return true if the value was replaced
2401 */
2402 public boolean replaceUnlinkedValue(final UnlinkedKey key, final Object oldValue, final Object newValue)
2403 {
2404 final ConcurrentHashMap<UnlinkedKey, Object> su = getStoreUnlinked();
2405 return(su.replace(key, oldValue, newValue));
2406 }
2407
2408 /**Remove any value associated with the supplied key in non-AEP-linked store.
2409 */
2410 public Object removeUnlinkedValue(final AEPLinkedKey key)
2411 { return(getStoreUnlinked().remove(key)); }
2412
2413 /**Store value against supplied key in non-AEP-linked store only if no value already mapped for that key.
2414 * A null value is used to clear any extant entry.
2415 * <p>
2416 * This is designed for good concurrency and as little locking as possible.
2417 * <p>
2418 * This may be cleared automatically if we become very short of memory.
2419 *
2420 * @return any previous value, or null if none
2421 */
2422 public Object putIfAbsentUnlinkedValue(final UnlinkedKey key, final Object value)
2423 {
2424 final ConcurrentHashMap<UnlinkedKey, Object> su = getStoreUnlinked();
2425 if(value == null)
2426 { return(su.remove(key)); }
2427 else
2428 { return(su.putIfAbsent(key, value)); }
2429 }
2430 //
2431 // /**The Scorer cache, currently not persistable/serialisable; never null. */
2432 // private transient ScorerCacheImpl scorers = new ScorerCacheImpl(this, logger);
2433
2434 /**The non-AEP-linked key for the Scorer cache; never null. */
2435 private static final UnlinkedKey scorerKey = new UnlinkedKey("scorerKey");
2436
2437 /**Get the Scorer cache; never null.
2438 * The cache may be discarded under extreme memory stress
2439 * and will be atomically (re)created as necessary.
2440 * <p>
2441 * We may restrict access in future.
2442 */
2443 public ScorerCacheIF getScorerCache()
2444 {
2445 ScorerCacheIF result;
2446 while(null == (result = (ScorerCacheIF) getUnlinkedValue(scorerKey)))
2447 { putIfAbsentUnlinkedValue(scorerKey, new ScorerCacheImpl(this, logger)); }
2448 return(result);
2449 }
2450
2451
2452 /**Deserialise.
2453 */
2454 private void readObject(final ObjectInputStream ois)
2455 throws IOException, ClassNotFoundException
2456 {
2457 // Default read of object.
2458 ois.defaultReadObject();
2459
2460 // Ensure that a (new) embedded Observable is in place.
2461 _observable = new EDVHObservable();
2462
2463 // Register the memory-release call-backs.
2464 MemoryTools.registerRecurrentEmergencyFreeHandle(_efh);
2465
2466 // Validate object state immediately.
2467 validateObject();
2468 }
2469
2470 /**Validate fields/state.
2471 * Called in the constructor and possibly after de-serialising.
2472 * <p>
2473 * Barf if something bad is found.
2474 * (Maybe allow some extra info in debug version.)
2475 */
2476 public void validateObject()
2477 throws InvalidObjectException
2478 {
2479 // Ensure that Observable is in place.
2480 if(_observable == null)
2481 { throw new InvalidObjectException("invalid null Observable"); }
2482 // Ensure that emergency-free callback is in place.
2483 if(_efh == null)
2484 { throw new InvalidObjectException("invalid null efh (emergency-free handle)"); }
2485 }
2486
2487 /**Unique Serialisation class ID generated by http://random.hd.org/. */
2488 private static final long serialVersionUID = -8932316265828567742L;
2489 }