001    /*
002    Copyright (c) 1996-2011, Damon Hart-Davis
003    All rights reserved.
004    
005    Redistribution and use in source and binary forms, with or without
006    modification, are permitted provided that the following conditions are
007    met:
008    
009      * Redistributions of source code must retain the above copyright
010        notice, this list of conditions and the following disclaimer.
011    
012      * Redistributions in binary form must reproduce the above copyright
013        notice, this list of conditions and the following disclaimer in the
014        documentation and/or other materials provided with the
015        distribution.
016    
017    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
018    IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
019    TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
020    PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
021    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
022    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
023    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
024    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
025    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
026    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
027    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
028    */
029    package org.hd.d.pg2k.svrCore.datasource;
030    
031    import java.io.BufferedInputStream;
032    import java.io.DataOutputStream;
033    import java.io.File;
034    import java.io.FileInputStream;
035    import java.io.FileNotFoundException;
036    import java.io.FileWriter;
037    import java.io.IOException;
038    import java.io.InputStream;
039    import java.io.InterruptedIOException;
040    import java.io.OutputStream;
041    import java.io.PrintWriter;
042    import java.io.RandomAccessFile;
043    import java.nio.ByteBuffer;
044    import java.security.DigestOutputStream;
045    import java.security.MessageDigest;
046    import java.text.SimpleDateFormat;
047    import java.util.ArrayList;
048    import java.util.BitSet;
049    import java.util.Calendar;
050    import java.util.Collections;
051    import java.util.Date;
052    import java.util.HashMap;
053    import java.util.HashSet;
054    import java.util.List;
055    import java.util.Map;
056    import java.util.Properties;
057    import java.util.Queue;
058    import java.util.Set;
059    import java.util.Vector;
060    import java.util.concurrent.ConcurrentHashMap;
061    import java.util.concurrent.ExecutionException;
062    import java.util.concurrent.Future;
063    import java.util.concurrent.LinkedBlockingQueue;
064    import java.util.concurrent.atomic.AtomicBoolean;
065    import java.util.concurrent.atomic.AtomicReference;
066    import java.util.concurrent.locks.ReentrantLock;
067    
068    import org.hd.d.pg2k.svrCore.AccessionData;
069    import org.hd.d.pg2k.svrCore.AllExhibitImmutableData;
070    import org.hd.d.pg2k.svrCore.AllExhibitProperties;
071    import org.hd.d.pg2k.svrCore.AllExhibitProperties.ExhibitDataSource;
072    import org.hd.d.pg2k.svrCore.CS8Bit;
073    import org.hd.d.pg2k.svrCore.CoreConsts;
074    import org.hd.d.pg2k.svrCore.ExhibitFile;
075    import org.hd.d.pg2k.svrCore.ExhibitName;
076    import org.hd.d.pg2k.svrCore.ExhibitPropsComputable;
077    import org.hd.d.pg2k.svrCore.ExhibitPropsGlobalImmutable;
078    import org.hd.d.pg2k.svrCore.ExhibitPropsLoadable;
079    import org.hd.d.pg2k.svrCore.ExhibitStaticAttr;
080    import org.hd.d.pg2k.svrCore.ExhibitThumbnails;
081    import org.hd.d.pg2k.svrCore.FileTools;
082    import org.hd.d.pg2k.svrCore.GenUtils;
083    import org.hd.d.pg2k.svrCore.MemoryTools;
084    import org.hd.d.pg2k.svrCore.Name;
085    import org.hd.d.pg2k.svrCore.Name.ExhibitFull;
086    import org.hd.d.pg2k.svrCore.ROByteArray;
087    import org.hd.d.pg2k.svrCore.Rnd;
088    import org.hd.d.pg2k.svrCore.Stratum;
089    import org.hd.d.pg2k.svrCore.TextUtils;
090    import org.hd.d.pg2k.svrCore.ThreadUtils;
091    import org.hd.d.pg2k.svrCore.Tuple;
092    import org.hd.d.pg2k.svrCore.MIME.ExhibitMIME;
093    import org.hd.d.pg2k.svrCore.props.GenProps;
094    import org.hd.d.pg2k.svrCore.props.LocalProps;
095    import org.hd.d.pg2k.svrCore.props.SecurityProps;
096    import org.hd.d.pg2k.svrCore.vars.BasicVarMgr;
097    import org.hd.d.pg2k.svrCore.vars.EventPeriod;
098    import org.hd.d.pg2k.svrCore.vars.EventVariableValue;
099    import org.hd.d.pg2k.svrCore.vars.SimpleVariableDefinition;
100    import org.hd.d.pg2k.svrCore.vars.SimpleVariableValue;
101    import org.hd.d.pg2k.svrCore.vars.SystemVariables;
102    
103    import ORG.hd.d.IsDebug;
104    
105    // FIXME: periodically/randomly (or upon demand) check for data file corruption.
106    
107    /**Exhibit pipeline stage that fetches its data directly from a filesystem.
108     * This emulates the way that the pre-PG2K Gallery accesses its data.
109     * <p>
110     * That this considers itself to be the definitive master data store
111     * for persistent system variables (particularly event histories)
112     * and will try to use the persistent data area to store them,
113     * in a robust way with a history.
114     * <p>
115     * Past event values returned by this routine are considered to be
116     * authoritative, and upstream consumers can cache such values
117     * knowing them to be definitive.
118     * <p>
119     * All requests for valid events/periods should be responded to
120     * as authoritative or not depending on the slot time as follows:
121     * <ul>
122     * <li>In the future: null are returned because no data *can* exist yet.
123     * <li>For the current interval: a non-authoritative, non-null response.
124     * <li>For any past interval: an authoritative response.
125     * </ul>
126     * Where the data for responses does not yet exist,
127     * it must be invented with no events to indicate that
128     * it definitely does not exist
129     * so as to allow negative cacheing by upstream users.
130     * <p>
131     * This is assumed not to need to filter out duplicate event/var updates
132     * (nor bad timestamps) which is slow and may not even matter here.
133     * Any network connection upstream of us should do such filtering.
134     * <p>
135     * We don't aim to be massively efficient with this since we hope the
136     * cacheing stage that should normally be downstream of us will compensate
137     * for our main inefficiencies.  We might, however, switch to some sort of
138     * non-blocking I/O in future so that failures of networked filesystems,
139     * (etc) internally won't stop Web service.
140     * <p>
141     * We locate configuration files and data using LocalProps.
142     */
143    public final class ExhibitDataFileSource implements SimpleExhibitPipelineIF
144        {
145        /**Create instance.
146         * Loads any persisted system variable values.
147         */
148        public ExhibitDataFileSource()
149            {
150            final File sysVarDir = _getEventHistoryStorageDir();
151            if(!sysVarDir.isDirectory())
152                { System.err.println("WARNING: event history storage dir not found: " + sysVarDir); }
153            else
154                {
155                // Don't block to load event histories if possible.
156                try { varMgr.loadEventHistories(sysVarDir, true); }
157                catch(final IOException e)
158                    { System.err.println("WARNING: event history could not be restored from: " + sysVarDir); }
159                }
160            }
161    
162        /**If true then when conserving power eliminate much activity that might force activity on the data area.
163         * This includes such things as checking status/flags,
164         * that might force an automount for example,
165         * using extra energy.
166         * <p>
167         * This may prove too conservative,
168         * eg refusing to look for new files at all even when explicitly indicated.
169         */
170        private static final boolean MINIMISE_FS_POWER = false;
171    
172        /**The cached exhibit-data file to read; never null.
173         * This is relative to the data directory.
174         * <p>
175         * We don't construct the exhibit static
176         * data ourselves when called by any of the SimpleExhibitPipelineIF methods,
177         * but instead read a GZIPed serialised AllExhibitProperties object at
178         * the file given by this value, throwing an IOException if it cannot
179         * be read.
180         * <p>
181         * The createStaticCacheFile() routine can be used to create this
182         * file and do as much precomputation and preparation of thumbnails,
183         * etc, as possible off-line.  Typically a command-line program
184         * would be used to run this each time the exhibits are changed,
185         * and the master will simply re-read the serialised file periodically.
186         * It is important in this case that the file be replaced atomically.
187         * The createStaticCacheFile() tries to create the file if it does not
188         * exist.
189         */
190        private static final File getStaticCacheFileName()
191            { return(new File(LocalProps.getDataDir(), CoreConsts.FS_DATA_CACHE_FILENAME)); }
192    
193        /**The name of the AEP longHash file; never null.
194         * Typically, removing this file,
195         * or specifying an old/extant AEP hash different to this,
196         * will force the AEP to be reconstructed from the filesystem.
197         * <p>
198         * This is relative to the data directory.
199         * <p>
200         * Never touches the filesystem; simply constructs the name.
201         */
202        private static final File getStaticHashFileName()
203            { return(new File(LocalProps.getDataDir(), CoreConsts.FS_DATA_HASH_FILENAME)); }
204    
205        /**Get the last/cached AEP longHash file content; null iff not present/readable.
206         * The elements are never null.
207         */
208        private static final Long getLastAEPHashIfAny()
209            {
210            final File staticHashFileName = getStaticHashFileName();
211            // If the hash file doesn't exist or is not readable or is zero length then ignore it.
212            if(!staticHashFileName.canRead() && (staticHashFileName.length() < 1)) { return(null); }
213    
214            try
215                {
216                // Expect the file to consist of one line containing the decimal longHash.
217                final long hash = Long.parseLong(FileTools.readTextFile(staticHashFileName).trim(), 10);
218                return(hash);
219                }
220            // In case of error, log it, and return null.
221            catch(final Exception e) { e.printStackTrace(); return(null); }
222            }
223    
224        /**Get the static attributes for a given exhibit; null if no such exhibit.
225         */
226        public ExhibitStaticAttr getStaticAttr(final ExhibitFull name)
227            throws IOException
228            {
229    //        // If we have a static cache then get attrs via that.
230    //        if(_getStaticCacheFileName() != null)
231    //            { return(getAllExhibitProperties(-1L).aeid.getStaticAttr(name)); }
232    //        // Else fetch directly from the filesystem.
233            return(_getStaticAttr(LocalProps.getDataDir(), name));
234            }
235    
236        /**Get the static attributes for a given exhibit; null if the named exhibit does not exist.
237         * <p>
238         * This computes a new uncached value on each call.
239         * <p>
240         * TODO: prevent this from blocking indefinitely, even with a filesystem hang.
241         */
242        private static ExhibitStaticAttr _getStaticAttr(final String fileRoot,
243                                                        final Name.ExhibitFull name)
244            throws IOException
245            {
246            if((fileRoot == null) || (name == null))
247                { throw new IllegalArgumentException(); }
248    
249            // With Name.ExhibitFull we know a priori that the value is 'safe', eg no ".." path components.
250            final String nameAsString = name.toString();
251            assert(!nameAsString.startsWith(".") && (nameAsString.indexOf("/.") == -1)) : "Dangerous name got through";
252    
253            final File file = new File(fileRoot, nameAsString);
254            logFSAccess("reading static attrs for: "+file, false);
255            if(!file.canRead())
256                { return(null); }
257    
258            return(new ExhibitStaticAttr(name,
259                                         file.length(),
260                                         file.lastModified()));
261            }
262    
263        /**Read a chunk of the raw exhibit binary into the supplied buffer.
264         * The start position and an implied maximum read length are supplied.
265         * The start must be non-negative and no larger than the exhibit data length.
266         * <p>
267         * The call may return less than the the buffer capacity,
268         * though will block until it has read at least one byte unless at EOF or for a zero-byte request;
269         * this will be clear from the state of the buffer.
270         * <p>
271         * If a zero-byte request is made then the file may not actually be accessed.
272         * <p>
273         * This goes directly to the filesystem for each call.
274         * <p>
275         * TODO: prevent this from blocking indefinitely, even with a filesystem hang.
276         */
277        public void getRawFile(final ByteBuffer buf, final Name.ExhibitFull exhibitName, final int position, final boolean dontCache)
278            throws IOException
279            {
280            if(null == exhibitName) { throw new IllegalArgumentException(); }
281    
282            // Use of Name.ExhibitFull means that we know a priori that the name is 'safe'.
283    
284    //        if(!ExhibitName.validNameSyntax(exhibitName))
285    //            { throw new IOException("invalid name (syntax)"); }
286    //
287    //        // Do some extra security checks of requested file's name.
288    //        if(exhibitName.startsWith(".") || (exhibitName.indexOf("/.") != -1))
289    //            { throw new IOException("invalid name (unsafe)"); }
290    
291            // Use of Name.ExhibitFull means that we know a priori that the name is 'safe'.
292            final String nameAsString = exhibitName.toString();
293            final File file = new File(LocalProps.getDataDir(), nameAsString);
294            logFSAccess("reading raw exhibit data from: "+file+", position="+position+", buf.remaining()="+buf.remaining(), false);
295            if(!file.exists() || !file.canRead())
296                { throw new FileNotFoundException(nameAsString); }
297    
298            // Do the read!
299            RandomAccessFile raf = null;
300            try
301                {
302                raf = new RandomAccessFile(file, "r");
303    //            raf.seek(position);
304    //            raf.readFully(buf, offset, len);
305                raf.getChannel().read(buf, position);
306                }
307            catch(final IOException e)
308                {
309                System.err.println("IOException reading exhibit bytes start="+position+" name="+exhibitName+": " + e.getMessage());
310                throw e; // Rethrow the error.
311                }
312            finally { if(raf != null) { raf.close(); } } // Release handle ASAP.
313            }
314    
315    
316        /**Gets all static exhibit data if its timestamp is not that specified.
317         * If the time specified is negative then the object will be returned
318         * unconditionally.
319         * <p>
320         * If no exhibits are currently installed a then default set with a zero
321         * timestamp is returned.
322         * <p>
323         * If the caller's copy appears to be up-to-date (eg the oldStamp
324         * matches that that we would have been returned) then null is returned.
325         *
326         * @throws InterruptedIOException  if another expensive call is already in progress; retry later
327         */
328        public AllExhibitImmutableData getAllExhibitImmutableData(final long oldStamp)
329            throws IOException
330            {
331    //        if(_getStaticCacheFileName() != null)
332    //            {
333    //            final AllExhibitImmutableData extant = getAllExhibitProperties(-1L).aeid;
334    //            // If caller has an up-to-date copy, return null.
335    //            if((oldStamp >= 0) && (oldStamp == extant.timestamp))
336    //                { return(null); }
337    //            return(extant);
338    //            }
339    
340            return(_getAllExhibitImmutableData(LocalProps.getDataDir(), oldStamp, true));
341            }
342    
343        /**Number of least-significant bytes of AllExhibitImmutableData timestamp to use as hash of full collection.
344         * Using a value of 1 would imply keeping about quarter-second timestamp
345         * precision (which does not exist in UNIX systems, typically).
346         * Using a value of 2 would imply about 1 minute precision.
347         * Using a value of 3 would imply about 5 hour precision.
348         * <p>
349         * Given that the Gallery is usually updated at intervals of at most
350         * hours to days normally, a value of 2 or 3 is probably optimal.
351         * <p>
352         * Must lie in the range 0 to 8 inclusive.
353         */
354        private static final int TS_LSbytes_HASH = 2;
355    
356        /**Gets all immutable exhibit data if its timestamp is not that specified.
357         * If the time specified is negative the object will be returned
358         * unconditionally.
359         * <p>
360         * This does not use any cache, and computes afresh on each call,
361         * and is thread-safe.
362         * <p>
363         * If no exhibits are currently installed then a default set with a zero
364         * timestamp is returned.
365         * <p>
366         * If the caller's copy appears to be up-to-date (eg the oldStamp
367         * matches that that we would have been returned) null is returned.
368         * <p>
369         * We make the timestamp we use for the whole collection be the
370         * most significant bits of the latest-stamped exhibit and the
371         * least significant bits of the collection timestamp XORed with a hash
372         * over all the (sorted) ExhibitStaticAttr entries.  We don't have many
373         * least significant bits, so we run a small-ish risk of missing some
374         * changes.  In fact, we are quite rough-and-ready about the hash too!
375         * <p>
376         * We know that many (typically 10) of the least significant bits of
377         * the timestamp would not carry information anyway, eg with a
378         * one-second granularity in the UNIX filesystem.
379         * <p>
380         * To avoid causing too much confusion with our somewhat-faked
381         * timestamp, we limit it to between 1 and the current time of day
382         * in the worst case, though because we ensure that the fake
383         * timestamp calculated is no later than the timestamp of the newest
384         * exhibit then unless there is lots of clock-skew between our host
385         * and the file server we should not see this latter clamp actually used.
386         * The latter limit, if needed, can make the system inefficient just after
387         * a new exhibit has been added since our fake timestamp may seem
388         * to change on every call.
389         *
390         * @param careful  if true, magic numbers of exhibits are checked and
391         *     other extra-careful checking is done; this should be the default
392         *     usage
393         */
394        public static AllExhibitImmutableData _getAllExhibitImmutableData(final String fileRoot,
395                                                                          final long oldStamp,
396                                                                          final boolean careful)
397            throws IOException
398            {
399            if(fileRoot == null)
400                { throw new IllegalArgumentException(); }
401            final File baseDir = new File(fileRoot);
402    
403    //        // Veto attempts to do this concurrently.
404    //        if(!_slow_op_lock.tryLock()) { throw new InterruptedIOException("already in progress: "+_slow_op_lock); }
405    //        try
406    //            {
407                // Load exhibit set and (efficiently) sort it.
408                // We access the underlying files in sorted order aiming to improve filesystem performance.
409                final List<Name.ExhibitFull> names = new ArrayList<Name.ExhibitFull>(ExhibitFile.getFilesystemBasedExhibitNames(
410                    baseDir, careful));
411                Collections.sort(names); // One-off sort of the names more efficient than incremental.
412    
413    if(ORG.hd.d.IsDebug.isDebug) { System.out.println("[Found "+(names.size())+" exhibits based at "+baseDir+".]"); }
414    
415                // Now iterate over the set in order,
416                // finding the newest exhibit,
417                // and computing a hash on all the static attr data.
418                // We also construct our map from names to attr at this point.
419                final Set<ExhibitStaticAttr> esas = new HashSet<ExhibitStaticAttr>(names.size() * 2);
420    
421                long newest = 1; // Newest exhibit's timestamp, but forced strictly positive.
422    
423                final MessageDigest md = GenUtils.getStandardDigest();
424                final DataOutputStream dos =
425                    new DataOutputStream(
426                        new DigestOutputStream(
427                            (new OutputStream(){ // Null output stream...
428                                @Override
429                                public final void write(final int b) { }
430                                @Override
431                                public final void write(final byte[] b, final int off, final int len) { }
432                                }),
433                            md));
434                for(final Name.ExhibitFull name : names)
435                    {
436                    final ExhibitStaticAttr esa = _getStaticAttr(fileRoot, name);
437                    if(null == esa) { continue; }
438    
439                    // We need to collect this ESAs...
440                    esas.add(esa);
441    
442                    // Take care of the timestamp...
443                    if(esa.timestamp > newest) { newest = esa.timestamp; }
444    
445                    // Update the hash...
446                    final ExhibitFull fn = esa.getExhibitFullName();
447                    dos.writeInt(fn.length());
448                    dos.write(fn.toByteArray());
449                    dos.writeLong(esa.length);
450                    dos.writeLong(esa.timestamp);
451                    dos.writeByte(1); // Acts as a nearly-unique separator.
452                    }
453    
454                dos.flush();
455                final byte[] hash = md.digest();
456    
457                // XOR lsbytes in `newest' until we run out of hash bytes...
458                // ...first subtracting enough from the real timestamp to
459                // make space for the hash
460                // (ie so the computed value is no newer than the newest exhibit)...
461                long fakeStamp = newest - (1 << (8 * TS_LSbytes_HASH));
462                // XOR in LS bytes...
463                for(int i = Math.min(hash.length, TS_LSbytes_HASH); --i >= 0; )
464                    { fakeStamp ^= ((long) (hash[i] & 0xff)) << (8 * i); }
465    
466                // Clamp the fake timestamp to within allowed limit values,
467                // though this should not actually be necessary.
468                // If there are no exhibits then force a zero timestamp.
469                final long synthStamp;
470                if(esas.size() == 0) { synthStamp = 0; }
471                else
472                    {
473                    synthStamp = Math.min(Math.min(fakeStamp, newest), System.currentTimeMillis());
474                    assert(synthStamp != 0);
475                    }
476    
477                // See if the client actually needs the value returned...
478                if((oldStamp >= 0) && (oldStamp == synthStamp))
479                    { return(null); } // No, client is up-to-date.
480    
481                // Now make the result!
482                final AllExhibitImmutableData result = new AllExhibitImmutableData(esas, synthStamp);
483    
484                // Warn loudly if duplicate short names were found...
485                final Name.ExhibitFull dups[] = result.getFullNamesWithDuplicateShortNames();
486                if(dups.length > 0)
487                    {
488                    for(int i = 0; i < dups.length; ++i)
489                        { System.err.println("WARNING: duplicate file component in: " + dups[i]); }
490                    }
491    
492                // Done!
493                return(result);
494    //            }
495    //        finally { _slow_op_lock.unlock(); }
496            }
497    
498        /**Hash (longHash) of last-loaded AEP and timestamp, or -1 if no AEP yet loaded.
499         * The elements are never null.
500         * <p>
501         * Is volatile to allow lockless (read) access.
502         * <p>
503         * Updated by _getAllExhibitProperties().
504         */
505        private volatile long _AEP_last_longHash = -1;
506    
507        /**Time before which we should not attempt to reload the AEP.
508         * Is volatile to allow lockless (read) access.
509         * <p>
510         * Updated by _getAllExhibitProperties().
511         * <p>
512         * Initially zero to force first recomputation.
513         */
514        private volatile long _AEP_nextLoad;
515    
516        /**Reentrant lock avoid concurrent slow operations being attempted; never null.
517         * Used by both the internal getAEP and getAEID routines.
518         * <p>
519         * Rather than block, concurrent attempts may be vetoed with IOException.
520         */
521        private final ReentrantLock _slow_op_lock = new ReentrantLock();
522    
523        /**Gets set of all exhibit properties if its hash is not that specified.
524         * If the hash specified is negative
525         * then the AEP will be loaded and returned unconditionally
526         * but this is likely to be expensive;
527         * call with the longHash of any extant item if at all possible.
528         * <p>
529         * If no exhibits are currently installed
530         * then a default empty set with a zero timestamp is returned.
531         * <p>
532         * If the caller's copy appears to be up-to-date
533         * (ie the oldHash matches that that would have been returned)
534         * then null is returned.
535         * <p>
536         * Note that a full load from the filesystem is likely to be slow/expensive.
537         *
538         * @throws InterruptedIOException  if another expensive call is already in progress
539         *     or we cannot otherwise currently attempt to (re)load an AEP;
540         *     retry later
541         */
542        public AllExhibitProperties getAllExhibitProperties(final long oldHash)
543            throws IOException
544            {
545            // Generally we want to be very careful when (re)loading the exhibit set
546            // but when (temporarily) short of power then this may be a little more reckless
547            // and hope to notice any subtle issues when the system is flush.
548            final boolean carefulLoad = !GenUtils.mustConservePower();
549    
550            return(_getAllExhibitProperties(oldHash, carefulLoad, false));
551            }
552    
553    
554        /**Minimum time between polls of the filesystem (ms) when not conserving power. */
555        private static final int MIN_POLL_MS = 7*60*1000;
556    
557        /**Minimum time between polls of the filesystem (ms) when conserving power. */
558        private static final int MIN_POLL_CONSERVING_MS = 121*60*1000;
559    
560        /**Gets set of all exhibit properties if its hash is not that specified.
561         * If the hash specified is negative
562         * then the AEP will be loaded and returned unconditionally
563         * but this is likely to be expensive;
564         * call with the longHash of any extant item if at all possible.
565         * <p>
566         * If no exhibits are currently installed
567         * then a default empty set with a zero timestamp is returned.
568         * <p>
569         * If the caller's copy appears to be up-to-date
570         * (eg the oldHash matches that that would have been returned)
571         * then null is returned.
572         * <p>
573         * This is more averse to checking/reloading the AEP if conserving power.
574         *
575         * @param careful  if true then magic numbers of exhibits are checked and
576         *     other extra-careful checking is done; this should be the default usage
577         * @param forceLoadFromFilesystem  if true, ignore any cached (hash) data and
578         *     always load data from the filesystem
579         *
580         * @throws InterruptedIOException  if another expensive call is already in progress
581         *     or we cannot otherwise currently attempt to (re)load an AEP;
582         *     retry later
583         */
584        private AllExhibitProperties _getAllExhibitProperties(final long oldHash,
585                                                              final boolean careful,
586                                                              final boolean forceLoadFromFilesystem)
587            throws IOException
588            {
589            // Warn if the caller may be forcing an unnecessary expensive full reload from the filesystem.
590    if(IsDebug.isDebug && !forceLoadFromFilesystem && (oldHash <= 0)) { System.err.println("WARNING: ExhibitDataFileSource._getAllExhibitProperties(): expensive 0 (empty AEP) or negative oldHash request"); }
591    
592            // When (temporarily) conserving power and NOT forcing a (re)load from the filesystem,
593            // then unless a negative oldHash has been specified
594            // (ie the caller already has a non-empty AEP to hand)
595            // and if the caller's existing hash is what we remember from our last AEP load
596            // then this routine doesn't recheck the filesystem at all
597            // unless enough time has elapsed since the last load attempt,
598            // so as to avoid even 'waking' (ie spinning/powering up) the filesystem.
599            // When conserving power we wait a while *after* 'next load' time,
600            // meaning also that when power is available we can very quickly test again.
601            // (If not conserving power then prodding the filesystem to ensure up-to-date is fine.)
602            final long startTime = System.currentTimeMillis();
603            if(!forceLoadFromFilesystem && (oldHash >= 0) &&
604                    (startTime < _AEP_nextLoad + (GenUtils.mustConservePower() ? (MIN_POLL_CONSERVING_MS - MIN_POLL_MS) : 0)))
605                {
606                // As far as we know the caller is up-to-date so return null.
607                if(_AEP_last_longHash == oldHash) { return(null); }
608                }
609    
610            // If (temporarily) conserving power and the caller has a non-empty AEP
611            // (that they can presumably work with for the time being)
612            // then veto trying to load/construct an AEP from the filesystem.
613            if(MINIMISE_FS_POWER && !forceLoadFromFilesystem && (oldHash > 0) && GenUtils.mustConservePower())
614                { throw new InterruptedIOException("conserving power; will not attempt to load/construct AEP while caller has non-empty AEP"); }
615    
616            // Try for reload from the filesystem (if not already busy)...
617            if(!_slow_op_lock.tryLock()) { throw new InterruptedIOException("already in progress: "+_slow_op_lock); }
618            try
619                {
620                // Iff there is a hash file containing the caller's specified (non-negative) oldHash
621                // then we assume that the caller's AEP is up-to-date.
622                // (After adding exhibits the hash file should be removed to force a later reload.)
623                final File staticHashFileName = getStaticHashFileName();
624                logFSAccess("checking cached AEP hash file " + staticHashFileName, false);
625                final Long hash = getLastAEPHashIfAny();
626                if(hash != null)
627                    {
628                    if(!forceLoadFromFilesystem && (oldHash >= 0) && (hash.longValue() == oldHash))
629                        {
630                        _AEP_last_longHash = oldHash; // Note the apparent longHash for the current exhibits.
631    
632                        // Simplified method to postpone next poll if no update was found...
633                        final long endTime = System.currentTimeMillis();
634                        final long waitTime = 101 + Math.max(MIN_POLL_MS, (CoreConsts.DEFAULT_TEMPORAL_SLACKNESS_S/4) * 1000);
635    
636                        // Postpone next check.
637                        final long nextUpdateTime = endTime + waitTime;
638                        _AEP_nextLoad = nextUpdateTime;
639    
640    if(ORG.hd.d.IsDebug.isDebug) { System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): caller is apparently up-to-date (see "+staticHashFileName+") with hash: "+oldHash+"; next check not before "+(new Date(nextUpdateTime))+".]"); }
641                        return(null);
642                        }
643                    }
644    
645                // Attempt to recover cache of entire AEP to avoid recomputing some very expensive state.
646                AllExhibitProperties oldProps = null;
647                final File staticCacheFileName = getStaticCacheFileName();
648                if(staticCacheFileName.canRead())
649                    {
650    System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): attempting to load old cache file to recover expensive-to-compute data: " + staticCacheFileName +" @ "+(new Date())+".]");
651                    try { oldProps = (AllExhibitProperties) FileTools.deserialiseFromFile(staticCacheFileName, true); }
652                    // Log but absorb any error, and do without cached data...
653                    catch(final Exception e) { e.printStackTrace(); }
654                    }
655    
656                // Try to load/construct the AEP...
657                try
658                    {
659    System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): starting AEP construction from filesystem @ "+(new Date())+".]");
660    
661                    // Turn back on any muted warnings.
662                    turnBackOnMutedWarnings();
663    
664                    final File baseDir = new File(LocalProps.getDataDir());
665                    logFSAccess("loading AEP from raw exhibit data in "+baseDir, false);
666    
667                    // Load the EPGI (should be quick: do it first).
668                    final ExhibitPropsGlobalImmutable epgi = ExhibitPropsGlobalImmutable.loadFromDataDir(baseDir);
669    if(ORG.hd.d.IsDebug.isDebug) { System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): loaded EPGI: "+epgi+"]"); }
670    
671    /* if(ORG.hd.d.IsDebug.isDebug) */ { System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): recomputing AllExhibitImmutableData... @ " + (new Date())+".]"); }
672                    // First, get immutable core data unconditionally...
673                    final AllExhibitImmutableData aeid =
674                        _getAllExhibitImmutableData(baseDir.getPath(), -1, careful);
675    
676                    // Create accession data for new exhibits...
677    /* if(ORG.hd.d.IsDebug.isDebug) */ { System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): creating new accession files @ "+(new Date())+".]"); }
678                    createAccessionFiles(aeid);
679    
680                    // Get sorted set of exhibit names.
681                    // We hope that iterating over a sorted set will result in
682                    // better filesystem performance than an unsorted set.
683                    final List<Name.ExhibitFull> sortedNames = aeid.getAllExhibitNamesSorted();
684    
685    /* if(ORG.hd.d.IsDebug.isDebug) */ { System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): recomputing loadedProperties over exhibit set of "+(sortedNames.size())+"... "+(new Date())+".]"); }
686    
687                    // Collect the loaded and computable properties.
688                    // We do this in parallel to maximise I/O throughput
689                    // (though this may in fact be quite CPU- and memory- intensive)
690                    // and use a ConcurrentHashMap as our working store as it is thread-safe.
691                    // We use the CPU-intensive thread pool
692                    // and wait for all work to complete.
693                    //
694                    // We recover all EPC data that we can from the previous/cached AEP if extant
695                    // to save *lots* of time and effort.
696                    final Map<Name.ExhibitFull,ExhibitPropsLoadable> loadedProps = new ConcurrentHashMap<Name.ExhibitFull, ExhibitPropsLoadable>(aeid.size() * 2);
697                    final Map<Name.ExhibitFull,ExhibitPropsComputable> computedProps = new ConcurrentHashMap<Name.ExhibitFull, ExhibitPropsComputable>(aeid.size() * 2);
698                    final List<Future<?>> tasks = new ArrayList<Future<?>>(MemoryTools.isMemoryStressed() ? 16 : sortedNames.size());
699                    final ExhibitDataSource ds = makeExhibitDataSource();
700                    final AllExhibitProperties c = oldProps;
701                    final boolean recoverFromCache = (null != c);
702                    for(final Name.ExhibitFull name : sortedNames)
703                        {
704                        // Create the task...
705                        final Runnable r = new Runnable(){
706                            public final void run()
707                                {
708                                // We don't store null/EMPTY EPL/EPC values.
709                                try
710                                    {
711                                    // Always reload EPL data
712                                    // else we will miss changes in description, etc.
713                                    final ExhibitPropsLoadable epl = ExhibitPropsLoadable.getLoadableProperties(name, baseDir);
714                                    if((epl != null) && !epl.equals(ExhibitPropsLoadable.EMPTY))
715                                        { loadedProps.put(name, epl); }
716    
717                                    // Recover expensive EPC data if possible...
718                                    ExhibitPropsComputable epc = null;
719                                    if(recoverFromCache)
720                                        { epc = c.getExhibitPropsComputable(name); }
721                                    if((null == epc) || epc.equals(ExhibitPropsComputable.EMPTY))
722                                        { epc = ExhibitPropsComputable.createExhibitPropsComputable(aeid.getStaticAttr(name), ds); }
723                                    if((epc != null) && !epc.equals(ExhibitPropsComputable.EMPTY))
724                                        { computedProps.put(name, epc); }
725                                    }
726                                catch(final Exception e)
727                                    { throw new RuntimeException("Could not create epl/epc for: " + name, e); }
728                                }
729                            };
730    
731                        // If we're currently pressed for resources then run this task synchronously...
732                        if(MemoryTools.isMemoryStressed() || !ThreadUtils.couldRunLowPriorityDiscardableTask())
733                            { try { r.run(); } catch(final Exception e) { throw new IOException(e); } }
734                        // ...else attempt to run the task concurrently for better throughput.
735                        else
736                            { tasks.add(ThreadUtils.computeIntensiveThreadPool.submit(r)); }
737                        }
738    
739                    // Wait for all asynchronous tasks to finish...
740                    for(final Future<?> task : tasks)
741                        {
742                        try { task.get(); }
743                        catch(final InterruptedException e)
744                            {
745                            final InterruptedIOException err = new InterruptedIOException(e.getMessage());
746                            err.initCause(e);
747                            throw err;
748                            }
749                        catch(final ExecutionException e)
750                            {
751                            final IOException err = new IOException(e.getMessage());
752                            err.initCause(e);
753                            throw err;
754                            }
755                        }
756    
757    /* if(ORG.hd.d.IsDebug.isDebug) */ { System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): constructing AEP... @ "+(new Date())+"]"); }
758    
759                    // Construct a new AEP from the data gathered.
760                    final AllExhibitProperties newProps = new AllExhibitProperties(null,
761                            epgi,
762                            aeid,
763                            loadedProps,
764                            computedProps,
765                            0);
766    
767    System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): compacting... @ "+(new Date())+"]");
768                    newProps.compact();
769    
770    System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): loaded and constructed new AEP with longHash="+newProps.longHash+" @ "+(new Date())+"]");
771    
772                    // Cache the longHash in memory...
773                    _AEP_last_longHash = newProps.longHash;
774                    // Save a hash file recording the AEP longHash (in decimal) if possible.
775                    // May not be possible if the filesystem is read-only for example.
776    System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): cacheing AEP hash to "+staticHashFileName+"... @ "+(new Date())+"]");
777                    try { FileTools.replacePublishedFile(staticHashFileName.getPath(), (new CS8Bit(Long.toString(newProps.longHash, 10))).toByteArray()); }
778                    catch(final IOException e) { e.printStackTrace(); }
779    
780                    // If the AEP has changed from the static cached version (if any) then try to save it for next time.
781                    if((oldProps == null) || (oldProps.longHash != newProps.longHash))
782                        {
783    System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): cacheing serialised AEP to "+staticCacheFileName+"... @ "+(new Date())+"]");
784                        try { FileTools.serialiseToFile(newProps, staticCacheFileName, true, !IsDebug.isDebug); }
785                        // Log but absorb any errors.
786                        catch(final Exception e) { e.printStackTrace(); }
787                        }
788    
789                    // If the caller really is up-to-date then return null (discarding the new AEP).
790                    if((oldHash >= 0) && (newProps.longHash == oldHash)) { return(null); }
791    
792                    // Return the newly-loaded AEP.
793                    return(newProps);
794                    }
795                finally
796                    {
797                    // Whether we succeeded or not...
798                    // work out the minimum wait before starting the next load.
799                    //
800                    // Usually, after a successful load,
801                    // the next load attempt should not happen within the normal slackness interval.
802                    //
803                    // This tries not to spend more than ~10% of real time doing loads if possible,
804                    // but caps the *minimum* wait before another attempt to a few hours.
805                    // We throw in a small random factor too, to help avoid collisions...
806                    //
807                    // The minimum polling interval is boosted when conserving power.
808                    final long endTime = System.currentTimeMillis();
809                    final long timeTaken = endTime - startTime;
810                    final long minWaitTime = Math.max(MIN_POLL_MS, CoreConsts.DEFAULT_TEMPORAL_SLACKNESS_S * (1000 / 10));
811                    final long waitTime = Math.max(minWaitTime, // Default/minimum wait time...
812                            Math.min(timeTaken << 3, // Limit effort to ~10% of wall-clock time if possible.
813                                     2 * 3600 * 1000L) + // Limit delay to next attempt to ~2 hours at worst.
814                        Rnd.fastRnd.nextInt(10001 + (CoreConsts.DEFAULT_TEMPORAL_SLACKNESS_S * 101)));
815    
816                    // Schedule next check.
817                    final long nextUpdateTime = endTime + waitTime;
818                    _AEP_nextLoad = nextUpdateTime;
819    /* if(ORG.hd.d.IsDebug.isDebug) */ { System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): done new AEP load: next check not before "+(waitTime/(1000 * 60))+"mins, at "+(new Date(nextUpdateTime))+".]"); }
820                    }
821                }
822            finally { _slow_op_lock.unlock(); }
823            }
824    
825        /**Wrap this instance as an ExhibitDataSource.
826         * Only really meant for internal use,
827         * but may be useful for low-level use of the file-based data.
828         */
829        public AllExhibitProperties.ExhibitDataSource makeExhibitDataSource()
830            {
831            return(new AllExhibitProperties.ExhibitDataSource(){
832                        @Override
833                        public final void getRawFile(final ByteBuffer buf, final ExhibitFull exhibitName, final int position)
834                            throws IOException
835                            { ExhibitDataFileSource.this.getRawFile(buf, exhibitName, position, false); }
836                        @Override
837                        public final boolean isExhibitFullyLoaded(final ExhibitStaticAttr esa)
838                            { return(true); }
839                        });
840            }
841    
842    
843        /**Gets the general properties as a GenProps object if its timestamp is not that specified.
844         * If the time specified is negative the object will be returned unconditionally.
845         * <p>
846         * If no props are currently installed/available a default set with a zero
847         * timestamp is returned.
848         * <p>
849         * If the caller's copy appears to be up-to-date (eg the oldStamp
850         * matches that that we would have been returned) null is returned.
851         * <p>
852         * This computes a new uncached value on each call.
853         */
854        public GenProps getGenProps(final long oldStamp)
855            throws IOException
856            {
857            final File file = new File(LocalProps.getConfDir(), CoreConsts.FS_CONF_SYSPROPS);
858    
859            // If the stamp presented is < 0 or different to that of the file,
860            // reread the file now.
861            final long newStamp = file.lastModified();
862            if((oldStamp < 0) || (oldStamp != newStamp))
863                {
864                try
865                    {
866                    final FileInputStream fis = new FileInputStream(file);
867                    try
868                        {
869                        final Properties props = FileTools.loadProperties(new BufferedInputStream(fis));
870                        return(new GenProps(props, newStamp));
871                        }
872                    finally { fis.close(); }
873                    }
874                catch(final IOException e)
875                    {
876                    e.printStackTrace(); // Dump a diagnostic...
877                    throw e; // Pass the error on.
878                    }
879                }
880    
881            return(null); // Caller's copy seems to be up-to-date.
882            }
883    
884        /**Gets the generic security properties as a Properties object if its timestamp is not that specified.
885         * If the time specified is negative the object will be returned unconditionally.
886         * <p>
887         * If no props are currently installed/available a default set with a zero
888         * timestamp is returned.
889         * <p>
890         * If the caller's copy appears to be up-to-date (eg the oldStamp
891         * matches that that would have been returned) null is returned.
892         * <p>
893         * This computes a new uncached value on each call.
894         * <p>
895         * These generic properties are fetchable over the network, for example,
896         * and need not be present locally at each host the JVM runs on.
897         */
898        public java.util.Properties getGenSecProps(final long oldStamp)
899            throws IOException
900            {
901            final SecurityProps sp =
902                SecurityProps.getSecurityPropsUncachedFromFilesystem(oldStamp);
903    
904            // If security props file has not changed, return null.
905            if(sp == null)
906                { return(null); }
907    
908            // Extract the relatively-safe "generic" component.
909            return(sp.getGenSecProps());
910            }
911    
912        /**Suffix appended to serialised thumbnail class data for each exhibit.
913         * Such files are beside the original exhibit file,
914         * are prefixed with FileTools.F_permPrefix,
915         * and suffixed with this (which is chosen so as not to look like an exhibit).
916         * <p>
917         * The file is a GZIPped, serialised, ExhibitThumbnails class if present.
918         */
919        public static final String THUMBNAIL_SUFFIX = ".thumbnails";
920    
921        /**If true, the serialised thumbnail files are compressed (GZIPped) on disc.
922         * This might cost a little time but might possibly save a little space.
923         * Note that thumbnails may in any case be GZIPped in transit to a slave server.
924         */
925        public static final boolean THUMBNAILS_ARE_GZIPPED = false;
926    
927        /**If true, we have warned about thumbnail directory absence.
928         * Starts, false; may be cleared periodically.
929         * Set true when we have warned.
930         * <p>
931         * Volatile to allow read/write without a lock.
932         * <p>
933         * Not critical to correct operation,
934         * so we can tolerate races and sloppy handling.
935         */
936        private volatile boolean _haveWarnedNoTNDir;
937    
938        /**Turn back on warnings we have muted.
939         * We do this occasionally so that persistent faults
940         * will show up in the log and don't just vanish.
941         * <p>
942         * Avoids locking unless really necessary.
943         */
944        private void turnBackOnMutedWarnings()
945            {
946            _haveWarnedNoTNDir = false;
947            }
948    
949    
950        /**Interval between logging filesystem accesses (ms); strictly positive.
951         * Should be short enough to reveal operations, possibly unwanted,
952         * that 'wake up' the filesystem (eg cause it to power-up, get mounted, etc).
953         * <p>
954         * Should be long enough to eliminate most uninteresting stuff
955         * such as the multiple accesses to load/examine one file.
956         * Keeping the interval to number a minute will most likely catch most 'wakeups'.
957         */
958        private static final int FS_LOG_INTERVAL_MS = 59000; // Just less than a minute.
959    
960        /**Count of filesystem access operations logged including those not displayed; never null.
961         * Accessed under _FSLogLock.
962         */
963        private static int countFSAccess;
964    
965        /**Time that last filesystem access was logged; initially zero.
966         * Accessed under _FSLogLock.
967         */
968        private static long lastLoggedFSAccess;
969    
970        /**Lock for logging filesystem access; never null. */
971        private static final ReentrantLock _FSLogLock = new ReentrantLock();
972    
973        /**Log access to data filesystem.
974         * Useful for tracing unwanted activity.
975         * <p>
976         * May limit output (eg to once every few minutes)
977         * to still give traceability of 'wakeup' operations.
978         *
979         * @param force if true then force printing
980         */
981        private static void logFSAccess(final String detail, final boolean force)
982            {
983            _FSLogLock.lock();
984            try
985                {
986                final int count = ++countFSAccess;
987    
988                // Discard this log message if too soon since the previous...
989                final long now = System.currentTimeMillis();
990                if(!force && (now < lastLoggedFSAccess + FS_LOG_INTERVAL_MS)) { return; }
991                lastLoggedFSAccess = now;
992    
993                // Write this long message.
994                System.out.println("[ExhibitDataFileSource: DATA FILESYSTEM ACCESS "+count+": "+detail+" @ "+(new Date(now))+"]");
995                }
996            finally { _FSLogLock.unlock(); }
997            }
998    
999    
1000        /**Gets the thumbnails for an exhibit; null if not (currently) available.
1001         * In this implementation we never try to create thumbnails
1002         * (we assume that the source directory is read only for example),
1003         * but we'll return one if we can find one.
1004         * <p>
1005         * Thumbnails are looked for by default in the exhibits directory
1006         * tree beside the source exhibits, but a LocalProps value
1007         * can override this so that source data can be separated
1008         * from derived data.
1009         *
1010         * @param create  if true then may try to create and cache missing thumbnails,
1011         *     but creating the thumbnail is only attempted
1012         *     if cacheing is likely to be successful
1013         *     because this should be a once-only operation per exhibit
1014         */
1015        public ExhibitThumbnails getThumbnails(final ExhibitFull name, final boolean create)
1016            throws IOException
1017            {
1018            logFSAccess("requesting pre-computed thumbnails for: "+name, false);
1019    
1020            // In the simple case where we know that thumbnails
1021            // definitely can't be created (from the type)
1022            // return the NO_THUMBNAILS value immediately,
1023            // ignoring the create parameter.
1024            // We never cache this (negative) value.
1025            final ExhibitMIME.ExhibitTypeParameters type = (ExhibitMIME.getInputFileType(name));
1026            if((type == null) || (type.handler == null) ||
1027               !type.canPossiblyCreateThumbnailOfSameMIMEType())
1028                { return(ExhibitThumbnails.NO_THUMBNAILS); }
1029    
1030            final String dataDir = LocalProps.getDataDir();
1031            final ExhibitStaticAttr esa = _getStaticAttr(dataDir, name); // Avoid tickling cache badly...
1032            // If the exhibit appears not to exist
1033            // then return null rather than NO_THUMBNAILS
1034            // (for example the exhibit may be in the process or being added).
1035            if(esa == null)
1036                { return(null); }
1037    
1038            // Construct the name of the thumbnails' file if it exists.
1039            final String relPath = thumbnailRelPath(name);
1040    
1041            // Compute full path from exhibit directory.
1042            final File topDir = new File(dataDir, LocalProps.getThumbnailRelDir());
1043    
1044            // If the top directory for the thumbnails does not exist,
1045            // then we definitely will not be returning thumbnails.
1046            // We also warn when this directory does not exist.
1047            final boolean noTNDir = !topDir.isDirectory();
1048            if(noTNDir)
1049                {
1050                if(!_haveWarnedNoTNDir)
1051                    {
1052                    System.err.println("ExhibitDataFileSource: WARNING: top-level thumbnail directory absent: " + topDir);
1053                    _haveWarnedNoTNDir = true;
1054                    }
1055    
1056                // We don't return ExhibitThumbnails.NO_THUMBNAILS,
1057                // since thumbnails may be available/generated later.
1058                return(null);
1059                }
1060    
1061            // If the thumbnail file seems readable, etc, then try to load it.
1062            // Any problem, eg an IOException, will be propagated to the caller.
1063            final File fullPath = new File(topDir, relPath);
1064            logFSAccess("reading thumbnail from: "+fullPath, false);
1065            if(fullPath.exists() && fullPath.canRead() && fullPath.isFile())
1066                {
1067                final ExhibitThumbnails result = (ExhibitThumbnails)
1068                    FileTools.deserialiseFromFile(fullPath, THUMBNAILS_ARE_GZIPPED);
1069                logFSAccess("read thumbnail from: "+fullPath+": "+result, ExhibitThumbnails.NO_THUMBNAILS.equals(result)); // Force result for NO_THUMBNAILS value...
1070                return(result);
1071                }
1072    
1073            // Only even consider thumbnail creation
1074            // if the thumbnail area appears to be writable.
1075            if(create && topDir.canWrite())
1076                {
1077                // TODO
1078                }
1079    
1080            // Can't seem to find a thumbnail file,
1081            // so tell the caller no thumbnails at the moment.
1082            // We don't return ExhibitThumbnails.NO_THUMBNAILS,
1083            // since thumbnails may be available/generated later.
1084            return(null);
1085            }
1086    
1087        /**Poll periodically.
1088         * Periodically save the system variables.
1089         */
1090        public void poll(final GenProps gp)
1091            {
1092            _saveSystemVariables(varMgr, false);
1093            }
1094    
1095        /**Save the system variables.
1096         * Mainly save the histories of persistent events.
1097         * <p>
1098         * We save these with a history so that in effect we have backups
1099         * if something bad happens
1100         * (though this implies that some other housekeeping will
1101         * have to clear up excess copies, etc,
1102         * in order to avoid space requirements growing without bound).
1103         * <p>
1104         * The data is saved in the persistent data area.
1105         * <p>
1106         * Reports but does not propagate exceptions.
1107         *
1108         * @param vars  set of variables to save
1109         * @param force  if true, force an immediate complete save
1110         */
1111        private void _saveSystemVariables(final BasicVarMgr vars,
1112                                          final boolean force)
1113            {
1114            // Return immediately if too soon to save again (postponed further when conserving energy)
1115            // and a save is not being forced.
1116            if(!force &&
1117               (_nextVarSaveEarliest + (GenUtils.mustConservePower() ? Math.max(CoreConsts.ASYNC_MIN_POWER_SAVE_NON_CRITICAL_DATA_FLUSH_MS, _VAR_SAVE_INTERVAL_MS*3) : 0) > System.currentTimeMillis()))
1118                { return; }
1119    
1120            // Keeps a history and attempts just to save changes...
1121            try
1122                {
1123                // First write out the persistent events as-is
1124                _appendToEventLogs();
1125    
1126                // Unless doing a forced save, eg at shut-down,
1127                // do an incremental changes-only save to be as quick as reasonably possible.
1128                // When forced, push out *all* stores, not just changed ones,
1129                // to minimise the chance of entirely 'losing' very-slowly-changing event sets.
1130                vars.saveEventHistories(_getEventHistoryStorageDir(), !force, true, !force);
1131                // The save seems to have completed fine, so postpone the next one.
1132                _nextVarSaveEarliest = System.currentTimeMillis() + _VAR_SAVE_INTERVAL_MS +
1133                    Rnd.fastRnd.nextInt(1 + (_VAR_SAVE_INTERVAL_MS/3));
1134                }
1135            catch(final IOException e)
1136                { e.printStackTrace(); }
1137            }
1138    
1139    
1140    
1141        /**Our private static instance of a GMT calendar.
1142         * No one must change the ZONE or DST offset.
1143         */
1144        private static final Calendar GMTCalendar = Calendar.getInstance();
1145        /**Initialise GMTCalendar. */
1146        static
1147            {
1148            // Set to GMT
1149            // i.e 0 Offset
1150            GMTCalendar.set(Calendar.ZONE_OFFSET, 0);
1151            // and 0 Daylight savings
1152            GMTCalendar.set(Calendar.DST_OFFSET, 0);
1153            }
1154    
1155        /**Format we use to insert into event history file name. */
1156        private static final SimpleDateFormat dateFmtEHFile =
1157            new SimpleDateFormat("yyyyMMdd");
1158        /**Set the Calendar for dateFmtEHFile to be the GMT calendar. */
1159        static { dateFmtEHFile.setCalendar(GMTCalendar); }
1160    
1161        /**Format we use for timestamp to insert into event log file name. */
1162        private static final SimpleDateFormat timestampFmtEHFile =
1163            new SimpleDateFormat("yyyyMMdd-HHmmss.SSS");
1164        /**Set the Calendar for dateFmtEHFile to be the GMT calendar. */
1165        static { timestampFmtEHFile.setCalendar(GMTCalendar); }
1166    
1167        /**Max log line length for event log. */
1168        private static final int MAX_EV_LOG_LINE_LENGTH = 4095;
1169    
1170        /**Private lock for _appendToEventLogs(). */
1171        private static final Object _aTWL_lock = new Object();
1172    
1173        /**Write out the accumulated event values to disc.
1174         * These are written to date-stamped files to allow automatic "rolling".
1175         * <p>
1176         * We keep a separate file for each named event,
1177         * and append events to each appropriate file in order
1178         * with a record of the form:
1179         * <pre>
1180         *     YYYYMMDD-HHmmss.SSS eventName eventValue
1181         * </pre>
1182         * with a null value being indicated by the value "null".
1183         * <p>
1184         * Note that the timestamp is that of the value,
1185         * not the time that the event is received or written.
1186         * <p>
1187         * We may cache file handles to reduce the cost of opening and closing
1188         * files for each event,
1189         * but in any case will have closed all log files opened by us
1190         * by the time this routine returns.
1191         * <p>
1192         * We synchronise on a private static lock to try to prevent two instances of this
1193         * running concurrently and leading to file corruption.
1194         */
1195        private void _appendToEventLogs()
1196            throws IOException
1197            {
1198            // Attempt to write out accumulated event values to the logs.
1199            final List<SimpleVariableValue> events = new ArrayList<SimpleVariableValue>(eventsToLog.size());
1200            // Move all events into working copy atomically to hold lock for short time.
1201            synchronized(eventsToLog) { events.addAll(eventsToLog); eventsToLog.clear(); }
1202    
1203            // If nothing to write then return immediately.
1204            final int nEvents = events.size();
1205            if(nEvents == 0) { return; }
1206    
1207    
1208    //System.out.println("[EVENTS TO LOG: " + nEvents + "...]");
1209    
1210            // Separate the events into groups (preserving order) by event name
1211            // so that we can do one file operation for all instances of one event.
1212            final Map<String, List<SimpleVariableValue>> groupedEvents = new HashMap<String, List<SimpleVariableValue>>();
1213            for(final SimpleVariableValue svv : events)
1214                {
1215                final String name = svv.getDef().getName();
1216                List<SimpleVariableValue> evvs = groupedEvents.get(name);
1217                if(evvs == null)
1218                    {
1219                    evvs = new ArrayList<SimpleVariableValue>();
1220                    groupedEvents.put(name, evvs);
1221                    }
1222                evvs.add(svv);
1223                }
1224    
1225            // Check that the log directory is writable.
1226            // Complain loudly and discard the events if not.
1227            final File logDir = _getEventLogDir();
1228            if((logDir == null) || !logDir.isDirectory() || !logDir.canWrite())
1229                {
1230                System.err.println("WARNING: persistent event log dir "+logDir+" not usable: discarding "+nEvents+" event(s).");
1231                return;
1232                }
1233    
1234            // Create/format the date string once, if we are going to use it.
1235            final String dateString = "." + dateFmtEHFile.format(new Date());
1236    
1237            // Any error accumulated so far...
1238            IOException err = null;
1239    
1240            synchronized(_aTWL_lock)
1241                {
1242                for(final String name : groupedEvents.keySet())
1243                    {
1244                    final StringBuilder sb = new StringBuilder(80);
1245                    sb.append("eventLog.");
1246                    sb.append(name);
1247                    sb.append(BasicVarMgr.EVENT_STORE_NAMETERM);
1248                    sb.append(dateString);
1249                    sb.append(".log");
1250                    final File f = new File(logDir, sb.toString());
1251    
1252                    try
1253                        {
1254                        final PrintWriter pw = new PrintWriter(new FileWriter(f, true)); // Append to existing file.
1255                        try
1256                            {
1257                            for(final SimpleVariableValue svv : groupedEvents.get(name))
1258                                {
1259                                final StringBuilder logLine = new StringBuilder(80);
1260    
1261                                logLine.append(timestampFmtEHFile.format(new Date(svv.getTimestamp())));
1262                                logLine.append(' ');
1263                                logLine.append(name);
1264                                logLine.append(' ');
1265                                logLine.append(svv.getValue());
1266    
1267                                // Truncate VERY long entries...
1268                                if(logLine.length() > MAX_EV_LOG_LINE_LENGTH)
1269                                    {
1270                                    logLine.setLength(MAX_EV_LOG_LINE_LENGTH - 3);
1271                                    logLine.append("..."); // Indicate truncation.
1272                                    }
1273    
1274                                // We sanitise the log line to ensure that it
1275                                // contains no control codes especially CRs or LFs
1276                                // that would disrupt the record-per-line format.
1277                                // Replace such toxic chars with something innocuous.
1278                                for(int i = logLine.length(); --i >= 0; )
1279                                    {
1280                                    final char ch = logLine.charAt(i);
1281                                    if((ch < 32) || (ch > 255) || (ch == 127))
1282                                        { logLine.setCharAt(i, ' '); }
1283                                    }
1284    
1285                                // Write entry in own line to EOF...
1286                                pw.println(logLine.toString());
1287                                }
1288    
1289                            // Flush to disc...
1290                            pw.flush();
1291                            }
1292                        finally { pw.close(); }
1293                        }
1294                    catch(final IOException e)
1295                        {
1296                        e.printStackTrace();
1297                        err = e; // Absorb error for now.  Rethrow later.
1298                        }
1299                    }
1300                }
1301    
1302            // If we encountered any error then (re)throw a sample one here.
1303            if(err != null) { throw err; }
1304            }
1305    
1306        /**Minimum interval between saves, ms; strictly positive.
1307         * Intended to avoid losing too much data at shut-down,
1308         * yet avoid wearing out the disc (or other storage such as Flash) and consuming CPU time
1309         * with redundant work.
1310         * <p>
1311         * Something from tens of seconds to a few minutes is probably about right.
1312         */
1313        private static final int _VAR_SAVE_INTERVAL_MS = 5*60*1000;
1314    
1315        /**Earliest time after which to allow the next event save (zero if no save yet completed).
1316         * Marked volatile to allow thread-safe lock-free access.
1317         */
1318        private transient volatile long _nextVarSaveEarliest;
1319    
1320    
1321        /**Get directory used for storing system-variable persistent event histories; never null.
1322         * Though never null, the indicated directory may not exist
1323         * or may otherwise not be usable.
1324         * <p>
1325         * We derive this from the LocalProps value.
1326         */
1327        private static File _getEventHistoryStorageDir()
1328            {
1329            return(new File(LocalProps.getPersistentStateDir(), "history"));
1330            }
1331    
1332        /**Get directory used for storing system-variable persistent event logs; never null.
1333         * Though never null, the indicated directory may not exist
1334         * or may otherwise not be usable.
1335         * <p>
1336         * We get derive this from the LocalProps value.
1337         */
1338        private static File _getEventLogDir()
1339            {
1340            return(new File(LocalProps.getPersistentStateDir(), "log"));
1341            }
1342    
1343        /**Our set of system variables.
1344         * Persistent values are loaded from disc on first attempt to
1345         * get and variable(s) if possible.
1346         * <p>
1347         * We expect this to be at the top of the data pipe on the master server,
1348         * and thus handle all the system non-local variables,
1349         * plus all the local variables of the master.
1350         * <p>
1351         * If any persistent values are set then they all get written to disc
1352         * as compressed serialised data on the next poll().
1353         * (Doing saves on each poll() helps reduce expensive disc writes by
1354         * grouping multiple updates into one save if they are happening rapidly.)
1355         * <p>
1356         * The current set or variables (including locals) is held in memory,
1357         * which also allows us to merge globals if required.
1358         * <p>
1359         * The varMgr is marked as an end-point so that it will
1360         * generate and return a unique local system ID.
1361         * <p>
1362         * This is assumed not to need to filter out duplicate updates
1363         * (nor bad timestamps) which is slow and may not even matter here.
1364         * Any network connection upstream of us should do such filtering.
1365         * <p>
1366         * We have the varMgr screen out repeats and bad timestamps, etc.
1367         */
1368        private final BasicVarMgr varMgr = new BasicVarMgr(true, true); // Endpoint.
1369    
1370        /**In-order List/Queue thread-safe list of persistent events to be logged; never null.
1371         * Remove items FIFO or lock the queue and remove all the items in one go.
1372         * <p>
1373         * We don't synchronously save these data.
1374         */
1375        private final Vector<SimpleVariableValue> eventsToLog = new Vector<SimpleVariableValue>();
1376    
1377        /**Set variable value; persistent values will eventually go to disc.
1378         *
1379         * @param newValue
1380         * @throws IOException
1381         */
1382        public void setVariable(final SimpleVariableValue newValue)
1383            throws IOException
1384            {
1385            varMgr.setVariable(newValue);
1386    
1387            // Capture any persistent event for later logging.
1388            if(newValue != null)
1389                {
1390                final SimpleVariableDefinition def = newValue.getDef();
1391                if(def.isEvent() && def.isPersistent() && SystemVariables.defs.contains(def))
1392                    { eventsToLog.add(newValue); }
1393                }
1394            }
1395    
1396        /**Set variable values; persistent values will eventually go to disc.
1397         *
1398         * @param newValues
1399         * @throws IOException
1400         */
1401        public int setVariables(final SimpleVariableValue[] newValues)
1402            throws IOException
1403            {
1404            final int n = varMgr.setVariables(newValues);
1405    
1406            // Capture any persistent events for later logging, in order.
1407            for(final SimpleVariableValue svv : newValues)
1408                {
1409                if(svv != null)
1410                    {
1411                    final SimpleVariableDefinition def = svv.getDef();
1412                    if(def.isEvent() && def.isPersistent() && SystemVariables.defs.contains(def))
1413                        { eventsToLog.add(svv); }
1414                    }
1415                }
1416    
1417            return(n);
1418            }
1419    
1420        /**Get variable value; persistent values may have come from disc.
1421         */
1422        public SimpleVariableValue getVariable(final SimpleVariableDefinition var)
1423            {
1424            return(varMgr.getVariable(var));
1425            }
1426    
1427        /**Get variable values; persistent values may have come from disc.
1428         */
1429        public SimpleVariableValue[] getVariables(final long changedSince)
1430            {
1431            return(varMgr.getVariables(changedSince));
1432            }
1433    
1434        /**Synchronise variable values.
1435         * This implementation <em>does not</em> force a flush to disc.
1436         */
1437        public void syncVariables(final boolean force)
1438            {
1439            // Nothing needed in this implementation.
1440            // In particular this does not flush values to disc.
1441            }
1442    
1443        /**Get the current partial, or previous full, event set at the specified interval; never null.
1444         * This is a simplified interface to return either the current event set
1445         * that is being collected, or the previous completed set.
1446         * <p>
1447         * The current set is the most timely, but may not contain enough data
1448         * to be meaningful if the new interval has just started.
1449         * <p>
1450         * The previous set is complete and thus most likely to have enough samples
1451         * to be useful, but is not completely current.
1452         * <p>
1453         * If a "previous" value returned from our local store is not marked
1454         * as authoritative, then we convert it to an authoritative value,
1455         * write that back to our local store and return it.
1456         * (The "current" value is generally not authoritative.)
1457         *
1458         * @param def  event definition (must be for an event); never null
1459         * @param intervalSelector  one of EVENT_INTERVAL_SELECTOR_xxx values
1460         * @param current  if true the current event set is returned,
1461         *     else the previous complete set is returned
1462         *
1463         * @return  requested event set; may be empty but never null if requested set not available
1464         */
1465        public EventVariableValue getEventValue(final SimpleVariableDefinition def,
1466                                                final EventPeriod intervalSelector,
1467                                                final boolean current)
1468            {
1469            final EventVariableValue eventValue = varMgr.getEventValue(def, intervalSelector, current);
1470            if(!current && !eventValue.isAuthoritative())
1471                {
1472                final EventVariableValue authValue = eventValue.makeAuthoritative();
1473                varMgr.setEventValue(authValue);
1474                return(authValue);
1475                }
1476            return(eventValue);
1477            }
1478    
1479        /**Get the specified event sets for the specified intervals; never null.
1480         * This allows retrieval of zero or more event sets for the specified
1481         * interval size.
1482         * <p>
1483         * Requests for more than SystemVariables.EVENT_SAMPLES_RETAINED in the
1484         * past (or for the future!) cannot be satisfied and data will not be
1485         * returned for them.
1486         * <p>
1487         * Usually not more than SystemVariables.EVENT_SAMPLES_RETAINED samples
1488         * will be returned in response to any one request as a safety measure.
1489         * <p>
1490         * (An implementation that is not an end-point may go upstream to fetch
1491         * missing values and cache them to satisfy future requests.)
1492         * <p>
1493         * If a "previous" value (older than the current interval)
1494         * returned from our local store is null or not marked
1495         * as authoritative, then we convert it to an authoritative value,
1496         * write that back to our local store and return it.
1497         * (The "current" value is generally not authoritative,
1498         * and entries in the future cannot be so.)
1499         *
1500         * @param def  event definition (must be for an event); never null
1501         * @param intervalSelector  one of EVENT_INTERVAL_SELECTOR_xxx values
1502         * @param intervalNumber  a time (as from System.currentTimeMillis())
1503         *     which identifies the first interval for which data is potentially
1504         *     required; if too far in the past or future then possibly no data
1505         *     will be available,
1506         *     zero is used to access the "all" bucket
1507         * @param whichValues  each true bit represents a slot for which data is
1508         *     required, bit 0 indicating data from the slot within which
1509         *     firstIntervalTime is located, bit 1 the previous slot, etc
1510         *
1511         * @return as many of the requested values as available,
1512         *     at least long enough to return all the available values,
1513         *     with [0] corresponding to bit 0 in the BitSet;
1514         *     may contain nulls or be zero-length but is never null
1515         */
1516        public EventVariableValue[] getEventValues(final SimpleVariableDefinition def,
1517                                                   final EventPeriod intervalSelector,
1518                                                   final long intervalNumber,
1519                                                   final BitSet whichValues)
1520            {
1521            // Compute currentInterval before getting values
1522            // to avoid a race which might make wrongly capture
1523            // an incomplete "current" value as authoritative and complete.
1524            final long currentInterval = intervalSelector.getIntervalNumber(System.currentTimeMillis());
1525    
1526            synchronized(varMgr) // Avoid races with new events coming in...
1527                {
1528                EventVariableValue[] result = varMgr.getEventValues(def, intervalSelector, intervalNumber, whichValues);
1529    
1530                // Ensure result array is large enough,
1531                // else copy into large enough one.
1532                final int wvl = (whichValues == null) ? 1 : whichValues.length();
1533                if(result.length < wvl)
1534                    {
1535                    final EventVariableValue newResult[] = new EventVariableValue[wvl];
1536                    System.arraycopy(result, 0, newResult, 0, result.length);
1537                    result = newResult;
1538                    }
1539    
1540                // For any values newer than the current interval,
1541                // and for which our local store had missing or non-authoritative values,
1542                // convert them to authoritative values
1543                // so that upstream users can cache them permanently.
1544                BitSet wv = whichValues;
1545                if(wv == null)
1546                    {
1547                    // Create synthetic BitSet for simplicity below.
1548                    wv = new BitSet(1);
1549                    wv.set(0);
1550                    }
1551                for(int i = wv.nextSetBit(0); i >= 0; i = wv.nextSetBit(i+1))
1552                    {
1553                    final long slotIntervalNumber = intervalNumber - i;
1554                    // We cannot make current/future events authoritative.
1555                    if(slotIntervalNumber >= currentInterval)
1556                        { continue; }
1557    
1558                    final EventVariableValue evv = result[i];
1559                    if((evv == null) || !evv.isAuthoritative())
1560                        {
1561                        // Make value authoritative,
1562                        // creating a new empty one if need be.
1563                        final EventVariableValue authValue = (evv != null) ? evv.makeAuthoritative() :
1564                            new EventVariableValue(true,
1565                                                   def,
1566                                                   intervalSelector,
1567                                                   slotIntervalNumber,
1568                                                   0, null, null);
1569    
1570                        assert(authValue.isAuthoritative());
1571                        assert((evv == null) || (authValue.getIntervalNumber() == evv.getIntervalNumber()));
1572                        assert((evv == null) || (authValue.getDef().equals(evv.getDef())));
1573    
1574                        // Return authoritative value to caller.
1575                        result[i] = authValue;
1576    
1577                        // Save in our local store.
1578                        varMgr.setEventValue(authValue);
1579    
1580    //if(IsDebug.isDebug) { System.out.println("Storing authoritative " + authValue); }
1581                        }
1582                    }
1583    
1584                return(result);
1585                }
1586            }
1587    
1588    
1589        /**Creates the name of a thumbnail starting at the thumbnail base directory from a full exhibit name.
1590         */
1591        private static String thumbnailRelPath(final Name.ExhibitFull exhibitName)
1592            {
1593            // Construct the name of the thumbnails file if it exists.
1594            return(ExhibitName.getDirComponent(exhibitName).toString() + File.separatorChar +
1595                    ExhibitName.getFileComponent(exhibitName) + THUMBNAIL_SUFFIX);
1596            }
1597    
1598    //    /**Creates thumbnails from all exhibits possible.
1599    //     * This is used to create exhibit thumbnails off-line,
1600    //     * taking as long and as much memory as required
1601    //     * and processing the resource-intensive thumbnail building sequentially.
1602    //     * <p>
1603    //     * These luxuries of non-real-time and lots of memory and CPU are denied
1604    //     * to the WAR/EAR servers when they try to compute thumbnails on the fly,
1605    //     * since there may be many simultaneous users and even attempted
1606    //     * Denial-of-Service attacks.
1607    //     * <p>
1608    //     * The thumbnails created here can be loaded by master servers,
1609    //     * and through it any slaves, thus making a much wider range of thumbnails
1610    //     * available than could be computed on the fly.
1611    //     * <p>
1612    //     * We will protest but terminate without an exception if the
1613    //     * thumbnail directory does not exist; we will assume that precomputed
1614    //     * thumbnails are not required.
1615    //     * <p>
1616    //     * We will only attempt to (*re) compute a thumbnail if:
1617    //     * <ul>
1618    //     * <li>The exhibit type is one that we can compute thumbnails for.
1619    //     * <li>The thumbnail file does not exist or is older than the exhibit file.
1620    //     * </ul>
1621    //     * <p>
1622    //     * This will attempt to put thumbnail files where the getThumbnails()
1623    //     * routine will expect to find them.
1624    //     */
1625    //    private static void createThumbnails(final AllExhibitProperties aep,
1626    //                                         final long stopBy)
1627    //        throws IOException
1628    //        {
1629    //        // Compute full path from exhibit directory.
1630    //        final File topDir = (new File(LocalProps.getDataDir(), LocalProps.getThumbnailRelDir())).getCanonicalFile();
1631    //
1632    //        // If the top directory for the thumbnails does not exist,
1633    //        // then we definitely will not be returning thumbnails.
1634    //        // We also warn when this directory does not exist.
1635    //        if(!topDir.isDirectory() || !topDir.canWrite())
1636    //            {
1637    //            // We might wish to limit frequency with which we report missing directory!
1638    //            System.err.println("Warning: top-level thumbnail directory absent: " + topDir);
1639    //            System.err.println("Aborting creation of thumbnails.");
1640    //            return;
1641    //            }
1642    //
1643    //        // Make an instance of the data retriever for use by the thumbnail generator.
1644    //        final ExhibitDataFileSource edfs = new ExhibitDataFileSource(aep);
1645    //        final AllExhibitProperties.ExhibitDataSource eds = edfs.makeExhibitDataSource();
1646    //
1647    //        // Now examine each exhibit in turn
1648    //        // (in sorted order to try to approximate a disc-friendly breadth-first search)
1649    //        // and if it is possible to build thumbnails
1650    //        // and they don't exist or are older than the exhibit,
1651    //        // try to build the thumbnails and save them.
1652    //        //
1653    //        // This tries to be robust and avoid blowing up if a single thumbnail build has problems.
1654    //        final List<Name.ExhibitFull> names = aep.aeid.getAllExhibitNamesSorted();
1655    //
1656    //        // Queue of tasks in progress.
1657    //        final Queue<Future<?>> tasks = new LinkedBlockingQueue<Future<?>>();
1658    //
1659    //        for(int i = names.size(); (--i >= 0) && (System.currentTimeMillis() <= stopBy); )
1660    //            {
1661    //            final Name.ExhibitFull name = names.get(i);
1662    //
1663    //            // In the simple case where we know that thumbnails
1664    //            // definitely can't be created
1665    //            // (from the type, or because there is no exhibit)
1666    //            // return the NO_THUMBNAILS value immediately,
1667    //            // ignoring the create parameter.
1668    //
1669    //            final ExhibitStaticAttr esa = aep.aeid.getStaticAttr(name);
1670    //            if(esa == null)
1671    //                { continue; }
1672    //
1673    //            final ExhibitMIME.ExhibitTypeParameters type = (ExhibitMIME.getInputFileType(esa.getCharSequence()));
1674    //            if((type == null) || (type.handler == null) ||
1675    //               !type.canPossiblyCreateThumbnailOfSameMIMEType())
1676    //                { continue; }
1677    //
1678    //            // These tasks are assumed to be heavy on CPU, and not mainly I/O-bound.
1679    //            tasks.add(ThreadUtils.computeIntensiveThreadPool.submit(new Runnable(){
1680    //                public final void run()
1681    //                    {
1682    //                    // Construct the name of the thumbnails file if it exists.
1683    //                    final String relPath = thumbnailRelPath(name);
1684    //
1685    //                    // If the thumbnail file seems to exist and is readable,
1686    //                    // and is newer than the exhibit,
1687    //                    // then assume that it is up-to-date.
1688    //                    final File fullPath = new File(topDir, relPath);
1689    //                    if(fullPath.exists() && fullPath.canRead() && fullPath.isFile() &&
1690    //                        (fullPath.lastModified() > esa.timestamp))
1691    //                        { return; }
1692    //
1693    //                    // OK, looks like we need to (re)build the thumbnails...
1694    //                    try {
1695    //                        // Ensure that the destination directory exists.
1696    //                        final File targetDir = fullPath.getParentFile();
1697    //                        if(!targetDir.exists())
1698    //                            { targetDir.mkdirs(); }
1699    //
1700    //                        // OK, attempt to generate thumbnails here, not under any lock.
1701    //                        final ExhibitThumbnails tns = type.handler.makeThumbnails(
1702    //                            esa, eds, aep,
1703    //                            true);  // Ignore resource limitations; this our best chance!
1704    //
1705    //                        // Attempt to save thumbnails (atomically), without additional compression.
1706    //                        FileTools.serialiseToFile(tns, fullPath, THUMBNAILS_ARE_GZIPPED, false);
1707    //                        }
1708    //                    catch(final ThreadDeath e)
1709    //                        { throw e; } // Don't intercept this one...
1710    //                    catch(final Throwable e)
1711    //                        {
1712    //                        System.err.println("ERROR: unable to build thumbnails for: " + name);
1713    //                        e.printStackTrace();
1714    //                        return; // Attempt to continue.
1715    //                        }
1716    //                    }
1717    //                }));
1718    //
1719    //            // Optimisation: release resources ASAP from any early completed tasks.
1720    //            Future<?> headTask;
1721    //            while(null != (headTask = tasks.peek()))
1722    //                { if(headTask.isDone()) { tasks.remove(); } }
1723    //            }
1724    //
1725    //        // Wait for all remaining tasks.
1726    //        while(!tasks.isEmpty())
1727    //            {
1728    //            try { tasks.remove().get(); }
1729    //            catch(final Exception e)
1730    //                {
1731    //                final IOException err = new IOException("did not complete");
1732    //                err.initCause(e);
1733    //                throw err;
1734    //                }
1735    //            }
1736    //        }
1737    
1738        /**Computes an AllExhibitProperties object and saves it to outputFile.
1739         * The object is saved serialised and GZIPed.
1740         * <p>
1741         * The save is avoided if we are not forcing a complete recompute
1742         * and the exhibit hash has not changed,
1743         * ie we try to avoid unnecessarily churning the filesystem
1744         * and/or the master server.
1745         * <p>
1746         * It almost certainly does not make sense for the
1747         * WAR_SYSPROPNAME_WARONLY_STATICCACHEFILE property to be set;
1748         * you could get a similar result by copying the file.
1749         * <p>
1750         * In passing, this may update ancillary state such as computable properties
1751         * and thumbnails.
1752         *
1753         * @param outputFile desired location of cache file; not null
1754         * @param quick  if true, attempt to be as quick as possible,
1755         *     else if false, magic numbers of exhibits are checked and
1756         *     other extra-careful checking is done;
1757         *     this should be the default usage
1758         * @param recompute  if true, recompute all derived data from scratch,
1759         *     eg do not reload any extant cache file
1760         *
1761         * @return AllExhibitProperties (non-null)
1762         */
1763        private static AllExhibitProperties createStaticCacheFile(final File outputFile,
1764                                                                  final boolean quick,
1765                                                                  final boolean recompute,
1766                                                                  final boolean fixaccessions)
1767            throws IOException
1768            {
1769    //        final long startTime = System.currentTimeMillis();
1770    
1771            // Submit task to update accessions concurrently.
1772            final Future<?> accessionsUpdate = ThreadUtils.nonCPUThreadPool.submit(new Runnable(){
1773                public final void run()
1774                    {
1775                    try
1776                        {
1777                        // Get a simple listing of exhibits on disc.
1778                        final AllExhibitImmutableData aeid = _getAllExhibitImmutableData(
1779                            LocalProps.getDataDir(),
1780                            -1,
1781                            !quick);
1782                        assert(aeid != null);
1783    
1784                        // Create/fix accessions files first.
1785                        createAccessionFiles(aeid);
1786                        }
1787                    catch(final IOException e)
1788                        {
1789                        e.printStackTrace();
1790                        throw new Error("failed to create accessions", e);
1791                        }
1792                    }
1793                });
1794    
1795            // If we're being quick then we can't be too careful...
1796            final boolean careful = !quick;
1797    
1798            // Wait for accessions update to complete
1799            // so that we can incorporate any new files in the AEP.
1800            try { accessionsUpdate.get(); }
1801            catch(final Exception e) { throw new Error(e); }
1802    
1803            // Create an instance.
1804            final ExhibitDataFileSource edfs = new ExhibitDataFileSource();
1805    
1806            // Now create and save the cache (if changed)...
1807            System.out.println("[ExhibitDataFileSource.createStaticCacheFile(): started: " +(new Date())+ ".]");
1808            // Get the properties...
1809            // Force reconstruction of AEP from filesystem.
1810            final AllExhibitProperties aep = edfs._getAllExhibitProperties(-1L, careful, true);
1811            System.out.println("[ExhibitDataFileSource.createStaticCacheFile(): *** exhibit count: " + aep.aeid.size() + ".]");
1812    
1813            return(aep);
1814            }
1815    
1816        /**Create accessions files.
1817         * @return true if any accessions files were created or fixed
1818         */
1819        private static boolean createAccessionFiles(final AllExhibitImmutableData aeid)
1820            throws IOException
1821            {
1822            final AtomicBoolean result = new AtomicBoolean(); // No updates/fixes/creations yet...
1823            final AtomicReference<String> couldNotMakeAccession = new AtomicReference<String>();
1824    
1825            final String dataDir = LocalProps.getDataDir();
1826    
1827            // Check accession files exist for all exhibits.
1828            logFSAccess("Adding accession files as necessary...", false);
1829    
1830            // Queue of tasks yet to complete...
1831            final Queue<Future<?>> tasks = new LinkedBlockingQueue<Future<?>>();
1832    
1833            // Do this in approximately sorted order so as to improve disc throughput...
1834            for(final Name.ExhibitFull exhibitName : aeid.getAllExhibitNamesSorted())
1835                {
1836                // These tasks are assumed to be light on CPU, and I/O-bound.
1837                tasks.add(ThreadUtils.nonCPUThreadPool.submit(new Runnable(){
1838                    public final void run()
1839                        {
1840                        try
1841                            {
1842                            final String newAccessionFilename = ExhibitPropsLoadable.relPathToNewAccession(exhibitName);
1843                            final File newAccessionFile = new File(dataDir, newAccessionFilename);
1844                            if(!newAccessionFile.exists())
1845                                {
1846                                // Compute the accession data...
1847                                final AccessionData ad = AccessionData.fromExhibitFile(new File(dataDir, exhibitName.toString()));
1848                                // Convert to (terse) XML.
1849                                final String xml = TextUtils.toXML(ad.getAsDOM(), false, true);
1850                                // Convert to UTF-8 text bytes ready to save.
1851                                final byte utf8[] = xml.getBytes(CoreConsts.FILE_ENCODING_UTF_8);
1852                                // Save new accessions file as near atomically as possible.
1853                                final String nAFS = newAccessionFile.toString();
1854                                logFSAccess("Creating accession file: "+nAFS, false);
1855                                if(FileTools.replacePublishedFile(nAFS, utf8, false))
1856                                    { result.set(true); }
1857                                }
1858                            }
1859                        catch(final IOException e)
1860                            {
1861                            // Whinge but continue if we encounter an error.
1862                            e.printStackTrace();
1863                            couldNotMakeAccession.set("IOException inspecting/making accession file for: " + exhibitName + ": " + e.getMessage());
1864                            }
1865                        }
1866                    }));
1867    
1868                // Optimisation: release resources ASAP from any early completed tasks.
1869                // NOTE: handles exceptions differently than final mop-up which may be undesirable.
1870                Future<?> headTask;
1871                while(null != (headTask = tasks.peek()))
1872                    { if(headTask.isDone()) { tasks.remove(); } }
1873                }
1874    
1875            // Wait for all remaining tasks.
1876            while(!tasks.isEmpty())
1877                {
1878                try { tasks.remove().get(); }
1879                catch(final Exception e)
1880                    {
1881                    final IOException err = new IOException("did not complete");
1882                    err.initCause(e);
1883                    throw err;
1884                    }
1885                }
1886    
1887            if(couldNotMakeAccession.get() != null)
1888                {
1889                System.err.println("ERROR: could not make at least one missing (new) accession file: ");
1890                System.err.println("    REASON: " + couldNotMakeAccession);
1891                }
1892    
1893            return(result.get());
1894            }
1895    
1896        /**Get requested Properties selected by key and versionID.
1897         * Fetches a Properties set unconditionally (versionID == -1)
1898         * else if the versionID presented is not current.
1899         *
1900         * @param key  selector (with possible embedded sub-key)
1901         *     for desired properties set; never null
1902         * @param versionID  if -1 then map is always returned if available,
1903         *     else must be non-negative and null is returned if the versionID
1904         *     presented matches that of the current version
1905         *     (ie if the caller has presumably got the up-to-date version);
1906         *     may be a timestamp or a hash or other value,
1907         *     and by convention is zero only for an empty properties set
1908         *
1909         * @return null, or Properties map guaranteed to contain only
1910         *     String keys and values
1911         */
1912        public java.util.Properties getProperties(final PropsKey key,
1913                                                  final long versionID)
1914            throws IOException
1915            {
1916            throw new IOException("NOT IMPLEMENTED");
1917            }
1918    
1919    
1920        /**Run from the command-line with a single argument (the output file name).
1921         * There is an optional <code>-quick</code> first argument,
1922         * which if present which attempts to cap the time spent in one run
1923         * (typically to no more than a few minutes)
1924         * so that any work to be done can be done incrementally
1925         * in several passes if need be.
1926         * <p>
1927         * When making a new static cache file we always try to reload the old one
1928         * (unless the <code>-recompute</code> flag is set, in which case we will not load it)
1929         * to save lots of time (especially for the computed properties)
1930         * and possibly preserve the hashNotChangedSince value.
1931         * <p>
1932         * If the <code>-fixaccessions</code> flag is passed then we try to fix existing
1933         * accessions files that are missing information, eg new checksum types,
1934         * though we will never try to "fix" an incorrect checksum for example
1935         * (that will always result in a warning on the console.)
1936         * <p>
1937         * If the <code>-checkexhibits</code> flag is true
1938         * then we try to verify that each exhibit's exhibit length, timestamp and hash
1939         * matches the values captured in the associated accession file.
1940         * We may check additional data (eg FEC data) in future.
1941         * Note that this option overrides any other options,
1942         * and avoids writing anything to disc.
1943         */
1944        public static void main(final String args[])
1945            {
1946            final long startTime = System.currentTimeMillis();
1947    
1948            if(args.length < 1)
1949                {
1950                System.err.println("Expects: " +
1951                    "[-quick | -recompute | -fixaccessions | -checkexhibits ] " +
1952                    "filename");
1953                System.exit(1);
1954                }
1955    
1956            boolean quick = false; // If true, use quicker exhibit-collection method.
1957            boolean recompute = false; // If true, force recompute from scratch; do not reload old cache file.
1958            boolean fixaccessions = false; // If true, fix existing accessions files.
1959            boolean checkexhibits = false; // If true, verify exhibit files are not corrupt.
1960            for(int i = args.length - 1; --i >= 0; )
1961                {
1962                if("-quick".equals(args[i])) { quick = true; }
1963                else if("-recompute".equals(args[i])) { recompute = true; }
1964                else if("-fixaccessions".equals(args[i])) { fixaccessions = true; }
1965                else if("-checkexhibits".equals(args[i])) { checkexhibits = true; }
1966                else
1967                    {
1968                    System.err.println("Unknown flag '" + args[i] + "'.");
1969                    System.exit(1);
1970                    return;
1971                    }
1972                }
1973    
1974            System.out.println("[ExhibitDataFileSource: START: " +(new Date(startTime))+ ".]");
1975    
1976            // Get the filename to load/save from/to; always last.
1977            final File staticCacheFilename = new File(args[args.length - 1]);
1978    
1979            try {
1980                if(checkexhibits)
1981                    {
1982                    checkExhibitData(quick, staticCacheFilename.getParentFile());
1983                    }
1984                else
1985                    {
1986                    // Create static cache file and related state.
1987                    final AllExhibitProperties aep = createStaticCacheFile(staticCacheFilename, quick, recompute, fixaccessions);
1988                    assert(aep != null);
1989    
1990    //                // Create any thumbnails that are missing.
1991    //                System.out.println("[ExhibitDataFileSource.createStaticCacheFile(): thumbnail building starting: " +(new Date())+ ".]");
1992    //                final long timeSoFar = Math.max(30000, 7 * (System.currentTimeMillis() - startTime));
1993    //                createThumbnails(aep, (!quick) ? Long.MAX_VALUE : (System.currentTimeMillis() + timeSoFar));
1994    //                System.out.println("[ExhibitDataFileSource.createStaticCacheFile(): thumbnail building finished: " +(new Date())+ ".]");
1995                    }
1996    
1997                final long endTime = System.currentTimeMillis();
1998                final long runTime = endTime - startTime;
1999                System.out.println("[ExhibitDataFileSource: END: " +(new Date(endTime))+ ": "+runTime+"ms.]");
2000                }
2001            catch(final Exception e)
2002                {
2003                e.printStackTrace();
2004                System.exit(1); // Exit with error code.
2005                }
2006            }
2007    
2008            /**Check exhibit data for corruption.
2009             * @param quick  take some optional speed-ups
2010             * @throws IOException  in case of difficulty
2011             */
2012            private static void checkExhibitData(final boolean quick,
2013                                             final File dataDir)
2014                throws IOException
2015                {
2016                    System.out.println("Checking exhibit data... (cache file ignored)");
2017                    final File baseDir = (dataDir != null) ? dataDir : new File(LocalProps.getDataDir());
2018                    // Get a simple listing of exhibits on disc.
2019                    final AllExhibitImmutableData aeid = _getAllExhibitImmutableData(
2020                        baseDir.getPath(),
2021                        -1,
2022                        !quick);
2023                    assert(aeid != null);
2024                    System.out.println("Exhibits found: " + aeid.length);
2025    
2026            // Collect existing accession data (as part of the loaded properties),
2027                    // and recompute the hashes from scratch to compare,
2028                    // and make sure that nothing has changed.
2029            // We do this in parallel to maximise I/O throughput,
2030            // and use a ConcurrentHashMap as our working store as it is thread-safe.
2031            // We get these using our non-CPU-bound thread pool
2032            // and wait for them all to complete.
2033                    final List<Name.ExhibitFull> sortedNames = aeid.getAllExhibitNamesSorted();
2034            final Map<Name.ExhibitFull,AccessionData> accessionMap = new ConcurrentHashMap<Name.ExhibitFull, AccessionData>(aeid.size() * 2);
2035            final List<Future<?>> tasks = new ArrayList<Future<?>>(sortedNames.size());
2036            for(final Name.ExhibitFull name : sortedNames)
2037                {
2038                tasks.add(ThreadUtils.nonCPUThreadPool.submit(new Runnable(){
2039                    public final void run()
2040                        {
2041                        // We don't store null/EMPTY values.
2042                        try
2043                            {
2044                            final ExhibitPropsLoadable epl = ExhibitPropsLoadable.getLoadableProperties(name, baseDir);
2045                            if((epl != null) && !epl.equals(ExhibitPropsLoadable.EMPTY) && (epl.getAccessionMetadata() != null))
2046                                {
2047                                    final AccessionData accessionMetadata = epl.getAccessionMetadata();
2048                                                            accessionMap.put(name, accessionMetadata);
2049    
2050                                                            // Compare the accession timestamp and length with the current file.
2051                                                            final ExhibitStaticAttr esa = aeid.getStaticAttr(name);
2052                                if(esa == null)
2053                                    { throw new Error("Missing exhibit: "+name); }
2054                                if(accessionMetadata.date == null)
2055                                    { System.err.println("Accession timestamp missing for: " + name); }
2056                                                            else if(accessionMetadata.date.longValue() != esa.timestamp)
2057                                                                { System.err.println("Accession timestamp does not match current file for: " + name + ": was/now " + new Date(accessionMetadata.date.longValue()) + '/' + new Date(esa.timestamp)); }
2058                                                            if(accessionMetadata.size == null)
2059                                                                { System.err.println("Accession length missing for: " + name); }
2060                                                            else if(accessionMetadata.size.longValue() != esa.length)
2061                                                                { System.err.println("Accession length does not match current file for: " + name + ": was/now " + accessionMetadata.size + '/' + esa.length); }
2062    
2063                                                            // Now recompute and compare the accession data hashes.
2064                                                            final InputStream is = new BufferedInputStream(new FileInputStream(new File(baseDir, name.toString())));
2065                                                            try
2066                                                                    {
2067                                                                    final Tuple.Pair<Integer,ROByteArray> hashes = AccessionData.computeFullFileHashes(is);
2068                                                                    if(!hashes.first.equals(accessionMetadata.hashCRC32))
2069                                                                    { System.err.println("Accession hash CRC32 changed for: " + name); }
2070                                                                    if(!hashes.second.equals(accessionMetadata.hashMD5))
2071                                                                { System.err.println("Accession hash MD5 changed for: " + name); }
2072                                                                    }
2073                                                            finally { is.close(); }
2074                                                            }
2075                            else
2076                                { System.err.println("No accession data available for: " + name); }
2077                            }
2078                        catch(final Exception e)
2079                            { throw new RuntimeException("Error loading accession data for: " + name, e); }
2080                        }
2081                    }));
2082                }
2083            // Wait for all tasks to finish...
2084            for(final Future<?> task : tasks)
2085                {
2086                try { task.get(); }
2087                catch(final InterruptedException e)
2088                    {
2089                    final InterruptedIOException err = new InterruptedIOException(e.getMessage());
2090                    err.initCause(e);
2091                    throw err;
2092                    }
2093                catch(final ExecutionException e)
2094                    {
2095                    final IOException err = new IOException(e.getMessage());
2096                    err.initCause(e);
2097                    throw err;
2098                    }
2099                }
2100            System.out.println("Accession data loaded for exhibits: " + accessionMap.size());
2101            if(accessionMap.size() != aeid.length)
2102                        { throw new IOException("Missing (or changed) accession data."); }
2103                }
2104    
2105            /**Assumes that this instance is the root/master and so returns Stratum.ROOT; non-null. */
2106        public Stratum getStratum() { return(Stratum.ROOT); }
2107    
2108        /**Shut down the data pipeline.
2109         * Saves any pending state (eg log entries and variables)
2110         * to disc if possible.
2111         * <p>
2112         * Has no upstream components to shut down
2113         * nor significant resources (memory) to release.
2114         */
2115        /* @Override */ public void destroy()
2116            {
2117            _saveSystemVariables(varMgr, true); // Force immediate save.
2118            }
2119        }