001 /*
002 Copyright (c) 1996-2011, Damon Hart-Davis
003 All rights reserved.
004
005 Redistribution and use in source and binary forms, with or without
006 modification, are permitted provided that the following conditions are
007 met:
008
009 * Redistributions of source code must retain the above copyright
010 notice, this list of conditions and the following disclaimer.
011
012 * Redistributions in binary form must reproduce the above copyright
013 notice, this list of conditions and the following disclaimer in the
014 documentation and/or other materials provided with the
015 distribution.
016
017 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
018 IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
019 TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
020 PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
021 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
022 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
023 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
024 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
025 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
026 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
027 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
028 */
029 package org.hd.d.pg2k.svrCore.datasource;
030
031 import java.io.BufferedInputStream;
032 import java.io.DataOutputStream;
033 import java.io.File;
034 import java.io.FileInputStream;
035 import java.io.FileNotFoundException;
036 import java.io.FileWriter;
037 import java.io.IOException;
038 import java.io.InputStream;
039 import java.io.InterruptedIOException;
040 import java.io.OutputStream;
041 import java.io.PrintWriter;
042 import java.io.RandomAccessFile;
043 import java.nio.ByteBuffer;
044 import java.security.DigestOutputStream;
045 import java.security.MessageDigest;
046 import java.text.SimpleDateFormat;
047 import java.util.ArrayList;
048 import java.util.BitSet;
049 import java.util.Calendar;
050 import java.util.Collections;
051 import java.util.Date;
052 import java.util.HashMap;
053 import java.util.HashSet;
054 import java.util.List;
055 import java.util.Map;
056 import java.util.Properties;
057 import java.util.Queue;
058 import java.util.Set;
059 import java.util.Vector;
060 import java.util.concurrent.ConcurrentHashMap;
061 import java.util.concurrent.ExecutionException;
062 import java.util.concurrent.Future;
063 import java.util.concurrent.LinkedBlockingQueue;
064 import java.util.concurrent.atomic.AtomicBoolean;
065 import java.util.concurrent.atomic.AtomicReference;
066 import java.util.concurrent.locks.ReentrantLock;
067
068 import org.hd.d.pg2k.svrCore.AccessionData;
069 import org.hd.d.pg2k.svrCore.AllExhibitImmutableData;
070 import org.hd.d.pg2k.svrCore.AllExhibitProperties;
071 import org.hd.d.pg2k.svrCore.AllExhibitProperties.ExhibitDataSource;
072 import org.hd.d.pg2k.svrCore.CS8Bit;
073 import org.hd.d.pg2k.svrCore.CoreConsts;
074 import org.hd.d.pg2k.svrCore.ExhibitFile;
075 import org.hd.d.pg2k.svrCore.ExhibitName;
076 import org.hd.d.pg2k.svrCore.ExhibitPropsComputable;
077 import org.hd.d.pg2k.svrCore.ExhibitPropsGlobalImmutable;
078 import org.hd.d.pg2k.svrCore.ExhibitPropsLoadable;
079 import org.hd.d.pg2k.svrCore.ExhibitStaticAttr;
080 import org.hd.d.pg2k.svrCore.ExhibitThumbnails;
081 import org.hd.d.pg2k.svrCore.FileTools;
082 import org.hd.d.pg2k.svrCore.GenUtils;
083 import org.hd.d.pg2k.svrCore.MemoryTools;
084 import org.hd.d.pg2k.svrCore.Name;
085 import org.hd.d.pg2k.svrCore.Name.ExhibitFull;
086 import org.hd.d.pg2k.svrCore.ROByteArray;
087 import org.hd.d.pg2k.svrCore.Rnd;
088 import org.hd.d.pg2k.svrCore.Stratum;
089 import org.hd.d.pg2k.svrCore.TextUtils;
090 import org.hd.d.pg2k.svrCore.ThreadUtils;
091 import org.hd.d.pg2k.svrCore.Tuple;
092 import org.hd.d.pg2k.svrCore.MIME.ExhibitMIME;
093 import org.hd.d.pg2k.svrCore.props.GenProps;
094 import org.hd.d.pg2k.svrCore.props.LocalProps;
095 import org.hd.d.pg2k.svrCore.props.SecurityProps;
096 import org.hd.d.pg2k.svrCore.vars.BasicVarMgr;
097 import org.hd.d.pg2k.svrCore.vars.EventPeriod;
098 import org.hd.d.pg2k.svrCore.vars.EventVariableValue;
099 import org.hd.d.pg2k.svrCore.vars.SimpleVariableDefinition;
100 import org.hd.d.pg2k.svrCore.vars.SimpleVariableValue;
101 import org.hd.d.pg2k.svrCore.vars.SystemVariables;
102
103 import ORG.hd.d.IsDebug;
104
105 // FIXME: periodically/randomly (or upon demand) check for data file corruption.
106
107 /**Exhibit pipeline stage that fetches its data directly from a filesystem.
108 * This emulates the way that the pre-PG2K Gallery accesses its data.
109 * <p>
110 * That this considers itself to be the definitive master data store
111 * for persistent system variables (particularly event histories)
112 * and will try to use the persistent data area to store them,
113 * in a robust way with a history.
114 * <p>
115 * Past event values returned by this routine are considered to be
116 * authoritative, and upstream consumers can cache such values
117 * knowing them to be definitive.
118 * <p>
119 * All requests for valid events/periods should be responded to
120 * as authoritative or not depending on the slot time as follows:
121 * <ul>
122 * <li>In the future: null are returned because no data *can* exist yet.
123 * <li>For the current interval: a non-authoritative, non-null response.
124 * <li>For any past interval: an authoritative response.
125 * </ul>
126 * Where the data for responses does not yet exist,
127 * it must be invented with no events to indicate that
128 * it definitely does not exist
129 * so as to allow negative cacheing by upstream users.
130 * <p>
131 * This is assumed not to need to filter out duplicate event/var updates
132 * (nor bad timestamps) which is slow and may not even matter here.
133 * Any network connection upstream of us should do such filtering.
134 * <p>
135 * We don't aim to be massively efficient with this since we hope the
136 * cacheing stage that should normally be downstream of us will compensate
137 * for our main inefficiencies. We might, however, switch to some sort of
138 * non-blocking I/O in future so that failures of networked filesystems,
139 * (etc) internally won't stop Web service.
140 * <p>
141 * We locate configuration files and data using LocalProps.
142 */
143 public final class ExhibitDataFileSource implements SimpleExhibitPipelineIF
144 {
145 /**Create instance.
146 * Loads any persisted system variable values.
147 */
148 public ExhibitDataFileSource()
149 {
150 final File sysVarDir = _getEventHistoryStorageDir();
151 if(!sysVarDir.isDirectory())
152 { System.err.println("WARNING: event history storage dir not found: " + sysVarDir); }
153 else
154 {
155 // Don't block to load event histories if possible.
156 try { varMgr.loadEventHistories(sysVarDir, true); }
157 catch(final IOException e)
158 { System.err.println("WARNING: event history could not be restored from: " + sysVarDir); }
159 }
160 }
161
162 /**If true then when conserving power eliminate much activity that might force activity on the data area.
163 * This includes such things as checking status/flags,
164 * that might force an automount for example,
165 * using extra energy.
166 * <p>
167 * This may prove too conservative,
168 * eg refusing to look for new files at all even when explicitly indicated.
169 */
170 private static final boolean MINIMISE_FS_POWER = false;
171
172 /**The cached exhibit-data file to read; never null.
173 * This is relative to the data directory.
174 * <p>
175 * We don't construct the exhibit static
176 * data ourselves when called by any of the SimpleExhibitPipelineIF methods,
177 * but instead read a GZIPed serialised AllExhibitProperties object at
178 * the file given by this value, throwing an IOException if it cannot
179 * be read.
180 * <p>
181 * The createStaticCacheFile() routine can be used to create this
182 * file and do as much precomputation and preparation of thumbnails,
183 * etc, as possible off-line. Typically a command-line program
184 * would be used to run this each time the exhibits are changed,
185 * and the master will simply re-read the serialised file periodically.
186 * It is important in this case that the file be replaced atomically.
187 * The createStaticCacheFile() tries to create the file if it does not
188 * exist.
189 */
190 private static final File getStaticCacheFileName()
191 { return(new File(LocalProps.getDataDir(), CoreConsts.FS_DATA_CACHE_FILENAME)); }
192
193 /**The name of the AEP longHash file; never null.
194 * Typically, removing this file,
195 * or specifying an old/extant AEP hash different to this,
196 * will force the AEP to be reconstructed from the filesystem.
197 * <p>
198 * This is relative to the data directory.
199 * <p>
200 * Never touches the filesystem; simply constructs the name.
201 */
202 private static final File getStaticHashFileName()
203 { return(new File(LocalProps.getDataDir(), CoreConsts.FS_DATA_HASH_FILENAME)); }
204
205 /**Get the last/cached AEP longHash file content; null iff not present/readable.
206 * The elements are never null.
207 */
208 private static final Long getLastAEPHashIfAny()
209 {
210 final File staticHashFileName = getStaticHashFileName();
211 // If the hash file doesn't exist or is not readable or is zero length then ignore it.
212 if(!staticHashFileName.canRead() && (staticHashFileName.length() < 1)) { return(null); }
213
214 try
215 {
216 // Expect the file to consist of one line containing the decimal longHash.
217 final long hash = Long.parseLong(FileTools.readTextFile(staticHashFileName).trim(), 10);
218 return(hash);
219 }
220 // In case of error, log it, and return null.
221 catch(final Exception e) { e.printStackTrace(); return(null); }
222 }
223
224 /**Get the static attributes for a given exhibit; null if no such exhibit.
225 */
226 public ExhibitStaticAttr getStaticAttr(final ExhibitFull name)
227 throws IOException
228 {
229 // // If we have a static cache then get attrs via that.
230 // if(_getStaticCacheFileName() != null)
231 // { return(getAllExhibitProperties(-1L).aeid.getStaticAttr(name)); }
232 // // Else fetch directly from the filesystem.
233 return(_getStaticAttr(LocalProps.getDataDir(), name));
234 }
235
236 /**Get the static attributes for a given exhibit; null if the named exhibit does not exist.
237 * <p>
238 * This computes a new uncached value on each call.
239 * <p>
240 * TODO: prevent this from blocking indefinitely, even with a filesystem hang.
241 */
242 private static ExhibitStaticAttr _getStaticAttr(final String fileRoot,
243 final Name.ExhibitFull name)
244 throws IOException
245 {
246 if((fileRoot == null) || (name == null))
247 { throw new IllegalArgumentException(); }
248
249 // With Name.ExhibitFull we know a priori that the value is 'safe', eg no ".." path components.
250 final String nameAsString = name.toString();
251 assert(!nameAsString.startsWith(".") && (nameAsString.indexOf("/.") == -1)) : "Dangerous name got through";
252
253 final File file = new File(fileRoot, nameAsString);
254 logFSAccess("reading static attrs for: "+file, false);
255 if(!file.canRead())
256 { return(null); }
257
258 return(new ExhibitStaticAttr(name,
259 file.length(),
260 file.lastModified()));
261 }
262
263 /**Read a chunk of the raw exhibit binary into the supplied buffer.
264 * The start position and an implied maximum read length are supplied.
265 * The start must be non-negative and no larger than the exhibit data length.
266 * <p>
267 * The call may return less than the the buffer capacity,
268 * though will block until it has read at least one byte unless at EOF or for a zero-byte request;
269 * this will be clear from the state of the buffer.
270 * <p>
271 * If a zero-byte request is made then the file may not actually be accessed.
272 * <p>
273 * This goes directly to the filesystem for each call.
274 * <p>
275 * TODO: prevent this from blocking indefinitely, even with a filesystem hang.
276 */
277 public void getRawFile(final ByteBuffer buf, final Name.ExhibitFull exhibitName, final int position, final boolean dontCache)
278 throws IOException
279 {
280 if(null == exhibitName) { throw new IllegalArgumentException(); }
281
282 // Use of Name.ExhibitFull means that we know a priori that the name is 'safe'.
283
284 // if(!ExhibitName.validNameSyntax(exhibitName))
285 // { throw new IOException("invalid name (syntax)"); }
286 //
287 // // Do some extra security checks of requested file's name.
288 // if(exhibitName.startsWith(".") || (exhibitName.indexOf("/.") != -1))
289 // { throw new IOException("invalid name (unsafe)"); }
290
291 // Use of Name.ExhibitFull means that we know a priori that the name is 'safe'.
292 final String nameAsString = exhibitName.toString();
293 final File file = new File(LocalProps.getDataDir(), nameAsString);
294 logFSAccess("reading raw exhibit data from: "+file+", position="+position+", buf.remaining()="+buf.remaining(), false);
295 if(!file.exists() || !file.canRead())
296 { throw new FileNotFoundException(nameAsString); }
297
298 // Do the read!
299 RandomAccessFile raf = null;
300 try
301 {
302 raf = new RandomAccessFile(file, "r");
303 // raf.seek(position);
304 // raf.readFully(buf, offset, len);
305 raf.getChannel().read(buf, position);
306 }
307 catch(final IOException e)
308 {
309 System.err.println("IOException reading exhibit bytes start="+position+" name="+exhibitName+": " + e.getMessage());
310 throw e; // Rethrow the error.
311 }
312 finally { if(raf != null) { raf.close(); } } // Release handle ASAP.
313 }
314
315
316 /**Gets all static exhibit data if its timestamp is not that specified.
317 * If the time specified is negative then the object will be returned
318 * unconditionally.
319 * <p>
320 * If no exhibits are currently installed a then default set with a zero
321 * timestamp is returned.
322 * <p>
323 * If the caller's copy appears to be up-to-date (eg the oldStamp
324 * matches that that we would have been returned) then null is returned.
325 *
326 * @throws InterruptedIOException if another expensive call is already in progress; retry later
327 */
328 public AllExhibitImmutableData getAllExhibitImmutableData(final long oldStamp)
329 throws IOException
330 {
331 // if(_getStaticCacheFileName() != null)
332 // {
333 // final AllExhibitImmutableData extant = getAllExhibitProperties(-1L).aeid;
334 // // If caller has an up-to-date copy, return null.
335 // if((oldStamp >= 0) && (oldStamp == extant.timestamp))
336 // { return(null); }
337 // return(extant);
338 // }
339
340 return(_getAllExhibitImmutableData(LocalProps.getDataDir(), oldStamp, true));
341 }
342
343 /**Number of least-significant bytes of AllExhibitImmutableData timestamp to use as hash of full collection.
344 * Using a value of 1 would imply keeping about quarter-second timestamp
345 * precision (which does not exist in UNIX systems, typically).
346 * Using a value of 2 would imply about 1 minute precision.
347 * Using a value of 3 would imply about 5 hour precision.
348 * <p>
349 * Given that the Gallery is usually updated at intervals of at most
350 * hours to days normally, a value of 2 or 3 is probably optimal.
351 * <p>
352 * Must lie in the range 0 to 8 inclusive.
353 */
354 private static final int TS_LSbytes_HASH = 2;
355
356 /**Gets all immutable exhibit data if its timestamp is not that specified.
357 * If the time specified is negative the object will be returned
358 * unconditionally.
359 * <p>
360 * This does not use any cache, and computes afresh on each call,
361 * and is thread-safe.
362 * <p>
363 * If no exhibits are currently installed then a default set with a zero
364 * timestamp is returned.
365 * <p>
366 * If the caller's copy appears to be up-to-date (eg the oldStamp
367 * matches that that we would have been returned) null is returned.
368 * <p>
369 * We make the timestamp we use for the whole collection be the
370 * most significant bits of the latest-stamped exhibit and the
371 * least significant bits of the collection timestamp XORed with a hash
372 * over all the (sorted) ExhibitStaticAttr entries. We don't have many
373 * least significant bits, so we run a small-ish risk of missing some
374 * changes. In fact, we are quite rough-and-ready about the hash too!
375 * <p>
376 * We know that many (typically 10) of the least significant bits of
377 * the timestamp would not carry information anyway, eg with a
378 * one-second granularity in the UNIX filesystem.
379 * <p>
380 * To avoid causing too much confusion with our somewhat-faked
381 * timestamp, we limit it to between 1 and the current time of day
382 * in the worst case, though because we ensure that the fake
383 * timestamp calculated is no later than the timestamp of the newest
384 * exhibit then unless there is lots of clock-skew between our host
385 * and the file server we should not see this latter clamp actually used.
386 * The latter limit, if needed, can make the system inefficient just after
387 * a new exhibit has been added since our fake timestamp may seem
388 * to change on every call.
389 *
390 * @param careful if true, magic numbers of exhibits are checked and
391 * other extra-careful checking is done; this should be the default
392 * usage
393 */
394 public static AllExhibitImmutableData _getAllExhibitImmutableData(final String fileRoot,
395 final long oldStamp,
396 final boolean careful)
397 throws IOException
398 {
399 if(fileRoot == null)
400 { throw new IllegalArgumentException(); }
401 final File baseDir = new File(fileRoot);
402
403 // // Veto attempts to do this concurrently.
404 // if(!_slow_op_lock.tryLock()) { throw new InterruptedIOException("already in progress: "+_slow_op_lock); }
405 // try
406 // {
407 // Load exhibit set and (efficiently) sort it.
408 // We access the underlying files in sorted order aiming to improve filesystem performance.
409 final List<Name.ExhibitFull> names = new ArrayList<Name.ExhibitFull>(ExhibitFile.getFilesystemBasedExhibitNames(
410 baseDir, careful));
411 Collections.sort(names); // One-off sort of the names more efficient than incremental.
412
413 if(ORG.hd.d.IsDebug.isDebug) { System.out.println("[Found "+(names.size())+" exhibits based at "+baseDir+".]"); }
414
415 // Now iterate over the set in order,
416 // finding the newest exhibit,
417 // and computing a hash on all the static attr data.
418 // We also construct our map from names to attr at this point.
419 final Set<ExhibitStaticAttr> esas = new HashSet<ExhibitStaticAttr>(names.size() * 2);
420
421 long newest = 1; // Newest exhibit's timestamp, but forced strictly positive.
422
423 final MessageDigest md = GenUtils.getStandardDigest();
424 final DataOutputStream dos =
425 new DataOutputStream(
426 new DigestOutputStream(
427 (new OutputStream(){ // Null output stream...
428 @Override
429 public final void write(final int b) { }
430 @Override
431 public final void write(final byte[] b, final int off, final int len) { }
432 }),
433 md));
434 for(final Name.ExhibitFull name : names)
435 {
436 final ExhibitStaticAttr esa = _getStaticAttr(fileRoot, name);
437 if(null == esa) { continue; }
438
439 // We need to collect this ESAs...
440 esas.add(esa);
441
442 // Take care of the timestamp...
443 if(esa.timestamp > newest) { newest = esa.timestamp; }
444
445 // Update the hash...
446 final ExhibitFull fn = esa.getExhibitFullName();
447 dos.writeInt(fn.length());
448 dos.write(fn.toByteArray());
449 dos.writeLong(esa.length);
450 dos.writeLong(esa.timestamp);
451 dos.writeByte(1); // Acts as a nearly-unique separator.
452 }
453
454 dos.flush();
455 final byte[] hash = md.digest();
456
457 // XOR lsbytes in `newest' until we run out of hash bytes...
458 // ...first subtracting enough from the real timestamp to
459 // make space for the hash
460 // (ie so the computed value is no newer than the newest exhibit)...
461 long fakeStamp = newest - (1 << (8 * TS_LSbytes_HASH));
462 // XOR in LS bytes...
463 for(int i = Math.min(hash.length, TS_LSbytes_HASH); --i >= 0; )
464 { fakeStamp ^= ((long) (hash[i] & 0xff)) << (8 * i); }
465
466 // Clamp the fake timestamp to within allowed limit values,
467 // though this should not actually be necessary.
468 // If there are no exhibits then force a zero timestamp.
469 final long synthStamp;
470 if(esas.size() == 0) { synthStamp = 0; }
471 else
472 {
473 synthStamp = Math.min(Math.min(fakeStamp, newest), System.currentTimeMillis());
474 assert(synthStamp != 0);
475 }
476
477 // See if the client actually needs the value returned...
478 if((oldStamp >= 0) && (oldStamp == synthStamp))
479 { return(null); } // No, client is up-to-date.
480
481 // Now make the result!
482 final AllExhibitImmutableData result = new AllExhibitImmutableData(esas, synthStamp);
483
484 // Warn loudly if duplicate short names were found...
485 final Name.ExhibitFull dups[] = result.getFullNamesWithDuplicateShortNames();
486 if(dups.length > 0)
487 {
488 for(int i = 0; i < dups.length; ++i)
489 { System.err.println("WARNING: duplicate file component in: " + dups[i]); }
490 }
491
492 // Done!
493 return(result);
494 // }
495 // finally { _slow_op_lock.unlock(); }
496 }
497
498 /**Hash (longHash) of last-loaded AEP and timestamp, or -1 if no AEP yet loaded.
499 * The elements are never null.
500 * <p>
501 * Is volatile to allow lockless (read) access.
502 * <p>
503 * Updated by _getAllExhibitProperties().
504 */
505 private volatile long _AEP_last_longHash = -1;
506
507 /**Time before which we should not attempt to reload the AEP.
508 * Is volatile to allow lockless (read) access.
509 * <p>
510 * Updated by _getAllExhibitProperties().
511 * <p>
512 * Initially zero to force first recomputation.
513 */
514 private volatile long _AEP_nextLoad;
515
516 /**Reentrant lock avoid concurrent slow operations being attempted; never null.
517 * Used by both the internal getAEP and getAEID routines.
518 * <p>
519 * Rather than block, concurrent attempts may be vetoed with IOException.
520 */
521 private final ReentrantLock _slow_op_lock = new ReentrantLock();
522
523 /**Gets set of all exhibit properties if its hash is not that specified.
524 * If the hash specified is negative
525 * then the AEP will be loaded and returned unconditionally
526 * but this is likely to be expensive;
527 * call with the longHash of any extant item if at all possible.
528 * <p>
529 * If no exhibits are currently installed
530 * then a default empty set with a zero timestamp is returned.
531 * <p>
532 * If the caller's copy appears to be up-to-date
533 * (ie the oldHash matches that that would have been returned)
534 * then null is returned.
535 * <p>
536 * Note that a full load from the filesystem is likely to be slow/expensive.
537 *
538 * @throws InterruptedIOException if another expensive call is already in progress
539 * or we cannot otherwise currently attempt to (re)load an AEP;
540 * retry later
541 */
542 public AllExhibitProperties getAllExhibitProperties(final long oldHash)
543 throws IOException
544 {
545 // Generally we want to be very careful when (re)loading the exhibit set
546 // but when (temporarily) short of power then this may be a little more reckless
547 // and hope to notice any subtle issues when the system is flush.
548 final boolean carefulLoad = !GenUtils.mustConservePower();
549
550 return(_getAllExhibitProperties(oldHash, carefulLoad, false));
551 }
552
553
554 /**Minimum time between polls of the filesystem (ms) when not conserving power. */
555 private static final int MIN_POLL_MS = 7*60*1000;
556
557 /**Minimum time between polls of the filesystem (ms) when conserving power. */
558 private static final int MIN_POLL_CONSERVING_MS = 121*60*1000;
559
560 /**Gets set of all exhibit properties if its hash is not that specified.
561 * If the hash specified is negative
562 * then the AEP will be loaded and returned unconditionally
563 * but this is likely to be expensive;
564 * call with the longHash of any extant item if at all possible.
565 * <p>
566 * If no exhibits are currently installed
567 * then a default empty set with a zero timestamp is returned.
568 * <p>
569 * If the caller's copy appears to be up-to-date
570 * (eg the oldHash matches that that would have been returned)
571 * then null is returned.
572 * <p>
573 * This is more averse to checking/reloading the AEP if conserving power.
574 *
575 * @param careful if true then magic numbers of exhibits are checked and
576 * other extra-careful checking is done; this should be the default usage
577 * @param forceLoadFromFilesystem if true, ignore any cached (hash) data and
578 * always load data from the filesystem
579 *
580 * @throws InterruptedIOException if another expensive call is already in progress
581 * or we cannot otherwise currently attempt to (re)load an AEP;
582 * retry later
583 */
584 private AllExhibitProperties _getAllExhibitProperties(final long oldHash,
585 final boolean careful,
586 final boolean forceLoadFromFilesystem)
587 throws IOException
588 {
589 // Warn if the caller may be forcing an unnecessary expensive full reload from the filesystem.
590 if(IsDebug.isDebug && !forceLoadFromFilesystem && (oldHash <= 0)) { System.err.println("WARNING: ExhibitDataFileSource._getAllExhibitProperties(): expensive 0 (empty AEP) or negative oldHash request"); }
591
592 // When (temporarily) conserving power and NOT forcing a (re)load from the filesystem,
593 // then unless a negative oldHash has been specified
594 // (ie the caller already has a non-empty AEP to hand)
595 // and if the caller's existing hash is what we remember from our last AEP load
596 // then this routine doesn't recheck the filesystem at all
597 // unless enough time has elapsed since the last load attempt,
598 // so as to avoid even 'waking' (ie spinning/powering up) the filesystem.
599 // When conserving power we wait a while *after* 'next load' time,
600 // meaning also that when power is available we can very quickly test again.
601 // (If not conserving power then prodding the filesystem to ensure up-to-date is fine.)
602 final long startTime = System.currentTimeMillis();
603 if(!forceLoadFromFilesystem && (oldHash >= 0) &&
604 (startTime < _AEP_nextLoad + (GenUtils.mustConservePower() ? (MIN_POLL_CONSERVING_MS - MIN_POLL_MS) : 0)))
605 {
606 // As far as we know the caller is up-to-date so return null.
607 if(_AEP_last_longHash == oldHash) { return(null); }
608 }
609
610 // If (temporarily) conserving power and the caller has a non-empty AEP
611 // (that they can presumably work with for the time being)
612 // then veto trying to load/construct an AEP from the filesystem.
613 if(MINIMISE_FS_POWER && !forceLoadFromFilesystem && (oldHash > 0) && GenUtils.mustConservePower())
614 { throw new InterruptedIOException("conserving power; will not attempt to load/construct AEP while caller has non-empty AEP"); }
615
616 // Try for reload from the filesystem (if not already busy)...
617 if(!_slow_op_lock.tryLock()) { throw new InterruptedIOException("already in progress: "+_slow_op_lock); }
618 try
619 {
620 // Iff there is a hash file containing the caller's specified (non-negative) oldHash
621 // then we assume that the caller's AEP is up-to-date.
622 // (After adding exhibits the hash file should be removed to force a later reload.)
623 final File staticHashFileName = getStaticHashFileName();
624 logFSAccess("checking cached AEP hash file " + staticHashFileName, false);
625 final Long hash = getLastAEPHashIfAny();
626 if(hash != null)
627 {
628 if(!forceLoadFromFilesystem && (oldHash >= 0) && (hash.longValue() == oldHash))
629 {
630 _AEP_last_longHash = oldHash; // Note the apparent longHash for the current exhibits.
631
632 // Simplified method to postpone next poll if no update was found...
633 final long endTime = System.currentTimeMillis();
634 final long waitTime = 101 + Math.max(MIN_POLL_MS, (CoreConsts.DEFAULT_TEMPORAL_SLACKNESS_S/4) * 1000);
635
636 // Postpone next check.
637 final long nextUpdateTime = endTime + waitTime;
638 _AEP_nextLoad = nextUpdateTime;
639
640 if(ORG.hd.d.IsDebug.isDebug) { System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): caller is apparently up-to-date (see "+staticHashFileName+") with hash: "+oldHash+"; next check not before "+(new Date(nextUpdateTime))+".]"); }
641 return(null);
642 }
643 }
644
645 // Attempt to recover cache of entire AEP to avoid recomputing some very expensive state.
646 AllExhibitProperties oldProps = null;
647 final File staticCacheFileName = getStaticCacheFileName();
648 if(staticCacheFileName.canRead())
649 {
650 System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): attempting to load old cache file to recover expensive-to-compute data: " + staticCacheFileName +" @ "+(new Date())+".]");
651 try { oldProps = (AllExhibitProperties) FileTools.deserialiseFromFile(staticCacheFileName, true); }
652 // Log but absorb any error, and do without cached data...
653 catch(final Exception e) { e.printStackTrace(); }
654 }
655
656 // Try to load/construct the AEP...
657 try
658 {
659 System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): starting AEP construction from filesystem @ "+(new Date())+".]");
660
661 // Turn back on any muted warnings.
662 turnBackOnMutedWarnings();
663
664 final File baseDir = new File(LocalProps.getDataDir());
665 logFSAccess("loading AEP from raw exhibit data in "+baseDir, false);
666
667 // Load the EPGI (should be quick: do it first).
668 final ExhibitPropsGlobalImmutable epgi = ExhibitPropsGlobalImmutable.loadFromDataDir(baseDir);
669 if(ORG.hd.d.IsDebug.isDebug) { System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): loaded EPGI: "+epgi+"]"); }
670
671 /* if(ORG.hd.d.IsDebug.isDebug) */ { System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): recomputing AllExhibitImmutableData... @ " + (new Date())+".]"); }
672 // First, get immutable core data unconditionally...
673 final AllExhibitImmutableData aeid =
674 _getAllExhibitImmutableData(baseDir.getPath(), -1, careful);
675
676 // Create accession data for new exhibits...
677 /* if(ORG.hd.d.IsDebug.isDebug) */ { System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): creating new accession files @ "+(new Date())+".]"); }
678 createAccessionFiles(aeid);
679
680 // Get sorted set of exhibit names.
681 // We hope that iterating over a sorted set will result in
682 // better filesystem performance than an unsorted set.
683 final List<Name.ExhibitFull> sortedNames = aeid.getAllExhibitNamesSorted();
684
685 /* if(ORG.hd.d.IsDebug.isDebug) */ { System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): recomputing loadedProperties over exhibit set of "+(sortedNames.size())+"... "+(new Date())+".]"); }
686
687 // Collect the loaded and computable properties.
688 // We do this in parallel to maximise I/O throughput
689 // (though this may in fact be quite CPU- and memory- intensive)
690 // and use a ConcurrentHashMap as our working store as it is thread-safe.
691 // We use the CPU-intensive thread pool
692 // and wait for all work to complete.
693 //
694 // We recover all EPC data that we can from the previous/cached AEP if extant
695 // to save *lots* of time and effort.
696 final Map<Name.ExhibitFull,ExhibitPropsLoadable> loadedProps = new ConcurrentHashMap<Name.ExhibitFull, ExhibitPropsLoadable>(aeid.size() * 2);
697 final Map<Name.ExhibitFull,ExhibitPropsComputable> computedProps = new ConcurrentHashMap<Name.ExhibitFull, ExhibitPropsComputable>(aeid.size() * 2);
698 final List<Future<?>> tasks = new ArrayList<Future<?>>(MemoryTools.isMemoryStressed() ? 16 : sortedNames.size());
699 final ExhibitDataSource ds = makeExhibitDataSource();
700 final AllExhibitProperties c = oldProps;
701 final boolean recoverFromCache = (null != c);
702 for(final Name.ExhibitFull name : sortedNames)
703 {
704 // Create the task...
705 final Runnable r = new Runnable(){
706 public final void run()
707 {
708 // We don't store null/EMPTY EPL/EPC values.
709 try
710 {
711 // Always reload EPL data
712 // else we will miss changes in description, etc.
713 final ExhibitPropsLoadable epl = ExhibitPropsLoadable.getLoadableProperties(name, baseDir);
714 if((epl != null) && !epl.equals(ExhibitPropsLoadable.EMPTY))
715 { loadedProps.put(name, epl); }
716
717 // Recover expensive EPC data if possible...
718 ExhibitPropsComputable epc = null;
719 if(recoverFromCache)
720 { epc = c.getExhibitPropsComputable(name); }
721 if((null == epc) || epc.equals(ExhibitPropsComputable.EMPTY))
722 { epc = ExhibitPropsComputable.createExhibitPropsComputable(aeid.getStaticAttr(name), ds); }
723 if((epc != null) && !epc.equals(ExhibitPropsComputable.EMPTY))
724 { computedProps.put(name, epc); }
725 }
726 catch(final Exception e)
727 { throw new RuntimeException("Could not create epl/epc for: " + name, e); }
728 }
729 };
730
731 // If we're currently pressed for resources then run this task synchronously...
732 if(MemoryTools.isMemoryStressed() || !ThreadUtils.couldRunLowPriorityDiscardableTask())
733 { try { r.run(); } catch(final Exception e) { throw new IOException(e); } }
734 // ...else attempt to run the task concurrently for better throughput.
735 else
736 { tasks.add(ThreadUtils.computeIntensiveThreadPool.submit(r)); }
737 }
738
739 // Wait for all asynchronous tasks to finish...
740 for(final Future<?> task : tasks)
741 {
742 try { task.get(); }
743 catch(final InterruptedException e)
744 {
745 final InterruptedIOException err = new InterruptedIOException(e.getMessage());
746 err.initCause(e);
747 throw err;
748 }
749 catch(final ExecutionException e)
750 {
751 final IOException err = new IOException(e.getMessage());
752 err.initCause(e);
753 throw err;
754 }
755 }
756
757 /* if(ORG.hd.d.IsDebug.isDebug) */ { System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): constructing AEP... @ "+(new Date())+"]"); }
758
759 // Construct a new AEP from the data gathered.
760 final AllExhibitProperties newProps = new AllExhibitProperties(null,
761 epgi,
762 aeid,
763 loadedProps,
764 computedProps,
765 0);
766
767 System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): compacting... @ "+(new Date())+"]");
768 newProps.compact();
769
770 System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): loaded and constructed new AEP with longHash="+newProps.longHash+" @ "+(new Date())+"]");
771
772 // Cache the longHash in memory...
773 _AEP_last_longHash = newProps.longHash;
774 // Save a hash file recording the AEP longHash (in decimal) if possible.
775 // May not be possible if the filesystem is read-only for example.
776 System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): cacheing AEP hash to "+staticHashFileName+"... @ "+(new Date())+"]");
777 try { FileTools.replacePublishedFile(staticHashFileName.getPath(), (new CS8Bit(Long.toString(newProps.longHash, 10))).toByteArray()); }
778 catch(final IOException e) { e.printStackTrace(); }
779
780 // If the AEP has changed from the static cached version (if any) then try to save it for next time.
781 if((oldProps == null) || (oldProps.longHash != newProps.longHash))
782 {
783 System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): cacheing serialised AEP to "+staticCacheFileName+"... @ "+(new Date())+"]");
784 try { FileTools.serialiseToFile(newProps, staticCacheFileName, true, !IsDebug.isDebug); }
785 // Log but absorb any errors.
786 catch(final Exception e) { e.printStackTrace(); }
787 }
788
789 // If the caller really is up-to-date then return null (discarding the new AEP).
790 if((oldHash >= 0) && (newProps.longHash == oldHash)) { return(null); }
791
792 // Return the newly-loaded AEP.
793 return(newProps);
794 }
795 finally
796 {
797 // Whether we succeeded or not...
798 // work out the minimum wait before starting the next load.
799 //
800 // Usually, after a successful load,
801 // the next load attempt should not happen within the normal slackness interval.
802 //
803 // This tries not to spend more than ~10% of real time doing loads if possible,
804 // but caps the *minimum* wait before another attempt to a few hours.
805 // We throw in a small random factor too, to help avoid collisions...
806 //
807 // The minimum polling interval is boosted when conserving power.
808 final long endTime = System.currentTimeMillis();
809 final long timeTaken = endTime - startTime;
810 final long minWaitTime = Math.max(MIN_POLL_MS, CoreConsts.DEFAULT_TEMPORAL_SLACKNESS_S * (1000 / 10));
811 final long waitTime = Math.max(minWaitTime, // Default/minimum wait time...
812 Math.min(timeTaken << 3, // Limit effort to ~10% of wall-clock time if possible.
813 2 * 3600 * 1000L) + // Limit delay to next attempt to ~2 hours at worst.
814 Rnd.fastRnd.nextInt(10001 + (CoreConsts.DEFAULT_TEMPORAL_SLACKNESS_S * 101)));
815
816 // Schedule next check.
817 final long nextUpdateTime = endTime + waitTime;
818 _AEP_nextLoad = nextUpdateTime;
819 /* if(ORG.hd.d.IsDebug.isDebug) */ { System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): done new AEP load: next check not before "+(waitTime/(1000 * 60))+"mins, at "+(new Date(nextUpdateTime))+".]"); }
820 }
821 }
822 finally { _slow_op_lock.unlock(); }
823 }
824
825 /**Wrap this instance as an ExhibitDataSource.
826 * Only really meant for internal use,
827 * but may be useful for low-level use of the file-based data.
828 */
829 public AllExhibitProperties.ExhibitDataSource makeExhibitDataSource()
830 {
831 return(new AllExhibitProperties.ExhibitDataSource(){
832 @Override
833 public final void getRawFile(final ByteBuffer buf, final ExhibitFull exhibitName, final int position)
834 throws IOException
835 { ExhibitDataFileSource.this.getRawFile(buf, exhibitName, position, false); }
836 @Override
837 public final boolean isExhibitFullyLoaded(final ExhibitStaticAttr esa)
838 { return(true); }
839 });
840 }
841
842
843 /**Gets the general properties as a GenProps object if its timestamp is not that specified.
844 * If the time specified is negative the object will be returned unconditionally.
845 * <p>
846 * If no props are currently installed/available a default set with a zero
847 * timestamp is returned.
848 * <p>
849 * If the caller's copy appears to be up-to-date (eg the oldStamp
850 * matches that that we would have been returned) null is returned.
851 * <p>
852 * This computes a new uncached value on each call.
853 */
854 public GenProps getGenProps(final long oldStamp)
855 throws IOException
856 {
857 final File file = new File(LocalProps.getConfDir(), CoreConsts.FS_CONF_SYSPROPS);
858
859 // If the stamp presented is < 0 or different to that of the file,
860 // reread the file now.
861 final long newStamp = file.lastModified();
862 if((oldStamp < 0) || (oldStamp != newStamp))
863 {
864 try
865 {
866 final FileInputStream fis = new FileInputStream(file);
867 try
868 {
869 final Properties props = FileTools.loadProperties(new BufferedInputStream(fis));
870 return(new GenProps(props, newStamp));
871 }
872 finally { fis.close(); }
873 }
874 catch(final IOException e)
875 {
876 e.printStackTrace(); // Dump a diagnostic...
877 throw e; // Pass the error on.
878 }
879 }
880
881 return(null); // Caller's copy seems to be up-to-date.
882 }
883
884 /**Gets the generic security properties as a Properties object if its timestamp is not that specified.
885 * If the time specified is negative the object will be returned unconditionally.
886 * <p>
887 * If no props are currently installed/available a default set with a zero
888 * timestamp is returned.
889 * <p>
890 * If the caller's copy appears to be up-to-date (eg the oldStamp
891 * matches that that would have been returned) null is returned.
892 * <p>
893 * This computes a new uncached value on each call.
894 * <p>
895 * These generic properties are fetchable over the network, for example,
896 * and need not be present locally at each host the JVM runs on.
897 */
898 public java.util.Properties getGenSecProps(final long oldStamp)
899 throws IOException
900 {
901 final SecurityProps sp =
902 SecurityProps.getSecurityPropsUncachedFromFilesystem(oldStamp);
903
904 // If security props file has not changed, return null.
905 if(sp == null)
906 { return(null); }
907
908 // Extract the relatively-safe "generic" component.
909 return(sp.getGenSecProps());
910 }
911
912 /**Suffix appended to serialised thumbnail class data for each exhibit.
913 * Such files are beside the original exhibit file,
914 * are prefixed with FileTools.F_permPrefix,
915 * and suffixed with this (which is chosen so as not to look like an exhibit).
916 * <p>
917 * The file is a GZIPped, serialised, ExhibitThumbnails class if present.
918 */
919 public static final String THUMBNAIL_SUFFIX = ".thumbnails";
920
921 /**If true, the serialised thumbnail files are compressed (GZIPped) on disc.
922 * This might cost a little time but might possibly save a little space.
923 * Note that thumbnails may in any case be GZIPped in transit to a slave server.
924 */
925 public static final boolean THUMBNAILS_ARE_GZIPPED = false;
926
927 /**If true, we have warned about thumbnail directory absence.
928 * Starts, false; may be cleared periodically.
929 * Set true when we have warned.
930 * <p>
931 * Volatile to allow read/write without a lock.
932 * <p>
933 * Not critical to correct operation,
934 * so we can tolerate races and sloppy handling.
935 */
936 private volatile boolean _haveWarnedNoTNDir;
937
938 /**Turn back on warnings we have muted.
939 * We do this occasionally so that persistent faults
940 * will show up in the log and don't just vanish.
941 * <p>
942 * Avoids locking unless really necessary.
943 */
944 private void turnBackOnMutedWarnings()
945 {
946 _haveWarnedNoTNDir = false;
947 }
948
949
950 /**Interval between logging filesystem accesses (ms); strictly positive.
951 * Should be short enough to reveal operations, possibly unwanted,
952 * that 'wake up' the filesystem (eg cause it to power-up, get mounted, etc).
953 * <p>
954 * Should be long enough to eliminate most uninteresting stuff
955 * such as the multiple accesses to load/examine one file.
956 * Keeping the interval to number a minute will most likely catch most 'wakeups'.
957 */
958 private static final int FS_LOG_INTERVAL_MS = 59000; // Just less than a minute.
959
960 /**Count of filesystem access operations logged including those not displayed; never null.
961 * Accessed under _FSLogLock.
962 */
963 private static int countFSAccess;
964
965 /**Time that last filesystem access was logged; initially zero.
966 * Accessed under _FSLogLock.
967 */
968 private static long lastLoggedFSAccess;
969
970 /**Lock for logging filesystem access; never null. */
971 private static final ReentrantLock _FSLogLock = new ReentrantLock();
972
973 /**Log access to data filesystem.
974 * Useful for tracing unwanted activity.
975 * <p>
976 * May limit output (eg to once every few minutes)
977 * to still give traceability of 'wakeup' operations.
978 *
979 * @param force if true then force printing
980 */
981 private static void logFSAccess(final String detail, final boolean force)
982 {
983 _FSLogLock.lock();
984 try
985 {
986 final int count = ++countFSAccess;
987
988 // Discard this log message if too soon since the previous...
989 final long now = System.currentTimeMillis();
990 if(!force && (now < lastLoggedFSAccess + FS_LOG_INTERVAL_MS)) { return; }
991 lastLoggedFSAccess = now;
992
993 // Write this long message.
994 System.out.println("[ExhibitDataFileSource: DATA FILESYSTEM ACCESS "+count+": "+detail+" @ "+(new Date(now))+"]");
995 }
996 finally { _FSLogLock.unlock(); }
997 }
998
999
1000 /**Gets the thumbnails for an exhibit; null if not (currently) available.
1001 * In this implementation we never try to create thumbnails
1002 * (we assume that the source directory is read only for example),
1003 * but we'll return one if we can find one.
1004 * <p>
1005 * Thumbnails are looked for by default in the exhibits directory
1006 * tree beside the source exhibits, but a LocalProps value
1007 * can override this so that source data can be separated
1008 * from derived data.
1009 *
1010 * @param create if true then may try to create and cache missing thumbnails,
1011 * but creating the thumbnail is only attempted
1012 * if cacheing is likely to be successful
1013 * because this should be a once-only operation per exhibit
1014 */
1015 public ExhibitThumbnails getThumbnails(final ExhibitFull name, final boolean create)
1016 throws IOException
1017 {
1018 logFSAccess("requesting pre-computed thumbnails for: "+name, false);
1019
1020 // In the simple case where we know that thumbnails
1021 // definitely can't be created (from the type)
1022 // return the NO_THUMBNAILS value immediately,
1023 // ignoring the create parameter.
1024 // We never cache this (negative) value.
1025 final ExhibitMIME.ExhibitTypeParameters type = (ExhibitMIME.getInputFileType(name));
1026 if((type == null) || (type.handler == null) ||
1027 !type.canPossiblyCreateThumbnailOfSameMIMEType())
1028 { return(ExhibitThumbnails.NO_THUMBNAILS); }
1029
1030 final String dataDir = LocalProps.getDataDir();
1031 final ExhibitStaticAttr esa = _getStaticAttr(dataDir, name); // Avoid tickling cache badly...
1032 // If the exhibit appears not to exist
1033 // then return null rather than NO_THUMBNAILS
1034 // (for example the exhibit may be in the process or being added).
1035 if(esa == null)
1036 { return(null); }
1037
1038 // Construct the name of the thumbnails' file if it exists.
1039 final String relPath = thumbnailRelPath(name);
1040
1041 // Compute full path from exhibit directory.
1042 final File topDir = new File(dataDir, LocalProps.getThumbnailRelDir());
1043
1044 // If the top directory for the thumbnails does not exist,
1045 // then we definitely will not be returning thumbnails.
1046 // We also warn when this directory does not exist.
1047 final boolean noTNDir = !topDir.isDirectory();
1048 if(noTNDir)
1049 {
1050 if(!_haveWarnedNoTNDir)
1051 {
1052 System.err.println("ExhibitDataFileSource: WARNING: top-level thumbnail directory absent: " + topDir);
1053 _haveWarnedNoTNDir = true;
1054 }
1055
1056 // We don't return ExhibitThumbnails.NO_THUMBNAILS,
1057 // since thumbnails may be available/generated later.
1058 return(null);
1059 }
1060
1061 // If the thumbnail file seems readable, etc, then try to load it.
1062 // Any problem, eg an IOException, will be propagated to the caller.
1063 final File fullPath = new File(topDir, relPath);
1064 logFSAccess("reading thumbnail from: "+fullPath, false);
1065 if(fullPath.exists() && fullPath.canRead() && fullPath.isFile())
1066 {
1067 final ExhibitThumbnails result = (ExhibitThumbnails)
1068 FileTools.deserialiseFromFile(fullPath, THUMBNAILS_ARE_GZIPPED);
1069 logFSAccess("read thumbnail from: "+fullPath+": "+result, ExhibitThumbnails.NO_THUMBNAILS.equals(result)); // Force result for NO_THUMBNAILS value...
1070 return(result);
1071 }
1072
1073 // Only even consider thumbnail creation
1074 // if the thumbnail area appears to be writable.
1075 if(create && topDir.canWrite())
1076 {
1077 // TODO
1078 }
1079
1080 // Can't seem to find a thumbnail file,
1081 // so tell the caller no thumbnails at the moment.
1082 // We don't return ExhibitThumbnails.NO_THUMBNAILS,
1083 // since thumbnails may be available/generated later.
1084 return(null);
1085 }
1086
1087 /**Poll periodically.
1088 * Periodically save the system variables.
1089 */
1090 public void poll(final GenProps gp)
1091 {
1092 _saveSystemVariables(varMgr, false);
1093 }
1094
1095 /**Save the system variables.
1096 * Mainly save the histories of persistent events.
1097 * <p>
1098 * We save these with a history so that in effect we have backups
1099 * if something bad happens
1100 * (though this implies that some other housekeeping will
1101 * have to clear up excess copies, etc,
1102 * in order to avoid space requirements growing without bound).
1103 * <p>
1104 * The data is saved in the persistent data area.
1105 * <p>
1106 * Reports but does not propagate exceptions.
1107 *
1108 * @param vars set of variables to save
1109 * @param force if true, force an immediate complete save
1110 */
1111 private void _saveSystemVariables(final BasicVarMgr vars,
1112 final boolean force)
1113 {
1114 // Return immediately if too soon to save again (postponed further when conserving energy)
1115 // and a save is not being forced.
1116 if(!force &&
1117 (_nextVarSaveEarliest + (GenUtils.mustConservePower() ? Math.max(CoreConsts.ASYNC_MIN_POWER_SAVE_NON_CRITICAL_DATA_FLUSH_MS, _VAR_SAVE_INTERVAL_MS*3) : 0) > System.currentTimeMillis()))
1118 { return; }
1119
1120 // Keeps a history and attempts just to save changes...
1121 try
1122 {
1123 // First write out the persistent events as-is
1124 _appendToEventLogs();
1125
1126 // Unless doing a forced save, eg at shut-down,
1127 // do an incremental changes-only save to be as quick as reasonably possible.
1128 // When forced, push out *all* stores, not just changed ones,
1129 // to minimise the chance of entirely 'losing' very-slowly-changing event sets.
1130 vars.saveEventHistories(_getEventHistoryStorageDir(), !force, true, !force);
1131 // The save seems to have completed fine, so postpone the next one.
1132 _nextVarSaveEarliest = System.currentTimeMillis() + _VAR_SAVE_INTERVAL_MS +
1133 Rnd.fastRnd.nextInt(1 + (_VAR_SAVE_INTERVAL_MS/3));
1134 }
1135 catch(final IOException e)
1136 { e.printStackTrace(); }
1137 }
1138
1139
1140
1141 /**Our private static instance of a GMT calendar.
1142 * No one must change the ZONE or DST offset.
1143 */
1144 private static final Calendar GMTCalendar = Calendar.getInstance();
1145 /**Initialise GMTCalendar. */
1146 static
1147 {
1148 // Set to GMT
1149 // i.e 0 Offset
1150 GMTCalendar.set(Calendar.ZONE_OFFSET, 0);
1151 // and 0 Daylight savings
1152 GMTCalendar.set(Calendar.DST_OFFSET, 0);
1153 }
1154
1155 /**Format we use to insert into event history file name. */
1156 private static final SimpleDateFormat dateFmtEHFile =
1157 new SimpleDateFormat("yyyyMMdd");
1158 /**Set the Calendar for dateFmtEHFile to be the GMT calendar. */
1159 static { dateFmtEHFile.setCalendar(GMTCalendar); }
1160
1161 /**Format we use for timestamp to insert into event log file name. */
1162 private static final SimpleDateFormat timestampFmtEHFile =
1163 new SimpleDateFormat("yyyyMMdd-HHmmss.SSS");
1164 /**Set the Calendar for dateFmtEHFile to be the GMT calendar. */
1165 static { timestampFmtEHFile.setCalendar(GMTCalendar); }
1166
1167 /**Max log line length for event log. */
1168 private static final int MAX_EV_LOG_LINE_LENGTH = 4095;
1169
1170 /**Private lock for _appendToEventLogs(). */
1171 private static final Object _aTWL_lock = new Object();
1172
1173 /**Write out the accumulated event values to disc.
1174 * These are written to date-stamped files to allow automatic "rolling".
1175 * <p>
1176 * We keep a separate file for each named event,
1177 * and append events to each appropriate file in order
1178 * with a record of the form:
1179 * <pre>
1180 * YYYYMMDD-HHmmss.SSS eventName eventValue
1181 * </pre>
1182 * with a null value being indicated by the value "null".
1183 * <p>
1184 * Note that the timestamp is that of the value,
1185 * not the time that the event is received or written.
1186 * <p>
1187 * We may cache file handles to reduce the cost of opening and closing
1188 * files for each event,
1189 * but in any case will have closed all log files opened by us
1190 * by the time this routine returns.
1191 * <p>
1192 * We synchronise on a private static lock to try to prevent two instances of this
1193 * running concurrently and leading to file corruption.
1194 */
1195 private void _appendToEventLogs()
1196 throws IOException
1197 {
1198 // Attempt to write out accumulated event values to the logs.
1199 final List<SimpleVariableValue> events = new ArrayList<SimpleVariableValue>(eventsToLog.size());
1200 // Move all events into working copy atomically to hold lock for short time.
1201 synchronized(eventsToLog) { events.addAll(eventsToLog); eventsToLog.clear(); }
1202
1203 // If nothing to write then return immediately.
1204 final int nEvents = events.size();
1205 if(nEvents == 0) { return; }
1206
1207
1208 //System.out.println("[EVENTS TO LOG: " + nEvents + "...]");
1209
1210 // Separate the events into groups (preserving order) by event name
1211 // so that we can do one file operation for all instances of one event.
1212 final Map<String, List<SimpleVariableValue>> groupedEvents = new HashMap<String, List<SimpleVariableValue>>();
1213 for(final SimpleVariableValue svv : events)
1214 {
1215 final String name = svv.getDef().getName();
1216 List<SimpleVariableValue> evvs = groupedEvents.get(name);
1217 if(evvs == null)
1218 {
1219 evvs = new ArrayList<SimpleVariableValue>();
1220 groupedEvents.put(name, evvs);
1221 }
1222 evvs.add(svv);
1223 }
1224
1225 // Check that the log directory is writable.
1226 // Complain loudly and discard the events if not.
1227 final File logDir = _getEventLogDir();
1228 if((logDir == null) || !logDir.isDirectory() || !logDir.canWrite())
1229 {
1230 System.err.println("WARNING: persistent event log dir "+logDir+" not usable: discarding "+nEvents+" event(s).");
1231 return;
1232 }
1233
1234 // Create/format the date string once, if we are going to use it.
1235 final String dateString = "." + dateFmtEHFile.format(new Date());
1236
1237 // Any error accumulated so far...
1238 IOException err = null;
1239
1240 synchronized(_aTWL_lock)
1241 {
1242 for(final String name : groupedEvents.keySet())
1243 {
1244 final StringBuilder sb = new StringBuilder(80);
1245 sb.append("eventLog.");
1246 sb.append(name);
1247 sb.append(BasicVarMgr.EVENT_STORE_NAMETERM);
1248 sb.append(dateString);
1249 sb.append(".log");
1250 final File f = new File(logDir, sb.toString());
1251
1252 try
1253 {
1254 final PrintWriter pw = new PrintWriter(new FileWriter(f, true)); // Append to existing file.
1255 try
1256 {
1257 for(final SimpleVariableValue svv : groupedEvents.get(name))
1258 {
1259 final StringBuilder logLine = new StringBuilder(80);
1260
1261 logLine.append(timestampFmtEHFile.format(new Date(svv.getTimestamp())));
1262 logLine.append(' ');
1263 logLine.append(name);
1264 logLine.append(' ');
1265 logLine.append(svv.getValue());
1266
1267 // Truncate VERY long entries...
1268 if(logLine.length() > MAX_EV_LOG_LINE_LENGTH)
1269 {
1270 logLine.setLength(MAX_EV_LOG_LINE_LENGTH - 3);
1271 logLine.append("..."); // Indicate truncation.
1272 }
1273
1274 // We sanitise the log line to ensure that it
1275 // contains no control codes especially CRs or LFs
1276 // that would disrupt the record-per-line format.
1277 // Replace such toxic chars with something innocuous.
1278 for(int i = logLine.length(); --i >= 0; )
1279 {
1280 final char ch = logLine.charAt(i);
1281 if((ch < 32) || (ch > 255) || (ch == 127))
1282 { logLine.setCharAt(i, ' '); }
1283 }
1284
1285 // Write entry in own line to EOF...
1286 pw.println(logLine.toString());
1287 }
1288
1289 // Flush to disc...
1290 pw.flush();
1291 }
1292 finally { pw.close(); }
1293 }
1294 catch(final IOException e)
1295 {
1296 e.printStackTrace();
1297 err = e; // Absorb error for now. Rethrow later.
1298 }
1299 }
1300 }
1301
1302 // If we encountered any error then (re)throw a sample one here.
1303 if(err != null) { throw err; }
1304 }
1305
1306 /**Minimum interval between saves, ms; strictly positive.
1307 * Intended to avoid losing too much data at shut-down,
1308 * yet avoid wearing out the disc (or other storage such as Flash) and consuming CPU time
1309 * with redundant work.
1310 * <p>
1311 * Something from tens of seconds to a few minutes is probably about right.
1312 */
1313 private static final int _VAR_SAVE_INTERVAL_MS = 5*60*1000;
1314
1315 /**Earliest time after which to allow the next event save (zero if no save yet completed).
1316 * Marked volatile to allow thread-safe lock-free access.
1317 */
1318 private transient volatile long _nextVarSaveEarliest;
1319
1320
1321 /**Get directory used for storing system-variable persistent event histories; never null.
1322 * Though never null, the indicated directory may not exist
1323 * or may otherwise not be usable.
1324 * <p>
1325 * We derive this from the LocalProps value.
1326 */
1327 private static File _getEventHistoryStorageDir()
1328 {
1329 return(new File(LocalProps.getPersistentStateDir(), "history"));
1330 }
1331
1332 /**Get directory used for storing system-variable persistent event logs; never null.
1333 * Though never null, the indicated directory may not exist
1334 * or may otherwise not be usable.
1335 * <p>
1336 * We get derive this from the LocalProps value.
1337 */
1338 private static File _getEventLogDir()
1339 {
1340 return(new File(LocalProps.getPersistentStateDir(), "log"));
1341 }
1342
1343 /**Our set of system variables.
1344 * Persistent values are loaded from disc on first attempt to
1345 * get and variable(s) if possible.
1346 * <p>
1347 * We expect this to be at the top of the data pipe on the master server,
1348 * and thus handle all the system non-local variables,
1349 * plus all the local variables of the master.
1350 * <p>
1351 * If any persistent values are set then they all get written to disc
1352 * as compressed serialised data on the next poll().
1353 * (Doing saves on each poll() helps reduce expensive disc writes by
1354 * grouping multiple updates into one save if they are happening rapidly.)
1355 * <p>
1356 * The current set or variables (including locals) is held in memory,
1357 * which also allows us to merge globals if required.
1358 * <p>
1359 * The varMgr is marked as an end-point so that it will
1360 * generate and return a unique local system ID.
1361 * <p>
1362 * This is assumed not to need to filter out duplicate updates
1363 * (nor bad timestamps) which is slow and may not even matter here.
1364 * Any network connection upstream of us should do such filtering.
1365 * <p>
1366 * We have the varMgr screen out repeats and bad timestamps, etc.
1367 */
1368 private final BasicVarMgr varMgr = new BasicVarMgr(true, true); // Endpoint.
1369
1370 /**In-order List/Queue thread-safe list of persistent events to be logged; never null.
1371 * Remove items FIFO or lock the queue and remove all the items in one go.
1372 * <p>
1373 * We don't synchronously save these data.
1374 */
1375 private final Vector<SimpleVariableValue> eventsToLog = new Vector<SimpleVariableValue>();
1376
1377 /**Set variable value; persistent values will eventually go to disc.
1378 *
1379 * @param newValue
1380 * @throws IOException
1381 */
1382 public void setVariable(final SimpleVariableValue newValue)
1383 throws IOException
1384 {
1385 varMgr.setVariable(newValue);
1386
1387 // Capture any persistent event for later logging.
1388 if(newValue != null)
1389 {
1390 final SimpleVariableDefinition def = newValue.getDef();
1391 if(def.isEvent() && def.isPersistent() && SystemVariables.defs.contains(def))
1392 { eventsToLog.add(newValue); }
1393 }
1394 }
1395
1396 /**Set variable values; persistent values will eventually go to disc.
1397 *
1398 * @param newValues
1399 * @throws IOException
1400 */
1401 public int setVariables(final SimpleVariableValue[] newValues)
1402 throws IOException
1403 {
1404 final int n = varMgr.setVariables(newValues);
1405
1406 // Capture any persistent events for later logging, in order.
1407 for(final SimpleVariableValue svv : newValues)
1408 {
1409 if(svv != null)
1410 {
1411 final SimpleVariableDefinition def = svv.getDef();
1412 if(def.isEvent() && def.isPersistent() && SystemVariables.defs.contains(def))
1413 { eventsToLog.add(svv); }
1414 }
1415 }
1416
1417 return(n);
1418 }
1419
1420 /**Get variable value; persistent values may have come from disc.
1421 */
1422 public SimpleVariableValue getVariable(final SimpleVariableDefinition var)
1423 {
1424 return(varMgr.getVariable(var));
1425 }
1426
1427 /**Get variable values; persistent values may have come from disc.
1428 */
1429 public SimpleVariableValue[] getVariables(final long changedSince)
1430 {
1431 return(varMgr.getVariables(changedSince));
1432 }
1433
1434 /**Synchronise variable values.
1435 * This implementation <em>does not</em> force a flush to disc.
1436 */
1437 public void syncVariables(final boolean force)
1438 {
1439 // Nothing needed in this implementation.
1440 // In particular this does not flush values to disc.
1441 }
1442
1443 /**Get the current partial, or previous full, event set at the specified interval; never null.
1444 * This is a simplified interface to return either the current event set
1445 * that is being collected, or the previous completed set.
1446 * <p>
1447 * The current set is the most timely, but may not contain enough data
1448 * to be meaningful if the new interval has just started.
1449 * <p>
1450 * The previous set is complete and thus most likely to have enough samples
1451 * to be useful, but is not completely current.
1452 * <p>
1453 * If a "previous" value returned from our local store is not marked
1454 * as authoritative, then we convert it to an authoritative value,
1455 * write that back to our local store and return it.
1456 * (The "current" value is generally not authoritative.)
1457 *
1458 * @param def event definition (must be for an event); never null
1459 * @param intervalSelector one of EVENT_INTERVAL_SELECTOR_xxx values
1460 * @param current if true the current event set is returned,
1461 * else the previous complete set is returned
1462 *
1463 * @return requested event set; may be empty but never null if requested set not available
1464 */
1465 public EventVariableValue getEventValue(final SimpleVariableDefinition def,
1466 final EventPeriod intervalSelector,
1467 final boolean current)
1468 {
1469 final EventVariableValue eventValue = varMgr.getEventValue(def, intervalSelector, current);
1470 if(!current && !eventValue.isAuthoritative())
1471 {
1472 final EventVariableValue authValue = eventValue.makeAuthoritative();
1473 varMgr.setEventValue(authValue);
1474 return(authValue);
1475 }
1476 return(eventValue);
1477 }
1478
1479 /**Get the specified event sets for the specified intervals; never null.
1480 * This allows retrieval of zero or more event sets for the specified
1481 * interval size.
1482 * <p>
1483 * Requests for more than SystemVariables.EVENT_SAMPLES_RETAINED in the
1484 * past (or for the future!) cannot be satisfied and data will not be
1485 * returned for them.
1486 * <p>
1487 * Usually not more than SystemVariables.EVENT_SAMPLES_RETAINED samples
1488 * will be returned in response to any one request as a safety measure.
1489 * <p>
1490 * (An implementation that is not an end-point may go upstream to fetch
1491 * missing values and cache them to satisfy future requests.)
1492 * <p>
1493 * If a "previous" value (older than the current interval)
1494 * returned from our local store is null or not marked
1495 * as authoritative, then we convert it to an authoritative value,
1496 * write that back to our local store and return it.
1497 * (The "current" value is generally not authoritative,
1498 * and entries in the future cannot be so.)
1499 *
1500 * @param def event definition (must be for an event); never null
1501 * @param intervalSelector one of EVENT_INTERVAL_SELECTOR_xxx values
1502 * @param intervalNumber a time (as from System.currentTimeMillis())
1503 * which identifies the first interval for which data is potentially
1504 * required; if too far in the past or future then possibly no data
1505 * will be available,
1506 * zero is used to access the "all" bucket
1507 * @param whichValues each true bit represents a slot for which data is
1508 * required, bit 0 indicating data from the slot within which
1509 * firstIntervalTime is located, bit 1 the previous slot, etc
1510 *
1511 * @return as many of the requested values as available,
1512 * at least long enough to return all the available values,
1513 * with [0] corresponding to bit 0 in the BitSet;
1514 * may contain nulls or be zero-length but is never null
1515 */
1516 public EventVariableValue[] getEventValues(final SimpleVariableDefinition def,
1517 final EventPeriod intervalSelector,
1518 final long intervalNumber,
1519 final BitSet whichValues)
1520 {
1521 // Compute currentInterval before getting values
1522 // to avoid a race which might make wrongly capture
1523 // an incomplete "current" value as authoritative and complete.
1524 final long currentInterval = intervalSelector.getIntervalNumber(System.currentTimeMillis());
1525
1526 synchronized(varMgr) // Avoid races with new events coming in...
1527 {
1528 EventVariableValue[] result = varMgr.getEventValues(def, intervalSelector, intervalNumber, whichValues);
1529
1530 // Ensure result array is large enough,
1531 // else copy into large enough one.
1532 final int wvl = (whichValues == null) ? 1 : whichValues.length();
1533 if(result.length < wvl)
1534 {
1535 final EventVariableValue newResult[] = new EventVariableValue[wvl];
1536 System.arraycopy(result, 0, newResult, 0, result.length);
1537 result = newResult;
1538 }
1539
1540 // For any values newer than the current interval,
1541 // and for which our local store had missing or non-authoritative values,
1542 // convert them to authoritative values
1543 // so that upstream users can cache them permanently.
1544 BitSet wv = whichValues;
1545 if(wv == null)
1546 {
1547 // Create synthetic BitSet for simplicity below.
1548 wv = new BitSet(1);
1549 wv.set(0);
1550 }
1551 for(int i = wv.nextSetBit(0); i >= 0; i = wv.nextSetBit(i+1))
1552 {
1553 final long slotIntervalNumber = intervalNumber - i;
1554 // We cannot make current/future events authoritative.
1555 if(slotIntervalNumber >= currentInterval)
1556 { continue; }
1557
1558 final EventVariableValue evv = result[i];
1559 if((evv == null) || !evv.isAuthoritative())
1560 {
1561 // Make value authoritative,
1562 // creating a new empty one if need be.
1563 final EventVariableValue authValue = (evv != null) ? evv.makeAuthoritative() :
1564 new EventVariableValue(true,
1565 def,
1566 intervalSelector,
1567 slotIntervalNumber,
1568 0, null, null);
1569
1570 assert(authValue.isAuthoritative());
1571 assert((evv == null) || (authValue.getIntervalNumber() == evv.getIntervalNumber()));
1572 assert((evv == null) || (authValue.getDef().equals(evv.getDef())));
1573
1574 // Return authoritative value to caller.
1575 result[i] = authValue;
1576
1577 // Save in our local store.
1578 varMgr.setEventValue(authValue);
1579
1580 //if(IsDebug.isDebug) { System.out.println("Storing authoritative " + authValue); }
1581 }
1582 }
1583
1584 return(result);
1585 }
1586 }
1587
1588
1589 /**Creates the name of a thumbnail starting at the thumbnail base directory from a full exhibit name.
1590 */
1591 private static String thumbnailRelPath(final Name.ExhibitFull exhibitName)
1592 {
1593 // Construct the name of the thumbnails file if it exists.
1594 return(ExhibitName.getDirComponent(exhibitName).toString() + File.separatorChar +
1595 ExhibitName.getFileComponent(exhibitName) + THUMBNAIL_SUFFIX);
1596 }
1597
1598 // /**Creates thumbnails from all exhibits possible.
1599 // * This is used to create exhibit thumbnails off-line,
1600 // * taking as long and as much memory as required
1601 // * and processing the resource-intensive thumbnail building sequentially.
1602 // * <p>
1603 // * These luxuries of non-real-time and lots of memory and CPU are denied
1604 // * to the WAR/EAR servers when they try to compute thumbnails on the fly,
1605 // * since there may be many simultaneous users and even attempted
1606 // * Denial-of-Service attacks.
1607 // * <p>
1608 // * The thumbnails created here can be loaded by master servers,
1609 // * and through it any slaves, thus making a much wider range of thumbnails
1610 // * available than could be computed on the fly.
1611 // * <p>
1612 // * We will protest but terminate without an exception if the
1613 // * thumbnail directory does not exist; we will assume that precomputed
1614 // * thumbnails are not required.
1615 // * <p>
1616 // * We will only attempt to (*re) compute a thumbnail if:
1617 // * <ul>
1618 // * <li>The exhibit type is one that we can compute thumbnails for.
1619 // * <li>The thumbnail file does not exist or is older than the exhibit file.
1620 // * </ul>
1621 // * <p>
1622 // * This will attempt to put thumbnail files where the getThumbnails()
1623 // * routine will expect to find them.
1624 // */
1625 // private static void createThumbnails(final AllExhibitProperties aep,
1626 // final long stopBy)
1627 // throws IOException
1628 // {
1629 // // Compute full path from exhibit directory.
1630 // final File topDir = (new File(LocalProps.getDataDir(), LocalProps.getThumbnailRelDir())).getCanonicalFile();
1631 //
1632 // // If the top directory for the thumbnails does not exist,
1633 // // then we definitely will not be returning thumbnails.
1634 // // We also warn when this directory does not exist.
1635 // if(!topDir.isDirectory() || !topDir.canWrite())
1636 // {
1637 // // We might wish to limit frequency with which we report missing directory!
1638 // System.err.println("Warning: top-level thumbnail directory absent: " + topDir);
1639 // System.err.println("Aborting creation of thumbnails.");
1640 // return;
1641 // }
1642 //
1643 // // Make an instance of the data retriever for use by the thumbnail generator.
1644 // final ExhibitDataFileSource edfs = new ExhibitDataFileSource(aep);
1645 // final AllExhibitProperties.ExhibitDataSource eds = edfs.makeExhibitDataSource();
1646 //
1647 // // Now examine each exhibit in turn
1648 // // (in sorted order to try to approximate a disc-friendly breadth-first search)
1649 // // and if it is possible to build thumbnails
1650 // // and they don't exist or are older than the exhibit,
1651 // // try to build the thumbnails and save them.
1652 // //
1653 // // This tries to be robust and avoid blowing up if a single thumbnail build has problems.
1654 // final List<Name.ExhibitFull> names = aep.aeid.getAllExhibitNamesSorted();
1655 //
1656 // // Queue of tasks in progress.
1657 // final Queue<Future<?>> tasks = new LinkedBlockingQueue<Future<?>>();
1658 //
1659 // for(int i = names.size(); (--i >= 0) && (System.currentTimeMillis() <= stopBy); )
1660 // {
1661 // final Name.ExhibitFull name = names.get(i);
1662 //
1663 // // In the simple case where we know that thumbnails
1664 // // definitely can't be created
1665 // // (from the type, or because there is no exhibit)
1666 // // return the NO_THUMBNAILS value immediately,
1667 // // ignoring the create parameter.
1668 //
1669 // final ExhibitStaticAttr esa = aep.aeid.getStaticAttr(name);
1670 // if(esa == null)
1671 // { continue; }
1672 //
1673 // final ExhibitMIME.ExhibitTypeParameters type = (ExhibitMIME.getInputFileType(esa.getCharSequence()));
1674 // if((type == null) || (type.handler == null) ||
1675 // !type.canPossiblyCreateThumbnailOfSameMIMEType())
1676 // { continue; }
1677 //
1678 // // These tasks are assumed to be heavy on CPU, and not mainly I/O-bound.
1679 // tasks.add(ThreadUtils.computeIntensiveThreadPool.submit(new Runnable(){
1680 // public final void run()
1681 // {
1682 // // Construct the name of the thumbnails file if it exists.
1683 // final String relPath = thumbnailRelPath(name);
1684 //
1685 // // If the thumbnail file seems to exist and is readable,
1686 // // and is newer than the exhibit,
1687 // // then assume that it is up-to-date.
1688 // final File fullPath = new File(topDir, relPath);
1689 // if(fullPath.exists() && fullPath.canRead() && fullPath.isFile() &&
1690 // (fullPath.lastModified() > esa.timestamp))
1691 // { return; }
1692 //
1693 // // OK, looks like we need to (re)build the thumbnails...
1694 // try {
1695 // // Ensure that the destination directory exists.
1696 // final File targetDir = fullPath.getParentFile();
1697 // if(!targetDir.exists())
1698 // { targetDir.mkdirs(); }
1699 //
1700 // // OK, attempt to generate thumbnails here, not under any lock.
1701 // final ExhibitThumbnails tns = type.handler.makeThumbnails(
1702 // esa, eds, aep,
1703 // true); // Ignore resource limitations; this our best chance!
1704 //
1705 // // Attempt to save thumbnails (atomically), without additional compression.
1706 // FileTools.serialiseToFile(tns, fullPath, THUMBNAILS_ARE_GZIPPED, false);
1707 // }
1708 // catch(final ThreadDeath e)
1709 // { throw e; } // Don't intercept this one...
1710 // catch(final Throwable e)
1711 // {
1712 // System.err.println("ERROR: unable to build thumbnails for: " + name);
1713 // e.printStackTrace();
1714 // return; // Attempt to continue.
1715 // }
1716 // }
1717 // }));
1718 //
1719 // // Optimisation: release resources ASAP from any early completed tasks.
1720 // Future<?> headTask;
1721 // while(null != (headTask = tasks.peek()))
1722 // { if(headTask.isDone()) { tasks.remove(); } }
1723 // }
1724 //
1725 // // Wait for all remaining tasks.
1726 // while(!tasks.isEmpty())
1727 // {
1728 // try { tasks.remove().get(); }
1729 // catch(final Exception e)
1730 // {
1731 // final IOException err = new IOException("did not complete");
1732 // err.initCause(e);
1733 // throw err;
1734 // }
1735 // }
1736 // }
1737
1738 /**Computes an AllExhibitProperties object and saves it to outputFile.
1739 * The object is saved serialised and GZIPed.
1740 * <p>
1741 * The save is avoided if we are not forcing a complete recompute
1742 * and the exhibit hash has not changed,
1743 * ie we try to avoid unnecessarily churning the filesystem
1744 * and/or the master server.
1745 * <p>
1746 * It almost certainly does not make sense for the
1747 * WAR_SYSPROPNAME_WARONLY_STATICCACHEFILE property to be set;
1748 * you could get a similar result by copying the file.
1749 * <p>
1750 * In passing, this may update ancillary state such as computable properties
1751 * and thumbnails.
1752 *
1753 * @param outputFile desired location of cache file; not null
1754 * @param quick if true, attempt to be as quick as possible,
1755 * else if false, magic numbers of exhibits are checked and
1756 * other extra-careful checking is done;
1757 * this should be the default usage
1758 * @param recompute if true, recompute all derived data from scratch,
1759 * eg do not reload any extant cache file
1760 *
1761 * @return AllExhibitProperties (non-null)
1762 */
1763 private static AllExhibitProperties createStaticCacheFile(final File outputFile,
1764 final boolean quick,
1765 final boolean recompute,
1766 final boolean fixaccessions)
1767 throws IOException
1768 {
1769 // final long startTime = System.currentTimeMillis();
1770
1771 // Submit task to update accessions concurrently.
1772 final Future<?> accessionsUpdate = ThreadUtils.nonCPUThreadPool.submit(new Runnable(){
1773 public final void run()
1774 {
1775 try
1776 {
1777 // Get a simple listing of exhibits on disc.
1778 final AllExhibitImmutableData aeid = _getAllExhibitImmutableData(
1779 LocalProps.getDataDir(),
1780 -1,
1781 !quick);
1782 assert(aeid != null);
1783
1784 // Create/fix accessions files first.
1785 createAccessionFiles(aeid);
1786 }
1787 catch(final IOException e)
1788 {
1789 e.printStackTrace();
1790 throw new Error("failed to create accessions", e);
1791 }
1792 }
1793 });
1794
1795 // If we're being quick then we can't be too careful...
1796 final boolean careful = !quick;
1797
1798 // Wait for accessions update to complete
1799 // so that we can incorporate any new files in the AEP.
1800 try { accessionsUpdate.get(); }
1801 catch(final Exception e) { throw new Error(e); }
1802
1803 // Create an instance.
1804 final ExhibitDataFileSource edfs = new ExhibitDataFileSource();
1805
1806 // Now create and save the cache (if changed)...
1807 System.out.println("[ExhibitDataFileSource.createStaticCacheFile(): started: " +(new Date())+ ".]");
1808 // Get the properties...
1809 // Force reconstruction of AEP from filesystem.
1810 final AllExhibitProperties aep = edfs._getAllExhibitProperties(-1L, careful, true);
1811 System.out.println("[ExhibitDataFileSource.createStaticCacheFile(): *** exhibit count: " + aep.aeid.size() + ".]");
1812
1813 return(aep);
1814 }
1815
1816 /**Create accessions files.
1817 * @return true if any accessions files were created or fixed
1818 */
1819 private static boolean createAccessionFiles(final AllExhibitImmutableData aeid)
1820 throws IOException
1821 {
1822 final AtomicBoolean result = new AtomicBoolean(); // No updates/fixes/creations yet...
1823 final AtomicReference<String> couldNotMakeAccession = new AtomicReference<String>();
1824
1825 final String dataDir = LocalProps.getDataDir();
1826
1827 // Check accession files exist for all exhibits.
1828 logFSAccess("Adding accession files as necessary...", false);
1829
1830 // Queue of tasks yet to complete...
1831 final Queue<Future<?>> tasks = new LinkedBlockingQueue<Future<?>>();
1832
1833 // Do this in approximately sorted order so as to improve disc throughput...
1834 for(final Name.ExhibitFull exhibitName : aeid.getAllExhibitNamesSorted())
1835 {
1836 // These tasks are assumed to be light on CPU, and I/O-bound.
1837 tasks.add(ThreadUtils.nonCPUThreadPool.submit(new Runnable(){
1838 public final void run()
1839 {
1840 try
1841 {
1842 final String newAccessionFilename = ExhibitPropsLoadable.relPathToNewAccession(exhibitName);
1843 final File newAccessionFile = new File(dataDir, newAccessionFilename);
1844 if(!newAccessionFile.exists())
1845 {
1846 // Compute the accession data...
1847 final AccessionData ad = AccessionData.fromExhibitFile(new File(dataDir, exhibitName.toString()));
1848 // Convert to (terse) XML.
1849 final String xml = TextUtils.toXML(ad.getAsDOM(), false, true);
1850 // Convert to UTF-8 text bytes ready to save.
1851 final byte utf8[] = xml.getBytes(CoreConsts.FILE_ENCODING_UTF_8);
1852 // Save new accessions file as near atomically as possible.
1853 final String nAFS = newAccessionFile.toString();
1854 logFSAccess("Creating accession file: "+nAFS, false);
1855 if(FileTools.replacePublishedFile(nAFS, utf8, false))
1856 { result.set(true); }
1857 }
1858 }
1859 catch(final IOException e)
1860 {
1861 // Whinge but continue if we encounter an error.
1862 e.printStackTrace();
1863 couldNotMakeAccession.set("IOException inspecting/making accession file for: " + exhibitName + ": " + e.getMessage());
1864 }
1865 }
1866 }));
1867
1868 // Optimisation: release resources ASAP from any early completed tasks.
1869 // NOTE: handles exceptions differently than final mop-up which may be undesirable.
1870 Future<?> headTask;
1871 while(null != (headTask = tasks.peek()))
1872 { if(headTask.isDone()) { tasks.remove(); } }
1873 }
1874
1875 // Wait for all remaining tasks.
1876 while(!tasks.isEmpty())
1877 {
1878 try { tasks.remove().get(); }
1879 catch(final Exception e)
1880 {
1881 final IOException err = new IOException("did not complete");
1882 err.initCause(e);
1883 throw err;
1884 }
1885 }
1886
1887 if(couldNotMakeAccession.get() != null)
1888 {
1889 System.err.println("ERROR: could not make at least one missing (new) accession file: ");
1890 System.err.println(" REASON: " + couldNotMakeAccession);
1891 }
1892
1893 return(result.get());
1894 }
1895
1896 /**Get requested Properties selected by key and versionID.
1897 * Fetches a Properties set unconditionally (versionID == -1)
1898 * else if the versionID presented is not current.
1899 *
1900 * @param key selector (with possible embedded sub-key)
1901 * for desired properties set; never null
1902 * @param versionID if -1 then map is always returned if available,
1903 * else must be non-negative and null is returned if the versionID
1904 * presented matches that of the current version
1905 * (ie if the caller has presumably got the up-to-date version);
1906 * may be a timestamp or a hash or other value,
1907 * and by convention is zero only for an empty properties set
1908 *
1909 * @return null, or Properties map guaranteed to contain only
1910 * String keys and values
1911 */
1912 public java.util.Properties getProperties(final PropsKey key,
1913 final long versionID)
1914 throws IOException
1915 {
1916 throw new IOException("NOT IMPLEMENTED");
1917 }
1918
1919
1920 /**Run from the command-line with a single argument (the output file name).
1921 * There is an optional <code>-quick</code> first argument,
1922 * which if present which attempts to cap the time spent in one run
1923 * (typically to no more than a few minutes)
1924 * so that any work to be done can be done incrementally
1925 * in several passes if need be.
1926 * <p>
1927 * When making a new static cache file we always try to reload the old one
1928 * (unless the <code>-recompute</code> flag is set, in which case we will not load it)
1929 * to save lots of time (especially for the computed properties)
1930 * and possibly preserve the hashNotChangedSince value.
1931 * <p>
1932 * If the <code>-fixaccessions</code> flag is passed then we try to fix existing
1933 * accessions files that are missing information, eg new checksum types,
1934 * though we will never try to "fix" an incorrect checksum for example
1935 * (that will always result in a warning on the console.)
1936 * <p>
1937 * If the <code>-checkexhibits</code> flag is true
1938 * then we try to verify that each exhibit's exhibit length, timestamp and hash
1939 * matches the values captured in the associated accession file.
1940 * We may check additional data (eg FEC data) in future.
1941 * Note that this option overrides any other options,
1942 * and avoids writing anything to disc.
1943 */
1944 public static void main(final String args[])
1945 {
1946 final long startTime = System.currentTimeMillis();
1947
1948 if(args.length < 1)
1949 {
1950 System.err.println("Expects: " +
1951 "[-quick | -recompute | -fixaccessions | -checkexhibits ] " +
1952 "filename");
1953 System.exit(1);
1954 }
1955
1956 boolean quick = false; // If true, use quicker exhibit-collection method.
1957 boolean recompute = false; // If true, force recompute from scratch; do not reload old cache file.
1958 boolean fixaccessions = false; // If true, fix existing accessions files.
1959 boolean checkexhibits = false; // If true, verify exhibit files are not corrupt.
1960 for(int i = args.length - 1; --i >= 0; )
1961 {
1962 if("-quick".equals(args[i])) { quick = true; }
1963 else if("-recompute".equals(args[i])) { recompute = true; }
1964 else if("-fixaccessions".equals(args[i])) { fixaccessions = true; }
1965 else if("-checkexhibits".equals(args[i])) { checkexhibits = true; }
1966 else
1967 {
1968 System.err.println("Unknown flag '" + args[i] + "'.");
1969 System.exit(1);
1970 return;
1971 }
1972 }
1973
1974 System.out.println("[ExhibitDataFileSource: START: " +(new Date(startTime))+ ".]");
1975
1976 // Get the filename to load/save from/to; always last.
1977 final File staticCacheFilename = new File(args[args.length - 1]);
1978
1979 try {
1980 if(checkexhibits)
1981 {
1982 checkExhibitData(quick, staticCacheFilename.getParentFile());
1983 }
1984 else
1985 {
1986 // Create static cache file and related state.
1987 final AllExhibitProperties aep = createStaticCacheFile(staticCacheFilename, quick, recompute, fixaccessions);
1988 assert(aep != null);
1989
1990 // // Create any thumbnails that are missing.
1991 // System.out.println("[ExhibitDataFileSource.createStaticCacheFile(): thumbnail building starting: " +(new Date())+ ".]");
1992 // final long timeSoFar = Math.max(30000, 7 * (System.currentTimeMillis() - startTime));
1993 // createThumbnails(aep, (!quick) ? Long.MAX_VALUE : (System.currentTimeMillis() + timeSoFar));
1994 // System.out.println("[ExhibitDataFileSource.createStaticCacheFile(): thumbnail building finished: " +(new Date())+ ".]");
1995 }
1996
1997 final long endTime = System.currentTimeMillis();
1998 final long runTime = endTime - startTime;
1999 System.out.println("[ExhibitDataFileSource: END: " +(new Date(endTime))+ ": "+runTime+"ms.]");
2000 }
2001 catch(final Exception e)
2002 {
2003 e.printStackTrace();
2004 System.exit(1); // Exit with error code.
2005 }
2006 }
2007
2008 /**Check exhibit data for corruption.
2009 * @param quick take some optional speed-ups
2010 * @throws IOException in case of difficulty
2011 */
2012 private static void checkExhibitData(final boolean quick,
2013 final File dataDir)
2014 throws IOException
2015 {
2016 System.out.println("Checking exhibit data... (cache file ignored)");
2017 final File baseDir = (dataDir != null) ? dataDir : new File(LocalProps.getDataDir());
2018 // Get a simple listing of exhibits on disc.
2019 final AllExhibitImmutableData aeid = _getAllExhibitImmutableData(
2020 baseDir.getPath(),
2021 -1,
2022 !quick);
2023 assert(aeid != null);
2024 System.out.println("Exhibits found: " + aeid.length);
2025
2026 // Collect existing accession data (as part of the loaded properties),
2027 // and recompute the hashes from scratch to compare,
2028 // and make sure that nothing has changed.
2029 // We do this in parallel to maximise I/O throughput,
2030 // and use a ConcurrentHashMap as our working store as it is thread-safe.
2031 // We get these using our non-CPU-bound thread pool
2032 // and wait for them all to complete.
2033 final List<Name.ExhibitFull> sortedNames = aeid.getAllExhibitNamesSorted();
2034 final Map<Name.ExhibitFull,AccessionData> accessionMap = new ConcurrentHashMap<Name.ExhibitFull, AccessionData>(aeid.size() * 2);
2035 final List<Future<?>> tasks = new ArrayList<Future<?>>(sortedNames.size());
2036 for(final Name.ExhibitFull name : sortedNames)
2037 {
2038 tasks.add(ThreadUtils.nonCPUThreadPool.submit(new Runnable(){
2039 public final void run()
2040 {
2041 // We don't store null/EMPTY values.
2042 try
2043 {
2044 final ExhibitPropsLoadable epl = ExhibitPropsLoadable.getLoadableProperties(name, baseDir);
2045 if((epl != null) && !epl.equals(ExhibitPropsLoadable.EMPTY) && (epl.getAccessionMetadata() != null))
2046 {
2047 final AccessionData accessionMetadata = epl.getAccessionMetadata();
2048 accessionMap.put(name, accessionMetadata);
2049
2050 // Compare the accession timestamp and length with the current file.
2051 final ExhibitStaticAttr esa = aeid.getStaticAttr(name);
2052 if(esa == null)
2053 { throw new Error("Missing exhibit: "+name); }
2054 if(accessionMetadata.date == null)
2055 { System.err.println("Accession timestamp missing for: " + name); }
2056 else if(accessionMetadata.date.longValue() != esa.timestamp)
2057 { System.err.println("Accession timestamp does not match current file for: " + name + ": was/now " + new Date(accessionMetadata.date.longValue()) + '/' + new Date(esa.timestamp)); }
2058 if(accessionMetadata.size == null)
2059 { System.err.println("Accession length missing for: " + name); }
2060 else if(accessionMetadata.size.longValue() != esa.length)
2061 { System.err.println("Accession length does not match current file for: " + name + ": was/now " + accessionMetadata.size + '/' + esa.length); }
2062
2063 // Now recompute and compare the accession data hashes.
2064 final InputStream is = new BufferedInputStream(new FileInputStream(new File(baseDir, name.toString())));
2065 try
2066 {
2067 final Tuple.Pair<Integer,ROByteArray> hashes = AccessionData.computeFullFileHashes(is);
2068 if(!hashes.first.equals(accessionMetadata.hashCRC32))
2069 { System.err.println("Accession hash CRC32 changed for: " + name); }
2070 if(!hashes.second.equals(accessionMetadata.hashMD5))
2071 { System.err.println("Accession hash MD5 changed for: " + name); }
2072 }
2073 finally { is.close(); }
2074 }
2075 else
2076 { System.err.println("No accession data available for: " + name); }
2077 }
2078 catch(final Exception e)
2079 { throw new RuntimeException("Error loading accession data for: " + name, e); }
2080 }
2081 }));
2082 }
2083 // Wait for all tasks to finish...
2084 for(final Future<?> task : tasks)
2085 {
2086 try { task.get(); }
2087 catch(final InterruptedException e)
2088 {
2089 final InterruptedIOException err = new InterruptedIOException(e.getMessage());
2090 err.initCause(e);
2091 throw err;
2092 }
2093 catch(final ExecutionException e)
2094 {
2095 final IOException err = new IOException(e.getMessage());
2096 err.initCause(e);
2097 throw err;
2098 }
2099 }
2100 System.out.println("Accession data loaded for exhibits: " + accessionMap.size());
2101 if(accessionMap.size() != aeid.length)
2102 { throw new IOException("Missing (or changed) accession data."); }
2103 }
2104
2105 /**Assumes that this instance is the root/master and so returns Stratum.ROOT; non-null. */
2106 public Stratum getStratum() { return(Stratum.ROOT); }
2107
2108 /**Shut down the data pipeline.
2109 * Saves any pending state (eg log entries and variables)
2110 * to disc if possible.
2111 * <p>
2112 * Has no upstream components to shut down
2113 * nor significant resources (memory) to release.
2114 */
2115 /* @Override */ public void destroy()
2116 {
2117 _saveSystemVariables(varMgr, true); // Force immediate save.
2118 }
2119 }