001 /*
002 Copyright (c) 1996-2012, Damon Hart-Davis
003 All rights reserved.
004
005 Redistribution and use in source and binary forms, with or without
006 modification, are permitted provided that the following conditions are
007 met:
008
009 * Redistributions of source code must retain the above copyright
010 notice, this list of conditions and the following disclaimer.
011
012 * Redistributions in binary form must reproduce the above copyright
013 notice, this list of conditions and the following disclaimer in the
014 documentation and/or other materials provided with the
015 distribution.
016
017 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
018 IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
019 TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
020 PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
021 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
022 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
023 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
024 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
025 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
026 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
027 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
028 */
029
030 package org.hd.d.pg2k.svrCore;
031
032 import java.io.BufferedInputStream;
033 import java.io.BufferedOutputStream;
034 import java.io.BufferedReader;
035 import java.io.ByteArrayInputStream;
036 import java.io.ByteArrayOutputStream;
037 import java.io.EOFException;
038 import java.io.File;
039 import java.io.FileFilter;
040 import java.io.FileInputStream;
041 import java.io.FileNotFoundException;
042 import java.io.FileOutputStream;
043 import java.io.IOException;
044 import java.io.InputStream;
045 import java.io.InputStreamReader;
046 import java.io.ObjectInputStream;
047 import java.io.ObjectOutputStream;
048 import java.io.OutputStream;
049 import java.nio.ByteBuffer;
050 import java.nio.ByteOrder;
051 import java.util.Collections;
052 import java.util.Properties;
053 import java.util.SortedMap;
054 import java.util.TreeMap;
055 import java.util.concurrent.locks.ReentrantReadWriteLock;
056 import java.util.zip.ZipEntry;
057 import java.util.zip.ZipException;
058 import java.util.zip.ZipInputStream;
059
060 import org.hd.d.pg2k.svrCore.Name.ExhibitFull;
061 import org.hd.d.pg2k.svrCore.datasource.SimpleExhibitPipelineIF;
062
063 /**This class has tools for common file operations.
064 * (Derived from old FileTools 1.100 01/02/03.)
065 */
066 public final class FileTools
067 {
068 /**Prevent creation of instances of this class. */
069 private FileTools() { }
070
071 /**Prefix used on temporary files, eg while doing atomic replacements.
072 * This used to be in GlobalParams but we may even need it
073 * while loading GlobalParams.
074 */
075 public static final String F_tmpPrefix = ".tmp.pg.";
076
077 /**Rough estimate of file-system allocation size for usage estimate (bytes); positive power of 2.
078 * We should separately round up both the file size and the filename size to
079 * a multiple of this in order to make a conservative estimate of
080 * filesystem usage when we are creating files, and especially if we
081 * cannot measure our actual usage.
082 * <p>
083 * Magnetic disc is often 512-byte blocks and/or 1kB fragments.
084 * Optical disc is often 1kB blocks.
085 * Flash memory SSD is often 2kB blocks.
086 * <p>
087 * Something around 512--8192 bytes is reasonable for many filesystems and media.
088 */
089 public static final int FS_EST_BLOCK_SIZE_BYTES = 1 << 12; // 4kB.
090
091
092 /**Get the extension of a file name, not including the leading dot.
093 * Returns null if no extension present.
094 */
095 public static final String getExtension(final CharSequence name)
096 {
097 final int pos = TextUtils.lastIndexOf(name, '.');
098 if(pos == -1) { return(null); }
099 final String extension = name.subSequence(pos + 1, name.length()).toString();
100 if(extension.indexOf(File.separatorChar) != -1)
101 { return(null); } // Whoops, was not really extension.
102 return(extension);
103 }
104
105
106 /**Private lock for runCmd. */
107 private static final Object _runCmdLock = new Object();
108
109 /**If _runCmdFix is true, we are working round a JDK 1.2 bugs in Process.
110 * In the JDK 1.2 bug, if some more than one thread runs a Process,
111 * even if not simultaneously, there is a small change of one
112 * spawned by a thread that is not the main thread hanging.
113 */
114 private static final boolean _runCmdFix = false;
115
116 /**Private object used for signalling within the _runCmdLock.
117 * Used as a signal between the runner thread and the runCmd() code
118 * that a new command is being posted or the result is
119 * ready.
120 */
121 private static final Object _runCmdSignal = new Object();
122 /**Private location for runCmd() where a new command's args are put.
123 * Null when no command currently running.
124 * <p>
125 * When this is set non-null by runCmd() it does a notifyAll() on
126 * _runCmdSignal, then waits on the same signal until
127 * _runCmdResult becomes non-null. This is cleared by
128 * the runner thread when it accepts and starts to run the
129 * new command.
130 * <p>
131 * Change under the _runCmdSignal lock.
132 */
133 private static String[] _runCmdArgs;
134 /**The result of running a command.
135 * Thgis can be Integer for an exit value,
136 * or IOException or a RuntimeException.
137 * <p>
138 * Set non-null by the runner thread (and a signal sent on
139 * _runCmdSignal) when the command terminates; cleared
140 * by runCmd() on accepting the result.
141 * <p>
142 * Change under the _runCmdSignal lock.
143 */
144 private static Object _runCmdResult;
145 /**The runner thread for commands.
146 * Assumed immortal.
147 */
148 private static final Thread _cmdRunner = !_runCmdFix ? null :
149 (new RunnerThread());
150 /**Initialise (as a daemon) and start the runner thread. */
151 static
152 {
153 if(_runCmdFix)
154 {
155 _cmdRunner.setDaemon(true); // Don't prevent process exiting.
156 _cmdRunner.start();
157 }
158 }
159
160 /**Runs a command, and returns the exit status.
161 * <EM>Does not expect the process to use stdin/stdout/stderr;
162 * the process may hang if it tries to use them.</EM>
163 * <p>
164 * Because of possible deadlock/hang problems in JDK1.2,
165 * I am serialising all sub-processing runs at the cost of
166 * some parallelism.
167 */
168 public static int runCmd(final String cmd[])
169 throws IOException
170 {
171 if((cmd == null) || (cmd.length < 1))
172 { throw new IllegalArgumentException(); }
173 synchronized(_runCmdLock)
174 {
175 if(_runCmdFix)
176 {
177 // Use elaborate work-round for JDK 1.2 bug.
178 //
179 // The args/result values should always be null,
180 // and the system quiescent when we get here, but the
181 // runner should be alive.
182 if((_runCmdArgs != null) || (_runCmdResult != null))
183 { throw new Error("INTERNAL ERROR: messed-up state"); }
184 if((_cmdRunner == null) || !_cmdRunner.isAlive())
185 { throw new Error("INTERNAL ERROR: runner missing or dead: " + _cmdRunner); }
186
187 // Set the args and signal the runner to wake it up.
188 synchronized(_runCmdSignal)
189 {
190 _runCmdArgs = cmd;
191 _runCmdSignal.notifyAll();
192 }
193
194 // Now sleep until signalled to collect the results
195 // (and set them to null when we get them).
196 Object result = null;
197 synchronized(_runCmdSignal)
198 {
199 while((result = _runCmdResult) == null)
200 {
201 try { _runCmdSignal.wait(10000); }
202 catch(final InterruptedException e) { } // Ignore.
203 if(!_cmdRunner.isAlive())
204 { throw new RuntimeException("INTERNAL ERROR: runner has died: " + _cmdRunner); }
205 }
206 _runCmdResult = null;
207 }
208
209 // Return our (non-null) result.
210 if(result instanceof Integer)
211 {
212 // Exit code.
213 return(((Integer) result).intValue());
214 }
215 if(result instanceof IOException)
216 {
217 // Expected IOException.
218 throw ((IOException) result);
219 }
220 throw new Error("unexpected error: " + result);
221 }
222
223 // The simple route...
224 final Runtime r = Runtime.getRuntime();
225 final Process p = r.exec(cmd);
226 p.getInputStream().close(); // Make sure it does not wait for stdin.
227 p.getOutputStream().close();
228 p.getErrorStream().close();
229
230 // We should really ensure we read and stdout/stderr generated.
231
232 for( ; ; ) // Loop until we have reaped the child...
233 {
234 try { return(p.waitFor()); }
235 catch(final InterruptedException e) { }
236 }
237 }
238 }
239
240 /**Recursively removes specified file/directory.
241 * Equivalent of UNIX "rm -rf file".
242 * <p>
243 * Fails if it cannot remove the target,
244 * but does not complain if the target does not exist.
245 */
246 public static void rmRecursively(final File fileOrDir)
247 throws IOException
248 { rmRecursively(fileOrDir, null); }
249
250 /**Recursively removes specified file/directory.
251 * Equivalent of UNIX "rm -rf file".
252 * <p>
253 * Fails if it cannot remove the target,
254 * but does not complain if the target does not exist.
255 *
256 * @param filter matches files/dirs to remove;
257 * null to remove all files/dirs from that specified
258 */
259 public static void rmRecursively(final File fileOrDir,
260 final FileFilter filter)
261 throws IOException
262 {
263 if(fileOrDir == null)
264 { throw new IllegalArgumentException(); }
265
266 // If target does not exist, return quietly.
267 if(!fileOrDir.exists())
268 { return; }
269
270 // If a directory then recursively deal with any contents first.
271 if(fileOrDir.isDirectory())
272 {
273 final String files[] = fileOrDir.list();
274 if(files != null)
275 {
276 for(int i = files.length; --i >= 0; )
277 {
278 final String file = files[i];
279 // Skip UNIX-style "." and ".." entries...
280 if(".".equals(file) || "..".equals(file))
281 { continue; }
282
283 // Remove/investigate any other entry...
284 rmRecursively(new File(fileOrDir, file), filter);
285 }
286 }
287 }
288
289 // Attempt to remove immediate target directly
290 // (unless a filter is supplied and this file is not accepted).
291 if((filter != null) && !filter.accept(fileOrDir))
292 { return; }
293
294 // Delete since no filter,
295 // or the file is accepted by the filter supplied...
296 fileOrDir.delete();
297
298 // Make sure the target has been removed.
299 if(fileOrDir.exists())
300 { throw new IOException("cannot remove: " + fileOrDir); }
301 }
302
303
304 /**Replaces an existing published file with a new one (see 3-arg version).
305 * Is verbose when it replaces the file.
306 */
307 public static boolean replacePublishedFile(final String name, final byte data[])
308 throws IOException
309 { return(replacePublishedFile(name, data, false)); }
310
311 /**Replaces an existing published file with a new one.
312 * This replaces (atomically if possible) the existing file (if any)
313 * of the given name, ensuring the correct permissions for
314 * a file to be published with a Web server (ie basically
315 * global read permissions), provided the following
316 * conditions are met:
317 * <p>
318 * <ul>
319 * <li>The filename extension is acceptable (not checked yet).
320 * <li>The data array is non-null and not zero-length.
321 * <li>The content of the data array is different to the file.
322 * <li>All the required permissions are available.
323 * </ul>
324 * <p>
325 * If the file is successfully replaced, true is returned.
326 * <p>
327 * If the file does not need replacing, false is returned
328 * (and the file is not replaced or touched).
329 * <p>
330 * If an error occurs, eg in the input data or during file
331 * operations, an IOException is thrown.
332 * <p>
333 * This routine enforces locking so that only one such
334 * operation may be performed at any one time. This does
335 * not avoid the possibility of externally-generated races.
336 * <p>
337 * The final file, once replaced, will be globally readable,
338 * and writable by us.
339 * <p>
340 * (If the final component of the file starts with ".",
341 * then the file will be accessible only by us.)
342 *
343 * @param quiet if true then only error messages will be output
344 */
345 public static boolean replacePublishedFile(final String name, final byte data[],
346 final boolean quiet)
347 throws IOException
348 {
349 if((name == null) || (name.length() == 0))
350 { throw new IOException("inappropriate file name"); }
351 final int length = data.length;
352 if((data == null) || (length == 0))
353 { throw new IOException("inappropriate file content"); }
354
355 final File extant = new File(name);
356
357 // Lock the critical external bits against read and write updates.
358 rPF_rwlock.writeLock().lock();
359 try
360 {
361 final File tempFile = makeTempFileNameInSameDirAsTarget(extant);
362
363 // Get extant file's length.
364 final long oldLength = extant.length();
365 // Should we overwrite it?
366 boolean overwrite = (oldLength < 1); // Missing or zero length.
367
368 // If length has changed, we should overwrite.
369 if(length != oldLength) { overwrite = true; }
370
371 // Now, if we haven't already decided to overwrite the file,
372 // check the content.
373 if(!overwrite)
374 {
375 try
376 {
377 final InputStream is = new BufferedInputStream(
378 new FileInputStream(extant));
379 try
380 {
381 final int l = length;
382 for(int i = 0; i < l; ++i)
383 {
384 if(data[i] != is.read())
385 { overwrite = true; break; }
386 }
387 }
388 finally
389 { is.close(); }
390 }
391 catch(final FileNotFoundException e) { overwrite = true; }
392 }
393
394 // OK, we don't want to overwrite, so return.
395 if(!overwrite) { return(false); }
396
397
398 // OVERWRITE OLD FILE WITH NEW...
399
400 try {
401 // Write new temp file...
402 // (Allow any IOException to terminate the function.)
403 FileOutputStream os = new FileOutputStream(tempFile);
404 try
405 {
406 os.write(data);
407 // Force to underlying media (eg fsync()).
408 os.flush();
409 os.getFD().sync();
410 }
411 finally { os.close(); }
412 os = null; // Help GC.
413 if(tempFile.length() != length)
414 { new IOException("temp file not written correctly"); }
415
416 // Attempt to atomically (or nearly so) replace extant file with tempfile.
417 return(_atomicishFileReplace(extant, tempFile, length, quiet));
418 }
419 finally // Tidy up...
420 {
421 tempFile.delete(); // Remove the temp file.
422 }
423 }
424 finally { rPF_rwlock.writeLock().unlock(); }
425
426 // Can't get here...
427 }
428
429 /**Attempt to atomically (or nearly so) replace extant file with tempfile.
430 * FIXME: Where renameTo() not atomic, rename old file to .bak version to preserve it in case of crash
431 */
432 private static boolean _atomicishFileReplace(final File extant, final File tempFile,
433 final int length, final boolean quiet)
434 throws IOException
435 {
436 final boolean globalRead = !extant.getName().startsWith(".");
437
438 // Ensure that the temp file has the correct read permissions.
439 tempFile.setReadable(true, !globalRead);
440 tempFile.setWritable(true, true);
441
442 // Warn if target does not have write perms, and try to add them.
443 // This should allow us to replace it with the new file.
444 final boolean alreadyExists = extant.exists();
445 if(alreadyExists && !extant.canWrite())
446 {
447 System.err.println("FileTools.replacePublishedFile(): "+
448 "WARNING: " + extant + " not writable.");
449 extant.setWritable(true, true);
450 if(!extant.canWrite())
451 {
452 throw new IOException("can't make target writable");
453 }
454 }
455
456 // (Atomically) move tempFile to extant file.
457 // Note that renameTo() may not be atomic
458 // and we may have to remove the target file first.
459 if(!tempFile.renameTo(extant))
460 {
461 // If the target already exists,
462 // then be prepared to explicitly delete it.
463 if(!alreadyExists || !extant.delete() || !tempFile.renameTo(extant))
464 { throw new IOException("renameTo/update of "+extant+" failed"); }
465 if(!quiet) { System.err.println("[WARNING: atomic replacement not possible for: " + extant + ": used explicit delete.]"); }
466 }
467
468 if((length >= 0) && (extant.length() != length))
469 { new IOException("update of "+extant+" failed"); }
470 extant.setReadable(true, !globalRead);
471 extant.setWritable(true, true);
472 if(!quiet) { System.err.println("["+(alreadyExists?"Updated":"Created")+" " + extant + "]"); }
473 return(true); // All seems OK.
474 }
475
476 /**Create a new temporary filename for the same directory as the extant file; never null.
477 * A file in the same directory is usually guaranteed to be in the same filesystem,
478 * and thus may often allow an atomic update/replace
479 * and in any case ensures that we cannot run out of space
480 * (barring other concurrent unrelated activity in the filesystem)
481 * when we try to replace the extant file with the temporary one.
482 * <p>
483 * To avoid internal races this should be generated and used
484 * within the scope of our internal 'filesystem update' lock.
485 * <p>
486 * The generated name starts with the 'temporary' prefix.
487 */
488 private static File makeTempFileNameInSameDirAsTarget(final File extant)
489 {
490 // Use a temporary file in the same directory (and thus the same filesystem)
491 // to avoid unexpectedly truncating the file when copying/moving it.
492 File tempFile;
493 for( ; ; )
494 {
495 tempFile = new File(extant.getParent(),
496 F_tmpPrefix +
497 Long.toString((Rnd.fastRnd.nextLong() >>> 1),
498 Character.MAX_RADIX) /* +
499 "." +
500 extant.getName() */ ); // Avoid making very long names...
501 if(tempFile.exists())
502 {
503 System.err.println("WARNING: FileTools.replacePublishedFile(): "+
504 "temporary file " + tempFile.getPath() +
505 " exists, looping...");
506 continue;
507 }
508 break;
509 }
510 return(tempFile);
511 }
512
513 /**Private lock for replacePublishedFile().
514 * We use a read/write lock to improve available concurrency.
515 * <p>
516 * TODO: We could extend this to a lock per distinct directory or filesystem.
517 */
518 private static final ReentrantReadWriteLock rPF_rwlock = new ReentrantReadWriteLock();
519
520 /**Makes a publicly-readable directory path if not already present.
521 * Like File.mkdirs(), but attempts to ensure that any directory
522 * component created by this routine is publicly readable
523 * and searchable, ie at least permissions read and execute
524 * for all. Final permissions will usually be 0755,.
525 * <p>
526 * Optionally, a new empty index.html file can be created
527 * inside any directory that is created as a simple Web security
528 * precaution, at least until something is put in its place.
529 * <p>
530 * If this routine fails it may have succeeded in creating some of the necessary
531 * parent directories.
532 * <p>
533 * This shares a lock with replacePublishedFile().
534 * <p>
535 * (This routine should maybe be merged with makeHTMLSubDirs(),
536 * though the relationship is not trivial.)
537 *
538 * @return <code>true</code> if and only if the directory was created,
539 * along with all necessary parent directories; <code>false</code>
540 * otherwise
541 */
542 public static boolean makePublishingDir(final File path)
543 throws IOException
544 {
545 rPF_rwlock.writeLock().lock();
546 try
547 {
548 if(path.exists()) { return(false); } // Nothing we can do...
549 if(path.mkdir())
550 {
551 path.setWritable(true, true);
552 path.setReadable(true, false);
553 path.setExecutable(true, false);
554 return(true);
555 }
556 final String parent = path.getParent();
557 if(parent == null) { return(false); }
558 if(!makePublishingDir(new File(parent))) { return(false); }
559 if(!path.mkdir()) { return(false); }
560 path.setWritable(true, true);
561 path.setReadable(true, false);
562 path.setExecutable(true, false);
563 return(true);
564 }
565 finally { rPF_rwlock.writeLock().unlock(); }
566 }
567
568 /**Read text file into a String.
569 * Reads a line at a time, trimming whitespace off either
570 * end and putting in a single new line at the end instead.
571 * <p>
572 * This treats the file as ISO-8859-1 8-bit data.
573 *
574 * @exception IOException in case of trouble
575 */
576 public static String readTextFile(final File f)
577 throws IOException
578 {
579 final StringBuilder sb = new StringBuilder((int) f.length());
580 BufferedReader br = null;
581 try {
582 br = new BufferedReader(
583 new InputStreamReader(
584 new FileInputStream(f), CoreConsts.FILE_ENCODING_8859_1));
585 String inputLine;
586 while((inputLine = br.readLine()) != null)
587 { sb.append(inputLine.trim()); sb.append('\n'); }
588 }
589 finally
590 {
591 // Try to close the file (possibly provoking an exception)...
592 if(br != null) { br.close(); br = null; }
593 }
594 return(sb.toString());
595 }
596
597 /**If true then try to stream (often large) serialised objects directly to the filesystem.
598 * If true the serialiseToFile() will always write something to the filesystem
599 * (and update the timestamp at least) even if nothing has changed,
600 * so the caller should probably try to avoid redundant/frequent calls.
601 */
602 public static final boolean STF_ALWAYS_STREAMS_TO_FILESYSTEM = true;
603
604 /**Given a file, serialises an object to it.
605 * This atomically replaces the target file if possible.
606 * <p>
607 * This may be streamed directly to the filesystem
608 * (though replacing and extant file atomically once fully written)
609 * to avoid running out of memory with very large objects.
610 * <p>
611 * The usual internal 'filesystem update' lock is help while this works.
612 *
613 * @param gzipped if true, the file is written GZIP-compressed
614 * to (usually) save significant space
615 * @param quiet if true, only outputs errors
616 *
617 * @throws IOException if something bad happens
618 * @return true if a file was replaced,
619 * false it was already present with the correct content
620 */
621 public static boolean serialiseToFile(final Object o,
622 final File filename, final boolean gzipped,
623 final boolean quiet)
624 throws IOException
625 {
626 if(!STF_ALWAYS_STREAMS_TO_FILESYSTEM)
627 {
628 // Use replacePublishedFile().
629 ObjectOutputStream oos = null;
630 try {
631 // Serialise to a byte[].
632 ByteArrayOutputStream baos = new ByteArrayOutputStream();
633 // We may gzip the stream to remove some redundancy.
634 final java.util.zip.GZIPOutputStream gos = gzipped ?
635 new java.util.zip.GZIPOutputStream(baos) : null;
636 oos = new ObjectOutputStream(gzipped ?
637 (OutputStream)gos : (OutputStream)baos);
638 oos.writeObject(o);
639 oos.flush();
640 if(gos != null) { gos.finish(); }
641 final byte data[] = baos.toByteArray();
642 baos = null; // Help GC as this may be large...
643
644 // Now save to file using pure Java APIs.
645 // Write-locked against other file replacements.
646 return(FileTools.replacePublishedFile(filename.getPath(), data, quiet));
647 }
648 finally
649 {
650 if(oos != null) { oos.close(); } // Free up OS resources.
651 }
652 }
653
654
655 // Stream directly to the filesystem
656 // which should save a lot of heap space
657 // especially for GZIPped output.
658
659 // Lock the critical external bits against read and write updates.
660 rPF_rwlock.writeLock().lock();
661 try
662 {
663 final File tempFile = makeTempFileNameInSameDirAsTarget(filename);
664
665 // REPLACE OLD FILE WITH NEW...
666
667 try {
668 // Write new temp file...
669 // (Allow any IOException to terminate the function.)
670 FileOutputStream os = new FileOutputStream(tempFile);
671 try
672 {
673 final BufferedOutputStream bos = new BufferedOutputStream(os, 1<<16); // Write big chunks for efficiency.
674
675 // We may gzip the stream to remove some redundancy.
676 final java.util.zip.GZIPOutputStream gos = gzipped ?
677 new java.util.zip.GZIPOutputStream(bos) : null;
678 final ObjectOutputStream oos = new ObjectOutputStream(gzipped ?
679 (OutputStream)gos : (OutputStream)bos);
680 oos.writeObject(o);
681 oos.flush();
682 if(gos != null) { gos.finish(); }
683
684 // Force to underlying media (eg fsync()).
685 bos.flush();
686 os.getFD().sync();
687 }
688 finally { os.close(); }
689 os = null; // Help GC.
690 if(tempFile.length() < 1) // Could use real minimum size...
691 { new IOException("temp file not written correctly"); }
692
693 // Attempt to atomically (or nearly so) replace extant file with tempfile.
694 return(_atomicishFileReplace(filename, tempFile, -1, quiet));
695 }
696 finally // Tidy up...
697 {
698 tempFile.delete(); // Remove the temp file.
699 }
700 }
701 finally { rPF_rwlock.writeLock().unlock(); }
702 }
703
704 // /**Minimum compressed serialised file size to insert a data pump.
705 // * With a data pump we will try to get the I/O and decompression in one thread
706 // * and the deserialisation in another.
707 // * <p>
708 // * The main value of this would be in overcoming I/O delays on large files
709 // * but some useful CPU concurrency may also be available.
710 // * <p>
711 // * We may not bother at any size on a uni-processor.
712 // */
713 // private static int MIN_DATAPUMP_SERGZ_FILESIZE = 1024 * 1024;
714
715 /**Given a file, deserialises an object from it.
716 * This buffers the input for efficiency.
717 *
718 * @param gzipped if true, the file is assumed to be in GZIP
719 * format and the stream is decompressed on the fly
720 * @throws IOException if something bad happens
721 */
722 public static Object deserialiseFromFile(final File filename, final boolean gzipped)
723 throws IOException
724 {
725 final long length = filename.length();
726 // final boolean useDataPump =
727 // (ThreadUtils.AVAILABLE_PROCESSORS > 1) && gzipped && (length >= MIN_DATAPUMP_SERGZ_FILESIZE);
728 final int bufSize = (int) Math.min(8192, length);
729
730 ObjectInputStream ois = null;
731
732 // Lock out concurrent published-file updates.
733 rPF_rwlock.readLock().lock();
734 try {
735 // Need to reload from disc.
736 InputStream is = new BufferedInputStream(new FileInputStream(
737 filename), bufSize);
738 if(gzipped) { is = new java.util.zip.GZIPInputStream(is, bufSize); }
739 // if(useDataPump) { is = GenUtils.dataPump(is, bufSize, true); }
740 ois = new ObjectInputStream(is);
741 return(ois.readObject());
742 }
743 catch(final ClassNotFoundException e)
744 { throw new IOException("ClassNotFoundException deserialising file ``"+filename+"'': " + e.getMessage()); }
745 catch(final Exception e)
746 {
747 final IOException err = new IOException("unexpected exception deserialising file ``"+filename+"'': " + e.getMessage(), e);
748 throw err;
749 }
750 finally
751 {
752 // Potentially allow updates again.
753 rPF_rwlock.readLock().unlock();
754
755 // Ensure that we release OS file-handling resources.
756 if(ois != null) { ois.close(); }
757 }
758 }
759
760 /**Load properties for the given InputStream.
761 * In case of any problem reading the stream for properties
762 * an IOException is thrown.
763 * <p>
764 * Also, if the property ``end'' is not defined with the
765 * value OK, an EOFException is thrown to indicate that the
766 * file was not read completely (eg if we catch it in the
767 * middle of a save). So make sure that files are defined
768 * with this property at the end.
769 *
770 * @throws IOException in case of general I/O problems
771 * @throws EOFException in case of missing <sample>end=OK</sample> value
772 */
773 public static Properties loadProperties(final InputStream is)
774 throws IOException
775 {
776 final Properties props = new Properties();
777 props.load(is);
778 if(!"OK".equals(props.getProperty("end")))
779 { throw new EOFException("missing end=OK at end of data: possibly corrupt or not fully saved?"); }
780 return(props);
781 }
782
783
784 /**Typical size in bytes/characters of (English, ASCII) text below which deflate/zlib compression unlikely to be very effective.
785 * This is for typical (English, ASCII) text given the overheads of
786 * the compression methods etc.
787 * <p>
788 * Avoiding trying to compress things smaller than this
789 * may save lots of CPU time without too much memory/space cost,
790 * and may allow better compression of aggregates.
791 */
792 public static final int TYPICAL_DEFLATE_MIN_TEXT_SIZE_COMPRESSABLE = 256;
793
794 /**Compress data (maximally deflate without zlib/gzip headers and footers) from byte[] to byte[]; never null.
795 * Equivalent to <code>ompressDeflatableData(in, 0, in.length)</code>.
796 * <p>
797 * This routine will not alter the content of the input array.
798 */
799 public static byte[] compressDeflatableData(final byte in[])
800 { return(compressDeflatableData(in, 0, in.length)); }
801
802 /**Compress data (maximally deflate without zlib/gzip headers and footers) from byte[] to byte[]; never null.
803 * This means that there is no checksum,
804 * so any integrity checking required had better be done elsewhere.
805 * <p>
806 * This may increase the size of the data in some cases,
807 * especially where the input array is short.
808 * <p>
809 * This routine will not alter the content of the input array.
810 */
811 public static byte[] compressDeflatableData(final byte in[], final int off, final int len)
812 {
813 try
814 {
815 // Data expansion is unusual except for short inputs.
816 // We assume that normally we get some compression
817 // and size the working buffer accordingly.
818 final ByteArrayOutputStream baos =
819 new ByteArrayOutputStream((3*in.length)/4 + 32);
820 final DefOutputStream cos =
821 new DefOutputStream(baos);
822 try
823 {
824 // Write uncompressed data to stream.
825 cos.write(in, off, len);
826 // Force everything out...
827 cos.finish();
828 cos.flush();
829 return(baos.toByteArray());
830 }
831 finally
832 {
833 cos.close(); // Free resources, including non-Java resources.
834 }
835 }
836 catch(final IOException e)
837 {
838 // Should never happen...
839 throw new Error("unexpected internal error", e);
840 }
841 }
842
843 /**Decompress deflated data (without zlib/gzip headers and footers) from byte[] to byte[].
844 * This means that there is no checksum,
845 * so any integrity checking required had better be done elsewhere.
846 * <p>
847 * This routine will not alter the content of the input array.
848 *
849 * @throws IOException in case of difficulty, such as corrupt data
850 */
851 public static byte[] decompressDeflatedData(final byte in[])
852 throws IOException
853 {
854 // Uncompress from the input stream...
855 final InputStream cis =
856 new DefInputStream(
857 new ByteArrayInputStream(in));
858 try
859 {
860 // Write to a new output stream in memory.
861 // Assume that a reasonable compression factor
862 // will have been obtained when sizing the
863 // internal buffer.
864 final int guessedExpandedSize = 1 + in.length * 2; // Always > 0.
865 final ByteArrayOutputStream boas = new ByteArrayOutputStream(guessedExpandedSize);
866 // Copy from input to output until EOF,
867 // using a small buffer for efficiency.
868 final byte buf[] = new byte[Math.min(512, guessedExpandedSize)];
869 for( ; ; )
870 {
871 final int n = cis.read(buf);
872 if(n == -1) { break; } // EOF.
873 boas.write(buf, 0, n);
874 }
875 // Use the uncompressed data!
876 final byte uncompressedData[] = boas.toByteArray();
877 boas.close();
878 return(uncompressedData);
879 }
880 finally { cis.close(); /* Release Java and non-Java resources. */ }
881 }
882
883 /**Rounds up a byte length to the next block size.
884 * Return is no smaller than input value.
885 * <p>
886 * Input value must be non-negative.
887 * <p>
888 * An input value of 0 results in a return value of 0.
889 */
890 public static long roundUpToFSBlockSize(final long length)
891 {
892 assert length >= 0;
893 return(((length + FS_EST_BLOCK_SIZE_BYTES - 1) / FS_EST_BLOCK_SIZE_BYTES) * FS_EST_BLOCK_SIZE_BYTES);
894 }
895
896 /**Thread to encapsulate execution of command. */
897 private static final class RunnerThread extends Thread {
898 public RunnerThread() {
899 super("FileTools runCmd runner");
900 }
901
902 @Override
903 public final void run()
904 {
905 //System.err.println("FileTools runCmd runner starting...");
906 for( ; ; )
907 {
908 try {
909 // Get new command...
910 String cmd[] = null;
911 synchronized(_runCmdSignal)
912 {
913 while((cmd = _runCmdArgs) == null)
914 {
915 try { _runCmdSignal.wait(); }
916 catch(final InterruptedException e) { } // Ignore.
917 }
918 _runCmdArgs = null;
919 }
920
921 //System.err.println("FileTools runner go... " + cmd[0]);
922
923 // Now run the command...
924 Object result = null;
925 try {
926 final Runtime r = Runtime.getRuntime();
927 final Process p = r.exec(cmd);
928 p.getInputStream().close(); // Make sure it does not wait for stdin.
929 p.getOutputStream().close();
930 p.getErrorStream().close();
931
932 // We should really ensure we read and stdout/stderr generated.
933
934 for( ; ; ) // Loop until we have reaped the child...
935 {
936 try {
937 result = new Integer(p.waitFor());
938 break;
939 }
940 catch(final InterruptedException e) { }
941 }
942 }
943 catch(final IOException e) { result = e; }
944 catch(final RuntimeException e) { result = e; }
945
946 // Now return the result, which we ensure is non-null.
947 if(result == null) { result = new Error("INTERNAL ERROR"); }
948 synchronized(_runCmdSignal)
949 {
950 _runCmdResult = result;
951 _runCmdSignal.notifyAll();
952 }
953 //System.err.println("FileTools runner done.");
954 }
955 catch(final Throwable t)
956 {
957 System.err.println("FileTools cmd runner caught: " + t.getMessage());
958 t.printStackTrace();
959 }
960 }
961 }
962 }
963
964 /**Wrap an exhibit as a stream; never null. */
965 public static AllExhibitProperties.ExhibitDataSource wrapExhibitAsStream(
966 final SimpleExhibitPipelineIF dataSource)
967 {
968 return(new AllExhibitProperties.ExhibitDataSource(){
969 @Override public final void getRawFile(final ByteBuffer buf, final ExhibitFull exhibitName, final int position) throws IOException
970 { dataSource.getRawFile(buf, exhibitName, position, false); }
971 });
972 }
973
974 /**Random-access read-only contiguous file accessor. */
975 public static interface RandomAccessData
976 {
977 /**Read up to the limit in the supplied buffer.
978 * If a full read is not possible (eg too close to EOF) then an exception is thrown.
979 */
980 public void readFully(final ByteBuffer buf, final int position) throws IOException;
981 /**Length of the data in bytes. */
982 public long length() throws IOException;
983 }
984
985 /**Wrap a single exhibit as RandomAccessData; never null. */
986 public static RandomAccessData wrapExhibitAsRandomAccessData(
987 final SimpleExhibitPipelineIF dataSource, final Name.ExhibitFull exhibitName)
988 {
989 return(new RandomAccessData(){
990 public final void readFully(final ByteBuffer buf, int position) throws IOException
991 {
992 for( ; ; )
993 {
994 final int remBefore = buf.remaining();
995 dataSource.getRawFile(buf, exhibitName, position, false);
996 final int remAfter = buf.remaining();
997 if(remAfter == 0) { return; /* All done. */ }
998 final int xferred = (remBefore - remAfter);
999 assert(xferred >= 0);
1000 if(xferred == 0) { throw new IOException("unable to complete read, possibly at EOF"); }
1001 position += xferred; // Allow for bytes just transferred.
1002 }
1003 }
1004 public final long length() throws IOException
1005 { return(dataSource.getStaticAttr(exhibitName).length); }
1006 });
1007 }
1008
1009 /**Wrap a byte[] as RandomAccessData; never null. */
1010 public static RandomAccessData wrapBytesAsRandomAccessData(final byte[] data)
1011 {
1012 return(new RandomAccessData(){
1013 public final void readFully(final ByteBuffer buf, final int position) throws IOException
1014 {
1015 if((buf == null) ||
1016 (position < 0) || (position + buf.remaining() > data.length))
1017 { throw new IllegalArgumentException(); }
1018 buf.put(data, position, buf.remaining());
1019 }
1020 public final long length() { return(data.length); }
1021 });
1022 }
1023
1024 /**Stand-alone blocking 'readFully()' for arbitrary InputStream.
1025 * The underlying call to is.read(b, off, len) is assumed to validate all arguments.
1026 */
1027 public static void readFully(final InputStream is,
1028 final byte[] b, int off, int len)
1029 throws IOException
1030 {
1031 while(len > 0)
1032 {
1033 final int n = is.read(b, off, len);
1034 if(n == -1) { throw new EOFException(); }
1035 off += n;
1036 len -= n;
1037 }
1038 }
1039
1040 /**Get ZipEntry and (read-only, uncompressed) data for a named file/entry; null if none.
1041 * A well-formed ZIP file should contain at least one entry,
1042 * but this may return an empty result if the input is not well-formed.
1043 * <p>
1044 * See http://infozip.sourceforge.net/ and http://www.zlib.net/ and
1045 * http://www.pkware.com/documents/casestudies/APPNOTE.TXT
1046 * <p>
1047 * The ZipEntry returned is a private copy.
1048 * <p>
1049 * This does not close the input stream.
1050 * <p>
1051 * Do this serially reading from the start of the archive.
1052 * Hideously inefficient for large archives,
1053 * especially if reading more than one entry.
1054 *
1055 * @param is input stream; must be open and non-null
1056 * @return map from file/entry name to offset of start of entry from start of stream; never null
1057 *
1058 * @throws IOException if the input is malformed
1059 */
1060 public static Tuple.Pair<ZipEntry,ROByteArray> getZIPEntry(final InputStream is,
1061 final String entryName)
1062 throws IOException
1063 {
1064 if(is == null) { throw new IllegalArgumentException(); }
1065 if(entryName == null) { throw new IllegalArgumentException(); }
1066
1067 final ZipInputStream zis = new ZipInputStream(is);
1068 ZipEntry ze;
1069 while(null != (ze = zis.getNextEntry()))
1070 {
1071 if(entryName.equals(ze.getName()))
1072 {
1073 // Got it!
1074 final long size = ze.getSize();
1075 if(size > Integer.MAX_VALUE) { throw new IOException("cannot represent size"); }
1076 final byte[] data;
1077 if(size != -1)
1078 {
1079 // We know the size so can do a single (efficient) allocation...
1080 data = new byte[(int) size];
1081 readFully(zis, data, 0, data.length);
1082 }
1083 else
1084 {
1085 // Read the data incrementally...
1086 final ByteArrayOutputStream buf = new ByteArrayOutputStream();
1087 final byte tmp[] = new byte[512]; // Efficient-ish block read size.
1088 // Transfer data in blocks in the hope of improving efficiency.
1089 for( ; ; )
1090 {
1091 final int n = zis.read(tmp);
1092 if(n < 1) { break; }
1093 buf.write(tmp, 0, n);
1094 }
1095 data = buf.toByteArray();
1096 }
1097 return(new Tuple.Pair<ZipEntry, ROByteArray>(ze, new ROByteArray(data)));
1098 }
1099 }
1100
1101 return(null); // Not found!
1102 }
1103
1104
1105 /**Immutable length/offset for a (32-bit) ZIP entry. */
1106 public static final class ZE
1107 {
1108 /**Length of (uncompressed) entry; non-negative. */
1109 public final int length;
1110 /**Offset from start of ZIP file; non-negative. */
1111 public final int offset;
1112 /**Create an instance. */
1113 public ZE(final int length, final int offset)
1114 {
1115 if((length < 0) || (offset < 0)) { throw new IllegalArgumentException(); }
1116 this.length = length;
1117 this.offset = offset;
1118 }
1119 }
1120
1121 /**Read (immutable) central directory from (end of) random-access 32-bit ZIP file; null if none.
1122 * This allows for fast random access into a ZIP file
1123 * (and quick rejection of attempts to fetch files not present at all).
1124 * <p>
1125 * The resultant map may have keys of mixed type
1126 * (using String keys for non-8-bit filenames, else more-compact representations)
1127 * but a lookup with any CharSequence key will work.
1128 *
1129 * @return map from entry/file name to (uncompressed) length and offset of entry from start of file,
1130 * or null if no central directory or otherwise apparently not a well-formed ZIP archive
1131 */
1132 public static SortedMap<CharSequence, ZE> getZIPEntriesLengthAndOffset(final RandomAccessData rad)
1133 throws IOException
1134 {
1135 if(rad == null) { throw new IllegalArgumentException(); }
1136
1137 // Length of entire resource.
1138 final long lengthL = rad.length();
1139 if(lengthL > Integer.MAX_VALUE) { throw new ZipException("Archive too large"); }
1140 final int length = (int) lengthL;
1141
1142 // END HEADER...
1143 // Create a (big-enough) little-endian buffer.
1144 final ByteBuffer endHdrBuf = ByteBuffer.allocate(ZipEntry.ENDHDR);
1145 endHdrBuf.order(ByteOrder.LITTLE_ENDIAN);
1146 final int endHdrPos = length - ZipEntry.ENDHDR;
1147 if(endHdrPos < 0) { throw new ZipException("Archive corrupt or too small"); }
1148 // Load from the data store...
1149 rad.readFully(endHdrBuf, endHdrPos);
1150 // Get ready to read/decode...
1151 endHdrBuf.flip();
1152 // Check the end-header signature.
1153 if(ZipEntry.ENDSIG != endHdrBuf.getInt()) { throw new ZipException("Bad end-header sig"); }
1154 // Get the entry count.
1155 final int count = endHdrBuf.getShort(ZipEntry.ENDTOT) & 0xffff; // Unsigned.
1156 //System.out.println("ENTRY COUNT = " + count);
1157 // Get the size of the central directory in bytes.
1158 final int cenSize = endHdrBuf.getInt(ZipEntry.ENDSIZ);
1159 //System.out.println("SIZE = " + cenSize);
1160 if(cenSize <= 0) { throw new ZipException("Bad CEN size"); }
1161 // Get the offset/position of the central directory (first member).
1162 final int cenOffset = endHdrBuf.getInt(ZipEntry.ENDOFF);
1163 //System.out.println("OFFSET = " + cenOffset);
1164 if(cenOffset <= 0) { throw new ZipException("Bad CEN offset"); }
1165
1166 final SortedMap<CharSequence, ZE> result = new TreeMap<CharSequence, ZE>(TextUtils.CASE_SENSITIVE_ORDER);
1167
1168 // CENTRAL DIRECTORY...
1169 final ByteBuffer cenBuf = ByteBuffer.allocate(cenSize);
1170 cenBuf.order(ByteOrder.LITTLE_ENDIAN);
1171 // Load from the data store...
1172 rad.readFully(cenBuf, cenOffset);
1173 // Get ready to read/decode...
1174 cenBuf.flip();
1175 // Process each entry.
1176 Name prev = null; // Helps reduce memory footprint of 8-bit names.
1177 for(int i = count; --i >= 0; )
1178 {
1179 final int initialPos = cenBuf.position();
1180 // Check the entry signature.
1181 if(ZipEntry.CENSIG != cenBuf.getInt(initialPos)) { throw new ZipException("Bad CEN (central-directory entry) sig "+(count-i)); }
1182 // Get uncompressed file length.
1183 final int fileLen = cenBuf.getInt(initialPos + ZipEntry.CENLEN);
1184 // Get name length.
1185 final int nameLen = cenBuf.getShort(initialPos + ZipEntry.CENNAM) & 0xffff;
1186 // Get extra data length.
1187 final int extraLen = cenBuf.getShort(initialPos + ZipEntry.CENEXT) & 0xffff;
1188 // Get comment length.
1189 final int commentLen = cenBuf.getShort(initialPos + ZipEntry.CENCOM) & 0xffff;
1190 // Get LOC offset.
1191 final int offset = cenBuf.getInt(initialPos + ZipEntry.CENOFF);
1192 // Pass over the CEN header...
1193 cenBuf.position(initialPos + ZipEntry.CENHDR);
1194 // Read name as UTF-8.
1195 final byte[] nameRaw = new byte[nameLen];
1196 cenBuf.get(nameRaw);
1197 // Convert to full String form.
1198 final String nameS = new String(nameRaw, CoreConsts.FILE_ENCODING_UTF_8);
1199 // Now attempt to convert to Name for 8-bit termini-sharing reduced footprint.
1200 // (We assume that adjacent entries' names are reasonably likely to share termini.)
1201 Name nameN = null;
1202 try { prev = nameN = Name.create(nameS, prev); }
1203 catch(final IllegalArgumentException e) { /* Ignore error from non-8-bit filename. */ }
1204
1205 // Skip any extra data.
1206 if(extraLen > 0) { cenBuf.position(cenBuf.position() + extraLen); }
1207 // Skip any comment data.
1208 if(commentLen > 0) { cenBuf.position(cenBuf.position() + commentLen); }
1209
1210 // Capture this new entry with the most efficient key available.
1211 result.put((nameN != null) ? nameN : nameS, new ZE(fileLen, offset));
1212 //System.out.println("ENTRY: "+name+" offset "+offset);
1213 }
1214
1215 // Wrap to make immutable.
1216 return(Collections.unmodifiableSortedMap(result));
1217 }
1218
1219 /**Compute available remaining usable bytes of space in filesystem containing given file/dir with specified reserve.
1220 * Estimates remaining free space in bytes if we are to avoid over-filling the target filesystem.
1221 * <p>
1222 * Result may be restricted by the JVM's user ID, quotas, etc.
1223 *
1224 * @return -ve if unknown, 0L if no space, else +ve usable bytes available
1225 */
1226 public static final long estimatedFreeSpaceBelowReserve(final File target, final int percentFree)
1227 {
1228 if(target == null) { throw new IllegalArgumentException(); }
1229 if((percentFree < 0) || (percentFree > 100)) { throw new IllegalArgumentException(); }
1230
1231 // System.out.println(target + ".totalSpace() = " + (target.getTotalSpace()));
1232 // System.out.println(target + ".getUsableSpace() = " + (target.getUsableSpace()));
1233
1234 final long totalSpace = target.getTotalSpace();
1235 if(totalSpace <= 0) { return(-1); /* Cannot extract useful info for this filesystem. */ }
1236
1237 final long freeSpace = target.getUsableSpace();
1238 if(freeSpace <= 0) { return(freeSpace); /* No space or unknown. */ }
1239
1240 // Compute the safety margin / reserve requested.
1241 // Rounds down the reserve margin...
1242 final long reserve = (percentFree * totalSpace) / 100;
1243 assert(reserve >= 0);
1244
1245 return(Math.max(0, freeSpace - reserve)); // Never returns -ve value.
1246 }
1247 }