001 /*
002 * Created by IntelliJ IDEA.
003 * User: d@hd.org
004 * Date: 06-Jun-02
005 * Time: 17:09:06
006
007 Copyright (c) 1996-2012, Damon Hart-Davis
008 All rights reserved.
009
010 Redistribution and use in source and binary forms, with or without
011 modification, are permitted provided that the following conditions are
012 met:
013
014 * Redistributions of source code must retain the above copyright
015 notice, this list of conditions and the following disclaimer.
016
017 * Redistributions in binary form must reproduce the above copyright
018 notice, this list of conditions and the following disclaimer in the
019 documentation and/or other materials provided with the
020 distribution.
021
022 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
023 IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
024 TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
025 PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
026 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
027 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
028 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
029 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
030 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
031 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
032 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
033
034 */
035
036 package org.hd.d.pg2k.webSvr.upload;
037
038 import java.io.BufferedInputStream;
039 import java.io.BufferedOutputStream;
040 import java.io.ByteArrayOutputStream;
041 import java.io.EOFException;
042 import java.io.File;
043 import java.io.FileInputStream;
044 import java.io.FileNotFoundException;
045 import java.io.FileOutputStream;
046 import java.io.IOException;
047 import java.io.InputStream;
048 import java.io.InvalidObjectException;
049 import java.io.OutputStream;
050 import java.io.OutputStreamWriter;
051 import java.io.RandomAccessFile;
052 import java.net.HttpURLConnection;
053 import java.net.URL;
054 import java.net.URLConnection;
055 import java.security.DigestOutputStream;
056 import java.security.MessageDigest;
057 import java.security.NoSuchAlgorithmException;
058 import java.util.Arrays;
059 import java.util.StringTokenizer;
060
061 import javax.servlet.ServletInputStream;
062 import javax.servlet.ServletOutputStream;
063 import javax.servlet.http.HttpServletRequest;
064 import javax.servlet.http.HttpSession;
065
066 import org.hd.d.pg2k.svrCore.AllExhibitImmutableData;
067 import org.hd.d.pg2k.svrCore.AllExhibitProperties;
068 import org.hd.d.pg2k.svrCore.CoreConsts;
069 import org.hd.d.pg2k.svrCore.ExhibitName;
070 import org.hd.d.pg2k.svrCore.FileTools;
071 import org.hd.d.pg2k.svrCore.Name;
072 import org.hd.d.pg2k.svrCore.SimpleLoggerIF;
073 import org.hd.d.pg2k.svrCore.MIME.ExhibitMIME;
074 import org.hd.d.pg2k.svrCore.datasource.ExhibitDataFileSource;
075 import org.hd.d.pg2k.svrCore.props.GenProps;
076 import org.hd.d.pg2k.svrCore.props.LocalProps;
077 import org.hd.d.pg2k.svrCore.uploader.UploadInfoBean;
078 import org.hd.d.pg2k.svrCore.uploader.UploaderConsts;
079 import org.hd.d.pg2k.svrCore.uploader.UploaderUtils;
080
081 /**Code snippets to support exhibit upload.
082 * Some of these are here just to keep the JSP files shorter, simpler,
083 * clearer, and faster to compile.
084 * <p>
085 * Thanks to <a href="mailto:bmarchal@pineapplesoft.com">Benoît Marchal</a>
086 * for example code to handle multipart/form-data HTTP POST data.
087 */
088 public final class HTTPUploaderUtils
089 {
090 /**Prevent instantiation. */
091 private HTTPUploaderUtils() { }
092
093
094 /**Returns ID if user is logged in for uploaded, else null.
095 * Can only be logged in if there is a session in place.
096 *
097 * @param request current HTTP request; never null
098 */
099 public static final String isLoggedInForUploads(final HttpServletRequest request)
100 {
101 if(request == null) { throw new IllegalArgumentException(); }
102
103 final HttpSession session = request.getSession(false); // Don't force a session.
104 final Object userIDo = (session == null) ? null :
105 session.getAttribute(UploaderConsts.USERID_FIELD_NAME);
106 // Do some very faint checking of validity here,
107 // but really non-null would do since validation should be done elsewhere.
108 final boolean authed = (userIDo instanceof String) &&
109 ExhibitName.validAuthorSyntax((String) userIDo);
110 if(authed) { return((String) userIDo); }
111
112 return(null); // Not logged in.
113 }
114
115 /**Accepts the body of a multipart/form-data POST request.
116 * This does not handle lots of complicated possible cases
117 * from complicated forms or strange/broken browsers.
118 * <p>
119 * In particular this only expects one file upload at a time.
120 * <p>
121 * This streams the data more-or-less directly to disc,
122 * so that huge uploads can be handled with fixed, modest memory.
123 * <p>
124 * This will verify the magic number as the file is uploaded,
125 * and abort the upload if it is too long.
126 * <p>
127 * The upload is to a temporary file beside (in the same directory as)
128 * the final destination (to avoid problems moving across filesystems)
129 * and is moved into place atomically if the upload is successful,
130 * and removed if unsuccessful.
131 * (Any that are left in place after a crash are easily identifiable
132 * for manual removal.)
133 * <p>
134 * This puts the uploaded file below destinationDir,
135 * creating any subdirectories needed,
136 * in a path of the form categoryDir/name eg <tt>art/cat-1-ANON.jpg</tt>.
137 * <p>
138 * This will not allow overwrite any existing file as a security
139 * precaution.
140 * <p>
141 * This does not serialise access to the upload area;
142 * this must be done externally if required.
143 * <p>
144 * We rely on UploadInfoBean only creating sensible and safe pathnames.
145 * <p>
146 * FIXME: we should probably validate the uploaded file content
147 * by trying whatever tests we have to hand before finally accepting it.
148 *
149 * @param request the HTTP POST request
150 * @param destinationDir the root directory to upload to
151 * @param maxUploadFileSize the maximum space that the uploaded file
152 * may consume (bytes); must be positive
153 * @param uib information on the bean to upload, must be non-null and
154 * be able to generate a valid unique name
155 * @throws IOException in case of I/O problems
156 * @throws InvalidObjectException if the upload has a bad magic number
157 */
158 public static void doUpload(final HttpServletRequest request,
159 final UploadInfoBean uib,
160 final int maxUploadFileSize,
161 final File destinationDir,
162 final SimpleLoggerIF logger)
163 throws IOException,
164 InvalidObjectException
165 {
166 if((maxUploadFileSize <= 0) ||
167 (request == null) ||
168 (uib == null) || !uib.enoughValidUniqueInfo() ||
169 (destinationDir == null) ||
170 (logger == null))
171 { throw new IllegalArgumentException(); }
172
173 if(!request.getContentType().startsWith("multipart/form-data"))
174 { throw new IOException("wrong request type"); }
175
176 String boundary =
177 request.getHeader("Content-Type");
178 boundary = boundary.substring(boundary.indexOf('=') + 1);
179 boundary = "--" + boundary;
180 //System.out.println("boundary='" + boundary + "'");
181 final ServletInputStream in = request.getInputStream();
182 doUpload(uib, destinationDir, in, boundary, maxUploadFileSize, logger);
183 }
184
185 /**Accepts the body of a multipart/form-data POST request.
186 * This does not handle lots of complicated possible cases
187 * from complicated forms or strange/broken browsers.
188 * <p>
189 * In particular this only expects one file upload at a time.
190 * <p>
191 * This streams the data more-or-less directly to disc,
192 * so that huge uploads can be handled with fixed, modest memory.
193 * <p>
194 * This will verify the magic number as the file is uploaded,
195 * and abort the upload if it is too long.
196 * <p>
197 * The upload is to a temporary file beside (in the same directory as)
198 * the final destination (to avoid problems moving across filesystems)
199 * and is moved into place atomically if the upload is successful,
200 * and removed if unsuccessful.
201 * (Any that are left in place after a crash are easily identifiable
202 * for manual removal.)
203 * <p>
204 * This puts the uploaded file below destinationDir,
205 * creating any subdirectories needed,
206 * in a path of the form categoryDir/name eg <tt>art/cat-1-ANON.jpg</tt>.
207 * <p>
208 * This will not allow overwrite any existing file as a security
209 * precaution.
210 * <p>
211 * This does not serialise access to the upload area;
212 * this must be done externally if required.
213 * <p>
214 * We rely on UploadInfoBean only creating sensible and safe pathnames.
215 * <p>
216 * FIXME: we should probably validate the uploaded file content
217 * by trying whatever tests we have to hand before finally accepting it.
218 *
219 * @param is the input stream from the HTTP POST request
220 * @param destinationDir the root directory to upload to
221 * @param maxUploadFileSize the maximum space that the uploaded file
222 * may consume (bytes); must be positive
223 * @param uib information on the bean to upload, must be non-null and
224 * be able to generate a valid unique name
225 * @param boundary the MIME multi-part boundary string
226 * (prefixed with lots of "-"s, exactly as will start a line)
227 * @throws IOException in case of I/O problems
228 * @throws InvalidObjectException if the upload has a bad magic number
229 */
230 public static void doUpload(final UploadInfoBean uib,
231 final File destinationDir,
232 final ServletInputStream is,
233 final String boundary,
234 final int maxUploadFileSize,
235 final SimpleLoggerIF logger)
236 throws IOException,
237 InvalidObjectException
238 {
239 if(logger == null)
240 { throw new IllegalArgumentException(); }
241
242 // Transcribe the upload stream if requested.
243 final ServletInputStream in = (transcriptName != null) ?
244 transcribeInputStream(is) : is;
245
246 final byte[] ibuf = new byte[RW_BLOCK_SIZE];
247 int state = DU_INITIAL_STATE;
248 String name = null;
249 String value = null;
250 String filename = null;
251
252 // // Collect any fields...
253 // final Map fields = new Hashtable();
254
255 // Get the type of the exhibit we are expecting to upload.
256 final String extn = uib.getSuffix();
257 assert("".equals(extn) || extn.startsWith("."));
258 final ExhibitMIME.ExhibitTypeParameters etp =
259 ExhibitMIME.getExhibitType(extn.substring(1));
260 assert(etp != null);
261
262 // We assume that UploadInfoBean always makes sensible and safe names.
263 assert ExhibitName.validNameSyntax(uib.getFullName());
264
265 // Generate the final target filename.
266 final File targetName = new File(destinationDir, uib.getFullName());
267 // Name of final directory in which exhibit file will go.
268 final File targetDir = targetName.getParentFile();
269 // Generate temporary name; the final component has the
270 // standard "temporary" component prefixed
271 // which is easy to recognise and ensures that the file does
272 // not have a valid exhibit file name.
273 final File tempFile = new File(targetDir,
274 FileTools.F_tmpPrefix + targetName.getName());
275
276 logger.log("Upload starting for file: dest = " + targetName);
277 logger.log(" [Temporary file " + tempFile + ".]");
278
279 // Make any parent directories needed.
280 targetDir.mkdirs();
281
282 // Abort if the target file or the temporary target already exist.
283 if(targetName.exists() || !tempFile.createNewFile())
284 { throw new IOException("target file/temporary already exists"); }
285 //FileOutputStream os = null;
286 int writeCount = 0; // Number of bytes.
287 int totalBytes = 0; // Bytes of exhibit written.
288
289 // Create a buffer to accumulate output into big chunks...
290 // We add a few hundred bytes on to the end to allow for
291 // the average interface between \n characters
292 // that will force readLine() to return,
293 // ie we aim to accommodate the RW_BLOCK_SIZE plus
294 // an full read without forcing obuf to be resized, mostly.
295 final ByteArrayOutputStream obuf =
296 new ByteArrayOutputStream(RW_BLOCK_SIZE + 1024);
297
298 RandomAccessFile os = null;
299 try {
300 // Open file for output...
301 // We only actually need pure write access...
302 os = new RandomAccessFile(tempFile, "rw");
303
304 // Read a block of bytes from the HTTP request...
305 int n; // = in.readLine(buf, 0, RW_BLOCK_SIZE);
306 while(-1 != (n = in.readLine(ibuf, 0, RW_BLOCK_SIZE))) // Until end of stream...
307 {
308
309 //System.out.print("Bytes read: ");
310 //for(int i = 0; i < n; ++i)
311 // {
312 // final int ib = buf[i] & 0xff;
313 // if((ib < 33) || (ib > 126)) { System.out.print(" 0x" + Integer.toHexString(ib)); }
314 // else { System.out.print(" " + (char)(ib)); }
315 // }
316 //System.out.println();
317
318 final String st = new String(ibuf, 0, n);
319 if(st.startsWith(boundary))
320 {
321 // Reset to fetch next field/value.
322 state = DU_INITIAL_STATE;
323 if(null != name)
324 {
325 // If it looks like we collected a parameter
326 // previously...
327 if(value != null)
328 {
329 // fields.put(name,
330 // value.substring(0,
331 // // -2 to remove CR/LF
332 // value.length() - 2));
333 }
334
335 // If it looks like we collected an exhibit...
336 else if(totalBytes >= 2)
337 {
338 // Trim off trailing CRLF.
339 // Adjust note of bytes written...
340 totalBytes -= 2;
341
342 // Flush out anything in the memory buffer.
343 // But don't bother with any of the trailing CRLF.
344 // This should usually avoid need to truncate file
345 // to explicitly remove the CRLF.
346 if(obuf.size() > 2)
347 {
348 final byte[] bn = obuf.toByteArray();
349 os.write(bn, 0, bn.length - 2); // Drop CRLF...
350 obuf.reset(); // Clear memory buffer...
351 ++writeCount; // Disc write done...
352 }
353
354 // Truncate to zap the trailing CRLF if need be.
355 // Should generally not be needed.
356 if(tempFile.length() > totalBytes)
357 {
358 os.setLength(totalBytes);
359 ++writeCount; // Disc write done...
360 }
361
362 // Prevent any further writing to the temp file...
363 os.close();
364 os = null;
365
366 // Validate file content...
367 // (Abort with InvalidObjectException if invalid.)
368 if(tempFile.length() < 1)
369 { throw new InvalidObjectException("exhibit file too short"); }
370 ExhibitMIME.checkMagicOK(etp, tempFile);
371 // FIXME: validate file content more thoroughly...
372
373 // And move it into position...
374 tempFile.renameTo(targetName);
375
376 logger.log("Uploaded file: source name = " + filename + ": dest = " + targetName + ": size = " + totalBytes);
377 logger.log(" Write count = " + writeCount + "; bytes/write = " + (totalBytes / Math.max(1, writeCount)));
378 }
379 // Done!
380
381 name = null;
382 value = null;
383 filename = null;
384 // contentType = null;
385 return;
386 }
387 }
388 else if(st.startsWith(
389 "Content-Disposition: form-data") &&
390 (state == DU_INITIAL_STATE))
391 {
392 final StringTokenizer tokenizer =
393 new StringTokenizer(st, ";=\"");
394 while(tokenizer.hasMoreTokens())
395 {
396 final String token = tokenizer.nextToken();
397 if(token.startsWith(" name"))
398 {
399 name = tokenizer.nextToken();
400 state = DU_GOT_PARAM_NAME;
401 }
402 else if(token.startsWith(" filename"))
403 {
404 filename = tokenizer.nextToken();
405 final StringTokenizer ftokenizer =
406 new StringTokenizer(filename, "\\/:");
407 filename = ftokenizer.nextToken();
408 while(ftokenizer.hasMoreTokens())
409 {
410 filename = ftokenizer.nextToken();
411 }
412 state = DU_GOT_FILE_NAME;
413 break;
414 }
415 }
416 }
417 else if(st.startsWith("Content-Type") &&
418 (state == DU_GOT_FILE_NAME))
419 {
420 // pos = st.indexOf(":");
421 // + 2 to remove the space
422 // - 2 to remove CR/LF
423 // contentType =
424 // st.substring(pos + 2, st.length() - 2);
425 }
426 // When we see end of filename parameter header
427 // start collecting actual file data.
428 else if(st.equals("\r\n") && (state == DU_GOT_FILE_NAME))
429 { state = DU_COLLECTING_FILE; }
430 else if(st.equals("\r\n") && (state == DU_GOT_PARAM_NAME))
431 { state = DU_COLLECTING_PARAM; }
432 else if(state == DU_COLLECTING_PARAM)
433 { value = ((value == null) ? st : value + st); }
434
435 // Append a block of uploaded bytes to the
436 // exhibit that we are collecting.
437 else if(state == DU_COLLECTING_FILE)
438 {
439 // If we would exceed the available space,
440 // abort...
441 if(os.getFilePointer() + n > maxUploadFileSize)
442 { throw new EOFException(); }
443
444 //System.out.println("Writing upload file block (bytes): " + n);
445
446 // Append next block to our in-memory buffer.
447 obuf.write(ibuf, 0, n);
448
449 // Logically extended the file...
450 totalBytes += n;
451
452 // If more than half the block size, flush out...
453 if(obuf.size() >= RW_BLOCK_SIZE)
454 {
455 final byte[] bn = obuf.toByteArray();
456 os.write(bn);
457 obuf.reset(); // Clear memory buffer...
458 ++writeCount; // Disc write done...
459
460 // As soon as we can check the magic number,
461 // do so, exactly once.
462 // (Abort with Exception if invalid.)
463 final int magicLength = etp.magic.length();
464 final long filePointer = os.getFilePointer();
465 if((filePointer >= magicLength) && // Now enough...
466 !(filePointer - bn.length >= magicLength)) // Not before.
467 { ExhibitMIME.checkMagicOK(etp, tempFile); }
468 }
469
470 }
471 else
472 {
473 throw new IOException("Unexpected state during upload; bytes just read: " + n);
474 }
475 }
476 }
477 finally
478 {
479 // Make sure that file is closed...
480 if(os != null) { try { os.close(); } catch(final IOException e) { } }
481
482 // Make sure that we remove any temporary file that we created
483 // (and didn't move to the real file).
484 tempFile.delete();
485 }
486 }
487
488 /**Writes the POST body for a multi-part MIME stream to upload an exhibit.
489 * This assumes that the other end has already been conveyed the
490 * content of the UploadInfoBean,
491 * and that the appropriate headers will be set for the body.
492 * <p>
493 * The non-null, non-empty, unique, not-appearing-in-the-data
494 * boundary String must be supplied.
495 * <p>
496 * We insist that the boundary marker is pure printable ASCII
497 * and at least 8 characters; it should usually be much longer.
498 * This should have a random component for safety.
499 * <p>
500 * The OutputStream should be buffered if at all possible, for efficiency.
501 *
502 * @param uib description of the the exhibit; never null
503 * @param os stream to write body to; never null
504 * @param data raw exhibit to upload, not altered by the routine; never null
505 * @param boundary unique boundary string; never empty or null
506 *
507 * @throws IOException
508 */
509 public static final void generateUploadPOSTBody(final UploadInfoBean uib,
510 final OutputStream os,
511 final byte data[],
512 final String boundary)
513 throws IOException
514 {
515 if((uib == null) || (os == null) ||
516 (boundary == null) || (boundary.length() == 0))
517 { throw new IllegalArgumentException(); }
518
519 // Check that the boundary marker looks superficially safe.
520 if(boundary.length() < 8)
521 { throw new IllegalArgumentException("boundary marker too short to be safe"); }
522 for(int i = boundary.length(); --i >= 0; )
523 {
524 final char c = boundary.charAt(i);
525 if((c < 33) || (c > 126))
526 { throw new IllegalArgumentException("boundary marker must be printable ASCII"); }
527 }
528
529 // Write prefix (pure printable ASCII).
530 final String prefix =
531 boundary + "\r\n" +
532 "Content-Disposition: form-data; name=\"" + UploaderConsts.exhibitFile + "\"; filename=\"autoupload\"\r\n" +
533 "Content-Type: application/octet-stream\r\n" +
534 "\r\n";
535 for(int i = 0; i < prefix.length(); ++i)
536 { os.write(prefix.charAt(i) & 0xff); }
537
538 // Write the actual exhibit data.
539 // Must be followed CRLF (which we prepend to the suffix)
540 // to reduce the number of separate writes.
541 os.write(data);
542
543 // Write trailer (pure printable ASCII).
544 final String suffix = "\r\n" + boundary + "--\r\n";
545 for(int i = 0; i < suffix.length(); ++i)
546 { os.write(suffix.charAt(i) & 0xff); }
547 }
548
549
550 /**If not null, relative pathname used to record transcript of exhibit upload.
551 * Generally only used for debugging.
552 * <p>
553 * Multiple uploads' transcripts will overwrite/corrupt one another.
554 * This is not intended for anything other than helping with testing.
555 * <p>
556 * Is null by default for release builds.
557 */
558 private static final File transcriptName = null;
559 // new File("uploadTranscript.log");
560
561 /**Take a transcript of the input stream.
562 * The transcriptName must be non-null (else no transcript will be made)
563 * and it must be a file that we can open write.
564 *
565 * @param is input stream to be transcribed
566 * @throws FileNotFoundException
567 */
568 private static ServletInputStream transcribeInputStream(final ServletInputStream is)
569 throws FileNotFoundException
570 {
571 if(transcriptName == null) { return(is); }
572
573 System.err.println("Writing upload transcript to: " + transcriptName);
574
575 final BufferedOutputStream bos = new BufferedOutputStream(
576 new FileOutputStream(transcriptName));
577
578 final ServletInputStream in = new ServletInputStream(){
579 /**Read a single byte; hideously inefficient. */
580 @Override
581 public int read()
582 throws IOException
583 {
584 final int b = is.read();
585 if(b >= 0) { bos.write(b); }
586 bos.flush();
587 return(b);
588 }
589
590 /**Read a block of bytes. */
591 @Override
592 public int read(final byte b[], final int off, final int len)
593 throws IOException
594 {
595 final int n = is.read(b, off, len);
596 if(n > 0) { bos.write(b, off, n); }
597 bos.flush();
598 return(n);
599 }
600
601 /**Read a whole array. */
602 @Override
603 public int read(final byte b[])
604 throws IOException
605 {
606 return(read(b, 0, b.length));
607 }
608
609 /**Read a whole line. */
610 @Override
611 public int readLine(final byte[] b, final int off, final int len)
612 throws IOException
613 {
614 final int n = is.readLine(b, off, len);
615 if(n > 0) { bos.write(b, off, n); }
616 bos.flush();
617 return(n);
618 }
619
620 /**Closes this input stream.
621 * Releases any system resources associated
622 * with the stream and the transcript.
623 * @exception IOException if an I/O error occurs.
624 */
625 @Override
626 public void close() throws IOException
627 {
628 bos.close();
629 super.close();
630 }
631 };
632
633 return(in);
634 }
635
636 // States for state machine in doUpload().
637 /**doUpload() initial state. */
638 private static final int DU_INITIAL_STATE = 0;
639 /**doUpload() got filename. */
640 private static final int DU_GOT_FILE_NAME = 1;
641 /**doUpload() got parameter name. */
642 private static final int DU_GOT_PARAM_NAME = 2;
643 /**doUpload() collecting file. */
644 private static final int DU_COLLECTING_FILE = 3;
645 /**doUpload() collecting parameter value. */
646 private static final int DU_COLLECTING_PARAM = 4;
647
648 /**Chunk size for reads/writes in upload; bigger is probably more efficient.
649 * Ideally (much) bigger than maximum magic number size so that we
650 * can read that from the first uploaded block.
651 * <p>
652 * Should probably be at least 512 to exceed minimum
653 * guaranteed Internet MTU and disc sector size,
654 * and probably a power of 2.
655 * <p>
656 * A good value is probably from 512 to 65536;
657 */
658 private final static int RW_BLOCK_SIZE = 64 * 1024;
659
660
661 /**Attempts to programmatically login at the given login URL with the user ID and password supplied.
662 * Assumed that any cookies/session required
663 * will automatically be held onto by the JVM.
664 *
665 * @param justPoll see if we remain logged in from previous attempt
666 * @param loginURL non-null URL (within our codebase)
667 * @param userID valid userID; never null
668 * @param passS valid password; never null
669 * @return true if successful, false if not
670 * @throws IOException in case of I/O problems
671 */
672 public static boolean doUploadLogin(final boolean justPoll,
673 final URL loginURL,
674 final String userID, final String passS)
675 throws IOException
676 {
677 // Regardless of expected status, try to force (idempotent) logout.
678 final URLConnection uc = loginURL.openConnection();
679
680 uc.setConnectTimeout(20000); // Don't wait indefinitely...
681 uc.setUseCaches(false);
682 uc.setDoOutput(true);
683 uc.setDoInput(true);
684 uc.setAllowUserInteraction(false);
685 uc.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
686 if(uc instanceof HttpURLConnection)
687 {
688 final HttpURLConnection hc = (HttpURLConnection) uc;
689 hc.setRequestMethod("POST");
690 }
691
692 // Do the login action if possible.
693 final OutputStreamWriter out = new OutputStreamWriter(
694 new BufferedOutputStream(uc.getOutputStream(), 512));
695 if(!justPoll)
696 {
697 out.write(UploaderConsts.USERID_FIELD_NAME);
698 out.write("=");
699 out.write(userID); // Should not need any encoding.
700 out.write("&");
701 out.write(UploaderConsts.PASSWD_FIELD_NAME);
702 out.write("=");
703 out.write(java.net.URLEncoder.encode(passS, "UTF-8")); // May need encoding...
704 out.flush();
705 out.close();
706 }
707
708 // Can we find the "logged-in-OK" header.
709 // (And are we logged in with the right ID?!)
710 final String header = uc.getHeaderField(UploaderConsts.LOGGED_IN_HEADER);
711 return(userID.equals(header));
712 }
713
714 /**Logout from the upload server if possible.
715 * @param loginURL non-null URL (within our codebase)
716 * @return true if successful, false if not
717 * @throws IOException in case of I/O problems
718 */
719 public static boolean doLogout(final URL loginURL)
720 throws IOException
721 {
722 // Regardless of expected status, try to force (idempotent) logout.
723 final URLConnection uc = loginURL.openConnection();
724
725 uc.setConnectTimeout(10000); // Don't wait too long...
726 uc.setUseCaches(false);
727 uc.setDoOutput(false);
728 uc.setDoInput(false);
729 uc.setAllowUserInteraction(false);
730 uc.setRequestProperty(UploaderConsts.LOGOUT_FIELD_NAME, "byebye");
731
732 // Do the logout action if possible.
733 uc.connect();
734
735 final HttpURLConnection hc = (HttpURLConnection) uc;
736 return(hc.getResponseCode() == 200);
737 }
738
739
740 /**Minimum space that must be left before we will allow an upload; strictly positive.
741 * Should be enough for an average-to-large typical exhibit, eg several MB.
742 */
743 public static final int MIN_BATCH_UPLOAD_SPACE = 10123456;
744
745
746 /**Run the server side of the batch upload protocol, after authentication.
747 * Can be called directly from servlet doPOST() method.
748 * <p>
749 * (Can also be used stand-alone for unit testing.)
750 * <p>
751 * This waits for a request from the client,
752 * then sends a FAIL response in case of immediate trouble,
753 * or collects the uploaded file,
754 * then sends an OK or FAIL response as appropriate.
755 *
756 * @param rawIS input stream from client; never null
757 * @param rawOS output stream to client; never null
758 * @param uploadDirS upload directory; valid and never null
759 * @param userID user ID authenticated with; valid syntax and never null
760 * @param userPass password authenticated with; valid syntax and never null
761 *
762 * @throws IOException in case of error
763 */
764 public static void runBatchUploadProtocolServerSide(final SimpleLoggerIF logger,
765 final AllExhibitProperties aep,
766 final GenProps gp,
767 final ServletInputStream rawIS,
768 final ServletOutputStream rawOS,
769 final String uploadDirS,
770 final String userID,
771 final String userPass)
772 throws IOException
773 {
774 if((aep == null) || (gp == null) ||
775 (rawIS == null) || (rawOS == null) ||
776 (uploadDirS == null) ||
777 (userID == null) || (userPass == null))
778 { throw new IllegalArgumentException(); }
779
780 final InputStream is = new BufferedInputStream(rawIS, UploaderUtils.BATCH_UPLOAD_BUFSIZE);
781 final OutputStream os = new BufferedOutputStream(rawOS, 1024);
782
783 try
784 {
785 // Collect list of all files currently in upload area.
786 final AllExhibitImmutableData aeidUploaded = ExhibitDataFileSource._getAllExhibitImmutableData(uploadDirS, -1L, false);
787
788 // Compute size consumed in total and by each user.
789 final long uploadedTotal = aeidUploaded.computeFileSpaceBytes(null);
790 final long uploadedByThisUser = aeidUploaded.computeFileSpaceBytes(userID);
791 final long uploadSpaceLeft = Math.max(0,
792 Math.min(LocalProps.getUploadMaxBytesPerUser() - uploadedByThisUser,
793 LocalProps.getUploadMaxBytesTotal() - uploadedTotal));
794
795 // // Don't allow upload if a reasonable amount of space not free.
796 // if(uploadSpaceLeft < MIN_BATCH_UPLOAD_SPACE)
797 // {
798 // UploaderUtils.sendResponseToBatchUploadClient(logger, os, false, lastExhibitName, "Not enough space to start upload ("+uploadSpaceLeft+"B vs "+MIN_BATCH_UPLOAD_SPACE+"B minimum).");
799 // return;
800 // }
801
802 // // Double-check credentials still OK.
803 // if(!SimplepassProps.isAuthorUploadPasswordCorrect(userID, userPass))
804 // {
805 // UploaderUtils.sendResponseToBatchUploadClient(logger, os, false, lastExhibitName, "Credentials expired.");
806 // return;
807 // }
808
809 // // Everything seems OK; tell the client so.
810 // UploaderUtils.sendResponseToBatchUploadClient(os, true, lastExhibitName, "Ready");
811 // os.flush(); // Force response back to client.
812
813 // Fetch next request from client.
814 final UploaderUtils.BatchUploadClientRequest clientRequest =
815 UploaderUtils.decodeRequestFromBatchUploadClient(is);
816
817 if(clientRequest == null)
818 {
819 // Client-requested connection termination; so quit now.
820 return;
821 }
822
823 // Note the requested exhibit for subsequent responses.
824 final String lastExhibitName = clientRequest.exhibitName;
825
826 // Check requested upload size, name, hash are OK.
827 final int exhibitFileSizeLimit = gp.getWEBSVR_MAX_EX_BYTES();
828 if(clientRequest.exhibitLength > exhibitFileSizeLimit)
829 {
830 UploaderUtils.sendResponseToBatchUploadClient(logger, os, false, lastExhibitName, "Exhibit too big for Gallery ("+exhibitFileSizeLimit+"B vs "+clientRequest.exhibitLength+"B requested).");
831 return;
832 }
833 if(clientRequest.exhibitLength > uploadSpaceLeft)
834 {
835 UploaderUtils.sendResponseToBatchUploadClient(logger, os, false, lastExhibitName, "Not enough space to start upload ("+uploadSpaceLeft+"B vs "+clientRequest.exhibitLength+"B requested).");
836 return;
837 }
838 final Name.ExhibitFull fullNameInUploadArea = aeidUploaded.getFullName(ExhibitName.getFileComponent(clientRequest.exhibitName).toString());
839 if(null != fullNameInUploadArea)
840 {
841 UploaderUtils.sendResponseToBatchUploadClient(logger, os, false, lastExhibitName, "Chosen name clashes with one already uploaded ("+fullNameInUploadArea+").");
842 return;
843 }
844 final Name.ExhibitFull fullNameInGallery = aep.aeid.getFullName(ExhibitName.getFileComponent(clientRequest.exhibitName).toString());
845 if(null != fullNameInGallery)
846 {
847 UploaderUtils.sendResponseToBatchUploadClient(logger, os, false, lastExhibitName, "Chosen name clashes with one already in the Gallery ("+fullNameInGallery+").");
848 return;
849 }
850 final Name.ExhibitFull nameInGalleryDopplehash = aep.getHashMD5ToName().get(clientRequest.hashMD5);
851 if(null != nameInGalleryDopplehash)
852 {
853 UploaderUtils.sendResponseToBatchUploadClient(logger, os, false, lastExhibitName, "Exhibit appears to be duplicate (MD5 hash same) of one already in the Gallery ("+nameInGalleryDopplehash+").");
854 return;
855 }
856
857 // Get the exhibit type.
858 // (Check that it is one that we recognise.)
859 final ExhibitMIME.ExhibitTypeParameters etp = ExhibitMIME.getInputFileType(clientRequest.exhibitName);
860 if(null == etp)
861 {
862 UploaderUtils.sendResponseToBatchUploadClient(logger, os, false, lastExhibitName, "Exhibit type unrecognised.");
863 return;
864 }
865
866 // Get the exhibit section/category.
867 // Reject the request if the section does not already exist.
868 final String category = ExhibitName.getCategoryComponent(clientRequest.exhibitName).toString();
869 if(aep.getCategoryExhibitCounts().get(category) == null)
870 {
871 UploaderUtils.sendResponseToBatchUploadClient(logger, os, false, lastExhibitName, "Exhibit category unrecognised.");
872 return;
873 }
874
875 final File destinationDir = new File(uploadDirS);
876
877 // Make sure the category directory exists
878 // else try to create it...
879 final File categoryDir = new File(destinationDir, category);
880 if(!categoryDir.isDirectory() || !categoryDir.canWrite())
881 {
882 logger.log(" [Making upload directory: " + categoryDir + ".]");
883 if(!categoryDir.mkdir() || !categoryDir.isDirectory() || !categoryDir.canWrite())
884 {
885 UploaderUtils.sendResponseToBatchUploadClient(logger, os, false, lastExhibitName, "Category upload directory cannot be made or is unusable: contact the maintainer.");
886 return;
887 }
888 }
889
890 // OK try to create the exhibit description and file
891 // if not already present,
892 // copy the data in as fast as possible,
893 // and at the end check that the hash is OK,
894 // else abort and tidy up.
895
896 final long uploadStart = System.currentTimeMillis();
897
898 // *** Actually do the upload! ***
899
900 // Generate the final target filename.
901 // Do not use any intermediate path supplied.
902 final File targetName = new File(categoryDir, ExhibitName.getFileComponent(clientRequest.exhibitName).toString());
903 // // Name of final directory in which exhibit file will go.
904 // final File targetDir = targetName.getParentFile();
905 // Generate temporary name; the final component has the
906 // standard "temporary" component prefixed
907 // which is easy to recognise and ensures that the file does
908 // not have a valid exhibit file name.
909 final File tempFile = new File(categoryDir,
910 FileTools.F_tmpPrefix + targetName.getName());
911
912 if(tempFile.exists())
913 {
914 UploaderUtils.sendResponseToBatchUploadClient(logger, os, false, lastExhibitName, "Exhibit upload apparently in progress already.");
915 return;
916 }
917
918 // Open file in non-append mode.
919 final MessageDigest digestMD5;
920 try { digestMD5 = MessageDigest.getInstance(CoreConsts.HASH_MD5); }
921 catch(final NoSuchAlgorithmException e) { throw new Error(e); } // Should not happen...
922 final FileOutputStream rawFileOutputStream = new FileOutputStream(tempFile, false); // Used to sync().
923 DigestOutputStream efos = new DigestOutputStream(rawFileOutputStream, digestMD5);
924 try
925 {
926 logger.log("Batch upload starting for file: dest = " + targetName);
927 logger.log(" [Temporary file " + tempFile + ".]");
928
929 // Copy the raw exhibit data one (large, efficient) block at a time.
930 final byte buf[] = new byte[(int) Math.min(clientRequest.exhibitLength, RW_BLOCK_SIZE)];
931 for(long bytesLeft = clientRequest.exhibitLength; bytesLeft > 0; )
932 {
933 final int maxXfer = (int) Math.min(buf.length, bytesLeft);
934 final int read = is.read(buf, 0, maxXfer);
935 if(read < 1) { throw new IOException("error reading data for exhibit: " + clientRequest.exhibitName); }
936 efos.write(buf, 0, read);
937 bytesLeft -= read;
938 }
939
940 // Flush out all the data.
941 efos.flush();
942 rawFileOutputStream.getFD().sync(); // Force to persistent store.
943
944 // Capture the hash.
945 final byte[] digestBytes = digestMD5.digest();
946
947 efos.close();
948 efos = null;
949
950 if(UploaderUtils.BATCH_UPLOAD_MSG_TERMINATOR != is.read())
951 {
952 UploaderUtils.sendResponseToBatchUploadClient(logger, os, false, lastExhibitName, "Missing terminator after exhibit data.");
953 return;
954 }
955
956 if(tempFile.length() != clientRequest.exhibitLength)
957 {
958 UploaderUtils.sendResponseToBatchUploadClient(logger, os, false, lastExhibitName, "Exhibit upload failed: write to local disc failed.");
959 return;
960 }
961
962 // Check the exhibit survived intact.
963 if(!Arrays.equals(digestBytes, clientRequest.hashMD5.toByteArray()))
964 {
965 UploaderUtils.sendResponseToBatchUploadClient(logger, os, false, lastExhibitName, "Exhibit hash wrong after upload.");
966 return;
967 }
968
969 // Check the magic number of the uploaded file.
970 final InputStream tfis = new BufferedInputStream(
971 new FileInputStream(tempFile), ExhibitMIME.getLongestMagicBytes());
972 try
973 {
974 if(!ExhibitMIME.magicOK(etp, tfis))
975 {
976 UploaderUtils.sendResponseToBatchUploadClient(logger, os, false, lastExhibitName, "Exhibit magic number wrong after upload.");
977 return;
978 }
979 }
980 finally { tfis.close(); } // Free resources...
981
982 // Move upload file into place.
983 if(!tempFile.renameTo(targetName))
984 {
985 UploaderUtils.sendResponseToBatchUploadClient(logger, os, false, lastExhibitName, "Exhibit upload failed: failed to move into final place.");
986 return;
987 }
988
989 // Generate the final description target filename.
990 final File targetDescName = new File(categoryDir,
991 ExhibitName.getFileComponent(clientRequest.exhibitName) +
992 CoreConsts.DESCRIPTION_FILE_SUFFIX);
993
994 // If there was a description, attempt to save it now.
995 final String desc = clientRequest.description;
996 if((desc != null) && (desc.length() > 0))
997 {
998 FileTools.replacePublishedFile(targetDescName.getPath(),
999 desc.getBytes(CoreConsts.FILE_ENCODING_8859_1),
1000 false); // Be noisy for now...
1001 }
1002
1003 final long uploadEnd = System.currentTimeMillis();
1004 final long uploadTime = uploadEnd - uploadStart;
1005
1006 // Compute and log effective upload bandwidth.
1007 final long uploadBps = ((clientRequest.exhibitLength * 1000) / Math.max(1, uploadTime));
1008 logger.log(" Exhibit upload completed, "+clientRequest.exhibitLength+"B, "+uploadTime+"ms: " + uploadBps + "Bps.");
1009
1010 // If we get here then it's probably all worked!
1011 // Tell the client...
1012 UploaderUtils.sendResponseToBatchUploadClient(logger, os, true, lastExhibitName, "Exhibit upload OK.");
1013 os.flush(); // Flush down the wire immediately...
1014 }
1015 finally
1016 {
1017 // Ensure that the temporary file is closed and removed.
1018 if(efos != null) { efos.close(); }
1019 tempFile.delete();
1020 }
1021 }
1022 finally
1023 {
1024 // Flush the output buffer...
1025 os.flush();
1026 }
1027 }
1028 }