001 /*
002 Copyright (c) 1996-2012, Damon Hart-Davis
003 All rights reserved.
004
005 Redistribution and use in source and binary forms, with or without
006 modification, are permitted provided that the following conditions are
007 met:
008
009 * Redistributions of source code must retain the above copyright
010 notice, this list of conditions and the following disclaimer.
011
012 * Redistributions in binary form must reproduce the above copyright
013 notice, this list of conditions and the following disclaimer in the
014 documentation and/or other materials provided with the
015 distribution.
016
017 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
018 IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
019 TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
020 PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
021 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
022 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
023 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
024 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
025 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
026 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
027 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
028 */
029
030 package org.hd.d.pg2k.test.dev;
031
032 import java.io.BufferedInputStream;
033 import java.io.ByteArrayInputStream;
034 import java.io.ByteArrayOutputStream;
035 import java.io.File;
036 import java.io.FileInputStream;
037 import java.io.IOException;
038 import java.io.InputStream;
039 import java.io.InvalidObjectException;
040 import java.util.Random;
041
042 import javax.servlet.ServletInputStream;
043
044 import junit.framework.TestCase;
045
046 import org.hd.d.pg2k.svrCore.AllExhibitProperties;
047 import org.hd.d.pg2k.svrCore.FileTools;
048 import org.hd.d.pg2k.svrCore.GenUtils;
049 import org.hd.d.pg2k.svrCore.Rnd;
050 import org.hd.d.pg2k.svrCore.SimpleLoggerIF;
051 import org.hd.d.pg2k.svrCore.datasource.ExhibitDataFileSource;
052 import org.hd.d.pg2k.svrCore.datasource.SimpleExhibitPipelineIF;
053 import org.hd.d.pg2k.svrCore.props.LocalProps;
054 import org.hd.d.pg2k.svrCore.uploader.UploadInfoBean;
055 import org.hd.d.pg2k.svrCore.uploader.UploaderConsts;
056 import org.hd.d.pg2k.svrCore.uploader.UploaderUtils;
057 import org.hd.d.pg2k.webSvr.upload.HTTPUploaderUtils;
058
059 /**Tests the exhibit upload mechanisms.
060 * May need to be running in a WAR container and in master mode
061 * for all tests to be operable
062 * (unless, exceptionally, the loopbackURL is pointing at another instance).
063 */
064 public final class ExhibitUploadTest extends TestCase
065 {
066 public ExhibitUploadTest(final String name)
067 {
068 super(name);
069 }
070
071 /**The upload directory, or null if none set. */
072 private File uploadDir;
073
074 /**Do any setup needed for the tests. */
075 @Override
076 protected void setUp()
077 throws Exception
078 {
079 // Empty and recreate the upload directory if defined.
080 final String uploadDirS = LocalProps.getUploadDir();
081 uploadDir = (uploadDirS == null) ? null : new File(uploadDirS);
082 if(uploadDir != null)
083 {
084 if(uploadDir.exists())
085 { FileTools.rmRecursively(uploadDir); }
086
087 // Try to ensure that it exists...
088 uploadDir.mkdirs();
089 }
090 }
091
092 /**Do any clearup needed after the tests. */
093 @Override
094 protected void tearDown()
095 throws Exception
096 {
097 // Zap the upload directory when done.
098 if(uploadDir != null)
099 {
100 if(uploadDir.exists())
101 { FileTools.rmRecursively(uploadDir); }
102 }
103 }
104
105
106 /**Test simple upload (of non-binary data) directly, not over HTTP.
107 * Generate a new random HTML text-fragment file,
108 * with a new, valid name,
109 * and upload it simulating a browser POST transaction,
110 * checking that it arrives intact in the upload directory.
111 */
112 public void testDirectUploadSimple()
113 throws Exception
114 {
115 // If no upload dir, we can't do the test.
116 if(uploadDir == null)
117 {
118 // System.err.println("WARNING: cannot use/create/empty upload dir so skipping test");
119 fail("WARNING: cannot use/create/empty upload dir so skipping test");
120 return;
121 }
122
123 // Set up a simple instance of a file data source.
124 final ExhibitDataFileSource edfs = new ExhibitDataFileSource(null);
125
126 final UploadInfoBean uib = _makeTestUploadUIB(edfs, ".htxt");
127
128 // Create the full name...
129 final String newExhibitName = uib.getFullName();
130
131 // Full name at which we expect upload to appear...
132 final File targetFileName = new File(uploadDir, newExhibitName);
133
134 // Ensure that it does not currently exist in the upload area...
135 assertFalse("File to upload must not exist in upload area before we start!",
136 targetFileName.exists());
137
138 // Generate a random HTML text fragment, legal as an exhibit.
139 final String exhibitText =
140 "<b>The cat sat on the mat " + rnd.nextInt() + " times.</b>";
141
142 // Chose a boundary string; would normally be chosen randomly...
143 final String boundary = "------------bdYG0aDRc9XTBFOYCIHkaq";
144
145 // Input lines from to servlet expected with the above posted.
146 final String POSTlines[] =
147 {
148 boundary,
149 "Content-Disposition: form-data; name=\"" + UploaderConsts.exhibitFile + "\"; filename=\"" + newExhibitName + "\"",
150 "Content-Type: application/octet-stream",
151 "",
152 exhibitText,
153 boundary + "--"
154 };
155
156 /*
157 ------------CZwvTEJ33GJISQKwfRxy4g
158 Content-Disposition: form-data; name="exhibitFile"; filename="sample.htxt"
159 Content-Type: application/octet-stream
160
161 <pre>
162 The cat sat on the mat.
163 </pre>
164 ------------CZwvTEJ33GJISQKwfRxy4g--
165 */
166
167 // This stream version allows *only* the line-read, not byte reads.
168 final ServletInputStream is = new ServletInputStream(){
169 int l;
170
171 @Override
172 public int readLine(final byte[] buf, final int off, final int len)
173 // throws IOException
174 {
175 if(l >= POSTlines.length)
176 { return(-1); } // EOF
177
178 // Object if buffer not big enough to read line in one go,
179 // or if offset is not zero.
180 // May refine this to be more forgiving later...
181 final String t = POSTlines[l++];
182 final int tl = t.length();
183 if((off != 0) || (tl+2 > len))
184 { throw new IllegalArgumentException("only simple uses permitted"); }
185
186 // Else copy lower bytes into place.
187 // We only use ASCII text in this test.
188 for(int i = tl; --i >= 0; )
189 { buf[i] = (byte) t.charAt(i); }
190 buf[tl] = '\r';
191 buf[tl+1] = '\n';
192
193 return(tl+2);
194 }
195
196 @Override
197 public int read() // throws IOException
198 { throw new IllegalStateException(); }
199 };
200
201 // Do upload...
202 HTTPUploaderUtils.doUpload(uib,
203 uploadDir,
204 is,
205 boundary,
206 Integer.MAX_VALUE,
207 GenUtils.systemOutLogger);
208
209 assertTrue("Upload destination directory must exist (or have been created)",
210 targetFileName.getParentFile().isDirectory());
211 assertTrue("File has not been uploaded and it should have been, to: " + targetFileName,
212 targetFileName.exists());
213 assertTrue("Target of upload must be a readable file",
214 targetFileName.isFile() && targetFileName.canRead());
215 assertEquals("Uploaded file must have the correct length",
216 exhibitText.length(), targetFileName.length());
217 // Ignore whitespace (such as line-end details) for simple content check.
218 assertEquals("Uploaded file must have the correct content",
219 exhibitText.trim(), FileTools.readTextFile(targetFileName).trim());
220 }
221
222 /**Makes a new random, legal UploadInfoBean for test purposes; never null.
223 * Creates a unique new random exhibit name for uploading to,
224 * picking one of the existing categories and authors from the current live
225 * data set.
226 * <p>
227 * May cause a jUnit assertion failure if it cannot do this successfully.
228 *
229 * @param p data pipeline for test data set;
230 * must have at least one author and one category we can use
231 * @param suffix must be legal non-null suffix for the file type
232 * to be uploaded, eg ".htxt" for HTML text, ".jpg" for a JPEG image
233 * @return correctly initialised UploadInfoBean; never null
234 * @throws IOException in case of difficulty with the data source
235 */
236 private static UploadInfoBean _makeTestUploadUIB(final SimpleExhibitPipelineIF p,
237 final String suffix)
238 throws IOException
239 {
240 // Collect the raw exhibit info from the underlying file system.
241 final AllExhibitProperties aep = p.getAllExhibitProperties(-1);
242
243 // Pretend that nothing else has been uploaded.
244 final AllExhibitProperties aepUpload = new AllExhibitProperties();
245
246 // Create a new, random but-legal exhibit name to upload as.
247 // Pick first extant author name.
248 final String author =
249 aep.getAuthorExhibitCounts().keySet().iterator().next();
250 // Pick first extant category.
251 final String category =
252 aep.getCategoryExhibitCounts().keySet().iterator().next();
253
254 // Synthesise information loaded into form.
255 final UploadInfoBean uib = new UploadInfoBean();
256 uib.setAep(aep);
257 uib.setUploadAeid(aepUpload.aeid);
258 uib.setCategory(category);
259 uib.setMainWords("new-test-text");
260 uib.setNumber(rnd.nextInt() >>> 1); // Non-negative number-in-series.
261 uib.setAuthor(author);
262 uib.setSuffix(suffix);
263
264 assertTrue("Must have enough info to make a full name",
265 uib.enoughInfo());
266 assertTrue("Must have enough valid info to make a full unique name",
267 uib.enoughValidUniqueInfo());
268
269 return(uib);
270 }
271
272 /**Minimum binary upload speed in Bps (bytes per second); strictly positive.
273 * The pre-200401020 algorithm could usually manage about 10kBps
274 * when uploading into a slow, remote (NFS-mounted) filesystem
275 * (with JDK 1.3.1 -server so that the JVM was not the bottleneck).
276 * So we enforce a minimum of at least 10kBps.
277 */
278 public static final int MIN_UPLOAD_SPEED_BYTESPERSEC = 10240;
279
280 /**Test direct binary upload (of a JPEG image) for correctness.
281 * We also measure and print the overall speed of upload which can be tuned,
282 * though the performance threshold enforced is low enough
283 * to allow even slow development machines to meet it.
284 */
285 public void testDirectUploadJPEG()
286 throws Exception
287 {
288 // If no upload dir, we can't do the test.
289 if(uploadDir == null)
290 {
291 // System.err.println("WARNING: cannot use/create/empty upload dir so skipping test");
292 fail("WARNING: cannot use/create/empty upload dir so skipping test: " + LocalProps.getUploadDir());
293 return;
294 }
295
296 // Set up a simple instance of a file data source.
297 final ExhibitDataFileSource edfs = new ExhibitDataFileSource(null);
298
299 final UploadInfoBean uib = _makeTestUploadUIB(edfs, ".jpg");
300
301 // Create the full name...
302 final String newExhibitName = uib.getFullName();
303
304 // Full name at which we expect upload to appear...
305 final File targetFileName = new File(uploadDir, newExhibitName);
306
307 // Ensure that it does not currently exist in the upload area...
308 assertFalse("File to upload must not exist in upload area before we start!",
309 targetFileName.exists());
310
311 // Create a new, random file-encoded JPEG.
312 // Large enough to provide a reasonable performance measure too.
313 final byte[] image = MediaHandlerTest.makeTestRGBTrueColourJPEGImage();
314 assertNotNull("Must be able to create a JPEG image", image);
315 assertTrue("Must be able to create non-zero-length JPEG image", 0 != image.length);
316 assertTrue("JPEG image must be reasonable size to test upload performance (>1sec at minimum acceptable speed): " +
317 image.length+" vs "+MIN_UPLOAD_SPEED_BYTESPERSEC,
318 image.length > MIN_UPLOAD_SPEED_BYTESPERSEC);
319 System.out.println("Test JPEG upload image size (bytes): " + image.length);
320
321 // Gather up output stream for POST command...
322 final ByteArrayOutputStream baos = new ByteArrayOutputStream(image.length + 1024);
323
324 // Choose suitable random boundary string.
325 final String boundary = "------------" + Long.toString(rnd.nextLong() >>> 1, 36);
326
327 // Actually generate the POST body bytes...
328 HTTPUploaderUtils.generateUploadPOSTBody(uib, baos, image, boundary);
329 final byte body[] = baos.toByteArray();
330 assertTrue("Extracted body cannot be zero length", body.length > 0);
331
332 final ServletInputStream is = new ServletInputStream(){
333 /**Position in body[]. */
334 private int pos;
335
336 /**Reads the next byte of data from the input stream, or -1 at EOF.
337 */
338 @Override
339 final public int read() // throws IOException
340 {
341 if(pos >= body.length) { return(-1); }
342 return(body[pos ++] & 0xff);
343 }
344 };
345
346 // Do upload (and time it)...
347 final long uploadStart = System.currentTimeMillis();
348 HTTPUploaderUtils.doUpload(uib,
349 uploadDir,
350 is,
351 boundary,
352 Integer.MAX_VALUE,
353 GenUtils.systemOutLogger);
354 final long uploadStop = System.currentTimeMillis();
355 final long uploadTime = uploadStop - uploadStart;
356
357 System.out.println("Upload time (ms): " + uploadTime);
358 System.out.println("Upload speed (kByte/s): " + (((float) image.length)/uploadTime));
359
360 assertTrue("Upload destination directory must exist (or have been created)",
361 targetFileName.getParentFile().isDirectory());
362 assertTrue("File has not been uploaded and it should have been, to: " + targetFileName,
363 targetFileName.exists());
364 assertTrue("Target of upload must be a readable file",
365 targetFileName.isFile() && targetFileName.canRead());
366 assertEquals("Uploaded file must have the correct length",
367 image.length, targetFileName.length());
368 // Check that the file has arrived intact.
369 final InputStream fis =
370 new BufferedInputStream(new FileInputStream(targetFileName));
371 try
372 {
373 for(int i = 0; i < image.length; ++i)
374 {
375 assertEquals("Uploaded file must match byte-by-byte (failed@"+i+")",
376 image[i] & 0xff, fis.read());
377 }
378 }
379 finally { fis.close(); } // Free resources...
380 }
381
382 /**Verify that bad uploads are rejected.
383 * In particular, test that:
384 * <ul>
385 * <li>The magic number is checked (relatively quickly if possible).
386 * <li>The entire file is verified, eg the image is loaded, if possible.
387 * </ul>
388 */
389 public void testDirectUploadBad()
390 throws Exception
391 {
392 // If no upload dir, we can't do the test.
393 if(uploadDir == null)
394 {
395 // System.err.println("WARNING: cannot use/create/empty upload dir so skipping test");
396 fail("WARNING: cannot use/create/empty upload dir so skipping test");
397 return;
398 }
399
400 // Set up a simple instance of a file data source.
401 final ExhibitDataFileSource edfs = new ExhibitDataFileSource(null);
402
403 final UploadInfoBean uib = _makeTestUploadUIB(edfs, ".jpg");
404
405 // Create the full name...
406 final String newExhibitName = uib.getFullName();
407
408 // Full name at which we expect upload to appear...
409 final File targetFileName = new File(uploadDir, newExhibitName);
410
411 // Ensure that it does not currently exist in the upload area...
412 assertFalse("File to upload must not exist in upload area before we start!",
413 targetFileName.exists());
414
415 // Create a random, bad JPEG (ie just random numbers).
416 final byte[] badImage = new byte[rnd.nextInt(987654)];
417 rnd.nextBytes(badImage);
418
419 // Gather up output stream for POST command...
420 final ByteArrayOutputStream baos = new ByteArrayOutputStream(badImage.length + 1024);
421
422 // Choose suitable random boundary string.
423 final String boundary = "------------" + Long.toString(rnd.nextLong() >>> 1, 36);
424
425 // Actually generate the POST body bytes...
426 HTTPUploaderUtils.generateUploadPOSTBody(uib, baos, badImage, boundary);
427 final byte body[] = baos.toByteArray();
428 assertTrue("Extracted body cannot be zero length", body.length > 0);
429
430 final ServletInputStream is = new ServletInputStream(){
431 /**Position in body[]. */
432 private int pos;
433
434 /**Reads the next byte of data from the input stream, or -1 at EOF.
435 */
436 @Override
437 final public int read() // throws IOException
438 {
439 if(pos >= body.length) { return(-1); }
440 return(body[pos ++] & 0xff);
441 }
442 };
443
444 // Make sure upload is vetoed because of bad magic number.
445 try
446 {
447 // Do upload (and time it)...
448 HTTPUploaderUtils.doUpload(uib,
449 uploadDir,
450 is,
451 boundary,
452 Integer.MAX_VALUE,
453 GenUtils.systemOutLogger);
454
455 // Various checks should have vetoed upload
456 fail("Allowed bad exhibit to be uploaded");
457 }
458 catch(final InvalidObjectException e)
459 {
460 // Good, want exhibit upload to be vetoed for bad magic number
461 }
462
463 // Ensure that bad file does not get uploaded.
464 assertFalse("Bad file must nto get uploaded",
465 targetFileName.exists());
466 }
467
468
469 /**Test send/receive of server response to batch-upload client.
470 */
471 public void testBatchUploaderResponseHandling()
472 throws Exception
473 {
474 // Resuable output buffer...
475 final ByteArrayOutputStream baos = new ByteArrayOutputStream();
476
477 // A random unique valid exhibit name.
478 final String exhibitName = "a/" + (Rnd.fastRnd.nextLong() >>> 1) + "-A.a";
479
480 // A random unique non-empty detail message.
481 final String detail = "detail: " + Rnd.fastRnd.nextLong();
482
483 final SimpleLoggerIF logger = null;
484
485 // Test a simple "OK" case.
486 baos.reset();
487 UploaderUtils.sendResponseToBatchUploadClient(logger, baos, true, exhibitName, detail);
488 assertEquals("Must be able to recover exhibit name",
489 exhibitName, UploaderUtils.decodeResponseToBatchUploadClient(new ByteArrayInputStream(baos.toByteArray())));
490 // Show that damaging the trailer causes rejection.
491 try
492 {
493 final byte[] buf = baos.toByteArray();
494 buf[buf.length-1] ^= (Rnd.fastRnd.nextInt() | 1); // Trailer byte definitely changed from correct value.
495 UploaderUtils.decodeResponseToBatchUploadClient(new ByteArrayInputStream(buf));
496 fail("Did not throw IOException from damaged OK message");
497 }
498 catch(final IOException e)
499 {
500 // Good, expected an error.
501 }
502
503 // Test a simple "FAIL" case.
504 baos.reset();
505 UploaderUtils.sendResponseToBatchUploadClient(logger, baos, false, exhibitName, detail);
506 try
507 {
508 UploaderUtils.decodeResponseToBatchUploadClient(new ByteArrayInputStream(baos.toByteArray()));
509 fail("Did not throw IOException from FAIL message");
510 }
511 catch(final IOException e)
512 {
513 // Good, expected an error.
514 assertTrue("Detail message not included in exception", e.getMessage().contains(detail));
515 }
516
517 // Make sure that some unpleasant random junk is discarded safely!
518 try
519 {
520 UploaderUtils.decodeResponseToBatchUploadClient(new InputStream(){
521 /**Returns random byte that could often be a valid start-of-message or text, ie semi-pathological! */
522 @Override
523 public final int read() // throws IOException
524 { return(rnd.nextBoolean() ? 'O' : (rnd.nextInt() & 0xff)); }
525 });
526 fail("Did not throw IOException from random garbage input");
527 }
528 catch(final IOException e)
529 {
530 // Good, expected an error.
531 }
532
533 // Make sure that EOF is rejected correctly.
534 try
535 {
536 UploaderUtils.decodeResponseToBatchUploadClient(new ByteArrayInputStream(new byte[0]));
537 fail("Did not throw IOException EOF");
538 }
539 catch(final IOException e)
540 {
541 // Good, expected an error.
542 }
543 }
544
545
546 /**Private source of OK pseudo-random numbers. */
547 private static final Random rnd = new Random();
548 }