001 /*
002 Copyright (c) 1996-2012, Damon Hart-Davis
003 All rights reserved.
004
005 Redistribution and use in source and binary forms, with or without
006 modification, are permitted provided that the following conditions are
007 met:
008
009 * Redistributions of source code must retain the above copyright
010 notice, this list of conditions and the following disclaimer.
011
012 * Redistributions in binary form must reproduce the above copyright
013 notice, this list of conditions and the following disclaimer in the
014 documentation and/or other materials provided with the
015 distribution.
016
017 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
018 IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
019 TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
020 PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
021 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
022 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
023 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
024 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
025 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
026 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
027 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
028 */
029 package org.hd.d.pg2k.webSvr.util;
030
031 import java.io.IOException;
032 import java.lang.management.ThreadInfo;
033 import java.lang.management.ThreadMXBean;
034 import java.net.InetAddress;
035 import java.net.UnknownHostException;
036 import java.util.ArrayList;
037 import java.util.Collections;
038 import java.util.Iterator;
039 import java.util.List;
040 import java.util.Map;
041 import java.util.Set;
042 import java.util.SortedMap;
043 import java.util.concurrent.ConcurrentHashMap;
044 import java.util.concurrent.atomic.AtomicInteger;
045
046 import javax.servlet.FilterChain;
047 import javax.servlet.FilterConfig;
048 import javax.servlet.ServletContext;
049 import javax.servlet.ServletException;
050 import javax.servlet.ServletRequest;
051 import javax.servlet.ServletResponse;
052 import javax.servlet.http.HttpServletRequest;
053 import javax.servlet.http.HttpServletResponse;
054
055 import org.hd.d.pg2k.svrCore.AddrTools;
056 import org.hd.d.pg2k.svrCore.CoreConsts;
057 import org.hd.d.pg2k.svrCore.GenUtils;
058 import org.hd.d.pg2k.svrCore.HostUtils;
059 import org.hd.d.pg2k.svrCore.MemoryTools;
060 import org.hd.d.pg2k.svrCore.MemoryTools.AutoExpirableFixedLifeBase;
061 import org.hd.d.pg2k.svrCore.MemoryTools.CacheMiniMap;
062 import org.hd.d.pg2k.svrCore.Rnd;
063 import org.hd.d.pg2k.svrCore.TextUtils;
064 import org.hd.d.pg2k.svrCore.ThreadUtils;
065 import org.hd.d.pg2k.svrCore.collections.LRUMapAutoSizeForHitRate;
066 import org.hd.d.pg2k.svrCore.location.GeoProximity;
067 import org.hd.d.pg2k.svrCore.location.GeoUtils;
068 import org.hd.d.pg2k.svrCore.location.GeoUtils.CCTLD;
069 import org.hd.d.pg2k.svrCore.location.LoadBalancingUtils;
070 import org.hd.d.pg2k.svrCore.props.GenProps;
071 import org.hd.d.pg2k.svrCore.props.GenPropsGenNames;
072 import org.hd.d.pg2k.svrCore.props.LocalProps;
073 import org.hd.d.pg2k.svrCore.stats.StatsLogger;
074 import org.hd.d.pg2k.svrCore.vars.SimpleVariableDefinition;
075 import org.hd.d.pg2k.svrCore.vars.SimpleVariablePipelineIF;
076 import org.hd.d.pg2k.svrCore.vars.SimpleVariableValue;
077 import org.hd.d.pg2k.svrCore.vars.SystemVariables;
078 import org.hd.d.pg2k.webSvr.exhibit.DataSourceBean;
079 import org.hd.d.pg2k.webSvr.virtualHosts.VirtualHosts;
080 import org.hd.d.tmf.ThroughputMonitorFilterBase;
081
082 import ORG.hd.d.IsDebug;
083
084 /**Filter to monitor throughput (output rate) to help regulate flow.
085 * Can be used in association with the compression filter to
086 * enable compression when our output pipe is likely to get congested.
087 * <p>
088 * (This filter has had a few miscellaneous bits of functionality thrown in for now:
089 * <ul>
090 * <li>This captures changes to the standard set of session attributes.
091 * <li>This redirects users to a canonical hostname
092 * that have arrived at a non-standard or deprecated one.
093 * </ul>)
094 * <p>
095 * This also makes a rough estimate of the number of users
096 * (without forcing a session) by counting the number of IP addresses
097 * used to visit the site through <em>this servlet container instance</em>
098 * (probably a slight overestimate). This ignores clients that are
099 * probably spiders (ie that never send a referring URL),
100 * and that appear to be one-off hotlinks from external sites.
101 * <p>
102 * A truly distributed front end will need a more sophisticated communication
103 * mechanism to share data efficiently between front-end instances.
104 * <p>
105 * TODO: This filter is not yet suitable for multi-JVM front-ends (clustering)
106 * or otherwise where the
107 * WAR container creates multiple instances of this filter
108 * for what is logically one front-end.
109 */
110 public final class ThroughputMonitorFilterPG2K extends ThroughputMonitorFilterBase
111 {
112 /**Request attribute (property name) in which we save the incoming requestURI.
113 * This is useful when we do a redirect internally.
114 */
115 public static final String PNAME_SAVED_REQ_URI = "tmf.savedReqURI";
116
117 /**If true, then we may try to force a preemptive GC when the system is idle. */
118 private static final boolean DO_PREEMPTIVE_GC = true;
119
120 /**Conversion factor from byte/second to GB/30day. */
121 private static final float BPS_TO_GBP30D = (3600 * 24 * 30) / 1.0e9f;
122
123 /**Ramp up time (ms) from zero bandwidth, eg at start-up or on recovery from overload; non-negative. */
124 private static final int RAMP_UP_TIME_MS = 10 * 60 * 1000;
125
126 /**Any GB-per-30-days traffic limit; strictly positive if set. */
127 private final int maxGBytesPer30D = Integer.getInteger(WebConsts.BANDWITDH_LIMIT_GBP30D_DEFAULT_PNAME, 0);
128
129 // Compute derived values for speed.
130 private final int maxBpsFor30D = Math.max(0, Math.round(maxGBytesPer30D / BPS_TO_GBP30D));
131 /**Unique prefix for exhibits. */
132 private static final String EX_PREFIX = '/' + WebConsts.BASE_PATH_EXHIBITS + '/';
133 /**Unique prefix for catalogue pages. */
134 private static final String CATPAGE_PREFIX = '/' + WebConsts.BASE_PATH_CATPAGE + '/';
135 /**Common (possibly ambiguous) prefix for catalogue page and search page. */
136 private static final String CATAREA_PREFIX = "/_c";
137
138 /**System event to which profiling/busy events are recorded. */
139 private static final SimpleVariableDefinition PROFILING_EVENT = SystemVariables.PERFMON_STRING_GLOBAL_EVENT;
140
141
142 /**Called when the system is idle according to the throughput monitor.
143 * We can use this to request a preemptive GC.
144 */
145 @Override protected void onIdle()
146 {
147 if(DO_PREEMPTIVE_GC)
148 {
149 final ServletContext ctxt = getConfig().getServletContext();
150 final DataSourceBean dsb = DataSourceBean.getApplicationInstance(ctxt);
151 if(MemoryTools.preemptiveGC())
152 {
153 try
154 {
155 dsb.setVariable(new SimpleVariableValue(
156 SystemVariables.GENSTATS_STRING_GLOBAL_EVENT,
157 "PreemptiveGC=" + LocalProps.getMirrorTag()));
158 }
159 catch(final Exception e)
160 {
161 e.printStackTrace(); // Absorb error, but whinge...
162 }
163 }
164 }
165 }
166
167 /**Unhook from servlet logger. */
168 @Override public void destroy()
169 {
170 // Force out stats...
171 StatsLogger.dumpSummary(statsIDTHRFGEN);
172 StatsLogger.dumpSummary(statsIDPERF);
173
174 // Stop using servlet logger.
175 logger.setContext(null);
176
177 super.destroy();
178 }
179
180 /**Hook into servlet logger. */
181 @Override public void init(final FilterConfig filterConfig)
182 {
183 logger.setContext(filterConfig.getServletContext()); // Use servlet.log().
184 super.init(filterConfig);
185
186 if(maxGBytesPer30D > 0)
187 {
188 final String message = "[Using per-30-day bandwidth limit of "+maxGBytesPer30D+"GB ("+(maxGBytesPer30D / BPS_TO_GBP30D)+"Bps).]";
189 System.out.println(message);
190 logger.log(message);
191 }
192 }
193
194
195 /**Our logger which falls back to System.out if servlet log not available; never null. */
196 private final WebUtils.ServletLoggerWithFallback logger = new WebUtils.ServletLoggerWithFallback();
197
198 /**The stats set to which we log general throughput events.
199 * The unique codes are the constants THRFGNAME_XXX.
200 */
201 private final StatsLogger.StatsConfig statsIDTHRFGEN =
202 new StatsLogger.StatsConfig("THROUGHPUT-GENERAL",
203 logger, // Use servlet log if poss.
204 false, // Only dump summaries...
205 12 * 3600, // About every 12 hours.
206 true); // Adaptive.
207
208 /**General stats event name prefix: client region by IP address. */
209 public static final String THRFNAMEPR_CLIENT_REGION = "client-region-";
210
211 /**General stats event name prefix: "sticky" client region by IP address. */
212 public static final String THRFNAMEPR_STICKY_CLIENT_REGION = "sticky-client-region-";
213
214 /**General stats event name prefix: client estimated proximity from IP address. */
215 public static final String THRFNAMEPR_CLIENT_PROXIMITY = "client-proximity-";
216
217 /**General stats event name prefix: client SPAMmer/compromised detected by given DNSRBL. */
218 public static final String THRFNAMEPR_DNSRBL_DETECTION = "client-in-DNSRBL-";
219
220 /**General stats event name prefix: coda processing time log(ms). */
221 public static final String THRFNAMEPR_CODATIME = "coda-time-";
222
223 /**General stats event name prefix: virtual host name used to fetch page. */
224 public static final String THRFNAMEPR_VHOST = "vhost-";
225
226 /**General stats event name: redirect issued to client because of unknown name. */
227 public static final String THRFNAME_REDIR_UNKNOWN = "redirect-unknown";
228
229 /**General stats event name: redirect issued to client to get them to local mirror. */
230 public static final String THRFNAME_REDIR_TO_LOCAL = "redirect-to-local";
231
232 /**General stats event name: redirect issued to client because of deprecated alias. */
233 public static final String THRFNAME_REDIR_ALIAS = "redirect-alias";
234
235 /**General stats event name: 1000 hits (GETs, POSTs, redirects, etc). */
236 public static final String THRFNAME_kHIT = "kHit";
237
238 /**General stats event name: catalogue page hits (GETs), local only, trimmed for space in logs, etc. */
239 public static final String THRFNAME_HIT_CAT_PAGE = "cat";
240
241 /**General stats event name: 1000 catalogue page hits (GETs). */
242 public static final String THRFNAME_kHIT_CAT_PAGE = "kHit-page-GET-catalogue";
243
244 /**General stats event name: 1000 "sticky client" page hits (GETs). */
245 public static final String THRFNAME_kHIT_STICKY_CLIENT_PAGE = "kHit-page-GET-byStickyClient";
246
247 /**The stats set to which we log performance/sampling events. */
248 private final StatsLogger.StatsConfig statsIDPERF =
249 new StatsLogger.StatsConfig("PERF",
250 logger, // Use servlet log if poss.
251 false, // Only dump summaries.
252 3600, // About every hour.
253 true); // Adaptive.
254
255
256 /**If false, be prepared to consume extra resources working out geographical location of client.
257 * Else (if true), just do quick lookup with internal/cached data.
258 */
259 private static final boolean DO_QUICK_GEO_LOOKUP = true;
260
261
262 /**If true, report any unexpected host name used to access the site. */
263 private static final boolean REPORT_UNEXPECTED_HOSTNAME = true;
264
265 /**If true, redirect users arriving with a deprecated host name.
266 * FIXME: possibly this functionality might be better moved elsewhere.
267 */
268 private static final boolean REDIRECT_DEPRECATED_HOSTNAME = true;
269
270
271 /**Returns expected approximate bytes-per-second per sticky client; strictly positive.
272 * Picks up our hard-wired value.
273 */
274 @Override protected int expectedBpsPerStickyClient()
275 {
276 return(WebConsts.DEFAULT_EXPECTED_USER_BW_BYTESPERSEC);
277 }
278
279 /**Time when we will next report the slow-changing parameters, initially 0 to force immediate send.
280 * Marked volatile for safe lock-free access,
281 * though in fact this is probably unnecessary.
282 */
283 private volatile long nextSendAllParams;
284
285 /**Last time reported bandwidth was (forced to) zero, eg because of overload.
286 * Set at start-up to allow gradual warm-up rather than sucking in loads of traffic and falling over.
287 * <p>
288 * Marked volatile for safe lock-free access,
289 * though in fact this is probably unnecessary.
290 */
291 private volatile long _lastForcedZero = System.currentTimeMillis();
292
293 /**Called on every tick, and passed the most important metrics.
294 *
295 * @param availableBpsPerClientBasic estimated available bytes/sec
296 * available for one new client (or an existing client); non-negative
297 * @param allVisitorCount approximate count of "live" visitors;
298 * non-negative
299 * @param stickyVisitorCount approximate count of "live" "sticky" visitors;
300 * non-negative
301 * @param ltbpss long-term bytes/sec smoothed throughput
302 * @param dailyUniqueVisitors estimated unique visitors per day;
303 * non-negative
304 * @param busyFractionSmoothed smoothed system "busy-ness" value
305 * ranging from 0.0 (idle) to 1.0 (busy)
306 * @param tooManyVisitors true if well over capacity by user count
307 * @param connectionTooBusy true if well over capacity by bandwidth
308 * @param systemTooBusy true if system busy fraction dangerously high
309 */
310 @Override
311 protected void tick(final int availableBpsPerClientBasic,
312 final int allVisitorCount,
313 final int stickyVisitorCount,
314 final float ltbpss,
315 final int dailyUniqueVisitors,
316 final float busyFractionSmoothed,
317 final boolean tooManyVisitors,
318 final boolean connectionTooBusy,
319 final boolean systemTooBusy)
320 {
321 final ServletContext ctxt = getConfig().getServletContext();
322 final DataSourceBean dsb = DataSourceBean.getApplicationInstance(ctxt);
323
324 // Set the light-load flag directly.
325 // We aim to be slightly conservative in this.
326 // In particular we want to see none of the danger flags set,
327 // and the smoothed "busy" value well under 50%.
328 // The overload flag is set elsewhere.
329 final boolean isLightlyLoaded = (!systemTooBusy) &&
330 (!connectionTooBusy) &&
331 (!tooManyVisitors) &&
332 (availableBpsPerClientBasic > 0) &&
333 (busyFractionSmoothed < 0.3f);
334 ctxt.setAttribute(WebConsts.BANDWIDTH_LIGHTLOAD_ATTR_NAME, isLightlyLoaded ? Boolean.TRUE : Boolean.FALSE);
335
336 // If this host is the master or is any host (temporarily) conserving power
337 // then report only a fraction of actual available bandwidth
338 // as we want to avoid attracting extra traffic/load if possible
339 // (and the master expects a heavy load serving mirrors rather than end-users).
340 // Note that we apply a similar/further "slowdown" factor for
341 // slaves with expensive bandwidth approaching their limits, etc,
342 // and this can be cumulative if required.
343 final boolean isLowPowerOrMaster = (!Boolean.TRUE.equals(dsb.isSlave())) ||
344 GenUtils.mustConservePower();
345 final int serverSlowdownFactor = LocalProps.getServerSlowdownFactor();
346 assert(serverSlowdownFactor > 0);
347 final int availableBpsPerClient;
348 if((!isLowPowerOrMaster) && (serverSlowdownFactor < 2))
349 {
350 // Usual case: full-speed primary mirror...
351 availableBpsPerClient = availableBpsPerClientBasic;
352 }
353 else
354 {
355 // If this is a master, or a slave with high-cost or scarce bandwidth or power,
356 // then effectively make this less attractive than any other equivalent-speed peer server
357 // within the same country-group by reducing the reported available bandwidth to scale.
358 //
359 // TODO: this should also not be too far from best/worst CO2/kWh electricity intensity ratio,
360 // where that is driving the low-power flag, ~10--25% in GB circa 2011.
361 if(!isLowPowerOrMaster)
362 { availableBpsPerClient = availableBpsPerClientBasic / serverSlowdownFactor; }
363 else
364 {
365 availableBpsPerClient = (availableBpsPerClientBasic * GeoProximity.CONTINENT.getCloseness()) /
366 (serverSlowdownFactor * GeoProximity.COUNTRYGROUP.getCloseness());
367 }
368 }
369
370 // Compute actual Bps (Bytes-per-second) since server start;
371 // we use the long-term smoothed value if this is not available.
372 final Object lifetimeBpsOutO = getValue(ATTR_NAME_SUFFIX_SERVER_LIFETIME_BYTES_PER_SEC_OUTBOUND);
373 final long lifetimeBpsOut = (!(lifetimeBpsOutO instanceof Number)) ? Math.round(ltbpss) :
374 (((Number)lifetimeBpsOutO).longValue());
375
376 // If lifetime bandwidth is close to (or above) any 30D limit
377 // then strictly limit what we report as available to what is left.
378 // Note that this threshold probably needs to be significantly below
379 // that imposed by any upstream traffic shaping
380 // if both are to be effective and behave as expected.
381 final boolean approachingPCMLimit = ((maxGBytesPer30D > 0) && (lifetimeBpsOut*1.2 > maxBpsFor30D));
382 // Conservative value so as to avoid attracting unsupportable traffic.
383 int reportedAvailableBpsPerClient = !approachingPCMLimit ?
384 availableBpsPerClient /* No "monthly" transfer limit need be enforced. */ :
385 (Math.min(availableBpsPerClient,
386 Math.max(0, (int) Math.round(maxBpsFor30D - lifetimeBpsOut*1.1))));
387
388 final double load = Math.max(busyFractionSmoothed, ThreadUtils.loadFraction());
389 final float lowThreshold = LocalProps.getLightLoadMax();
390 // If not currently lightly loaded
391 // then scale down reported bandwidth to zero as we approach/pass high threshold.
392 // Use max of system load average and calculated load from HTTP-request completion time.
393 if(!isLightlyLoaded || (load >= lowThreshold))
394 {
395 final float highThreshold = LocalProps.getHeavyLoadMin();
396 // If heavily loaded then knock available bandwidth down to zero to deter new clients.
397 if(systemTooBusy || connectionTooBusy|| tooManyVisitors ||
398 (load >= highThreshold))
399 { reportedAvailableBpsPerClient = 0; } // Overload event!
400 else
401 {
402 // The less CPU head-room is left, the more that the reported available bandwidth is knocked down.
403 final double spareCPU = Math.min(1, (highThreshold - load) / (highThreshold - lowThreshold));
404 assert(spareCPU >= 0);
405 final int reducedAvailableBpsPerClient = (int) Math.floor(reportedAvailableBpsPerClient * spareCPU);
406 assert(reducedAvailableBpsPerClient >= 0);
407 assert(reducedAvailableBpsPerClient <= reportedAvailableBpsPerClient);
408 reportedAvailableBpsPerClient = reducedAvailableBpsPerClient;
409 }
410 }
411
412 // If calculated available bandwidth so far is now zero for whatever reason
413 // then restart the slow-start flag.
414 final long now = System.currentTimeMillis();
415 if(reportedAvailableBpsPerClient <= 0)
416 { _lastForcedZero = now; }
417
418
419 // If this instance is still warming up or recovering from an overload
420 // then further scale down the bandwidth to allow a linear warm-up.
421 // Don't treat this as a bad event in itself, to avoid getting stuck at zero.
422 final long warmTime; // Take snapshot.
423 if((reportedAvailableBpsPerClient > 0) &&
424 ((warmTime = (now - _lastForcedZero)) < RAMP_UP_TIME_MS))
425 {
426 // The longer the warm-up time so far, the higher the bandwidth allowed.
427 final double scaleFactor = Math.max(0, warmTime) / (double) RAMP_UP_TIME_MS;
428 assert(scaleFactor >= 0);
429 assert(scaleFactor <= 1);
430 final int reducedAvailableBpsPerClient = (int) Math.floor(reportedAvailableBpsPerClient * scaleFactor);
431 assert(reducedAvailableBpsPerClient >= 0);
432 assert(reducedAvailableBpsPerClient <= reportedAvailableBpsPerClient);
433 reportedAvailableBpsPerClient = reducedAvailableBpsPerClient;
434 }
435
436 // If it looks as if we don't have enough residual bandwidth per client
437 // then clamp our reported available bandwidth to zero to deter new clients.
438 // Don't treat this as a bad event in itself, just generate less noise.
439 if(reportedAvailableBpsPerClient < expectedBpsPerStickyClient()/2)
440 { reportedAvailableBpsPerClient = 0; }
441
442
443 // Post updates to system variables.
444 // We need not necessarily set every one of these on each tick,
445 // especially those that only meaningfully change slowly,
446 // though the cost of doing so is probably not excessive.
447 //
448 // IOExceptions may veto setting later values,
449 // so set more important and/or fast-changing ones first.
450 try
451 {
452 // Predicted bandwidth available to a new client.
453 // Used to help distributed load-balancing algorithm.
454 // Conservative to avoid overload.
455 dsb.setVariable(new SimpleVariableValue(
456 SystemVariables.ThroughputMonitorFilter_AVAIL_BPS_PER_CLIENT,
457 Integer.valueOf(reportedAvailableBpsPerClient)));
458 // For now our "sticky" count is our normal user count.
459 dsb.setVariable(new SimpleVariableValue(
460 SystemVariables.ThroughputMonitorFilter_STICKY_CLIENT_COUNT,
461 Integer.valueOf(stickyVisitorCount)));
462
463 if(now >= nextSendAllParams)
464 {
465 // All-users count, including spiders, hotlinkers, etc.
466 dsb.setVariable(new SimpleVariableValue(
467 SystemVariables.ThroughputMonitorFilter_CLIENT_COUNT,
468 Integer.valueOf(allVisitorCount)));
469 // Long-term bandwidth.
470 dsb.setVariable(new SimpleVariableValue(
471 SystemVariables.ThroughputMonitorFilter_ltBps,
472 new Integer((int) ltbpss))); // Integer is compact and simple.
473 // Smoothed busy-fraction value.
474 dsb.setVariable(new SimpleVariableValue(
475 SystemVariables.ThroughputMonitorFilter_BUSY_FRACTION,
476 new Float(busyFractionSmoothed))); // Float is compact and has enough precision.
477 // Estimated yearly unique "sticky" human users.
478 dsb.setVariable(new SimpleVariableValue(
479 SystemVariables.ThroughputMonitorFilter_YEARLY_UNIQUE_VISITORS,
480 new Float(dailyUniqueVisitors * 365.242f))); // No more than ~6SF of precision sensible!
481 // Note the version of the AEP that we are currently using locally.
482 // This can be used to decide which mirrors are in sync.
483 dsb.setVariable(new SimpleVariableValue(
484 SystemVariables.ThroughputMonitorFilter_AEP_LONGHASH,
485 (new Long(dsb.getAllExhibitProperties(-1).longHash))));
486 // Set our mirror tag if we have one (else null)...
487 // Note that we may be a mirror and master simultaneously.
488 dsb.setVariable(new SimpleVariableValue(
489 SystemVariables.ThroughputMonitorFilter_ACTIVE_MIRROR_NAME,
490 LocalProps.getMirrorTag()));
491
492 // Put off next full send a while,
493 // but before the previous values actually expire...
494 nextSendAllParams = now + SystemVariables.MAX_VALUE_DISTRIBUTION_LATENCY_MS/3;
495 }
496 }
497 catch(final IllegalArgumentException e)
498 { System.err.println("ThroughputMonitorFilter: IllegalArgumentException: failed to set system variable: " + e.getMessage()); }
499 catch(final IOException e)
500 { System.err.println("ThroughputMonitorFilter: IOException: failed to set system variable: " + e.getMessage()); }
501
502
503 // Attempt slow/continuous system CPU profile if so requested.
504 try
505 {
506 final GenProps gp = dsb.getGenProps(-1);
507 final SlowProfileLevel slowProfileEnabled = slowProfileEnabled(gp);
508 if(slowProfileEnabled != SlowProfileLevel.NONE)
509 { logLiveThreads(dsb, statsIDPERF, -1, slowProfileEnabled); }
510 }
511 catch(final Exception e)
512 {
513 e.printStackTrace();
514 }
515 }
516
517 /**Allow a hard-wired (local) profiling from start-up if GenPropsGenNames.GPGEN_KEY_debugFlag is non-null in system properties. */
518 private static final boolean permSlowProfile;
519 static
520 {
521 boolean pSP = false;
522 try { pSP = Boolean.getBoolean(GenPropsGenNames.GPGEN_KEY_debugFlag); }
523 catch(final Exception e) { /* No access; soldier on without whingeing. */ }
524 permSlowProfile = pSP;
525 if(pSP) { System.out.println("*** SLOW PROFILING FORCED ON"); }
526 }
527
528 /**Level of slow-profiling and its logging; NONE if off, LOCAL if local only, GLOBAL if local and global. */
529 public static enum SlowProfileLevel { NONE, LOCAL, GLOBAL; };
530
531 /**If true then slow profiling is enabled in system properties; never null.
532 * True if:
533 * * Permanent slow profiling enabled in system properties
534 * * Local slow profiling enabled in local properties
535 * * Global slow profiling enabled in global properties either with "true" or with our mirror name.
536 * @param gp global properties; never null
537 */
538 private SlowProfileLevel slowProfileEnabled(final GenProps gp)
539 {
540 final String debugProp = gp.getGen().get(GenPropsGenNames.GPGEN_KEY_debugFlag);
541 if(null != debugProp)
542 {
543 // GenProps "true" value means "all instances log globally".
544 if("true".equals(debugProp)) { return(SlowProfileLevel.GLOBAL); }
545
546 // GenProps non-null value other than "true" means
547 // that just the instance(s) with matching mirror tag should log globally.
548 if(debugProp.equals(LocalProps.getMirrorTag())) { return(SlowProfileLevel.GLOBAL); }
549 }
550
551 // System property set enables local logging.
552 if(permSlowProfile) { return(SlowProfileLevel.LOCAL); }
553
554 // Non-null value in local props enables local logging.
555 if(null != LocalProps.getGen().get(GenPropsGenNames.GPGEN_KEY_debugFlag)) { return(SlowProfileLevel.LOCAL); }
556
557 // No slow-profiling at all.
558 return(SlowProfileLevel.NONE);
559 }
560
561 /**Don't dump busy threads again before this time; initially zero so first dump is immediately as required.
562 * Volatile for thread-safety and atomicity of access.
563 */
564 private long dontDumpThreadsBefore;
565
566 /**Make a note of the thread state in a (busy) system.
567 * Automatically log the "busy" methods to the global distributed logging store,
568 * and extra detail to the System.err.
569 *
570 * @param connCount current active-connection count; non-negative
571 */
572 @Override protected void logBusyThreads(final int connCount)
573 {
574 final long dumpStart = System.currentTimeMillis();
575
576 // Don't print if too soon (but do always record events).
577 final boolean dontPrint = (System.currentTimeMillis() < dontDumpThreadsBefore);
578
579 try
580 {
581 final ServletContext servletContext = getConfig().getServletContext();
582 final ServletContext ctxt = servletContext;
583 final DataSourceBean dsb = DataSourceBean.getApplicationInstance(ctxt);
584 final SimpleVariablePipelineIF vars = dsb;
585 final GenProps gp = dsb.getGenProps(-1);
586
587 // Note that this host system is busy.
588 vars.setVariable(new SimpleVariableValue(PROFILING_EVENT,
589 "SysBusy=" + LocalProps.getMirrorTag()));
590
591 if(!dontPrint)
592 { servletContext.log("WARNING: SYSTEM TOO BUSY: ACTIVE CONNECTION COUNT: " + connCount); }
593
594 // Don't log threads again if slow profiling is already enabled.
595 final SlowProfileLevel slowProfileEnabled = slowProfileEnabled(gp);
596 if(slowProfileEnabled == SlowProfileLevel.NONE)
597 { logLiveThreads(dsb, statsIDPERF, connCount, SlowProfileLevel.GLOBAL); }
598 }
599 catch(final Exception e)
600 { logger.log("unexpected Exception", e); }
601 finally
602 {
603 // If we just printed a full dump
604 // then postpone the next one a while.
605 if(!dontPrint)
606 {
607 final long dumpEnd = System.currentTimeMillis();
608 final long dumpTime = dumpEnd - dumpStart;
609 // Limit time spent dumping to no more than about 1%,
610 // but dump at least once per minute in any case.
611 dontDumpThreadsBefore = System.currentTimeMillis() +
612 10000 + Math.min(50000, 100*dumpTime);
613 }
614 }
615 }
616
617 /**If true then only show top-most method in log rather than entire trace... */
618 private static final boolean LOG_ONLY_TOPMOST = true;
619
620 /**If true then omit threads in native code, ie not actively burning CPU in the JVM. */
621 private static final boolean LOG_ONLY_INJVM = true;
622
623 /**Log running threads to the global event record and/or to the servlet stats/log as appropriate.
624 * @param dsb current data source bean; never null
625 * @param statsLogger local stats recorder/logger; may be null if its local stats not required
626 * @param connCount number of open inbound connections if non-negative
627 * @param loggingLevel level/scope of logging output
628 * (always logs locally unless NONE, logs global only if GLOBAL); never null
629 *
630 * Primarily made public for testing.
631 */
632 public static void logLiveThreads(final DataSourceBean dsb,
633 final StatsLogger.StatsConfig statsLogger,
634 final int connCount,
635 final SlowProfileLevel loggingLevel)
636 throws IOException
637 {
638 if(null == dsb) { throw new IllegalArgumentException(); }
639 if(null == loggingLevel) { throw new IllegalArgumentException(); }
640
641 final boolean logLocally = (loggingLevel != SlowProfileLevel.NONE);
642 final boolean logGlobally = (loggingLevel == SlowProfileLevel.GLOBAL);
643
644 // Now note the state of the threads.
645 final Map<Thread, StackTraceElement[]> threads = Thread.getAllStackTraces();
646 boolean loggedThreadCount = false;
647 for(final Thread t : threads.keySet())
648 {
649 final Thread.State state = t.getState();
650
651 // Omit this current thread's detail as uninteresting...
652 if(t == Thread.currentThread())
653 { continue; }
654 // Omit threads in probably-uninteresting
655 // (non-CPU/IO-using, non-blocked) states...
656 if((state == Thread.State.NEW) ||
657 (state == Thread.State.TIMED_WAITING) ||
658 (state == Thread.State.WAITING) || /* WAITING might be indefinitely blocked or deadlocked. */
659 (state == Thread.State.TERMINATED))
660 { continue; }
661 // Omit some uninteresting (eg system) threads quickly...
662 final StackTraceElement[] stackTraceElements = threads.get(t);
663 // Zero-length stack, eg like for a signal handler...
664 if(0 == stackTraceElements.length)
665 { continue; }
666 // In native code (so not actively burning CPU cycles in Java)...
667 final StackTraceElement topOfStack = stackTraceElements[0];
668 if(LOG_ONLY_INJVM && (null != topOfStack) && topOfStack.isNativeMethod())
669 { continue; }
670
671 // If we haven't logged the live thread count yet then do so here.
672 // This means that on a quiet system we shouldn't need to log anything at all.
673 if(logLocally && !loggedThreadCount)
674 {
675 dsb.log("System activity: LIVE THREAD COUNT: " + threads.size() + ((connCount < 0) ? "" : (", connection count " + connCount)));
676 loggedThreadCount = true;
677 }
678
679
680 if(logLocally) { dsb.log("System activity: ACTIVE THREAD: " + t.getName() + ": " + state + ", stack depth " + stackTraceElements.length); }
681 // Log stack entries for PG2K code only.
682 boolean loggedEvent = false;
683 for(final StackTraceElement ste : stackTraceElements)
684 {
685 if(ste == null)
686 { continue; }
687 if(loggedEvent && !logLocally)
688 { break; /* Nothing more to do... */ }
689 final String cn = ste.getClassName();
690 final boolean ourCode = cn.startsWith("org.hd.d.pg2k.");
691 // Don't show 3rd-party methods above the first instance
692 // of our code in the stack trace.
693 if(!ourCode && loggedEvent)
694 { continue; }
695 final String mn = ste.getMethodName();
696 final int ln = ste.getLineNumber();
697 if(ourCode && !loggedEvent)
698 {
699 final String methodDetail = ("Busy=" + cn + "." + mn); // Leave out line number for more consistent stats.
700 final String threadState = "BusyThreadState=" + state;
701
702 // Note the state of the Thread running our method.
703 if(logGlobally)
704 { dsb.setVariable(new SimpleVariableValue(PROFILING_EVENT, threadState)); }
705 if(logLocally)
706 { if(null != statsLogger) { StatsLogger.captureDataPoint(statsLogger, threadState); } }
707
708 // Log the event...
709 // Do so after the thread state
710 // as this is a slightly more interesting "last value"
711 // to see in the stats.
712 if(logGlobally)
713 { dsb.setVariable(new SimpleVariableValue(PROFILING_EVENT, methodDetail)); }
714 if(logLocally)
715 {
716 dsb.log("System activity: METHOD: " + ((ln >= 0) ? (methodDetail + ("(line:" + ln + ")")) : methodDetail));
717 if(null != statsLogger) { StatsLogger.captureDataPoint(statsLogger, methodDetail); }
718 }
719
720 // Print any lock on which we are blocked, and by whom...
721 if(logLocally)
722 {
723 final long tid = t.getId();
724 final ThreadMXBean tMXB =
725 java.lang.management.ManagementFactory.getThreadMXBean();
726 final ThreadInfo tInfo = tMXB.getThreadInfo(tid);
727 if(tInfo != null)
728 {
729 final String lockName = tInfo.getLockName();
730 if(lockName != null)
731 { dsb.log("System activity: MONITOR: " + lockName); }
732 final String lockerName = tInfo.getLockOwnerName();
733 if(lockerName != null)
734 { dsb.log("System activity: BLOCKING THREAD: " + lockerName); }
735 }
736 }
737
738 loggedEvent = true;
739 if(LOG_ONLY_TOPMOST) { break;}
740 }
741 }
742 }
743 }
744
745 /**Allow start-up fast execution sampling for performance tuning. */
746 private static final boolean STARTUP_EXECUTION_SAMPLING_ENABLED = true;
747
748 /**Start-up sampling time (ms); strictly positive. */
749 private static final int STARTUP_EXECUTION_SAMPLING_MS = 180000;
750
751 /**Maximum number of (top) sampled sites to dump; strictly positive. */
752 private static final int STARTUP_EXECUTION_SAMPLING_MAX_DUMP = 100;
753
754 /**Start sampling ASAP, if enabled and requested at run-time. */
755 static
756 {
757 if(STARTUP_EXECUTION_SAMPLING_ENABLED &&
758 Boolean.getBoolean("org.hd.d.pg2k.profileStartup"))
759 {
760 final long sampleStartTime = System.currentTimeMillis();
761 final long stopBy = sampleStartTime + STARTUP_EXECUTION_SAMPLING_MS;
762 final Thread t = new Thread("startup sampling thread"){
763 @Override public final void run()
764 {
765 final ConcurrentHashMap<StackTraceElement,AtomicInteger> counts =
766 new ConcurrentHashMap<StackTraceElement, AtomicInteger>(10001);
767 System.out.println("[Sampling started...]");
768 do
769 {
770 // Sample loop.
771 _perfSampleThreadsQuick(counts);
772 // Approx 100Hz sample rate should be OK.
773 try { Thread.sleep(10); }
774 catch(final InterruptedException e)
775 {
776 e.printStackTrace();
777 break; // Stop sampling...
778 }
779 } while(System.currentTimeMillis() < stopBy);
780
781 GenUtils.dumpPerfSamples("START-UP ACTIVITY SAMPLES (10ms)", counts, null, STARTUP_EXECUTION_SAMPLING_MAX_DUMP, GenUtils.systemOutLogger);
782 }
783 };
784 t.setDaemon(true);
785 t.setPriority(Thread.MAX_PRIORITY); // Highest-possible priority.
786 t.start();
787 }
788 }
789
790 /**Quickly sample running threads for performance logging, eg at start-up.
791 * This is likely to be called very frequently,
792 * so should be very fast and efficient.
793 * <p>
794 * This logs the top-most (current) activation record for every live thread
795 * (non-NEW, non-TERMINATED) in the system as a count in the table.
796 * <p>
797 * The calling thread is omitted.
798 * <p>
799 * It is safe for other threads to read the counts map while it is
800 * being updated in here: the map is always kept in a consistent state.
801 *
802 * @param counts caller-supplied map to be updated with counts of samples
803 * at each execution site; never null
804 */
805 private static void _perfSampleThreadsQuick(final ConcurrentHashMap<StackTraceElement,AtomicInteger> counts)
806 {
807 // Now note the state of the threads.
808 final Map<Thread, StackTraceElement[]> threads = Thread.getAllStackTraces();
809 for(final Thread t : threads.keySet())
810 {
811 final Thread.State state = t.getState();
812 // Omit this current thread's sample as uninteresting...
813 if(t == Thread.currentThread())
814 { continue; }
815 // Omit sample for threads in probably-uninteresting states...
816 if((state == Thread.State.NEW) ||
817 (state == Thread.State.TERMINATED))
818 { continue; }
819 // Get stack trace.
820 final StackTraceElement[] stackTraceElements = threads.get(t);
821 // Skip threads with an empty or strange stack.
822 final StackTraceElement top;
823 if((stackTraceElements == null) ||
824 (stackTraceElements.length < 1) ||
825 (null == (top = stackTraceElements[0])))
826 { continue; }
827
828 // Get existing counter for this execution site,
829 // or (atomically) create a new zero one.
830 AtomicInteger count;
831 while(null == (count = counts.get(top)))
832 { counts.putIfAbsent(top, new AtomicInteger()); }
833
834 // Increment counter.
835 count.incrementAndGet();
836 }
837 }
838
839 // /**Private session attribute name which we use to avoid recording multiple new hits/conversions for one client.
840 // * We DO NOT create a session to store this,
841 // * but if a session is available then we check with this.
842 // */
843 // private static final String SESSION_STICKY_LOGGED_ATTRNAME = "org.hd.d.pg2k.webSvr.util.ThroughputMonitorFilterPG2K.ALREADY_NOTED_AS_STICKY";
844
845 /**Returns true if we deem this a hit to be from a "sticky" client.
846 * "Sticky" means:
847 * <ul>
848 * <li>A GET request.
849 * <li>Not a spider/bot or a hot-link or a prefetch
850 * (ie a human seems to have clicked from another of our pages).
851 * <li>Not a SPAMmer or compromised machine.
852 * <li>A request for a text page (not an exhibit/graphic) not top-level,
853 * eg a catalogue page (as the real meat of the site).
854 * </ul>
855 * <p>
856 * This means that we should capture hits by a user on their
857 * second and subsequent page views.
858 *
859 * @param isGET true if the method is GET
860 * @param requestH the HTTP request; never null
861 * @param clientIP the client's IP address; never null
862 *
863 * @return true iff this hit is to be treated as from a "sticky" client
864 */
865 @Override
866 protected boolean isStickyClient(final boolean isGET,
867 final HttpServletRequest requestH,
868 final InetAddress clientIP)
869 {
870 assert(requestH != null);
871 assert(clientIP != null);
872
873 // Reject non-catalogue-area hits
874 // (disallow top-level pages, allow cat/search pages).
875 // We do this check first because it should be relatively fast.
876 final String reqURI = requestH.getRequestURI();
877 if((reqURI == null) ||
878 // (reqURI.lastIndexOf('/') <= 0) ||
879 !reqURI.startsWith(CATAREA_PREFIX))
880 { return(false); }
881
882 // Reject non-human visitors or new visitors or precache requests...
883 if(!isGET ||
884 WebUtils.requestProbablyFromSpider(requestH) ||
885 WebUtils.isPrecacheRequest(requestH) ||
886 (null != WebUtils.requestProbablyReferredFromExternalSite(requestH)))
887 { return(false); }
888
889 // Reject visitors from compromised machines
890 // and those from bad neighbourhoods listed in DNS BLs.
891 // This may be slow, so we do it as the last check.
892 if(null != AddrTools.isAddrBlackholed(getDNSBLs(), clientIP, false))
893 { return(false); }
894
895 // Note the "sticky client" page hit.
896 stickyClientPageHit(clientIP);
897 return(true); // Yes, is sticky.
898 }
899
900 /**Counter of "sticky" hits on site pages (rather than graphics).
901 * These are batched up and submitted to stats in bundles of 1000 (kHits).
902 * This value should remain between 0 and around 1000.
903 * <p>
904 * We use an AtomicInteger to avoid having to take a lock.
905 */
906 private final AtomicInteger stickyClientHitCounter = new AtomicInteger();
907
908 /**Note a page (eg HTML) hit from a sticky client.
909 * @param clientIP client's IP address, never null.
910 */
911 protected void stickyClientPageHit(final InetAddress clientIP)
912 {
913 // Record another 1000 hits...
914 if(stickyClientHitCounter.incrementAndGet() >= 1000)
915 {
916 stickyClientHitCounter.addAndGet(-1000);
917 StatsLogger.captureDataPoint(statsIDTHRFGEN,
918 THRFNAME_kHIT_STICKY_CLIENT_PAGE);
919 final DataSourceBean dsb = DataSourceBean.getApplicationInstance(getConfig().getServletContext());
920 try
921 {
922 dsb.setVariable(new SimpleVariableValue(
923 SystemVariables.GENSTATS_STRING_GLOBAL_EVENT,
924 THRFNAME_kHIT_STICKY_CLIENT_PAGE));
925 }
926 catch(final IOException e)
927 {
928 e.printStackTrace(); // Absorb and report errors.
929 }
930 }
931
932 // Record the hit from a "sticky" client by location.
933 // Use a quick lookup to avoid delaying the user unduly.
934 // dsb.setVariable(new SimpleVariableValue(
935 // SystemVariables.GENSTATS_STRING_GLOBAL_EVENT,
936 // MemoryTools.intern("StickyHitFrom=" + GeoUtils.getRegionByAddress(clientIP, true))));
937 final String regionByAddress = GeoUtils.getRegionByAddress(clientIP, true);
938 StatsLogger.captureDataPoint(statsIDTHRFGEN,
939 THRFNAMEPR_STICKY_CLIENT_REGION + regionByAddress);
940 }
941
942 /**Counter of all (successful) hits.
943 * These are batched up and submitted to stats in bundles of 1000 (kHits).
944 * This value should remain between 0 and around 1000.
945 * <p>
946 * We use an AtomicInteger to avoid having to take a lock.
947 */
948 private final AtomicInteger hitCounter = new AtomicInteger();
949
950 /**Counter of all catalogue-page GET hits.
951 * These are batched up and submitted to stats in bundles of 1000 (kHits).
952 * This value should remain between 0 and around 1000.
953 * <p>
954 * We use an AtomicInteger to avoid having to take a lock.
955 */
956 private final AtomicInteger catPageHitCounter = new AtomicInteger();
957
958 /**If true, check clients in DNS BL <em>before</em> each access if status is not known.
959 * This can severely impact performance as seen by the user
960 * if used for (for example) all HTML pages
961 * so a "lazy" approach is generally preferred,
962 * but if before downloads of expensive items (eg exhibits) then an eager approach may be better.
963 */
964 private final static boolean CHECK_DNSRBL_EARLY = true;
965
966 /**Name of attribute that we use to avoid duplicate application of this filter on one filter chain. */
967 private static final String ANTI_DUP_ATTR_NAME = "org.hd.d.pg2k.webSvr.util.ThroughputMonitorFilterPG2K.DUPFLAG";
968
969 /**Called to handle each (HTTP) request.
970 * Hooks into the requests to measure throughput and other stats.
971 * <p>
972 * We may also redirect a client if a non-preferred (or unrecognised)
973 * hostname is used to contact the server.
974 * <p>
975 * We call super.doFilter() to get all the basic throughput measurement
976 * tasks done.
977 */
978 @Override public void doFilter(final ServletRequest request,
979 final ServletResponse response,
980 final FilterChain chain)
981 throws IOException, ServletException
982 {
983 // Avoid duplicate application of this filter,
984 // eg during servlet redirection.
985 if(null != request.getAttribute(ANTI_DUP_ATTR_NAME))
986 {
987 // Avoid counting the same traffic more than once.
988 if(IsDebug.isDebug) { System.err.println("WARNING: avoiding duplicate TMF application to " + request); }
989 chain.doFilter(request, response);
990 return;
991 }
992 request.setAttribute(ANTI_DUP_ATTR_NAME, Boolean.TRUE);
993
994 if(!(response instanceof HttpServletResponse) ||
995 !(request instanceof HttpServletRequest))
996 {
997 // Cannot handle non-HTTP traffic, so don't even wrap it.
998 chain.doFilter(request, response);
999 return;
1000 }
1001
1002 final HttpServletRequest httpServletRequest = (HttpServletRequest) request;
1003 final HttpServletResponse httpServletResponse = (HttpServletResponse) response;
1004
1005 // Record another hit...
1006 if(hitCounter.incrementAndGet() >= 1000)
1007 {
1008 hitCounter.addAndGet(-1000);
1009 StatsLogger.captureDataPoint(statsIDTHRFGEN, THRFNAME_kHIT);
1010 }
1011
1012 // We can monitor this if it is an HTTP request
1013 // and, possibly, only if it is a GET request.
1014 if(!ThroughputMonitorFilterBase.MONITOR_GET_TRAFFIC_ONLY ||
1015 "GET".equalsIgnoreCase(httpServletRequest.getMethod()))
1016 {
1017 final InetAddress clientIP =
1018 InetAddress.getByName(request.getRemoteAddr());
1019
1020 // Save the original incoming request URI
1021 // for components that need to see it.
1022 final String reqURI = httpServletRequest.getRequestURI();
1023 httpServletRequest.setAttribute(PNAME_SAVED_REQ_URI, reqURI);
1024
1025 // We may block accesses from "bad" clients
1026 // (SPAMmers' spiders or compromised machines)
1027 // to all but top-level (including error) pages.
1028 // More than one '/' in the URI means a non-top-level page.
1029 // We may allow some content through anyway for efficiency
1030 // and/or allow the top-level page(s) to show properly.
1031 final BLRecord statusInRBL = inBL.get(clientIP);
1032 final FilterConfig config = getConfig();
1033 assert(config != null);
1034
1035 // final boolean isNotTopLevel = (reqURI != null) &&
1036 // (reqURI.indexOf('/') != reqURI.lastIndexOf('/'));
1037 // final boolean checkBL = isNotTopLevel && /* Allow any top-level content. */
1038 // !reqURI.startsWith(TN_PREFIX) && /* Allow thumbnails. */
1039 // !reqURI.startsWith(STATIC_PREFIX); /* Allow static content. */
1040
1041 // Check client's bona fides before attempts to download exhibits.
1042 final boolean checkBL = (reqURI != null) && reqURI.startsWith(EX_PREFIX);
1043 if(checkBL)
1044 {
1045 // Does this look like a "bad" (SPAMmer/compromised) client?
1046 // We decide so if we have any entry in cache of this client address
1047 // or a positive result from a quick BL check.
1048 // We ignore the cache-entry expiry time for this purpose,
1049 // relying on coda processing to expunge or recheck old records.
1050 // This may only happen when a non-blocked resource is accessed
1051 // AFTER the BL entry for this address has been dropped.
1052 String BL = null;
1053 if(statusInRBL != null)
1054 { BL = statusInRBL.blName; }
1055 else if(CHECK_DNSRBL_EARLY)
1056 {
1057 // If no cache entry then do a QUICK check here
1058 // and cache the result if it is bad.
1059 BL = AddrTools.isAddrBlackholed(getDNSBLs(), clientIP, true);
1060 if(BL != null)
1061 {
1062 inBL.put(clientIP, new BLRecord(DNSBL_RESULT_CACHE_MS, BL));
1063 }
1064 }
1065
1066 // Looks like a bad 'un!
1067 if(BL != null)
1068 {
1069 StatsLogger.captureDataPoint(statsIDTHRFGEN,
1070 ThroughputMonitorFilterPG2K.THRFNAMEPR_DNSRBL_DETECTION + BL);
1071 // This is interesting enough to log centrally...
1072 final DataSourceBean dsb = DataSourceBean.getApplicationInstance(config.getServletContext());
1073 dsb.setVariable(new SimpleVariableValue(
1074 SystemVariables.GENSTATS_STRING_GLOBAL_EVENT,
1075 ThroughputMonitorFilterPG2K.THRFNAMEPR_DNSRBL_DETECTION + BL));
1076 // Note a few more details of this request...
1077 logger.log("WARNING: client "+AddrTools.doReverseLookup(clientIP, true)+'/'+clientIP+" is in DNSRBL "+BL+" and requested "+TextUtils.sanitiseForXML(httpServletRequest.getRequestURL().toString(), 512, true));
1078 // TODO: maybe redirect to a "your machine has been pwnd" page.
1079 httpServletResponse.sendError(HttpServletResponse.SC_FORBIDDEN,
1080 "WARNING: get help from your ISP or administrator: your machine may be INFECTED with a VIRUS or TROJAN, or your IP address may have been used to send SPAM: your IP address is listed in a DNS BL (blocklist)");
1081 return;
1082 }
1083 }
1084
1085
1086 // Note the hit by (normalised) host name.
1087 // Optionally also redirect away from deprecated host names...
1088 // Possibly allow bounce to "local" mirror.
1089 final String serverName = request.getServerName();
1090 // Our preferred form of the hostname is all-lower-case.
1091 final String serverNameLC = serverName.toLowerCase();
1092 final String normalisedServerName =
1093 HostUtils.normaliseVirtualHostName(serverNameLC);
1094 final VirtualHosts.VirtualHost vHost =
1095 VirtualHosts.getVirtualHostDetails(serverNameLC, null);
1096 StatsLogger.captureDataPoint(statsIDTHRFGEN,
1097 ThroughputMonitorFilterPG2K.THRFNAMEPR_VHOST +
1098 ((vHost == null) ? "OTHER" : normalisedServerName));
1099 // Report unexpected/unrecognised host name used to access the site,
1100 // though quietly tolerate (common) literal IP addresses.
1101 final boolean unacceptableHostname = ((vHost == null) &&
1102 !AddrTools.IPV4_PATTERN_COMMON.matcher(serverNameLC).matches() &&
1103 !AddrTools.IPV6_PATTERN_COMMON_IN_URL.matcher(serverNameLC).matches());
1104 if(unacceptableHostname)
1105 {
1106 if(ThroughputMonitorFilterPG2K.REPORT_UNEXPECTED_HOSTNAME)
1107 { logger.log("Unexpected hostname used to access this site: "+serverName+", clientIP="+clientIP); }
1108 // Regard these requests as a bit suspect, and thus low-priority.
1109 request.setAttribute("LowPriConn", Boolean.TRUE);
1110 }
1111
1112 final String mirrorTag = LocalProps.getMirrorTag();
1113 final String mirrorName;
1114
1115 // Optionally redirect user from deprecated/non-canonical hostname.
1116 // Usually we redirect to the canonical name,
1117 // but if we are a mirror and the inbound is for a mirror in the same country,
1118 // then redirect to our mirror URL,
1119 // all so as to try to keep mirror juggling from hurting SEs and end users.
1120 // Never redirect a robots.txt request so we can handle it elsewhere.
1121 // We may have to be more flexible with cloud-hosted instances, eg during testing.
1122 if(ThroughputMonitorFilterPG2K.REDIRECT_DEPRECATED_HOSTNAME &&
1123 !WebConsts.ROBOTS_PAGE.equals(reqURI))
1124 {
1125 // If we don't recognise the host name at all,
1126 // then usually PERMANENTLY redirect to our main/default name,
1127 // preserving the URI (but not any non-standard port).
1128 if(null == vHost)
1129 {
1130 if(!unacceptableHostname)
1131 {
1132 // Probably numeric, so accept as-is...
1133 }
1134 // However, if this is a cloud instance, for now let the URL name stand as-is...
1135 else if(LocalProps.isCloudMirrorInstance())
1136 {
1137 // FIXME: in future, redirect to proper mirror name for our tag if DNS extant, etc.
1138 logger.log("Avoiding redirect on unexpected URL for cloud instance");
1139 }
1140 else // Unexpected name for non-cloud instance over which we should have more control,
1141 {
1142 StatsLogger.captureDataPoint(statsIDTHRFGEN,
1143 ThroughputMonitorFilterPG2K.THRFNAME_REDIR_UNKNOWN);
1144 final String canonURI = "http://" +
1145 VirtualHosts.VIRTUAL_HOST_DEFAULT.normalisedName +
1146 ((reqURI == null) ? "/" : reqURI);
1147 // Permanent (301, SE-friendly) redirect.
1148 httpServletResponse.setStatus(HttpServletResponse.SC_MOVED_PERMANENTLY);
1149 httpServletResponse.setHeader("Location", canonURI);
1150 return;
1151 }
1152 }
1153 // As an optimisation when fielding a request for an old/deprecated mirror name,
1154 // if this instance is a mirror,
1155 // and a (valid) mirror name is being used that isn't ours,
1156 // then redirect to our own mirror name, preserving the URI.
1157 // We expect to end up servicing this redirected request sooner or later.
1158 //
1159 // This used to require also that the request mirror name is for the same country as our mirror name,
1160 // but now this will accept any mirror traffic pointed at it,
1161 // but only PERMANENTLY redirect those for the same country.
1162 else if((null != mirrorTag) &&
1163 CoreConsts.MAIN_DATA_HOST.equals(vHost.normalisedName) &&
1164 HostUtils.isMirrorName(serverNameLC) &&
1165 !serverNameLC.equals(mirrorName = HostUtils.makeMirrorNameGeneric(mirrorTag)))
1166 {
1167 StatsLogger.captureDataPoint(statsIDTHRFGEN,
1168 ThroughputMonitorFilterPG2K.THRFNAME_REDIR_ALIAS);
1169 final String requestURI = reqURI;
1170 final String canonURI = "http://" +
1171 mirrorName +
1172 ((requestURI == null) ? "/" : requestURI);
1173 final boolean isSameCountry = mirrorTag.substring(0, 2).equals(HostUtils.getMirrorCC(serverNameLC).code);
1174 // Permanent (301, SE-friendly) redirect if same country.
1175 final boolean permRedir = isSameCountry;
1176 final int redirCode = permRedir ? HttpServletResponse.SC_MOVED_PERMANENTLY : HttpServletResponse.SC_MOVED_TEMPORARILY;
1177 (httpServletResponse).setStatus(redirCode);
1178 (httpServletResponse).setHeader("Location", canonURI);
1179 logger.log("WARNING: redirected ("+redirCode+") from inbound server name "+serverName+" to mirror "+mirrorName+" for client="+clientIP);
1180 return;
1181 }
1182 // If a deprecated alias is being used,
1183 // or the alias/name is not lower-case,
1184 // and the name is not the master (special) name,
1185 // then PERMANENTLY redirect to the main/canonical alias,
1186 // preserving the URI (but not any non-standard port),
1187 // providing that the preferred name exists in DNS!
1188 // (This should also purge duplicate/dead aliases/mirrors in search engines
1189 // once we are redirecting them elsewhere.)
1190 else if(!serverName.equals(serverNameLC) ||
1191 (!serverNameLC.equals(vHost.normalisedName) &&
1192 !HostUtils.isMasterName(serverNameLC) &&
1193 ((null != mirrorTag) && !serverNameLC.equals("mirror-" + mirrorTag + '.' + vHost.normalisedName))))
1194 {
1195 final String preferredAlias = vHost.normalisedName;
1196 try
1197 {
1198 // Abort redirection if target hostname/alias
1199 // does not appear to be in DNS.
1200 // Not completely robust because of Java DNS cacheing,
1201 // and potentially expensive,
1202 // but may catch some cock-ups.
1203 InetAddress.getByName(preferredAlias);
1204
1205 StatsLogger.captureDataPoint(statsIDTHRFGEN,
1206 ThroughputMonitorFilterPG2K.THRFNAME_REDIR_ALIAS);
1207 final String requestURI = reqURI;
1208 final String canonURI = "http://" +
1209 preferredAlias +
1210 ((requestURI == null) ? "/" : requestURI);
1211
1212 // ((HttpServletResponse) response).sendRedirect(canonURI); // Temp redirect.
1213
1214 // Permanent (301, SE-friendly) redirect.
1215 (httpServletResponse).setStatus(HttpServletResponse.SC_MOVED_PERMANENTLY);
1216 (httpServletResponse).setHeader("Location", canonURI);
1217 logger.log("WARNING: redirected (perm:301) from inbound server name "+serverName+" to preferred "+preferredAlias+" for client="+clientIP);
1218 return;
1219 }
1220 catch(final UnknownHostException e)
1221 {
1222 // Whoops! Our preferred alias is not even in DNS!
1223 logger.log("ERROR: preferred server alias not in DNS, aborted redirect: " + preferredAlias);
1224 }
1225 }
1226 }
1227
1228 // Capture changes to standard session attributes.
1229 // Should only do this on GET requests
1230 // to avoid interfering with things such as file uploads (POST).
1231 final boolean isGET = "GET".equalsIgnoreCase(httpServletRequest.getMethod());
1232 if(isGET)
1233 { SessionVarBean.updateSessionVarsFromRequest(httpServletRequest); }
1234
1235
1236 // Optionally redirect (bounce) the user to a "local" (wrt the client/browser) mirror
1237 // for performance and load-balancing,
1238 // but only extremely carefully and conservatively.
1239 // Only do this for GETs and with no extant session, for example.
1240 // Only do this on internal referrals on relatively stable items
1241 // (and never at the home/error pages)
1242 // so that the user gets to see at least one page!
1243 //
1244 // Only bounce if we know the user's country, and it is not this mirror's,
1245 // and EITHER:
1246 // * There is a live mirror in the same country as this user, OR
1247 // * The user has no proximity to us and the "optimal" mirror is closer to them.
1248 if(WebConsts.ALLOW_BOUNCE_TO_LOCAL_MIRROR &&
1249 isGET /* Do not redirect anything other than GETs. */ &&
1250 (null != mirrorTag) /* We must have a tag to tell if we are already "close" to the user. */ &&
1251 (serverName.equals(CoreConsts.MAIN_DATA_HOST) /* ||
1252 serverName.equals(VirtualHosts.VIRTUAL_HOST_SMLPX.normalisedName) */ ) /* On a generic/main URL. */ &&
1253 (null != reqURI) &&
1254 (reqURI.startsWith(CATPAGE_PREFIX) || reqURI.startsWith(EX_PREFIX)) /* Cat page or exhibit. */ &&
1255 (null == httpServletRequest.getSession(false)) /* User has no session. */ &&
1256 Rnd.fastRnd.nextBoolean() /* Prevent indefinite bouncing of a user request. */ &&
1257 !WebUtils.requestProbablyFromSpider(httpServletRequest) /* Never bounce a spider/bot. */ &&
1258 (null == WebUtils.requestProbablyReferredFromExternalSite(httpServletRequest)) /* Only bounce on internal clicks (eg not on first hit). */ )
1259 {
1260 // Only consider bouncing the user
1261 // if the user is not already close to us
1262 // (ie in their country based on a FAST lookup for the user).
1263 final String regionByAddress = GeoUtils.getRegionByAddress(clientIP, true);
1264 final boolean validCcTLDForUser = CCTLD.isSyntaticallyValidCcTLD(regionByAddress);
1265 if(validCcTLDForUser &&
1266 !mirrorTag.startsWith(regionByAddress))
1267 {
1268 // If we manage to find a good local mirror for the user
1269 // then we set it below and bounce the user to it with a temp redirect.
1270 String localMirrorTag = null;
1271
1272 // Get list of active mirrors in the user's country.
1273 // We do not take into account whether
1274 // the mirrors are on the same AEP version as us
1275 // which means that there is a small chance that
1276 // we will redirect the user to a non-existent page (404).
1277 // We will eliminate dead mirrors later.
1278 final DataSourceBean dsb = DataSourceBean.getApplicationInstance(config.getServletContext());
1279 final SortedMap<String,Long> mirrors = LoadBalancingUtils.getActiveMirrors(dsb, false);
1280 // Weed out those not in the same country as the user.
1281 for(final Iterator<String> tagIt = mirrors.keySet().iterator(); tagIt.hasNext(); )
1282 {
1283 final String tag = tagIt.next();
1284 if(!tag.startsWith(regionByAddress)) { tagIt.remove(); }
1285 }
1286 // Order remaining mirrors best-first by bandwidth
1287 // and eliminate mirrors not known to be alive and responsive.
1288 final List<String> bestFirstMirrors = LoadBalancingUtils.orderMirrorTagsBestBandwidthFirst(mirrors);
1289
1290 if(IsDebug.isDebug) { logger.log("candidate for redirection to closer mirror: client="+clientIP+", region="+regionByAddress+"; same-country mirrors="+bestFirstMirrors.size()); }
1291
1292 // If there are any available servers in the user's county
1293 // then select that which has the highest available bandwidth.
1294 if(!bestFirstMirrors.isEmpty())
1295 { localMirrorTag = bestFirstMirrors.get(0); }
1296
1297 // If we haven't yet found the ideal local mirror,
1298 // and we have no proximity (NONE) to the user,
1299 // then bounce them to the "optimal" mirror if significantly closer to them.
1300 // This has to be within a closely-connected group of countries (ie COUNTRYGROUP),
1301 // not just in the huge area controlled by an address registry (ie CONTINENT).
1302 if((null == localMirrorTag) &&
1303 (GeoProximity.NONE == GeoUtils.computeProximity(regionByAddress, new GeoUtils.CCTLD(mirrorTag.substring(0, 2)))))
1304 {
1305 final String bestMirrorHostAndPort = MirrorSelectionUtils.chooseMirrorHostForLowLatency(httpServletRequest, dsb);
1306 final CCTLD putativeOptimalMirrorCCTLD = HostUtils.getMirrorCC(bestMirrorHostAndPort);
1307 if((null != putativeOptimalMirrorCCTLD) &&
1308 (GeoUtils.computeProximity(regionByAddress, putativeOptimalMirrorCCTLD).getCloseness() >= GeoProximity.COUNTRYGROUP.getCloseness()))
1309 {
1310 // OK, "optimal" mirror is fairly close to user, so bounce them to it.
1311 localMirrorTag = HostUtils.getMirrorTag(bestMirrorHostAndPort);
1312 }
1313 }
1314
1315 // If we found a better/close mirror for the user
1316 // then bounce them to it.
1317 if(null != localMirrorTag)
1318 {
1319 // There is a mirror for the user
1320 // in their own country (and it's not us)
1321 // so redirect (temporarily) to the local mirror,
1322 // preserving the URI.
1323 final String newURL = "http://" + LoadBalancingUtils.makeMirrorNameFromTag(localMirrorTag) + reqURI;
1324 ((HttpServletResponse) response).sendRedirect(newURL); // Temp redirect.
1325
1326 // Try to do some of the logging
1327 // AFTER pushing the redirect to the user
1328 // to try to minimise overall latency.
1329 StatsLogger.captureDataPoint(statsIDTHRFGEN,
1330 ThroughputMonitorFilterPG2K.THRFNAME_REDIR_TO_LOCAL);
1331 final String message = "INFO: redirected (temp:302) from inbound server name "+serverName+" to local "+TextUtils.sanitiseForXML(newURL, 256, true)+" for "+regionByAddress+" client="+clientIP;
1332 logger.log(message);
1333 // Record which country of the bounced user.
1334 dsb.setVariable(new SimpleVariableValue(
1335 SystemVariables.GENSTATS_STRING_GLOBAL_EVENT,
1336 "bounced-to-local-mirror=" + regionByAddress));
1337 return;
1338 }
1339 }
1340 }
1341
1342
1343 // Pass this request to the main monitor code to track.
1344 super.doFilter(request, response, chain); // Pass it up...
1345
1346
1347 // Do some processing after the request has been handled.
1348 _doCoda(config, httpServletRequest, isGET, reqURI, statusInRBL, clientIP, checkBL);
1349 }
1350 else
1351 {
1352 // We can't handle this request, so pass it up the chain.
1353 chain.doFilter(request, response);
1354 }
1355 }
1356
1357 /**Handle coda processing, ie after servicing body of request.
1358 * Processing started after the bulk of the request has been dealt with
1359 * may be less visible to the end user.
1360 */
1361 private void _doCoda(final FilterConfig config,
1362 final HttpServletRequest requestH,
1363 final boolean isGET,
1364 final String reqURI,
1365 final BLRecord statusInRBL,
1366 final InetAddress clientIP,
1367 final boolean checkBL)
1368 throws IOException
1369 {
1370 // Note when we start the wind-down...
1371 final long codaStart = System.currentTimeMillis();
1372
1373 final DataSourceBean dsb = DataSourceBean.getApplicationInstance(config.getServletContext());
1374 final boolean isSpider = WebUtils.requestProbablyFromSpider(requestH);
1375
1376 // If this looks like a catalogue-page isGET request then count it as such.
1377 // Do so only AFTER the response completes to minimise latency.
1378 // We use reqURI rather than pathInfo
1379 // to try to avoid counting some (internal) redirects.
1380 final boolean isCatPageGet = isGET && !isSpider &&
1381 (reqURI != null) &&
1382 (reqURI.endsWith("ml")) /* Eg .html or .wml */ &&
1383 (reqURI.startsWith(CATPAGE_PREFIX));
1384 if(isCatPageGet)
1385 {
1386 // Log the cat-page hit locally only.
1387 dsb.setVariable(new SimpleVariableValue(
1388 SystemVariables.GENSTATS_STRING_LOCAL_EVENT,
1389 THRFNAME_HIT_CAT_PAGE));
1390
1391 if(catPageHitCounter.incrementAndGet() >= 1000)
1392 {
1393 // Record another 1000 cat-page hits...
1394 catPageHitCounter.addAndGet(-1000);
1395 StatsLogger.captureDataPoint(statsIDTHRFGEN, THRFNAME_kHIT_CAT_PAGE);
1396 // Log the cat-page kHit centrally too.
1397 dsb.setVariable(new SimpleVariableValue(
1398 SystemVariables.GENSTATS_STRING_GLOBAL_EVENT,
1399 THRFNAME_kHIT_CAT_PAGE));
1400 }
1401 }
1402
1403
1404 // Note the hit by approximate geographical region...
1405 // Do so only AFTER the response completes to minimise latency,
1406 // and only for requests that did not abort with an exception.
1407 // We may be prepared to spend some time trying to look this up
1408 // dynamically, hoping that DNS cacheing will keep the cost down.
1409 final String regionByAddress = GeoUtils.getRegionByAddress(clientIP, ThroughputMonitorFilterPG2K.DO_QUICK_GEO_LOOKUP);
1410 StatsLogger.captureDataPoint(statsIDTHRFGEN,
1411 ThroughputMonitorFilterPG2K.THRFNAMEPR_CLIENT_REGION + regionByAddress);
1412
1413 // Note the rough proximity of the client so that
1414 // we can monitor how well geo-sensitive load balancing is doing.
1415 // We can only do this if we know our physical location
1416 // (eg if this server is a mirror and thus has an embedded ccTLD).
1417 final String mirrorTag = LocalProps.getMirrorTag();
1418 if(mirrorTag != null)
1419 {
1420 final GeoUtils.CCTLD cc = new GeoUtils.CCTLD(mirrorTag.substring(0, 2));
1421 final GeoProximity proximity = GeoUtils.computeProximityByAddress(clientIP, cc, ThroughputMonitorFilterPG2K.DO_QUICK_GEO_LOOKUP);
1422 StatsLogger.captureDataPoint(statsIDTHRFGEN,
1423 ThroughputMonitorFilterPG2K.THRFNAMEPR_CLIENT_PROXIMITY + proximity);
1424 }
1425
1426 // Look for access by a compromised machine.
1427 // Do this AFTER handling the request
1428 // to reduce the impact on legitimate clients,
1429 // especially on their first page load(s).
1430 //
1431 // Do we have any valid (non-expired) info on this client?
1432 // Atomically lock out other lookup activity if no such info,
1433 // while we start a lookup ourself.
1434 //
1435 // Avoid making the effort (and the DNS traffic!) for exempt URIs.
1436 if(checkBL)
1437 {
1438 final boolean unknownRBLStatus;
1439 synchronized(inBL)
1440 {
1441 final BLRecord after = inBL.get(clientIP);
1442 unknownRBLStatus = (after == null) || after.hasExpired();
1443 // If unknown then create a temporary record
1444 // so as to lock out wasteful concurrent lookups
1445 // while we are running one of our own.
1446 if(unknownRBLStatus)
1447 {
1448 // Preserve old status (if available) for a while,
1449 // or give client benefit of the doubt
1450 // during the lookup.
1451 inBL.put(clientIP, new BLRecord(20000 + Rnd.fastRnd.nextInt(20000),
1452 (after == null) ? null : after.blName));
1453 }
1454 }
1455 // Find out RBL status of this client if we don't know...
1456 // Spin this off in a separate thread if possible for speed,
1457 // using the thread pool for I/O-bound work.
1458 if(unknownRBLStatus)
1459 {
1460 ThreadUtils.nonCPUThreadPool.submit(new Runnable(){
1461 public final void run()
1462 {
1463 final Set<String> BLs = getDNSBLs();
1464 if(IsDebug.isDebug) { logger.log("[Running DNSRBL check on " + clientIP + " in " + (new ArrayList<String>(BLs)) + "...]"); }
1465 // Do a fairly fast-but-thorough check.
1466 final String DNSBLServer = AddrTools.isAddrBlackholed(BLs,
1467 clientIP,
1468 false);
1469 if(DNSBLServer != null)
1470 {
1471 // This definitely seems to be a bad client,
1472 // so record this with a long-life entry in cache.
1473 // We make the expiry time a little harder to guess
1474 // (using goodRnd) for the black hats out there...
1475 // Mean expiry time is DNSBL_RESULT_CACHE_MS.
1476 inBL.put(clientIP, new BLRecord((DNSBL_RESULT_CACHE_MS/2) +
1477 (GenUtils.mustConservePowerExtreme() ? Rnd.fastRnd : Rnd.goodRnd).nextInt(DNSBL_RESULT_CACHE_MS), DNSBLServer));
1478 }
1479 else
1480 {
1481 // If the client does not seem to be in any RBL,
1482 // then cache a negative entry for a short(er) time.
1483 // Mean expiry time is < DNSBL_RESULT_CACHE_MS.
1484 inBL.put(clientIP, new BLRecord((DNSBL_RESULT_CACHE_MS/5) + Rnd.fastRnd.nextInt(DNSBL_RESULT_CACHE_MS), null));
1485 }
1486 }
1487 });
1488 }
1489 }
1490
1491 // Record how long coda processing takes, if significant.
1492 final long codaEnd = System.currentTimeMillis();
1493 final long codaTime = codaEnd - codaStart;
1494 if(codaTime > 1)
1495 {
1496 StatsLogger.captureDataPoint(statsIDTHRFGEN,
1497 ThroughputMonitorFilterPG2K.THRFNAMEPR_CODATIME + GenUtils.log2Approx(codaTime));
1498 }
1499 }
1500
1501 /**Get the set of DNS RBLs to screen traffic, empty if none; never null. */
1502 private Set<String> getDNSBLs()
1503 {
1504 final FilterConfig config = getConfig();
1505 if(null == config) { return(Collections.emptySet()); } // Servlet shutting down.
1506
1507 final ServletContext context = config.getServletContext();
1508 try
1509 {
1510 final GenProps gp = DataSourceBean.getApplicationInstance(context).getGenProps(-1);
1511 return(gp.getDNSBLs());
1512 }
1513 catch(final Exception e)
1514 {
1515 context.log("unexpected error in getDNSBLs()", e); // Whinge but continue...
1516 return(Collections.emptySet());
1517 }
1518 }
1519
1520 /**Approximate time we cache a positive or negative DNS BL lookup value for; strictly positive.
1521 * A value of the order of an hour or so
1522 * probably matches the policy of the underlying DNS BLs reasonably well.
1523 */
1524 private static final int DNSBL_RESULT_CACHE_MS = 3600 * 1000 +
1525 Rnd.fastRnd.nextInt(1800 * 1000);
1526
1527 /**Immutable DNS block-list record, with a definite expiry time and the String name of the list for a bad client. */
1528 private static final class BLRecord extends AutoExpirableFixedLifeBase
1529 {
1530 /**Name of the list for a bad client, null for a good client (for -ve cacheing). */
1531 final String blName;
1532 /**Create an instance. */
1533 BLRecord(final int msLifetime, final String blName)
1534 {
1535 super(msLifetime);
1536 this.blName = blName;
1537 }
1538 }
1539
1540 /**LRU set of clients currently known to be in a DNS BL.
1541 * A mapping is kept from the full/exact client IP address
1542 * to the String name of the BL for a bad client
1543 * (or to null for a good client not in any BL).
1544 * <p>
1545 * A thread-safe LRU map is used to limit memory use
1546 * (it does not grow if it can discard an old/expired entry).
1547 * The underlying DNS service should also be cacheing for us,
1548 * so a cache-miss here is probably not desperately expensive.
1549 * We allow a large range in capacity to help cope with DDoS-like
1550 * bursts of distinct clients (ie distinct remote IP addresses),
1551 * while whittling down memory consumption in quieter times.
1552 * <p>
1553 * Instances with larger heaps will get larger maximum sizes
1554 * by about one extra slot per total-heap MB at creation of this map.
1555 * <p>
1556 * We're prepared to discard this entirely under severe memory stress.
1557 */
1558 private final CacheMiniMap<InetAddress, BLRecord> inBL =
1559 LRUMapAutoSizeForHitRate.<InetAddress, BLRecord>create(0.01f, 0,
1560 301 + Rnd.fastRnd.nextInt(123) + Math.max(0,(int)(Runtime.getRuntime().totalMemory()>>20)),
1561 LRUMapAutoSizeForHitRate.DEFAULT_LOAD_FACTOR, "inBL");
1562
1563
1564 /**Exclude some traffic from connection-responsiveness monitoring.
1565 * Exclude stuff that we expect to be unavoidably slow,
1566 * or that is non-interactive and need not be fast.
1567 *
1568 * @return true to ask for this request not to be monitored
1569 */
1570 @Override protected boolean dontMonitorMe(final HttpServletRequest httpServletRequest)
1571 {
1572 // Ignore tunnel traffic from mirrors/peers.
1573 if("POST".equalsIgnoreCase(httpServletRequest.getMethod()))
1574 { return(true); }
1575
1576 return(false); // The default is to monitor all connections.
1577 }
1578 }
1579