001 /*
002 Copyright (c) 1996-2012, Damon Hart-Davis
003 All rights reserved.
004
005 Redistribution and use in source and binary forms, with or without
006 modification, are permitted provided that the following conditions are
007 met:
008
009 * Redistributions of source code must retain the above copyright
010 notice, this list of conditions and the following disclaimer.
011
012 * Redistributions in binary form must reproduce the above copyright
013 notice, this list of conditions and the following disclaimer in the
014 documentation and/or other materials provided with the
015 distribution.
016
017 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
018 IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
019 TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
020 PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
021 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
022 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
023 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
024 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
025 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
026 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
027 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
028 */
029 package org.hd.d.pg2k.svrCore;
030
031 import java.net.Inet4Address;
032 import java.net.InetAddress;
033 import java.net.UnknownHostException;
034 import java.util.ArrayList;
035 import java.util.Arrays;
036 import java.util.Collection;
037 import java.util.Collections;
038 import java.util.Iterator;
039 import java.util.List;
040 import java.util.concurrent.Callable;
041 import java.util.concurrent.Future;
042 import java.util.regex.Pattern;
043
044 import org.hd.d.pg2k.svrCore.MemoryTools.RecurrentEmergencyFreeHandle;
045 import org.xbill.DNS.ARecord;
046 import org.xbill.DNS.Cache;
047 import org.xbill.DNS.Credibility;
048 import org.xbill.DNS.DClass;
049 import org.xbill.DNS.ExtendedResolver;
050 import org.xbill.DNS.Lookup;
051 import org.xbill.DNS.Name;
052 import org.xbill.DNS.PTRRecord;
053 import org.xbill.DNS.RRset;
054 import org.xbill.DNS.Record;
055 import org.xbill.DNS.ReverseMap;
056 import org.xbill.DNS.SetResponse;
057 import org.xbill.DNS.TextParseException;
058 import org.xbill.DNS.Type;
059
060 import ORG.hd.d.IsDebug;
061
062 /**This class has tools for IP-address and DNS manipulation.
063 *
064 * @author Damon Hart-Davis
065 */
066 public final class AddrTools
067 {
068 /**Prevent creation of instances of this class. */
069 private AddrTools() { }
070
071 /**If true, use DNSJava to do DNS lookups where possible, else use Java's built-in DNS resolver.
072 * DNSJava does not retain all results indefinitely unlike Java's DNS resolver,
073 * and allows us to use separate caches with different characteristics for different query types
074 * which can make it much more robust for our purposes.
075 */
076 public static final boolean USE_DNSJAVA = true;
077
078 /**Pattern to match common literal representation of IPv4 addresses; not null. */
079 public static final Pattern IPV4_PATTERN_COMMON = Pattern.compile(
080 "(([01]?[0-9][0-9]?|2[0-4][0-9]|25[0-5])\\.){3}([01]?[0-9][0-9]?|2[0-4][0-9]|25[0-5])");
081
082 /**Pattern to match basic unshortened literal representation of IPv6 addresses (case-insensitive); not null. */
083 public static final Pattern IPV6_PATTERN_BASIC = Pattern.compile(
084 "([0-9a-f]{1,4}:){7}[0-9a-f]{1,4}", Pattern.CASE_INSENSITIVE);
085 /**Pattern to match common unshortened representation of IPv6 addresses (case-insensitive) in URL; not null. */
086 public static final Pattern IPV6_PATTERN_BASIC_IN_URL = Pattern.compile(
087 "\\[(" + IPV6_PATTERN_BASIC.pattern() + ")\\]", Pattern.CASE_INSENSITIVE);
088
089 /* See for full IPv6 regex: http://stackoverflow.com/questions/53497/regular-expression-that-matches-valid-ipv6-addresses
090 (\A([0-9a-f]{1,4}:){1,1}(:[0-9a-f]{1,4}){1,6}\Z)|
091 (\A([0-9a-f]{1,4}:){1,2}(:[0-9a-f]{1,4}){1,5}\Z)|
092 (\A([0-9a-f]{1,4}:){1,3}(:[0-9a-f]{1,4}){1,4}\Z)|
093 (\A([0-9a-f]{1,4}:){1,4}(:[0-9a-f]{1,4}){1,3}\Z)|
094 (\A([0-9a-f]{1,4}:){1,5}(:[0-9a-f]{1,4}){1,2}\Z)|
095 (\A([0-9a-f]{1,4}:){1,6}(:[0-9a-f]{1,4}){1,1}\Z)|
096 (\A(([0-9a-f]{1,4}:){1,7}|:):\Z)|
097 (\A:(:[0-9a-f]{1,4}){1,7}\Z)|
098 (\A((([0-9a-f]{1,4}:){6})(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3})\Z)|
099 (\A(([0-9a-f]{1,4}:){5}[0-9a-f]{1,4}:(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3})\Z)|
100 (\A([0-9a-f]{1,4}:){5}:[0-9a-f]{1,4}:(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}\Z)|
101 (\A([0-9a-f]{1,4}:){1,1}(:[0-9a-f]{1,4}){1,4}:(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}\Z)|
102 (\A([0-9a-f]{1,4}:){1,2}(:[0-9a-f]{1,4}){1,3}:(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}\Z)|
103 (\A([0-9a-f]{1,4}:){1,3}(:[0-9a-f]{1,4}){1,2}:(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}\Z)|
104 (\A([0-9a-f]{1,4}:){1,4}(:[0-9a-f]{1,4}){1,1}:(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}\Z)|
105 (\A(([0-9a-f]{1,4}:){1,5}|:):(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}\Z)|
106 (\A:(:[0-9a-f]{1,4}){1,5}:(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}\Z)
107 */
108
109 /**Pattern to match common (not all) optionally-shortened literal representation of IPv6 addresses (case-insensitive); not null. */
110 public static final Pattern IPV6_PATTERN_COMMON = Pattern.compile(
111 "("+IPV6_PATTERN_BASIC.pattern()+")|" + // eg FEDC:BA98:7654:3210:FEDC:BA98:7654:3210
112 "(:(:[0-9a-f]{1,4}){1,7})|" + // eg ::1
113 "(([0-9a-f]{1,4}:){1,1}(:[0-9a-f]{1,4}){1,6})|" +
114 "(([0-9a-f]{1,4}:){1,2}(:[0-9a-f]{1,4}){1,5})|" +
115 "(([0-9a-f]{1,4}:){1,3}(:[0-9a-f]{1,4}){1,4})|" +
116 "(([0-9a-f]{1,4}:){1,4}(:[0-9a-f]{1,4}){1,3})|" +
117 "(([0-9a-f]{1,4}:){1,5}(:[0-9a-f]{1,4}){1,2})|" +
118 "(([0-9a-f]{1,4}:){1,6}(:[0-9a-f]{1,4}){1,1})",
119 Pattern.CASE_INSENSITIVE);
120 /**Pattern to match common (not all) optionally-shortened representation of IPv6 addresses (case-insensitive) in URL; not null. */
121 public static final Pattern IPV6_PATTERN_COMMON_IN_URL = Pattern.compile(
122 "\\[(" + IPV6_PATTERN_COMMON.pattern() + ")\\]", Pattern.CASE_INSENSITIVE);
123
124 /**Fixed empty search path for lookups. */
125 private static final Name[] EMPTY_SEARCH_PATH = new Name[0];
126
127 /**Approximate total maximum timeout for DNS resolution in seconds; strictly positive.
128 * Quite short for the the geo-lookup (etc)
129 * as we usually only get limited value from a small number of lookups,
130 * and we do not want to block interactive activity for an extended period.
131 * <p>
132 * A value of a few seconds to a few tens of seconds should normally suffice,
133 * and puts a reasonable cap on the cost of a "full" lookup.
134 */
135 private static final int RESOLVER_QUICK_TIMEOUT_S = 4;
136
137 /**Maximum retries of resolution against each DNS server/resolver; strictly positive.
138 * A value greater than 1 is more robust,
139 * ie allows for the odd lost packet,
140 * and may get good results because the retry may gain the benefit
141 * of async cacheing upstream between calls on one particular resolver.
142 * <p>
143 * A value of 1 or 2 is typical.
144 */
145 private static final int RESOLVER_QUICK_RETRIES = 2;
146
147 /**Default maximum time to positively cache a record for (seconds); strictly positive.
148 * Limiting this to a few hours at most can limit memory load
149 * when there are many users.
150 */
151 private static final int DEFAULT_DNS_CACHE_S = 3601 + Rnd.fastRnd.nextInt(1800);
152
153 /**Default maximum time to negatively cache a record for (seconds); strictly positive.
154 * Setting this to a few tens of seconds
155 * (plus a few interleaved "failed lookup" intervals)
156 * can improve performance.
157 */
158 private static final int DEFAULT_DNS_NCACHE_S = 31 + Rnd.fastRnd.nextInt(23) +
159 (7 * RESOLVER_QUICK_TIMEOUT_S);
160
161 /**Our ExtendedResolver with the normal servers but a short timeout.
162 * The DNS servers will be deduced once at initialisation.
163 */
164 private static final ExtendedResolver RESOLVER_QUICK;
165
166 /**Initialise our resolver. */
167 static
168 {
169 try {
170 RESOLVER_QUICK = new ExtendedResolver();
171 RESOLVER_QUICK.setLoadBalance(true); // Be nice to upstream resolvers.
172 RESOLVER_QUICK.setRetries(RESOLVER_QUICK_RETRIES);
173 final int resolverCount = RESOLVER_QUICK.getResolvers().length; // Resolvers found.
174 // Set the timeout per resolver to get the approximate maximum time specified.
175 RESOLVER_QUICK.setTimeout(Math.max(1, RESOLVER_QUICK_TIMEOUT_S /
176 (RESOLVER_QUICK_RETRIES * Math.max(1, resolverCount))));
177 }
178 catch(final UnknownHostException e)
179 {
180 e.printStackTrace();
181 throw new Error(e);
182 }
183 }
184
185 /**If true then we randomise the order in which we do RBL lookups.
186 * We do this to better spread the load if several of them could
187 * answer the question.
188 * <p>
189 * This does not have to be "very" random,
190 * indeed, round-robin would probably do just as well.
191 */
192 private static final boolean RANDOMISE_RBL_LOOKUP_ORDER = true;
193
194 /**Maximum reasonable cache size under normal circumstances; strictly positive. */
195 private static final int MAX_CACHE_SIZE_LIMIT = 32768;
196
197 /**Minimum reasonable cache size under normal circumstances; strictly positive (greater than 1). */
198 private static final int MIN_CACHE_SIZE_LIMIT = 256;
199
200 /**Maximum DNS cache size; strictly positive.
201 * Approx 1 entry per 64kB of current total heap (eg 4096 entries @ 256MB),
202 * but capped to something bearable since upstream resolvers should cache stuff for us too.
203 * <p>
204 * Sensitive also to remaining free heap space.
205 */
206 private static int computeMaxCacheSize()
207 {
208 return(Math.min(MAX_CACHE_SIZE_LIMIT,
209 (int) Math.max(MIN_CACHE_SIZE_LIMIT,
210 (MemoryTools.percentFreeWithinTarget() * (Runtime.getRuntime().totalMemory() >> 16)) >> 8)));
211 }
212
213 /**The shared DNS cache (for IN records); never null.
214 * This is created is tuned for our purposes
215 * (for example shorter timeouts and longer negative cacheing than default).
216 * It may be discarded and recreated if we are short of memory;
217 * hopefully caching by boundary DNS resolvers should minimise
218 * the impact if we do this.
219 * <p>
220 * This is not handed out to random callers,
221 * but they can give us a Lookup object and we will set the cache
222 * and other relevant parameters,
223 * and return the result to them.
224 * <p>
225 * This is intended to be highly threadable for many concurrent lookups.
226 */
227 private static final Cache _cacheIN;
228 /**Emergency-free handle kept to avoid it being GCed. */
229 private static final CacheREFH _cacheREFH;
230 /**Initialise DNS cache. */
231 static
232 {
233 final Cache newCache = new Cache(DClass.IN);
234 newCache.setMaxCache(DEFAULT_DNS_CACHE_S);
235 newCache.setMaxNCache(DEFAULT_DNS_NCACHE_S);
236 newCache.setMaxEntries(computeMaxCacheSize()); // Set to suit (current) heap state...
237 _cacheIN = newCache;
238
239 final CacheREFH refh = new CacheREFH(newCache);
240 _cacheREFH = refh;
241 // Register this cache instance to be cleared/slimmed when system is under memory stress.
242 MemoryTools.registerRecurrentEmergencyFreeHandle(refh);
243 }
244
245 /**Return access to the DNS cache; never null.
246 * If short of memory then we may automatically discard/clear the cache.
247 * <p>
248 * This cache may have several parameters tuned for geo-lookup purposes,
249 * such as negative/positive cache time-limits.
250 * <p>
251 * Adding belt-and-braces clear of cache if memory under great pressure.
252 */
253 private static Cache _getDNSCache()
254 {
255 final Cache cache = _cacheIN;
256
257 // Clamp cache to sub-normal maximum size (asynchronously) if memory is stressed,
258 // extending the limit again once less stressed.
259 // Should be relatively cheap.
260 if((cache.getSize() > MIN_CACHE_SIZE_LIMIT) && MemoryTools.isMemoryStressed())
261 { cache.setMaxEntries(MIN_CACHE_SIZE_LIMIT-1); }
262 else if((cache.getMaxEntries() < MIN_CACHE_SIZE_LIMIT) && !MemoryTools.isMemoryStressed())
263 { cache.setMaxEntries(computeMaxCacheSize()); }
264
265 return cache;
266 }
267
268 /**Do (quick) blocking lookup in our shared and tuned cache and resolver.
269 * This should work better for us than the built-in Java implementation
270 * and we should build a useful but size-bounded cache unless
271 * the JVM gets very low on memory, in which case we may discard it and start again.
272 * <p>
273 * The caller should <em>not</em> extract/sequester/inspect
274 * the cache/resolver/etc properties that this sets in the Lookup parameter,
275 * so this routine is only package-visible.
276 * <p>
277 * FIXME: As of 20060703 this was the most expensive consumer of time for doHTML.jsp.
278 */
279 static Record[] doQuickLookupWithTunedCache(final Lookup lookup)
280 {
281 //final long startTime = System.currentTimeMillis();
282
283 lookup.setCache(_getDNSCache());
284 lookup.setSearchPath(EMPTY_SEARCH_PATH);
285 lookup.setResolver(RESOLVER_QUICK);
286 final Record[] result = lookup.run();
287
288 //final long time = System.currentTimeMillis() - startTime;
289 //if(time > 1) /* Ignore noise. */ { (new Throwable("time: "+time+"ms")).printStackTrace(); }
290
291 return(result);
292 }
293
294 /**Do local lookup in our shared and tuned cache only; never null.
295 * Do not do DNS resolution; return nothing if there is nothing in cache.
296 * <p>
297 * The Name supplied is looked up exactly as-is,
298 * so had better be absolute for example.
299 * <p>
300 * We only make this package-visible in case the DNS java classes
301 * can leak state.
302 */
303 static SetResponse doLookupInTunedCacheOnly(final String name,
304 final int type,
305 final int minCred)
306 throws TextParseException
307 { return(_getDNSCache().lookupRecords(Name.fromString(name), type, minCred)); }
308
309 /**Return true if we can find an A record for the given name quickly.
310 * Uses our shared/tuned cache and does a quick lookup
311 * (ie times out quickly).
312 *
313 * @throws IllegalArgumentException if argument is null or unparsable.
314 */
315 public static boolean hasARecordQuick(final String name)
316 {
317 if(name == null)
318 { throw new IllegalArgumentException(); }
319
320 try
321 {
322 final Lookup lookup = new Lookup(name, Type.A);
323 final Record[] result = doQuickLookupWithTunedCache(lookup);
324
325 // If we got some results, there is an A record..
326 return((result != null) && (result.length > 0));
327 }
328 catch(final TextParseException e)
329 {
330 throw new IllegalArgumentException("bad DNS name: " + name);
331 }
332 }
333
334 // /**Convert an (IPv4) InetAddress to dotted-quad form.
335 // * Avoid doing any DNS lookup if at all possible.
336 // *
337 // * @param addr parameter to convert to dotted-quad;
338 // * must be non-null IPv4 address
339 // *
340 // * @throws IllegalArgumentException if addr is null or not an IPv4 address
341 // */
342 // public static String toStringDottedQuad(final InetAddress addr)
343 // throws IllegalArgumentException
344 // {
345 // if(addr == null)
346 // { throw new IllegalArgumentException(); }
347 //
348 // final byte raw[] = addr.getAddress();
349 //
350 // if(raw.length != 4)
351 // { throw new IllegalArgumentException("not an IPv4 address"); }
352 //
353 // final StringBuilder sb = new StringBuilder(15);
354 // for(int i = 0; i < 4; ++i)
355 // {
356 // if(sb.length() != 0) { sb.append('.'); }
357 // sb.append(raw[i] & 0xff);
358 // }
359 //
360 // return(sb.toString());
361 // }
362
363 /**Convert an IPv4 address to reverse dotted-quad form with a trailing dot.
364 * This can be useful for RBL and reverse lookups.
365 * <p>
366 * Avoid doing any DNS lookup if at all possible.
367 *
368 * @param addr parameter to convert to reverse dotted-quad;
369 * must be non-null IPv4 address
370 *
371 * @throws IllegalArgumentException if addr is null or not an IPv4 address
372 */
373 public static String toStringReverseDottedQuad(final Inet4Address addr)
374 throws IllegalArgumentException
375 {
376 if(addr == null)
377 { throw new IllegalArgumentException(); }
378
379 final byte raw[] = addr.getAddress();
380
381 assert(raw.length == 4);
382 // if(raw.length != 4)
383 // { throw new IllegalArgumentException("not an IPv4 address"); }
384
385 final StringBuilder sb = new StringBuilder(16);
386 for(int i = raw.length; --i >= 0; )
387 {
388 sb.append(raw[i] & 0xff).append('.');
389 }
390
391 return(sb.toString());
392 }
393
394 /**Do reverse lookup on IP address to get the name, returns null if lookup fails.
395 * This can optionally verify the name by trying to look up the name it
396 * found and making sure that at least one of the IPs returned matches.
397 * <p>
398 * This avoids the built-in JVM lookup routines to avoid the default
399 * infinite cacheing that it does for class-loading safety.
400 * <p>
401 * Blocks until it has an answer or times out.
402 *
403 * @param addr IP address to look up as numeric address; never null
404 * @param verify if true, attempt to verify with forward lookup of
405 * the putative result, returning null if no suitable match is found
406 */
407 public static String doReverseLookup(final String addr,
408 final boolean verify)
409 {
410 if(addr == null)
411 { throw new IllegalArgumentException(); }
412
413 try { return(doReverseLookup(InetAddress.getByName(addr), verify)); }
414 // Return null for an unparsable address.
415 catch(final UnknownHostException e) { return(null); }
416 }
417
418 /**Do reverse lookup on IP address to get the name, returns null if lookup fails.
419 * This can optionally verify the name by trying to look up the name it
420 * found and making sure that at least one of the IPs returned matches.
421 * <p>
422 * This avoids the built-in JVM lookup routines to avoid the default
423 * infinite cacheing that it does for class-loading safety.
424 * <p>
425 * Blocks until it has an answer or times out.
426 *
427 * @param addr IP address to look up; never null
428 * @param verify if true, attempt to verify with forward lookup of
429 * the putative result, returning null if no suitable match is found
430 */
431 public static String doReverseLookup(final InetAddress addr,
432 final boolean verify)
433 {
434 if(addr == null)
435 { throw new IllegalArgumentException(); }
436
437 // // Special case for loopback address
438 // // to avoid going to DNS
439 // // which should enhance speed and robustness.
440 // //
441 // // This does not need verification.
442 // if(isLoopbackAddress(addr))
443 // { return(DEFAULT_IPV4_LOOPBACK_CANON_NAME); }
444
445 // Options.set("verbose"); // Make DNS java talkative.
446
447 //System.err.println("Doing reverse lookup for address: " + toStringDottedQuad(addr));
448 try
449 {
450 // Do the reverse lookupRev...
451 // final String result = Address.getHostName(addr);
452 // Use our tuned cache to do it.
453 final Name name = ReverseMap.fromAddress(addr);
454 final Lookup lookupRev = new Lookup(name, Type.PTR);
455 final Record recordsRev[] = doQuickLookupWithTunedCache(lookupRev);
456 if((recordsRev == null) || (recordsRev.length < 1))
457 { throw new UnknownHostException("unknown address: " + addr.getHostAddress()); }
458 final PTRRecord ptr = (PTRRecord) recordsRev[0];
459 final String result = ptr.getTarget().toString();
460
461 //System.err.println(" Reverse lookupRev result was: " + result);
462
463 // If not verifying, return the result immediately.
464 if(!verify) { return(result); }
465
466 // If verifying, check against all addresses for the forward lookup.
467 // final InetAddress addrs[] = Address.getAllByName(result);
468 // Use our tuned cache.
469 final InetAddress[] addrs = lookupARecords(result, false);
470 //System.err.println(" Verifying, forward lookupRev answers: " + addrs.length);
471 for(int i = addrs.length; --i >= 0; )
472 {
473 // If an address matched then the address is verified.
474 if(addr.equals(addrs[i])) { return(result); }
475 }
476
477 return(null); // Failed to verify the address.
478 }
479 catch(final UnknownHostException e)
480 {
481 // Lookup failed.
482 //System.err.println(" Lookup failed: " + e.getMessage());
483 return(null);
484 }
485 catch(final TextParseException e)
486 {
487 //System.err.println(" Lookup failed: " + e.getMessage());
488 return(null);
489 }
490 }
491
492 /**Look up IPv4 addresses for the given hostname; never null nor empty.
493 * If the host name supplied is not already absolute (trailing '.')
494 * then one will be appended before lookup to avoid ambiguity/confusion!.
495 * <p>
496 * Non-blocking lookup may return only first viable answer.
497 *
498 * @param hostname (absolute) hostname to look up in DNS; non-null
499 * @param nonBlocking attempt to make this non-blocking if at all possible
500 * and if a value isn't immediately available issue a background request instead if possible
501 * @return non-null, non-empty set of IPv4 addresses for the hostname
502 * @throws TextParseException for a malformed hostname
503 * @throws UnknownHostException if no A records can be found
504 */
505 public static InetAddress[] lookupARecords(final String hostname, final boolean nonBlocking)
506 throws TextParseException, UnknownHostException
507 {
508 if(null == hostname) { throw new IllegalArgumentException(); }
509
510 final String hostnameAbsolute = hostname.endsWith(".") ? hostname : (hostname + ".");
511
512 if(nonBlocking)
513 {
514 final SetResponse r = doLookupInTunedCacheOnly(hostnameAbsolute, Type.A, Credibility.NORMAL);
515 // If the answer is definitively negative then nothing else to be done.
516 if(r.isNXDOMAIN())
517 { throw new UnknownHostException("no such host: " + hostnameAbsolute); }
518
519 // Return first viable answer.
520 final RRset[] rrsa = r.answers();
521 if(rrsa != null)
522 {
523 for(final RRset rrs : rrsa)
524 {
525 if(rrs.size() > 0)
526 {
527 final Record record = rrs.first();
528 if(record.getType() == Type.A) {
529 return (new InetAddress[]{ ((ARecord) record).getAddress() });
530 }
531 }
532 }
533 }
534
535 // Couldn't find any answer at all in the cache.
536 // Try to force a lookup in the background to get the results cached.
537 final Lookup lookupFwd = new Lookup(hostnameAbsolute, Type.A);
538 ThreadUtils.nonCPUThreadPoolDiscardable.execute(new Runnable()
539 {
540 @Override public void run() { doQuickLookupWithTunedCache(lookupFwd); }
541 });
542 // Tell the caller that we have no immediate answer.
543 throw new UnknownHostException("host not known from cache: " + hostnameAbsolute);
544 }
545
546 // Do blocking lookup.
547 final Lookup lookupFwd = new Lookup(hostnameAbsolute, Type.A);
548 final Record recordsFwd[] = doQuickLookupWithTunedCache(lookupFwd);
549 if((recordsFwd == null) || (recordsFwd.length < 1))
550 { throw new UnknownHostException("unknown host: " + hostnameAbsolute); }
551 final InetAddress[] addrs = new InetAddress[recordsFwd.length];
552 for(int i = addrs.length; --i >= 0; )
553 {
554 final ARecord a = (ARecord) recordsFwd[i];
555 addrs[i] = a.getAddress();
556 }
557 return addrs;
558 }
559
560 /**Check an (IPv4) IP address with the RBL servers for being "black-holed".
561 * For an IPv4 address of the numerical form a.b.c.d,
562 * list does a lookup for an A record for d.c.b.a.RBL.domain,
563 * and if it finds one, the client address is "bad".
564 * <p>
565 * We may do these checks serially, or in parallel for speed.
566 * We may pick one RBL server at random each time for speed.
567 * The first suitable A record returned stops the search.
568 * <p>
569 * We may randomise the order in which we do lookup to spread
570 * the load amongst the RBL servers more fairly.
571 * <p>
572 * FIXME: For now, always returns null for non-IPv4 addresses.
573 *
574 * @param quick if true, check just one of the BLs at random
575 *
576 * @return name of one BL server that black-holed address if so,
577 * null if address is OK
578 *
579 * @throws IllegalArgumentException if given a null address
580 */
581 public static final String isAddrBlackholed(final Collection<String> extantRBLServers,
582 final InetAddress addr,
583 final boolean quick)
584 throws IllegalArgumentException
585 {
586 if((addr == null) || (extantRBLServers == null))
587 { throw new IllegalArgumentException(); }
588
589 // Can only do this for IPv4 addresses for now.
590 if(!(addr instanceof Inet4Address)) { return(null); }
591
592 final byte rawAddr[] = addr.getAddress();
593
594 assert(rawAddr.length == 4);
595 // if(rawAddr.length != 4)
596 // { return(null); }
597
598 // Return null immediately if no servers available.
599 final int nServers = extantRBLServers.size();
600 if(nServers == 0) { return(null); }
601
602 // Create the reverse dotted-quad address plus trailing dot.
603 final String prefix = toStringReverseDottedQuad((Inet4Address) addr);
604
605 // If there is exactly one DNS BL server then check it immediately.
606 if(nServers == 1) { return(_checkOneAddrInDNSBL(prefix, extantRBLServers.iterator().next())); }
607
608 // The iterator for the RBL servers to test.
609 final Iterator<String> serverIt;
610
611 if(quick)
612 {
613 // Try just one of the RBL servers at random for speed.
614 serverIt = Collections.singletonList(
615 ((extantRBLServers instanceof List) ? (List<String>)extantRBLServers : new ArrayList<String>(extantRBLServers)).
616 get(Rnd.fastRnd.nextInt(nServers))).iterator();
617 return(_checkOneAddrInDNSBL(prefix, serverIt.next()));
618 }
619 else if(RANDOMISE_RBL_LOOKUP_ORDER)
620 {
621 // Try all RBL servers, but in a random order.
622 final List<String> l = new ArrayList<String>(extantRBLServers);
623 Collections.shuffle(l, Rnd.fastRnd);
624 serverIt = l.iterator();
625 }
626 else
627 {
628 // Try all RBL servers in the order given.
629 serverIt = extantRBLServers.iterator();
630 }
631
632 // Check the DNS BL servers concurrently using our I/O-bound thread pool.
633 final List<Future<String>> pending = new ArrayList<Future<String>>(nServers);
634 while(serverIt.hasNext())
635 {
636 final String server = serverIt.next();
637 ThreadUtils.nonCPUThreadPool.submit(new Callable<String>(){
638 public final String call() throws Exception
639 { return(_checkOneAddrInDNSBL(prefix, server)); }
640 });
641 }
642
643 // Collect the results.
644 for(final Future<String> f : pending)
645 {
646 try
647 {
648 final String server = f.get();
649 // As soon as we find a DNS BL that lists this client,
650 // stop and return the DNS BL server name,
651 // since the client is blacklisted!
652 if(null != server) { return(server); }
653 }
654 catch(final Exception e)
655 {
656 // Log the error, but absorb/ignore it.
657 e.printStackTrace();
658 }
659 }
660
661 return(null); // No, this address is fine.
662 }
663
664 /**Look up one address in DNSBL.
665 */
666 private static String _checkOneAddrInDNSBL(final String prefix, final String server)
667 {
668 final String fqdn = prefix + server + '.'; // Make absolute.
669
670 // If we can find an A record,
671 // then this address is black-holed.
672 return(hasARecordQuick(fqdn) ? server : null);
673 }
674
675 /**Emergency-free handle for the DNS cache. */
676 private static final class CacheREFH implements RecurrentEmergencyFreeHandle
677 {
678 /**Handle on DNS cache; never null. */
679 private final Cache newCache;
680
681 private CacheREFH(final Cache newCache)
682 {
683 assert(null != newCache);
684 this.newCache = newCache;
685 }
686
687 @Override public void run()
688 {
689 final int size = newCache.getSize();
690 // Clear cache entirely when more than about double
691 // the heap-sensitive current target maximum.
692 if(size > (computeMaxCacheSize()<<1))
693 { newCache.clearCache(); }
694 // Log *after* liberating some space.
695 if(IsDebug.isDebug) { System.err.println("AddrTools DNS cache size (vs max) was " + size + ", now "+newCache.getSize()+" (max="+newCache.getMaxEntries()+")"); }
696 }
697
698 @Override public String toString()
699 {
700 return("AddrTools DNS cache size " + newCache.getSize());
701 }
702 }
703
704
705 /**Immutable store of a non-null, non-zero-length unsigned-byte IP(v4) address prefix.
706 */
707 public static final class AddrPrefix implements Comparable<AddrPrefix>,
708 MemoryTools.Internable
709 {
710 /**Create an instance from a non-null, non-zero-length byte array.
711 * Takes a copy of the prefix data.
712 * @param addrPrefix prefix of address; non-null
713 */
714 public AddrPrefix(final byte addrPrefix[])
715 {
716 if((addrPrefix == null) || (addrPrefix.length < 1))
717 { throw new IllegalArgumentException(); }
718 prefix = addrPrefix.clone();
719 }
720
721 /**Create an instance from the prefix of a non-null, non-zero-length byte array. */
722 public AddrPrefix(final byte addrPrefix[], final int len)
723 {
724 if((addrPrefix == null) ||
725 (len < 1) || (len > addrPrefix.length))
726 { throw new IllegalArgumentException(); }
727 prefix = new byte[len];
728 for(int i = len; --i >= 0; ) { prefix[i] = addrPrefix[i]; }
729 }
730
731 /**Create a shorter (but non-zero-length) instance from an existing prefix. */
732 public AddrPrefix(final AddrPrefix ap, final int len)
733 {
734 if((ap == null) ||
735 (len < 1) || (len >= ap.prefix.length))
736 { throw new IllegalArgumentException(); }
737 prefix = new byte[len];
738 for(int i = len; --i >= 0; ) { prefix[i] = ap.prefix[i]; }
739 }
740
741 /**Create an instance by parsing a padded dotted prefix format.
742 * This is tolerant and does not insist that the octets
743 * are padded with zeros on the left.
744 * <p>
745 * Can be performance-critical, especially at start-up,
746 * so hand-crafted for speed and to avoid creating unnecessary objects.
747 *
748 * @param s dotted prefix of form n(.n)* eg 127.0.0 or 10.0 or 10.000;
749 * there must be at least one octet,
750 * and each octet must be between 1 and 3 digits long
751 *
752 * @throws IllegalArgumentException if the value is unparsable
753 * or not at least one octet
754 */
755 public AddrPrefix(final String s)
756 throws IllegalArgumentException
757 {
758 if(s == null)
759 { throw new IllegalArgumentException(); }
760 final int sl = s.length();
761 if(sl < 1)
762 { throw new IllegalArgumentException(); }
763
764 // Count the internal delimiter dots.
765 // (There should not be dots in the first or last positions,
766 // nor adjacent to one another.)
767 int nDots = 0;
768 for(int i = sl-1; --i > 0; )
769 { if('.' == s.charAt(i)) { ++nDots; --i; } }
770 final int prefixLength = 1 + nDots;
771 final byte[] d = new byte[prefixLength];
772
773 // Work backwards through the array
774 // collecting one octet per loop.
775 int index = prefixLength;
776 for(int i = sl; --i >= 0; )
777 {
778 final char lastDigit = s.charAt(i--);
779 if((lastDigit < '0') || (lastDigit > '9'))
780 { throw new IllegalArgumentException("last octet digit invalid: '" + lastDigit + "' at index " +(i+1) + " in \"" + s + "\" with prefixLength="+prefixLength); }
781 int octet = lastDigit - '0';
782
783 // We're done if we run out of input String
784 // (and this must be the first octet).
785 if(i < 0) { d[0] = (byte)octet; break; }
786 final char digit2 = s.charAt(i);
787 // We're done for this this octet if we hit a dot.
788 if('.' == digit2) { d[--index] = (byte)octet; continue; }
789 if((digit2 < '0') || (digit2 > '9'))
790 { throw new IllegalArgumentException("digit2 invalid"); }
791 octet += 10 * (digit2 - '0');
792
793 // We're done if we run out of input String
794 // (and this must be the first octet).
795 if(i <= 0) { d[0] = (byte)octet; break; }
796 final char digit3 = s.charAt(--i);
797 // We're done for this this octet if we hit a dot.
798 if('.' == digit3) { d[--index] = (byte)octet; continue; }
799 if((digit3 < '0') || (digit3 > '2'))
800 { throw new IllegalArgumentException("digit3 invalid"); }
801 octet += 100 * (digit3 - '0');
802 if(octet > 255)
803 { throw new IllegalArgumentException("octet > 255"); }
804 d[--index] = (byte)octet;
805
806 // Check for presence of delimiter dot if input not finished.
807 if((i > 1) && ('.' != s.charAt(--i)))
808 { throw new IllegalArgumentException("missing ."); }
809 }
810
811 prefix = d;
812 }
813
814 /**Address prefix, non-zero-length, non-null. */
815 private final byte prefix[];
816
817 /**Get the prefix length, in bytes; strictly positive. */
818 public int length()
819 { return(prefix.length); }
820
821 /**Returns true if the argument is a strict prefix of this value. */
822 public boolean isStrictPrefix(final AddrPrefix other)
823 {
824 final int apLen = other.prefix.length;
825 if(apLen >= prefix.length) { return(false); }
826 for(int i = apLen; --i >= 0; )
827 { if(other.prefix[i] != prefix[i]) { return(false); } }
828 return(true);
829 }
830
831 /**Sorted as if in a dictionary with the bytes treated as unsigned. */
832 public int compareTo(final AddrPrefix other)
833 {
834 final int minLen = Math.min(prefix.length, other.prefix.length);
835 for(int i = 0; i < minLen; ++i)
836 {
837 final int diff = (prefix[i] & 0xff) - (other.prefix[i] & 0xff);
838 if(diff != 0) { return(diff); }
839 }
840
841 // Equal in their common portion, so the shorter one sorts first.
842 return(prefix.length - other.prefix.length);
843 }
844
845 /**Compute hash based on the assumption that most prefixes are 1--3 bytes long.
846 * All bytes should be involved in the hash for it to be effective
847 * since prefixes are likely to be dense in maps, etc.
848 */
849 @Override
850 public int hashCode()
851 {
852 final int length = prefix.length;
853 // For CPU efficiency, and for the quality of the generated hash,
854 // deal with the common short prefixes specially.
855 // In each case, we assume that the last byte is the most "noisy".
856 switch(length)
857 {
858 case 1: { return(prefix[0]); }
859 case 2: { return(prefix[1] ^ ((131+prefix[0]) * 271)); }
860 case 3: { return(prefix[2] + ((prefix[0]-129) * 82193) + (prefix[1] * 263)); }
861 }
862 // Default (general) case, using all prefix bytes.
863 return(Arrays.hashCode(prefix));
864 }
865
866 /**Equal if the prefixes are the same length and have the same content. */
867 @Override
868 public boolean equals(final Object obj)
869 {
870 if(this == obj) { return(true); }
871 if(!(obj instanceof AddrPrefix)) { return(false); }
872 final AddrPrefix other = ((AddrPrefix)obj);
873 final int length = prefix.length;
874 if(length != other.prefix.length) { return(false); }
875 for(int i = length; --i >= 0; )
876 { if(prefix[i] != other.prefix[i]) { return(false); } }
877 return(true); // Identical.
878 }
879
880 /**Get a copy of the byte[] prefix data; never null. */
881 public byte[] toByteArray()
882 { return(prefix.clone()); }
883
884 /**Print as padded dotted prefix; never null. */
885 public String toPaddedDottedPrefix()
886 {
887 final StringBuilder sb = new StringBuilder(prefix.length * 4);
888 final StringBuilder octet = new StringBuilder(3);
889 for(int i = 0; i < prefix.length; ++i)
890 {
891 octet.setLength(0);
892 octet.append(prefix[i] & 0xff);
893 while(octet.length() < 3) { octet.insert(0, '0'); }
894 if(sb.length() > 0) { sb.append('.'); }
895 sb.append(octet);
896 }
897 return(sb.toString());
898 }
899
900 /**Human-readable representation; currently equivalent to toPaddedDottedPrefix(); never null. */
901 @Override
902 public String toString()
903 { return(toPaddedDottedPrefix()); }
904 }
905 }