001 /*
002 Copyright (c) 1996-2012, Damon Hart-Davis
003 All rights reserved.
004
005 Redistribution and use in source and binary forms, with or without
006 modification, are permitted provided that the following conditions are
007 met:
008
009 * Redistributions of source code must retain the above copyright
010 notice, this list of conditions and the following disclaimer.
011
012 * Redistributions in binary form must reproduce the above copyright
013 notice, this list of conditions and the following disclaimer in the
014 documentation and/or other materials provided with the
015 distribution.
016
017 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
018 IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
019 TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
020 PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
021 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
022 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
023 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
024 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
025 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
026 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
027 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
028 */
029 package org.hd.d.pg2k.svrCore.vars;
030
031 import java.util.Arrays;
032 import java.util.Collections;
033 import java.util.EnumSet;
034 import java.util.HashMap;
035 import java.util.Map;
036 import java.util.SortedSet;
037 import java.util.TreeSet;
038
039 import org.hd.d.pg2k.svrCore.CoreConsts;
040 import org.hd.d.pg2k.svrCore.MemoryTools;
041
042
043 /**Immutable enumeration of distributed system variables.
044 * To avoid too much recursive definition,
045 * variable names (etc) should be defined here and used by the setters, etc.
046 */
047 public final class SystemVariables
048 {
049 /**Maximum latency we allow in distributing variable values (ms); positive.
050 * Downstream and upstream caches should probably be refreshed at
051 * a fraction of this delay (say one quarter) to ensure that the
052 * round-trip from one slave VM, to the master,
053 * and back to the same or another VM is not much more than this.
054 * <p>
055 * This value should be identical for all participants in a system.
056 * <p>
057 * A value of a few seconds to a few minutes is probably reasonable
058 * to balance network traffic against prompt response to events that affect
059 * the whole distributed system.
060 * <p>
061 * We base this on the maximum permitted inter-peer clock skew.
062 */
063 public static final int MAX_VALUE_DISTRIBUTION_LATENCY_MS = 2 * CoreConsts.MAX_PEER_CLOCK_SKEW_MS;
064
065 /**Maximum interval between variable updates for system to be considered live (ms).
066 * This is between any two variable updates, not simply of the same value.
067 * <p>
068 * If it longer than this between any variable updates for a given server instance,
069 * then other participants may consider that instance to be "dead"/resting,
070 * and discard knowledge of it and/or variable values automatically.
071 * <p>
072 * There are other ways of finding out how old data from a particular
073 * system is for filtering, but this is a back-stop.
074 * <p>
075 * Should be long enough to survive a temporarily-disconnected system,
076 * eg due to a short network outage or a down or disconnected master,
077 * but short enough to avoid filling up variable stores/caches
078 * with useless stale data from dead subsystems.
079 * <p>
080 * Must be much longer than MAX_VALUE_DISTRIBUTION_LATENCY_MS.
081 * <p>
082 * A value of the order of minutes to hours is probably good;
083 * and we might add in some small multiple of the normal inter-system skew
084 * that we are normally prepared to tolerate.
085 */
086 public static final int MAX_QUIET_SYSTEM_VAR_LIFE_MS =
087 Math.max(50 * MAX_VALUE_DISTRIBUTION_LATENCY_MS,
088 30 * CoreConsts.DEFAULT_TEMPORAL_SLACKNESS_S * 1000);
089
090
091 /**Maximum number of samples retained at each interval; strictly positive.
092 * A power of two for efficiency,
093 * and enough so that at the longest interval we have enough samples
094 * to cover at least a year.
095 * <p>
096 * Most periods/intervals are chosen so that the full span of samples at one interval
097 * covers a useful time, ie from over an hour to over a year from VSHORT to VLONG.
098 * Indeed, the span is designed to overlap a useful cycle time by a little
099 * so that this time last hour/day/year etc can be easily compared with the same place
100 * last time round the cycle.
101 */
102 public static final int EVENT_SAMPLES_RETAINED = 512;
103
104 /**Very-short-term event interval in milliseconds.
105 * Of the order of a few seconds.
106 * <p>
107 * (One use is to make real-time testing of the event system possible.)
108 */
109 public static final int EVENT_INTERVAL_VSHORT_TERM_MS = 10 * 1000; // 10s
110
111 /**Short-term event interval in milliseconds.
112 * Of the order of a few minutes.
113 */
114 public static final int EVENT_INTERVAL_SHORT_TERM_MS = 5 * 60 * 1000; // 5m
115
116 /**Medium-term event interval in milliseconds.
117 * An integer multiple of the next-shorter interval.
118 * <p>
119 * Of the order of a few tens of minutes.
120 */
121 public static final int EVENT_INTERVAL_MEDIUM_TERM_MS = 30 * 60 * 1000; // 30m
122
123 /**Long-term event interval in milliseconds.
124 * An integer multiple of the next-shorter interval.
125 * <p>
126 * Of the order of a few hours.
127 */
128 public static final int EVENT_INTERVAL_LONG_TERM_MS = 2 * 3600 * 1000; // 2h
129
130 /**Very-long-term event interval in milliseconds.
131 * An integer multiple of the next-shorter interval.
132 * <p>
133 * Of the order of tens of hours to days.
134 */
135 public static final int EVENT_INTERVAL_VLONG_TERM_MS = 24 * 3600 * 1000; // 1D
136
137
138 /**A read-only, local, "null" variable used to do "keep alives" in variable stores.
139 * This contains no data,
140 * and may be sent automatically by a slave system
141 * to tell the master and other slaves that it is still alive.
142 */
143 public static final SimpleVariableDefinition KEEP_ALIVE =
144 new SimpleVariableDefinition("keepAlive",
145 SimpleVariableDefinition.TYPE_NONE,
146 false, // global
147 false, // non-persistent (we want a new one on each restart)
148 false, // read/write nominally, though there is no data
149 false, 0, null);
150
151
152 /**The read-only, local, String variable for the local system ID.
153 * This value of this (local) variable should be unique to each system,
154 * and not propagated out of each system directly.
155 */
156 public static final SimpleVariableDefinition LOCAL_SYS_ID =
157 new SimpleVariableDefinition("localSysID",
158 SimpleVariableDefinition.TYPE_IID,
159 true, // local (purely for the local machine)
160 false, // non-persistent (we want a new one on each restart)
161 true, // read-only (the tunnel end-point should inject this)
162 false, 0, null);
163
164
165 /**A read-write global Number variable for testing. */
166 public static final SimpleVariableDefinition TEST_NUMBER_GLOBAL =
167 new SimpleVariableDefinition("test.NUMBER.global",
168 SimpleVariableDefinition.TYPE_NUMBER,
169 false, // global
170 false, // non-persistent
171 false, // read/write
172 false, 0, null);
173
174 /**A second read-write global Number variable for testing. */
175 public static final SimpleVariableDefinition TEST_NUMBER_GLOBAL2 =
176 new SimpleVariableDefinition("test.NUMBER.global2",
177 SimpleVariableDefinition.TYPE_NUMBER,
178 false, // global
179 false, // non-persistent
180 false, // read/write
181 false, 0, null);
182
183 // /**A special read-write global Number variable for testing (for the wait routine only). */
184 // public static final SimpleVariableDefinition TEST_NUMBER_GLOBAL_WAIT =
185 // new SimpleVariableDefinition("test.NUMBER.globalwait",
186 // SimpleVariableDefinition.TYPE_NUMBER,
187 // false, // global
188 // false, // non-persistent
189 // false // read/write
190 // );
191
192 /**A read-write local Number variable for testing. */
193 public static final SimpleVariableDefinition TEST_NUMBER_LOCAL =
194 new SimpleVariableDefinition("test.NUMBER.local",
195 SimpleVariableDefinition.TYPE_NUMBER,
196 true, // local
197 false, // non-persistent
198 false, // read/write
199 false, 0, null);
200
201 /**A second read-write local Number variable for testing. */
202 public static final SimpleVariableDefinition TEST_NUMBER_LOCAL2 =
203 new SimpleVariableDefinition("test.NUMBER.local2",
204 SimpleVariableDefinition.TYPE_NUMBER,
205 true, // local
206 false, // non-persistent
207 false, // read/write
208 false, 0, null);
209
210 /**A read-write local String variable for testing. */
211 public static final SimpleVariableDefinition TEST_STRING_LOCAL =
212 new SimpleVariableDefinition("test.STRING.local",
213 SimpleVariableDefinition.TYPE_STRING,
214 true, // local
215 false, // non-persistent
216 false, // read/write
217 false, 0, null);
218
219 /**A read-write global String variable for testing. */
220 public static final SimpleVariableDefinition TEST_STRING_GLOBAL =
221 new SimpleVariableDefinition("test.STRING.global",
222 SimpleVariableDefinition.TYPE_STRING,
223 false, // global
224 false, // non-persistent
225 false, // read/write
226 false, 0, null);
227
228 /**A read-write persistent global String event variable for testing. */
229 public static final SimpleVariableDefinition TEST_STRING_GLOBAL_EVENT =
230 new SimpleVariableDefinition("test.STRING.global.event",
231 SimpleVariableDefinition.TYPE_STRING,
232 false, // global
233 true, // persistent
234 false, // read/write
235 true, 100, null); // event with max 100x different values...
236
237
238
239
240
241
242 /**Maximum number of different performance-monitoring (String) stats recordable.
243 * This should be high enough to see a reasonably long tail of data,
244 * but not so huge as to cripple the system upon transfer or for space.
245 */
246 public static final int PERFMON_MAX_EVENTS = 203;
247
248 /**Generic read-write non-persistent global String event variable for performance monitoring.
249 * Note that this is valid for longer event periods.
250 * <p>
251 * This is non-persistent since the data is ephemeral
252 * and has little value once fixes have been made, etc.
253 * <p>
254 * We expect this to be very quiet most of the time,
255 * except during heavy load or when background sampling is enabled.
256 */
257 public static final SimpleVariableDefinition PERFMON_STRING_GLOBAL_EVENT =
258 new SimpleVariableDefinition("perfmon",
259 SimpleVariableDefinition.TYPE_STRING,
260 false, // global
261 false, // non-persistent
262 false, // read/write
263 true, // event
264 PERFMON_MAX_EVENTS,
265 EnumSet.of(EventPeriod.VLONG, EventPeriod.LONG)); // Long periods.
266
267
268
269 /**Maximum number of different generic (String) stats recordable.
270 * This should be high enough to accommodate many disjoint sets of values,
271 * but not so huge as to cripple the system upon transfer or for space.
272 */
273 public static final int GENSTATS_MAX_EVENTS = 2003;
274
275 /**Generic read-write persistent global String event variable for general stats.
276 * This is used for stats events that do not otherwise have their own event variable.
277 * <p>
278 * Note that this is valid for all event periods.
279 */
280 public static final SimpleVariableDefinition GENSTATS_STRING_GLOBAL_EVENT =
281 new SimpleVariableDefinition("pg2k.genstats.STRING.global.event",
282 SimpleVariableDefinition.TYPE_STRING,
283 false, // global
284 true, // persistent
285 false, // read/write
286 true, // event
287 GENSTATS_MAX_EVENTS,
288 null); // All event periods.
289
290 /**Generic read-write persistent local String event variable for general stats.
291 * This is used for stats events that do not otherwise have their own event variable.
292 * <p>
293 * Note that this is valid for all event periods.
294 * <p>
295 * Likely to receive overlap of events with GENSTATS_STRING_GLOBAL_EVENT.
296 */
297 public static final SimpleVariableDefinition GENSTATS_STRING_LOCAL_EVENT =
298 new SimpleVariableDefinition("pg2k.genstats.STRING.local.event",
299 SimpleVariableDefinition.TYPE_STRING,
300 true, // local
301 true, // persistent
302 false, // read/write
303 true, // event
304 GENSTATS_MAX_EVENTS,
305 null); // All event periods.
306
307
308
309 /**ExhibitDataSimpleCache pseudo-variable name prefix. */
310 public static final String ExhibitDataSimpleCache_PSEUDOVAR_PREFIX =
311 "pg2k.ExhibitDataSimpleCache.";
312
313 /**ExhibitDataSimpleCache cache-percent-full pseudo-variable definition.
314 * In the range [0, 100] indicating approximately
315 * what percentage of the available cache space is currently in use.
316 * <p>
317 * If there is more than one ExhibitDataSimpleCache in a local pipeline
318 * then behaviour is undefined as all instances will update this value.
319 * <p>
320 * Main attributes:
321 * <ul>
322 * <li>Local.
323 * <li>Of type Number.
324 * <li>Non-persistent.
325 * </ul>
326 */
327 public static final SimpleVariableDefinition ExhibitDataSimpleCache_CACHE_AVAIL_SPACE_PERCENT_USED =
328 new SimpleVariableDefinition(ExhibitDataSimpleCache_PSEUDOVAR_PREFIX + "percentUsed",
329 SimpleVariableDefinition.TYPE_NUMBER);
330
331 /**ExhibitDataSimpleCache entries-fully-cached pseudo-variable definition.
332 * In the range [0, 100] indicating approximately
333 * what percentage of the available exhibits are fully cached.
334 * <p>
335 * If there is more than one ExhibitDataSimpleCache in a local pipeline
336 * then behaviour is undefined as all instances will update this value.
337 * <p>
338 * Main attributes:
339 * <ul>
340 * <li>Local.
341 * <li>Of type Number.
342 * <li>Non-persistent.
343 * </ul>
344 */
345 public static final SimpleVariableDefinition ExhibitDataSimpleCache_EXHIBITS_FULLY_CACHED_PERCENT =
346 new SimpleVariableDefinition(ExhibitDataSimpleCache_PSEUDOVAR_PREFIX + "percentFullyCached",
347 SimpleVariableDefinition.TYPE_NUMBER);
348
349 /**ExhibitDataSimpleCache cached-exhibit count.
350 * In the range [0, +inf[ indicating how many exhibit
351 * are fully or partially cached.
352 * <p>
353 * If there is more than one ExhibitDataSimpleCache in a local pipeline
354 * then behaviour is undefined as all instances will update this value.
355 * <p>
356 * Main attributes:
357 * <ul>
358 * <li>Local.
359 * <li>Of type Number.
360 * <li>Non-persistent.
361 * </ul>
362 */
363 public static final SimpleVariableDefinition ExhibitDataSimpleCache_CACHED_EXHIBIT_COUNT =
364 new SimpleVariableDefinition(ExhibitDataSimpleCache_PSEUDOVAR_PREFIX + "cachedCount",
365 SimpleVariableDefinition.TYPE_NUMBER);
366
367
368
369
370
371
372 /**ThroughputMonitorFilter pseudo-variable name prefix. */
373 public static final String ThroughputMonitorFilter_PSEUDOVAR_PREFIX =
374 "pg2k.ThroughputMonitorFilter.";
375
376 /**ThroughputMonitorFilter user-count value.
377 * In the range [0,+inf[ indicating approximately
378 * how many users are on the system including spiders, hotlinkers, etc.
379 * <p>
380 * The global map of values can be summed to give the total on the
381 * entire distributed system.
382 * <p>
383 * Main attributes:
384 * <ul>
385 * <li>Globalmap.
386 * <li>Of type Number.
387 * <li>Non-persistent.
388 * </ul>
389 */
390 public static final SimpleVariableDefinition ThroughputMonitorFilter_CLIENT_COUNT =
391 new SimpleVariableDefinition(ThroughputMonitorFilter_PSEUDOVAR_PREFIX + "clientCount",
392 SimpleVariableDefinition.TYPE_NUMBER,
393 false, // Global map.
394 false, // Not persistent.
395 false, false, 0, null); // Read/write.
396
397 /**ThroughputMonitorFilter "sticky" user-count value.
398 * In the range [0,+inf[ indicating approximately
399 * how many distinct "sticky" human users are on the system.
400 * <p>
401 * A "sticky" user is typically one that views at least a couple of pages
402 * and stays around for a few minutes.
403 * <p>
404 * The global map of values can be summed to give the total on the
405 * entire distributed system.
406 * <p>
407 * Main attributes:
408 * <ul>
409 * <li>Globalmap.
410 * <li>Of type Number.
411 * <li>Non-persistent.
412 * </ul>
413 */
414 public static final SimpleVariableDefinition ThroughputMonitorFilter_STICKY_CLIENT_COUNT =
415 new SimpleVariableDefinition(ThroughputMonitorFilter_PSEUDOVAR_PREFIX + "stickyClientCount",
416 SimpleVariableDefinition.TYPE_NUMBER,
417 false, // Global map.
418 false, // Not persistent.
419 false, false, 0, null); // Read/write.
420
421 /**ThroughputMonitorFilter long-term (smoothed) bytes-per-second value.
422 * In the range [0,+inf[ indicating approximately
423 * the total download bandwidth on the entire system.
424 * <p>
425 * This is expressed as an integral quantity for simplicity/robustness,
426 * as an Integer if possible for compactness on the wire.
427 * <p>
428 * The global map of values can be summed to give the total on the
429 * entire distributed system.
430 * <p>
431 * Main attributes:
432 * <ul>
433 * <li>Globalmap.
434 * <li>Of type Number.
435 * <li>Non-persistent.
436 * </ul>
437 */
438 public static final SimpleVariableDefinition ThroughputMonitorFilter_ltBps =
439 new SimpleVariableDefinition(ThroughputMonitorFilter_PSEUDOVAR_PREFIX + "ltBps",
440 SimpleVariableDefinition.TYPE_NUMBER,
441 false, // Global map.
442 false, // Not persistent.
443 false, false, 0, null); // Read/write.
444
445 /**ThroughputMonitorFilter yearly-unique-visitors value.
446 * In the range [0,+inf[ indicating approximately
447 * the computed number of unique visitors per year system-wide
448 * based on current traffic.
449 * <p>
450 * The global map of values can be summed to give the total on the
451 * entire distributed system.
452 * <p>
453 * Main attributes:
454 * <ul>
455 * <li>Globalmap.
456 * <li>Of type Number.
457 * <li>Non-persistent.
458 * </ul>
459 */
460 public static final SimpleVariableDefinition ThroughputMonitorFilter_YEARLY_UNIQUE_VISITORS =
461 new SimpleVariableDefinition(ThroughputMonitorFilter_PSEUDOVAR_PREFIX + "yearlyUniqueVisitors",
462 SimpleVariableDefinition.TYPE_NUMBER,
463 false, // Global map.
464 false, // Not persistent.
465 false, false, 0, null); // Read/write.
466
467 /**ThroughputMonitorFilter busy fraction (0.0 means idle, 1.0 flat-out busy).
468 * In the range [0,1] indicating approximately
469 * the fraction of available capacity in use.
470 * <p>
471 * Main attributes:
472 * <ul>
473 * <li>Globalmap.
474 * <li>Of type Number.
475 * <li>Non-persistent.
476 * </ul>
477 */
478 public static final SimpleVariableDefinition ThroughputMonitorFilter_BUSY_FRACTION =
479 new SimpleVariableDefinition(ThroughputMonitorFilter_PSEUDOVAR_PREFIX + "busyFraction",
480 SimpleVariableDefinition.TYPE_NUMBER,
481 false, // Global map.
482 false, // Not persistent.
483 false, // Read/write.
484 false, 0, null); // Not an event.
485
486
487
488 /**ThroughputMonitorFilter available-bandwidth-per-client value in Bps (byte/second).
489 * In the range [0,+inf[ indicating approximately
490 * the the available bandwidth to a new client
491 * based on current traffic.
492 * <p>
493 * Will be forced to zero if the system is busy or otherwise unwilling
494 * to accept new clients.
495 * <p>
496 * Main attributes:
497 * <ul>
498 * <li>Globalmap.
499 * <li>Of type Number.
500 * <li>Non-persistent.
501 * </ul>
502 */
503 public static final SimpleVariableDefinition ThroughputMonitorFilter_AVAIL_BPS_PER_CLIENT =
504 new SimpleVariableDefinition(ThroughputMonitorFilter_PSEUDOVAR_PREFIX + "availableBpsPerClient",
505 SimpleVariableDefinition.TYPE_NUMBER,
506 false, // Global map.
507 false, // Not persistent.
508 false, // Read/write.
509 false, 0, null); // Not an event.
510
511
512 /**ThroughputMonitorFilter current AEP hash (a Long).
513 * Set by each TMF to indicate which AEP version its server is using.
514 * <p>
515 * Can be used in load balancing to avoid sending traffic to a mirror
516 * which is not on the same version of the AEP.
517 * <p>
518 * Main attributes:
519 * <ul>
520 * <li>Globalmap.
521 * <li>Of type Number (ie a Long).
522 * <li>Non-persistent.
523 * </ul>
524 */
525 public static final SimpleVariableDefinition ThroughputMonitorFilter_AEP_LONGHASH =
526 new SimpleVariableDefinition(ThroughputMonitorFilter_PSEUDOVAR_PREFIX + "AEPLHash",
527 SimpleVariableDefinition.TYPE_NUMBER,
528 false, // Global map.
529 false, // Not persistent.
530 false, // Read/write.
531 false, 0, null); // Not an event.
532
533
534
535 /**ThroughputMonitorFilter mirror name value.
536 * The string short name of this mirror (CC-suffix) if this machine is a mirror.
537 * <p>
538 * The global map of values can be used to find which mirrors are active.
539 * <p>
540 * Main attributes:
541 * <ul>
542 * <li>Globalmap.
543 * <li>Of type String.
544 * <li>Non-persistent.
545 * </ul>
546 */
547 public static final SimpleVariableDefinition ThroughputMonitorFilter_ACTIVE_MIRROR_NAME =
548 new SimpleVariableDefinition(ThroughputMonitorFilter_PSEUDOVAR_PREFIX + "activeMirrorName",
549 SimpleVariableDefinition.TYPE_STRING,
550 false, // Global map.
551 false, // Not persistent.
552 false, false, 0, null); // Read/write.
553
554
555
556
557
558 /**ThroughputMonitorFilter pseudo-variable name prefix. */
559 public static final String TunnelServlet_PSEUDOVAR_PREFIX =
560 "pg2k.TunnelServlet.";
561
562 /**TunnelServlet slave/client/mirror remote-addresses value (as Strings).
563 * String value of each tunnel client (slave) addr as seen (and injected by)
564 * the tunnel servlet.
565 * <p>
566 * These addresses are one of:
567 * <ul>
568 * <li>null for local tunnels
569 * <li>dotted-quad IPv4 addresses for HTTP-tunnelled clients over IPv4
570 * </ul>
571 * <p>
572 * Main attributes:
573 * <ul>
574 * <li>Globalmap.
575 * <li>Of type String.
576 * <li>Non-persistent.
577 * </ul>
578 */
579 public static final SimpleVariableDefinition TunnelServlet_SLAVE_ADDRS =
580 new SimpleVariableDefinition(TunnelServlet_PSEUDOVAR_PREFIX + "slaveIPs",
581 SimpleVariableDefinition.TYPE_STRING,
582 false, // Global map.
583 false, // Not persistent.
584 false, false, 0, null); // Read/write.
585
586
587
588 /**Maximum number of different exhibit name events recorded by most access-pattern stats.
589 * Should be large enough not to miss too many one-off events
590 * (else we may never see the second-and-subsequent hits for the same value)
591 * yet small enough to reasonably limit memory consumption for the stats.
592 * <p>
593 * We pick a prime-ish value for a larf.
594 */
595 public static final int MAX_DIFF_EXHIBIT_NAME_VALUES = 1001;
596
597 /**A read-write persistent global String event variable for recording catalogue page views by exhibit.
598 * Should ideally reflect access by (sane) humans,
599 * thus excluding obvious spiders, repeat clickers, etc, if possible!
600 * This may well record a representative sampling rather than all events,
601 * so rank is more reliable than absolute count.
602 * <p>
603 * The event values are exhibit short names (eg no directory path),
604 * since they save a little memory, should be unique,
605 * and may even be a bit more long-lasting than the full name.
606 * <p>
607 * We are only interested in the longest-term stats.
608 */
609 public static final SimpleVariableDefinition ACCESSPATTERN_CAT_PAGE_VIEW =
610 new SimpleVariableDefinition("AccessPattern.catPageView",
611 SimpleVariableDefinition.TYPE_STRING,
612 false, // global
613 true, // persistent
614 false, // read/write
615 true, MAX_DIFF_EXHIBIT_NAME_VALUES, EnumSet.of(EventPeriod.VLONG));
616
617 /**A read-write persistent global String event variable for recording completed downloads by exhibit.
618 * Should ideally reflect access by (sane) humans,
619 * thus excluding obvious spiders, repeat clickers, etc, if possible!
620 * This may well record a representative sampling rather than all events,
621 * so rank is more reliable than absolute count.
622 * <p>
623 * The event values are exhibit short names (eg no directory path),
624 * since they save a little memory, should be unique,
625 * and may even be a bit more long-lasting than the full name.
626 * <p>
627 * We are only interested in the longest-term stats.
628 */
629 public static final SimpleVariableDefinition ACCESSPATTERN_COMPLETED_DOWNLOAD =
630 new SimpleVariableDefinition("AccessPattern.download.completed",
631 SimpleVariableDefinition.TYPE_STRING,
632 false, // global
633 true, // persistent
634 false, // read/write
635 true, MAX_DIFF_EXHIBIT_NAME_VALUES, EnumSet.of(EventPeriod.VLONG));
636
637 /**A read-write persistent local String event variable for recording completed downloads by exhibit.
638 * Should ideally reflect access by (sane) humans,
639 * thus excluding obvious spiders, repeat clickers, etc, if possible!
640 * This may well record a representative sampling rather than all events,
641 * so rank is more reliable than absolute count.
642 * <p>
643 * The event values are exhibit short names (eg no directory path),
644 * since they save a little memory, should be unique,
645 * and may even be a bit more long-lasting than the full name.
646 * <p>
647 * We are only interested in the longest-term stats.
648 */
649 public static final SimpleVariableDefinition ACCESSPATTERN_COMPLETED_DOWNLOAD_LOCAL =
650 new SimpleVariableDefinition("AccessPattern.download.completed.local",
651 SimpleVariableDefinition.TYPE_STRING,
652 true, // local
653 true, // persistent
654 false, // read/write
655 true, MAX_DIFF_EXHIBIT_NAME_VALUES, EnumSet.of(EventPeriod.VLONG));
656
657
658 /**A read-write persistent global String event variable for recording ad click-throughs by exhibit.
659 * Notes a click through to a paid ad from a catalogue page,
660 * where we can capture this information.
661 * <p>
662 * At best this is likely to be a sampling.
663 * <p>
664 * Should ideally reflect access by (sane) humans,
665 * thus excluding obvious spiders, repeat clickers, etc, if possible!
666 * This may well record a representative sampling rather than all events,
667 * so rank is more reliable than absolute count.
668 * <p>
669 * The event values are exhibit short names (eg no directory path),
670 * since they save a little memory, should be unique,
671 * and may even be a bit more long-lasting than the full name.
672 * <p>
673 * We are only interested in the longest-term stats.
674 */
675 public static final SimpleVariableDefinition ACCESSPATTERN_CLICKTHROUGH =
676 new SimpleVariableDefinition("AccessPattern.clickthru",
677 SimpleVariableDefinition.TYPE_STRING,
678 false, // global
679 true, // persistent
680 false, // read/write
681 true, MAX_DIFF_EXHIBIT_NAME_VALUES, EnumSet.of(EventPeriod.VLONG));
682
683
684 /**A read-write persistent global String event variable for recording ad click-throughs by canonicalised URI.
685 * Notes a click through to a paid ad from any (valid) page.
686 * <p>
687 * At best this is likely to be a sampling.
688 * <p>
689 * Should ideally reflect access by (sane) humans,
690 * thus excluding obvious spiders, repeat clickers, etc, if possible!
691 * This may well record a representative sampling rather than all events,
692 * so rank is more reliable than absolute count.
693 * <p>
694 * The URIs are lower-cased (since some are case-insensitive)
695 * and limited in length.
696 * <p>
697 * We allow quite a number of unique values as information is likely
698 * to be quite sparse.
699 * <p>
700 * We are only interested in the longest-term stats.
701 */
702 public static final SimpleVariableDefinition ACCESSPATTERN_CLICKTHROUGH_BY_URI =
703 new SimpleVariableDefinition("AccessPattern.clickthru.byURI",
704 SimpleVariableDefinition.TYPE_STRING,
705 false, // global
706 true, // persistent
707 false, // read/write
708 true, 3*MAX_DIFF_EXHIBIT_NAME_VALUES, EnumSet.of(EventPeriod.VLONG));
709
710
711
712 /**A read-write persistent global String event variable for recording hotlinking to our exhibits by referrer host.
713 * This notes where an external site links directly to one of our exhibits (or thumbnails)
714 * rather than to one of our HTML pages.
715 * <p>
716 * We only really need to record the top offenders, not every minor infraction!
717 * <p>
718 * This is sometimes OK, eg from a bona-fide image search engine,
719 * but is often bad, eg bandwidth theft by idle Web site builders/bloggers,
720 * so we need to keep an eye on it.
721 * <p>
722 * Entries are sanitized and normalised to limit noise and maximise utility.
723 * <p>
724 * We are only interested in the longest-term stats.
725 */
726 public static final SimpleVariableDefinition ACCESSPATTERN_EX_HOTLINK_REF_HOST =
727 new SimpleVariableDefinition("AccessPattern.hotlinker",
728 SimpleVariableDefinition.TYPE_STRING,
729 false, // global
730 true, // persistent
731 false, // read/write
732 true, 256, EnumSet.of(EventPeriod.VLONG));
733
734
735
736
737 /**Maximum number of different exhibit name events recorded for votes.
738 * We regard votes as quite valuable,
739 * so will accommodate a larger set of these than for many other events.
740 */
741 public static final int MAX_DIFF_VOTE_VALUES = 1001;
742 /* 1 | (3 * MAX_DIFF_EXHIBIT_NAME_VALUES); */
743
744 /**A read-write persistent global String event variable for recording positive votes by exhibit.
745 * Notes a "thumbs-up" vote for the names exhibit.
746 * <p>
747 * Should ideally reflect access by (sane) humans,
748 * thus excluding obvious spiders, repeat clickers, etc, if possible!
749 * This may well record a representative sampling rather than all events,
750 * so rank is more reliable than absolute count.
751 * <p>
752 * The event values are exhibit short names (eg no directory path),
753 * since they save a little memory, should be unique,
754 * and may even be a bit more long-lasting than the full name.
755 * <p>
756 * We are only interested in the longest-term stats.
757 */
758 public static final SimpleVariableDefinition VOTE_PRO =
759 new SimpleVariableDefinition("Vote.pro",
760 SimpleVariableDefinition.TYPE_STRING,
761 false, // global
762 true, // persistent
763 false, // read/write
764 true, // event
765 MAX_DIFF_VOTE_VALUES, EnumSet.of(EventPeriod.VLONG));
766
767 /**A read-write persistent global String event variable for recording negative votes by exhibit.
768 * Notes a "thumbs-down" vote for the names exhibit.
769 * <p>
770 * Should ideally reflect access by (sane) humans,
771 * thus excluding obvious spiders, repeat clickers, etc, if possible!
772 * This may well record a representative sampling rather than all events,
773 * so rank is more reliable than absolute count.
774 * <p>
775 * The event values are exhibit short names (eg no directory path),
776 * since they save a little memory, should be unique,
777 * and may even be a bit more long-lasting than the full name.
778 * <p>
779 * We are only interested in the longest-term stats.
780 */
781 public static final SimpleVariableDefinition VOTE_CON =
782 new SimpleVariableDefinition("Vote.con",
783 SimpleVariableDefinition.TYPE_STRING,
784 false, // global
785 true, // persistent
786 false, // read/write
787 true, // event
788 MAX_DIFF_VOTE_VALUES, EnumSet.of(EventPeriod.VLONG));
789
790 /**A read-write persistent global String event variable for recording vote comments by exhibit.
791 * Contains the (short) exhibit name
792 * followed (after a space) by a sanitised version of a user's comment.
793 * <p>
794 * This has a maximum of "0" different values since all we are interested in
795 * is the audit trail, from which we may hand-pick various items.
796 * <p>
797 * We are only interested in the longest-term stats.
798 */
799 public static final SimpleVariableDefinition VOTE_COMMENT =
800 new SimpleVariableDefinition("Vote.comment",
801 SimpleVariableDefinition.TYPE_STRING,
802 false, // global
803 true, // persistent
804 false, // read/write
805 true, 0, EnumSet.of(EventPeriod.VLONG));
806
807
808 /**Maximum number of Scorer global "events"; strictly positive.
809 * The global event store exists for interchange of "best" Scorers between servers,
810 * whereas the local event store additionally acts as persistent storage,
811 * and thus the local store should be (much) larger than the global one.
812 * (A large global store also implies significant network transmission costs.)
813 */
814 public static final int AI_SCORER_GLOBAL_MAX_EVENTS = 2003;
815
816 /**Interchange between separate Gallery servers of the "best" Scorers/parameters.
817 * Note that this is valid for all event periods.
818 */
819 public static final SimpleVariableDefinition AI_SCORER_STRING_GLOBAL_EVENT =
820 new SimpleVariableDefinition("pg2k.ai.scorer.global.event",
821 SimpleVariableDefinition.TYPE_STRING,
822 false, // global
823 true, // persistent
824 false, // read/write
825 true, // event
826 AI_SCORER_GLOBAL_MAX_EVENTS,
827 null); // All event periods.
828
829 /**Persistent storage of a large selection of the "best" Scorers/parameters.
830 * This is larger than the global set since we persist a larger set of Scorers
831 * for a deep local "gene pool", and we don't pay to send this over a network.
832 * <p>
833 * Note that this is valid for all event periods.
834 * <p>
835 * Likely to receive overlap of events with AI_SCORER_STRING_GLOBAL_EVENT.
836 */
837 public static final SimpleVariableDefinition AI_SCORER_STRING_LOCAL_EVENT =
838 new SimpleVariableDefinition("pg2k.ai.scorer.local.event",
839 SimpleVariableDefinition.TYPE_STRING,
840 true, // local
841 true, // persistent
842 false, // read/write
843 true, // event
844 4*AI_SCORER_GLOBAL_MAX_EVENTS,
845 null); // All event periods.
846
847
848
849
850 /**Private array of system variable definitions; non-null. */
851 private static final SimpleVariableDefinition vars[] =
852 {
853 // "System" variables...
854 KEEP_ALIVE,
855 LOCAL_SYS_ID,
856
857 // Generic stats.
858 GENSTATS_STRING_GLOBAL_EVENT,
859 GENSTATS_STRING_LOCAL_EVENT,
860
861 // Sub-system-specific stats...
862 ThroughputMonitorFilter_CLIENT_COUNT,
863 ThroughputMonitorFilter_STICKY_CLIENT_COUNT,
864 ThroughputMonitorFilter_ltBps,
865 ThroughputMonitorFilter_YEARLY_UNIQUE_VISITORS,
866 ThroughputMonitorFilter_ACTIVE_MIRROR_NAME,
867 ThroughputMonitorFilter_AVAIL_BPS_PER_CLIENT,
868 ThroughputMonitorFilter_BUSY_FRACTION,
869 ThroughputMonitorFilter_AEP_LONGHASH,
870
871 // System performance monitoring.
872 PERFMON_STRING_GLOBAL_EVENT,
873
874 ExhibitDataSimpleCache_CACHE_AVAIL_SPACE_PERCENT_USED,
875 ExhibitDataSimpleCache_EXHIBITS_FULLY_CACHED_PERCENT,
876 ExhibitDataSimpleCache_CACHED_EXHIBIT_COUNT,
877
878 TunnelServlet_SLAVE_ADDRS,
879
880 ACCESSPATTERN_CAT_PAGE_VIEW,
881 ACCESSPATTERN_COMPLETED_DOWNLOAD,
882 ACCESSPATTERN_COMPLETED_DOWNLOAD_LOCAL,
883 ACCESSPATTERN_CLICKTHROUGH,
884 ACCESSPATTERN_CLICKTHROUGH_BY_URI,
885 ACCESSPATTERN_EX_HOTLINK_REF_HOST,
886
887 VOTE_PRO,
888 VOTE_CON,
889 VOTE_COMMENT,
890
891 AI_SCORER_STRING_LOCAL_EVENT,
892 AI_SCORER_STRING_GLOBAL_EVENT,
893
894 // Values used for unit testing.
895 // In normal use these should never get set
896 // and never be transmitted across tunnels, etc,
897 // so should not consume significant resources.
898 TEST_NUMBER_GLOBAL, TEST_NUMBER_GLOBAL2,
899 TEST_NUMBER_LOCAL, TEST_NUMBER_LOCAL2,
900 TEST_STRING_GLOBAL,
901 TEST_STRING_LOCAL,
902 TEST_STRING_GLOBAL_EVENT,
903 };
904 /**Internalise/canonicalise all the legal system variables.
905 * This means that we should be able to eliminate duplicates
906 * easily when, for example, deserialising over the wire,
907 * thus saving memory at the cost of a small amount of time.
908 */
909 static
910 {
911 for(final SimpleVariableDefinition def : vars)
912 { MemoryTools.intern(def); }
913 }
914
915 /**Immutable SortedSet of system SimpleVariableDefinition items. */
916 public static final SortedSet<SimpleVariableDefinition> defs =
917 Collections.unmodifiableSortedSet(new TreeSet<SimpleVariableDefinition>(Arrays.asList(vars)));
918
919 /**Immutable Map from String variable name to SimpleVariableDefinition. */
920 public static final Map<String,SimpleVariableDefinition> nameToDef;
921 /**Initialise nameToDef. */
922 static
923 {
924 final HashMap<String,SimpleVariableDefinition> m = new HashMap<String,SimpleVariableDefinition>(1 + 2 * vars.length);
925 for(int i = vars.length; --i >= 0; )
926 {
927 final SimpleVariableDefinition def = vars[i];
928 m.put(def.getName(), def);
929 }
930 nameToDef = Collections.unmodifiableMap(m);
931 }
932
933 /**Immutable Set of String names of variables with slightly sensitive content.
934 * Values in these variables should possibly not be published as freely
935 * as other system variables, maybe because it contains unedited user input.
936 * <p>
937 * However, no data in the system is secret or sensitive in a legal sense,
938 * so this is only a weak hint!
939 */
940 public static final SortedSet<String> sensitiveVars =
941 Collections.unmodifiableSortedSet(new TreeSet<String>(Arrays.asList(new String[]{
942 ACCESSPATTERN_CLICKTHROUGH.getName(),
943 ACCESSPATTERN_CLICKTHROUGH_BY_URI.getName(),
944 VOTE_COMMENT.getName(),
945 })));
946 }