001/* 002 * Copyright 2008-2018 Ping Identity Corporation 003 * All Rights Reserved. 004 */ 005/* 006 * Copyright (C) 2008-2018 Ping Identity Corporation 007 * 008 * This program is free software; you can redistribute it and/or modify 009 * it under the terms of the GNU General Public License (GPLv2 only) 010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only) 011 * as published by the Free Software Foundation. 012 * 013 * This program is distributed in the hope that it will be useful, 014 * but WITHOUT ANY WARRANTY; without even the implied warranty of 015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 016 * GNU General Public License for more details. 017 * 018 * You should have received a copy of the GNU General Public License 019 * along with this program; if not, see <http://www.gnu.org/licenses>. 020 */ 021package com.unboundid.ldap.sdk.examples; 022 023 024 025import java.io.IOException; 026import java.io.OutputStream; 027import java.io.Serializable; 028import java.text.ParseException; 029import java.util.ArrayList; 030import java.util.LinkedHashMap; 031import java.util.LinkedHashSet; 032import java.util.List; 033import java.util.Random; 034import java.util.concurrent.CyclicBarrier; 035import java.util.concurrent.atomic.AtomicBoolean; 036import java.util.concurrent.atomic.AtomicLong; 037 038import com.unboundid.ldap.sdk.Control; 039import com.unboundid.ldap.sdk.LDAPConnection; 040import com.unboundid.ldap.sdk.LDAPConnectionOptions; 041import com.unboundid.ldap.sdk.LDAPException; 042import com.unboundid.ldap.sdk.ResultCode; 043import com.unboundid.ldap.sdk.Version; 044import com.unboundid.ldap.sdk.controls.AssertionRequestControl; 045import com.unboundid.ldap.sdk.controls.PermissiveModifyRequestControl; 046import com.unboundid.ldap.sdk.controls.PostReadRequestControl; 047import com.unboundid.ldap.sdk.controls.PreReadRequestControl; 048import com.unboundid.util.ColumnFormatter; 049import com.unboundid.util.FixedRateBarrier; 050import com.unboundid.util.FormattableColumn; 051import com.unboundid.util.HorizontalAlignment; 052import com.unboundid.util.LDAPCommandLineTool; 053import com.unboundid.util.ObjectPair; 054import com.unboundid.util.OutputFormat; 055import com.unboundid.util.RateAdjustor; 056import com.unboundid.util.ResultCodeCounter; 057import com.unboundid.util.ThreadSafety; 058import com.unboundid.util.ThreadSafetyLevel; 059import com.unboundid.util.ValuePattern; 060import com.unboundid.util.WakeableSleeper; 061import com.unboundid.util.args.ArgumentException; 062import com.unboundid.util.args.ArgumentParser; 063import com.unboundid.util.args.BooleanArgument; 064import com.unboundid.util.args.ControlArgument; 065import com.unboundid.util.args.FileArgument; 066import com.unboundid.util.args.FilterArgument; 067import com.unboundid.util.args.IntegerArgument; 068import com.unboundid.util.args.StringArgument; 069 070import static com.unboundid.util.Debug.*; 071import static com.unboundid.util.StaticUtils.*; 072 073 074 075/** 076 * This class provides a tool that can be used to perform repeated modifications 077 * in an LDAP directory server using multiple threads. It can help provide an 078 * estimate of the modify performance that a directory server is able to 079 * achieve. The target entry DN may be a value pattern as described in the 080 * {@link ValuePattern} class. This makes it possible to modify a range of 081 * entries rather than repeatedly updating the same entry. 082 * <BR><BR> 083 * Some of the APIs demonstrated by this example include: 084 * <UL> 085 * <LI>Argument Parsing (from the {@code com.unboundid.util.args} 086 * package)</LI> 087 * <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util} 088 * package)</LI> 089 * <LI>LDAP Communication (from the {@code com.unboundid.ldap.sdk} 090 * package)</LI> 091 * <LI>Value Patterns (from the {@code com.unboundid.util} package)</LI> 092 * </UL> 093 * <BR><BR> 094 * All of the necessary information is provided using command line arguments. 095 * Supported arguments include those allowed by the {@link LDAPCommandLineTool} 096 * class, as well as the following additional arguments: 097 * <UL> 098 * <LI>"-b {entryDN}" or "--targetDN {baseDN}" -- specifies the DN of the 099 * entry to be modified. This must be provided. It may be a simple DN, 100 * or it may be a value pattern to express a range of entry DNs.</LI> 101 * <LI>"-A {name}" or "--attribute {name}" -- specifies the name of the 102 * attribute to modify. Multiple attributes may be modified by providing 103 * multiple instances of this argument. At least one attribute must be 104 * provided.</LI> 105 * <LI>"-l {num}" or "--valueLength {num}" -- specifies the length in bytes to 106 * use for the values of the target attributes. If this is not provided, 107 * then a default length of 10 bytes will be used.</LI> 108 * <LI>"-C {chars}" or "--characterSet {chars}" -- specifies the set of 109 * characters that will be used to generate the values to use for the 110 * target attributes. It should only include ASCII characters. Values 111 * will be generated from randomly-selected characters from this set. If 112 * this is not provided, then a default set of lowercase alphabetic 113 * characters will be used.</LI> 114 * <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of 115 * concurrent threads to use when performing the modifications. If this 116 * is not provided, then a default of one thread will be used.</LI> 117 * <LI>"-i {sec}" or "--intervalDuration {sec}" -- specifies the length of 118 * time in seconds between lines out output. If this is not provided, 119 * then a default interval duration of five seconds will be used.</LI> 120 * <LI>"-I {num}" or "--numIntervals {num}" -- specifies the maximum number of 121 * intervals for which to run. If this is not provided, then it will 122 * run forever.</LI> 123 * <LI>"--iterationsBeforeReconnect {num}" -- specifies the number of modify 124 * iterations that should be performed on a connection before that 125 * connection is closed and replaced with a newly-established (and 126 * authenticated, if appropriate) connection.</LI> 127 * <LI>"-r {modifies-per-second}" or "--ratePerSecond {modifies-per-second}" 128 * -- specifies the target number of modifies to perform per second. It 129 * is still necessary to specify a sufficient number of threads for 130 * achieving this rate. If this option is not provided, then the tool 131 * will run at the maximum rate for the specified number of threads.</LI> 132 * <LI>"--variableRateData {path}" -- specifies the path to a file containing 133 * information needed to allow the tool to vary the target rate over time. 134 * If this option is not provided, then the tool will either use a fixed 135 * target rate as specified by the "--ratePerSecond" argument, or it will 136 * run at the maximum rate.</LI> 137 * <LI>"--generateSampleRateFile {path}" -- specifies the path to a file to 138 * which sample data will be written illustrating and describing the 139 * format of the file expected to be used in conjunction with the 140 * "--variableRateData" argument.</LI> 141 * <LI>"--warmUpIntervals {num}" -- specifies the number of intervals to 142 * complete before beginning overall statistics collection.</LI> 143 * <LI>"--timestampFormat {format}" -- specifies the format to use for 144 * timestamps included before each output line. The format may be one of 145 * "none" (for no timestamps), "with-date" (to include both the date and 146 * the time), or "without-date" (to include only time time).</LI> 147 * <LI>"-Y {authzID}" or "--proxyAs {authzID}" -- Use the proxied 148 * authorization v2 control to request that the operation be processed 149 * using an alternate authorization identity. In this case, the bind DN 150 * should be that of a user that has permission to use this control. The 151 * authorization identity may be a value pattern.</LI> 152 * <LI>"--suppressErrorResultCodes" -- Indicates that information about the 153 * result codes for failed operations should not be displayed.</LI> 154 * <LI>"-c" or "--csv" -- Generate output in CSV format rather than a 155 * display-friendly format.</LI> 156 * </UL> 157 */ 158@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE) 159public final class ModRate 160 extends LDAPCommandLineTool 161 implements Serializable 162{ 163 /** 164 * The serial version UID for this serializable class. 165 */ 166 private static final long serialVersionUID = 2709717414202815822L; 167 168 169 170 // Indicates whether a request has been made to stop running. 171 private final AtomicBoolean stopRequested; 172 173 // The argument used to indicate whether to generate output in CSV format. 174 private BooleanArgument csvFormat; 175 176 // Indicates that the tool should use the increment modification type instead 177 // of replace. 178 private BooleanArgument increment; 179 180 // Indicates that modify requests should include the permissive modify request 181 // control. 182 private BooleanArgument permissiveModify; 183 184 // The argument used to indicate whether to suppress information about error 185 // result codes. 186 private BooleanArgument suppressErrorsArgument; 187 188 // The argument used to indicate that a generic control should be included in 189 // the request. 190 private ControlArgument control; 191 192 // The argument used to specify a variable rate file. 193 private FileArgument sampleRateFile; 194 195 // The argument used to specify a variable rate file. 196 private FileArgument variableRateData; 197 198 // Indicates that modify requests should include the assertion request control 199 // with the specified filter. 200 private FilterArgument assertionFilter; 201 202 // The argument used to specify the collection interval. 203 private IntegerArgument collectionInterval; 204 205 // The increment amount to use when performing an increment instead of a 206 // replace. 207 private IntegerArgument incrementAmount; 208 209 // The argument used to specify the number of modify iterations on a 210 // connection before it is closed and re-established. 211 private IntegerArgument iterationsBeforeReconnect; 212 213 // The argument used to specify the number of intervals. 214 private IntegerArgument numIntervals; 215 216 // The argument used to specify the number of threads. 217 private IntegerArgument numThreads; 218 219 // The argument used to specify the seed to use for the random number 220 // generator. 221 private IntegerArgument randomSeed; 222 223 // The target rate of modifies per second. 224 private IntegerArgument ratePerSecond; 225 226 // The number of values to include in the replace modification. 227 private IntegerArgument valueCount; 228 229 // The argument used to specify the length of the values to generate. 230 private IntegerArgument valueLength; 231 232 // The number of warm-up intervals to perform. 233 private IntegerArgument warmUpIntervals; 234 235 // The argument used to specify the name of the attribute to modify. 236 private StringArgument attribute; 237 238 // The argument used to specify the set of characters to use when generating 239 // values. 240 private StringArgument characterSet; 241 242 // The argument used to specify the DNs of the entries to modify. 243 private StringArgument entryDN; 244 245 // Indicates that modify requests should include the post-read request control 246 // to request the specified attribute. 247 private StringArgument postReadAttribute; 248 249 // Indicates that modify requests should include the pre-read request control 250 // to request the specified attribute. 251 private StringArgument preReadAttribute; 252 253 // The argument used to specify the proxied authorization identity. 254 private StringArgument proxyAs; 255 256 // The argument used to specify the timestamp format. 257 private StringArgument timestampFormat; 258 259 // The thread currently being used to run the searchrate tool. 260 private volatile Thread runningThread; 261 262 // A wakeable sleeper that will be used to sleep between reporting intervals. 263 private final WakeableSleeper sleeper; 264 265 266 267 /** 268 * Parse the provided command line arguments and make the appropriate set of 269 * changes. 270 * 271 * @param args The command line arguments provided to this program. 272 */ 273 public static void main(final String[] args) 274 { 275 final ResultCode resultCode = main(args, System.out, System.err); 276 if (resultCode != ResultCode.SUCCESS) 277 { 278 System.exit(resultCode.intValue()); 279 } 280 } 281 282 283 284 /** 285 * Parse the provided command line arguments and make the appropriate set of 286 * changes. 287 * 288 * @param args The command line arguments provided to this program. 289 * @param outStream The output stream to which standard out should be 290 * written. It may be {@code null} if output should be 291 * suppressed. 292 * @param errStream The output stream to which standard error should be 293 * written. It may be {@code null} if error messages 294 * should be suppressed. 295 * 296 * @return A result code indicating whether the processing was successful. 297 */ 298 public static ResultCode main(final String[] args, 299 final OutputStream outStream, 300 final OutputStream errStream) 301 { 302 final ModRate modRate = new ModRate(outStream, errStream); 303 return modRate.runTool(args); 304 } 305 306 307 308 /** 309 * Creates a new instance of this tool. 310 * 311 * @param outStream The output stream to which standard out should be 312 * written. It may be {@code null} if output should be 313 * suppressed. 314 * @param errStream The output stream to which standard error should be 315 * written. It may be {@code null} if error messages 316 * should be suppressed. 317 */ 318 public ModRate(final OutputStream outStream, final OutputStream errStream) 319 { 320 super(outStream, errStream); 321 322 stopRequested = new AtomicBoolean(false); 323 sleeper = new WakeableSleeper(); 324 } 325 326 327 328 /** 329 * Retrieves the name for this tool. 330 * 331 * @return The name for this tool. 332 */ 333 @Override() 334 public String getToolName() 335 { 336 return "modrate"; 337 } 338 339 340 341 /** 342 * Retrieves the description for this tool. 343 * 344 * @return The description for this tool. 345 */ 346 @Override() 347 public String getToolDescription() 348 { 349 return "Perform repeated modifications against " + 350 "an LDAP directory server."; 351 } 352 353 354 355 /** 356 * Retrieves the version string for this tool. 357 * 358 * @return The version string for this tool. 359 */ 360 @Override() 361 public String getToolVersion() 362 { 363 return Version.NUMERIC_VERSION_STRING; 364 } 365 366 367 368 /** 369 * Indicates whether this tool should provide support for an interactive mode, 370 * in which the tool offers a mode in which the arguments can be provided in 371 * a text-driven menu rather than requiring them to be given on the command 372 * line. If interactive mode is supported, it may be invoked using the 373 * "--interactive" argument. Alternately, if interactive mode is supported 374 * and {@link #defaultsToInteractiveMode()} returns {@code true}, then 375 * interactive mode may be invoked by simply launching the tool without any 376 * arguments. 377 * 378 * @return {@code true} if this tool supports interactive mode, or 379 * {@code false} if not. 380 */ 381 @Override() 382 public boolean supportsInteractiveMode() 383 { 384 return true; 385 } 386 387 388 389 /** 390 * Indicates whether this tool defaults to launching in interactive mode if 391 * the tool is invoked without any command-line arguments. This will only be 392 * used if {@link #supportsInteractiveMode()} returns {@code true}. 393 * 394 * @return {@code true} if this tool defaults to using interactive mode if 395 * launched without any command-line arguments, or {@code false} if 396 * not. 397 */ 398 @Override() 399 public boolean defaultsToInteractiveMode() 400 { 401 return true; 402 } 403 404 405 406 /** 407 * Indicates whether this tool should provide arguments for redirecting output 408 * to a file. If this method returns {@code true}, then the tool will offer 409 * an "--outputFile" argument that will specify the path to a file to which 410 * all standard output and standard error content will be written, and it will 411 * also offer a "--teeToStandardOut" argument that can only be used if the 412 * "--outputFile" argument is present and will cause all output to be written 413 * to both the specified output file and to standard output. 414 * 415 * @return {@code true} if this tool should provide arguments for redirecting 416 * output to a file, or {@code false} if not. 417 */ 418 @Override() 419 protected boolean supportsOutputFile() 420 { 421 return true; 422 } 423 424 425 426 /** 427 * Indicates whether this tool should default to interactively prompting for 428 * the bind password if a password is required but no argument was provided 429 * to indicate how to get the password. 430 * 431 * @return {@code true} if this tool should default to interactively 432 * prompting for the bind password, or {@code false} if not. 433 */ 434 @Override() 435 protected boolean defaultToPromptForBindPassword() 436 { 437 return true; 438 } 439 440 441 442 /** 443 * Indicates whether this tool supports the use of a properties file for 444 * specifying default values for arguments that aren't specified on the 445 * command line. 446 * 447 * @return {@code true} if this tool supports the use of a properties file 448 * for specifying default values for arguments that aren't specified 449 * on the command line, or {@code false} if not. 450 */ 451 @Override() 452 public boolean supportsPropertiesFile() 453 { 454 return true; 455 } 456 457 458 459 /** 460 * Indicates whether the LDAP-specific arguments should include alternate 461 * versions of all long identifiers that consist of multiple words so that 462 * they are available in both camelCase and dash-separated versions. 463 * 464 * @return {@code true} if this tool should provide multiple versions of 465 * long identifiers for LDAP-specific arguments, or {@code false} if 466 * not. 467 */ 468 @Override() 469 protected boolean includeAlternateLongIdentifiers() 470 { 471 return true; 472 } 473 474 475 476 /** 477 * {@inheritDoc} 478 */ 479 @Override() 480 protected boolean logToolInvocationByDefault() 481 { 482 return true; 483 } 484 485 486 487 /** 488 * Adds the arguments used by this program that aren't already provided by the 489 * generic {@code LDAPCommandLineTool} framework. 490 * 491 * @param parser The argument parser to which the arguments should be added. 492 * 493 * @throws ArgumentException If a problem occurs while adding the arguments. 494 */ 495 @Override() 496 public void addNonLDAPArguments(final ArgumentParser parser) 497 throws ArgumentException 498 { 499 String description = "The DN of the entry to modify. It may be a simple " + 500 "DN or a value pattern to specify a range of DN (e.g., " + 501 "\"uid=user.[1-1000],ou=People,dc=example,dc=com\"). See " + 502 ValuePattern.PUBLIC_JAVADOC_URL + " for complete details about the " + 503 "value pattern syntax. This must be provided."; 504 entryDN = new StringArgument('b', "entryDN", true, 1, "{dn}", description); 505 entryDN.setArgumentGroupName("Modification Arguments"); 506 entryDN.addLongIdentifier("entry-dn", true); 507 parser.addArgument(entryDN); 508 509 510 description = "The name of the attribute to modify. Multiple attributes " + 511 "may be specified by providing this argument multiple " + 512 "times. At least one attribute must be specified."; 513 attribute = new StringArgument('A', "attribute", true, 0, "{name}", 514 description); 515 attribute.setArgumentGroupName("Modification Arguments"); 516 parser.addArgument(attribute); 517 518 519 description = "The length in bytes to use when generating values for the " + 520 "replace modifications. If this is not provided, then a " + 521 "default length of ten bytes will be used."; 522 valueLength = new IntegerArgument('l', "valueLength", true, 1, "{num}", 523 description, 1, Integer.MAX_VALUE, 10); 524 valueLength.setArgumentGroupName("Modification Arguments"); 525 valueLength.addLongIdentifier("value-length", true); 526 parser.addArgument(valueLength); 527 528 529 description = "The number of values to include in replace " + 530 "modifications. If this is not provided, then a default " + 531 "of one value will be used."; 532 valueCount = new IntegerArgument(null, "valueCount", false, 1, "{num}", 533 description, 0, Integer.MAX_VALUE, 1); 534 valueCount.setArgumentGroupName("Modification Arguments"); 535 valueCount.addLongIdentifier("value-count", true); 536 parser.addArgument(valueCount); 537 538 539 description = "Indicates that the tool should use the increment " + 540 "modification type rather than the replace modification " + 541 "type."; 542 increment = new BooleanArgument(null, "increment", 1, description); 543 increment.setArgumentGroupName("Modification Arguments"); 544 parser.addArgument(increment); 545 546 547 description = "The amount by which to increment values when using the " + 548 "increment modification type. The amount may be negative " + 549 "if values should be decremented rather than incremented. " + 550 "If this is not provided, then a default increment amount " + 551 "of one will be used."; 552 incrementAmount = new IntegerArgument(null, "incrementAmount", false, 1, 553 null, description, Integer.MIN_VALUE, 554 Integer.MAX_VALUE, 1); 555 incrementAmount.setArgumentGroupName("Modification Arguments"); 556 incrementAmount.addLongIdentifier("increment-amount", true); 557 parser.addArgument(incrementAmount); 558 559 560 description = "The set of characters to use to generate the values for " + 561 "the modifications. It should only include ASCII " + 562 "characters. If this is not provided, then a default set " + 563 "of lowercase alphabetic characters will be used."; 564 characterSet = new StringArgument('C', "characterSet", true, 1, "{chars}", 565 description, 566 "abcdefghijklmnopqrstuvwxyz"); 567 characterSet.setArgumentGroupName("Modification Arguments"); 568 characterSet.addLongIdentifier("character-set", true); 569 parser.addArgument(characterSet); 570 571 572 description = "Indicates that modify requests should include the " + 573 "assertion request control with the specified filter."; 574 assertionFilter = new FilterArgument(null, "assertionFilter", false, 1, 575 "{filter}", description); 576 assertionFilter.setArgumentGroupName("Request Control Arguments"); 577 assertionFilter.addLongIdentifier("assertion-filter", true); 578 parser.addArgument(assertionFilter); 579 580 581 description = "Indicates that modify requests should include the " + 582 "permissive modify request control."; 583 permissiveModify = new BooleanArgument(null, "permissiveModify", 1, 584 description); 585 permissiveModify.setArgumentGroupName("Request Control Arguments"); 586 permissiveModify.addLongIdentifier("permissive-modify", true); 587 parser.addArgument(permissiveModify); 588 589 590 description = "Indicates that modify requests should include the " + 591 "pre-read request control with the specified requested " + 592 "attribute. This argument may be provided multiple times " + 593 "to indicate that multiple requested attributes should be " + 594 "included in the pre-read request control."; 595 preReadAttribute = new StringArgument(null, "preReadAttribute", false, 0, 596 "{attribute}", description); 597 preReadAttribute.setArgumentGroupName("Request Control Arguments"); 598 preReadAttribute.addLongIdentifier("pre-read-attribute", true); 599 parser.addArgument(preReadAttribute); 600 601 602 description = "Indicates that modify requests should include the " + 603 "post-read request control with the specified requested " + 604 "attribute. This argument may be provided multiple times " + 605 "to indicate that multiple requested attributes should be " + 606 "included in the post-read request control."; 607 postReadAttribute = new StringArgument(null, "postReadAttribute", false, 0, 608 "{attribute}", description); 609 postReadAttribute.setArgumentGroupName("Request Control Arguments"); 610 postReadAttribute.addLongIdentifier("post-read-attribute", true); 611 parser.addArgument(postReadAttribute); 612 613 614 description = "Indicates that the proxied authorization control (as " + 615 "defined in RFC 4370) should be used to request that " + 616 "operations be processed using an alternate authorization " + 617 "identity. This may be a simple authorization ID or it " + 618 "may be a value pattern to specify a range of " + 619 "identities. See " + ValuePattern.PUBLIC_JAVADOC_URL + 620 " for complete details about the value pattern syntax."; 621 proxyAs = new StringArgument('Y', "proxyAs", false, 1, "{authzID}", 622 description); 623 proxyAs.setArgumentGroupName("Request Control Arguments"); 624 proxyAs.addLongIdentifier("proxy-as", true); 625 parser.addArgument(proxyAs); 626 627 628 description = "Indicates that modify requests should include the " + 629 "specified request control. This may be provided multiple " + 630 "times to include multiple request controls."; 631 control = new ControlArgument('J', "control", false, 0, null, description); 632 control.setArgumentGroupName("Request Control Arguments"); 633 parser.addArgument(control); 634 635 636 description = "The number of threads to use to perform the " + 637 "modifications. If this is not provided, a single thread " + 638 "will be used."; 639 numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}", 640 description, 1, Integer.MAX_VALUE, 1); 641 numThreads.setArgumentGroupName("Rate Management Arguments"); 642 numThreads.addLongIdentifier("num-threads", true); 643 parser.addArgument(numThreads); 644 645 646 description = "The length of time in seconds between output lines. If " + 647 "this is not provided, then a default interval of five " + 648 "seconds will be used."; 649 collectionInterval = new IntegerArgument('i', "intervalDuration", true, 1, 650 "{num}", description, 1, 651 Integer.MAX_VALUE, 5); 652 collectionInterval.setArgumentGroupName("Rate Management Arguments"); 653 collectionInterval.addLongIdentifier("interval-duration", true); 654 parser.addArgument(collectionInterval); 655 656 657 description = "The maximum number of intervals for which to run. If " + 658 "this is not provided, then the tool will run until it is " + 659 "interrupted."; 660 numIntervals = new IntegerArgument('I', "numIntervals", true, 1, "{num}", 661 description, 1, Integer.MAX_VALUE, 662 Integer.MAX_VALUE); 663 numIntervals.setArgumentGroupName("Rate Management Arguments"); 664 numIntervals.addLongIdentifier("num-intervals", true); 665 parser.addArgument(numIntervals); 666 667 description = "The number of modify iterations that should be processed " + 668 "on a connection before that connection is closed and " + 669 "replaced with a newly-established (and authenticated, if " + 670 "appropriate) connection. If this is not provided, then " + 671 "connections will not be periodically closed and " + 672 "re-established."; 673 iterationsBeforeReconnect = new IntegerArgument(null, 674 "iterationsBeforeReconnect", false, 1, "{num}", description, 0); 675 iterationsBeforeReconnect.setArgumentGroupName("Rate Management Arguments"); 676 iterationsBeforeReconnect.addLongIdentifier("iterations-before-reconnect", 677 true); 678 parser.addArgument(iterationsBeforeReconnect); 679 680 description = "The target number of modifies to perform per second. It " + 681 "is still necessary to specify a sufficient number of " + 682 "threads for achieving this rate. If neither this option " + 683 "nor --variableRateData is provided, then the tool will " + 684 "run at the maximum rate for the specified number of " + 685 "threads."; 686 ratePerSecond = new IntegerArgument('r', "ratePerSecond", false, 1, 687 "{modifies-per-second}", description, 688 1, Integer.MAX_VALUE); 689 ratePerSecond.setArgumentGroupName("Rate Management Arguments"); 690 ratePerSecond.addLongIdentifier("rate-per-second", true); 691 parser.addArgument(ratePerSecond); 692 693 final String variableRateDataArgName = "variableRateData"; 694 final String generateSampleRateFileArgName = "generateSampleRateFile"; 695 description = RateAdjustor.getVariableRateDataArgumentDescription( 696 generateSampleRateFileArgName); 697 variableRateData = new FileArgument(null, variableRateDataArgName, false, 1, 698 "{path}", description, true, true, true, 699 false); 700 variableRateData.setArgumentGroupName("Rate Management Arguments"); 701 variableRateData.addLongIdentifier("variable-rate-data", true); 702 parser.addArgument(variableRateData); 703 704 description = RateAdjustor.getGenerateSampleVariableRateFileDescription( 705 variableRateDataArgName); 706 sampleRateFile = new FileArgument(null, generateSampleRateFileArgName, 707 false, 1, "{path}", description, false, 708 true, true, false); 709 sampleRateFile.setArgumentGroupName("Rate Management Arguments"); 710 sampleRateFile.addLongIdentifier("generate-sample-rate-file", true); 711 sampleRateFile.setUsageArgument(true); 712 parser.addArgument(sampleRateFile); 713 parser.addExclusiveArgumentSet(variableRateData, sampleRateFile); 714 715 description = "The number of intervals to complete before beginning " + 716 "overall statistics collection. Specifying a nonzero " + 717 "number of warm-up intervals gives the client and server " + 718 "a chance to warm up without skewing performance results."; 719 warmUpIntervals = new IntegerArgument(null, "warmUpIntervals", true, 1, 720 "{num}", description, 0, Integer.MAX_VALUE, 0); 721 warmUpIntervals.setArgumentGroupName("Rate Management Arguments"); 722 warmUpIntervals.addLongIdentifier("warm-up-intervals", true); 723 parser.addArgument(warmUpIntervals); 724 725 description = "Indicates the format to use for timestamps included in " + 726 "the output. A value of 'none' indicates that no " + 727 "timestamps should be included. A value of 'with-date' " + 728 "indicates that both the date and the time should be " + 729 "included. A value of 'without-date' indicates that only " + 730 "the time should be included."; 731 final LinkedHashSet<String> allowedFormats = new LinkedHashSet<String>(3); 732 allowedFormats.add("none"); 733 allowedFormats.add("with-date"); 734 allowedFormats.add("without-date"); 735 timestampFormat = new StringArgument(null, "timestampFormat", true, 1, 736 "{format}", description, allowedFormats, "none"); 737 timestampFormat.addLongIdentifier("timestamp-format", true); 738 parser.addArgument(timestampFormat); 739 740 description = "Indicates that information about the result codes for " + 741 "failed operations should not be displayed."; 742 suppressErrorsArgument = new BooleanArgument(null, 743 "suppressErrorResultCodes", 1, description); 744 suppressErrorsArgument.addLongIdentifier("suppress-error-result-codes", 745 true); 746 parser.addArgument(suppressErrorsArgument); 747 748 description = "Generate output in CSV format rather than a " + 749 "display-friendly format"; 750 csvFormat = new BooleanArgument('c', "csv", 1, description); 751 parser.addArgument(csvFormat); 752 753 description = "Specifies the seed to use for the random number generator."; 754 randomSeed = new IntegerArgument('R', "randomSeed", false, 1, "{value}", 755 description); 756 randomSeed.addLongIdentifier("random-seed", true); 757 parser.addArgument(randomSeed); 758 759 760 // The incrementAmount argument can only be used if the increment argument 761 // is provided. 762 parser.addDependentArgumentSet(incrementAmount, increment); 763 764 765 // Neither the valueLength nor valueCount arguments can be used if the 766 // increment argument is provided. 767 parser.addExclusiveArgumentSet(increment, valueLength); 768 parser.addExclusiveArgumentSet(increment, valueCount); 769 } 770 771 772 773 /** 774 * Indicates whether this tool supports creating connections to multiple 775 * servers. If it is to support multiple servers, then the "--hostname" and 776 * "--port" arguments will be allowed to be provided multiple times, and 777 * will be required to be provided the same number of times. The same type of 778 * communication security and bind credentials will be used for all servers. 779 * 780 * @return {@code true} if this tool supports creating connections to 781 * multiple servers, or {@code false} if not. 782 */ 783 @Override() 784 protected boolean supportsMultipleServers() 785 { 786 return true; 787 } 788 789 790 791 /** 792 * Retrieves the connection options that should be used for connections 793 * created for use with this tool. 794 * 795 * @return The connection options that should be used for connections created 796 * for use with this tool. 797 */ 798 @Override() 799 public LDAPConnectionOptions getConnectionOptions() 800 { 801 final LDAPConnectionOptions options = new LDAPConnectionOptions(); 802 options.setUseSynchronousMode(true); 803 return options; 804 } 805 806 807 808 /** 809 * Performs the actual processing for this tool. In this case, it gets a 810 * connection to the directory server and uses it to perform the requested 811 * modifications. 812 * 813 * @return The result code for the processing that was performed. 814 */ 815 @Override() 816 public ResultCode doToolProcessing() 817 { 818 runningThread = Thread.currentThread(); 819 820 try 821 { 822 return doToolProcessingInternal(); 823 } 824 finally 825 { 826 runningThread = null; 827 } 828 829 } 830 831 832 /** 833 * Performs the actual processing for this tool. In this case, it gets a 834 * connection to the directory server and uses it to perform the requested 835 * modifications. 836 * 837 * @return The result code for the processing that was performed. 838 */ 839 private ResultCode doToolProcessingInternal() 840 { 841 // If the sample rate file argument was specified, then generate the sample 842 // variable rate data file and return. 843 if (sampleRateFile.isPresent()) 844 { 845 try 846 { 847 RateAdjustor.writeSampleVariableRateFile(sampleRateFile.getValue()); 848 return ResultCode.SUCCESS; 849 } 850 catch (final Exception e) 851 { 852 debugException(e); 853 err("An error occurred while trying to write sample variable data " + 854 "rate file '", sampleRateFile.getValue().getAbsolutePath(), 855 "': ", getExceptionMessage(e)); 856 return ResultCode.LOCAL_ERROR; 857 } 858 } 859 860 861 // Determine the random seed to use. 862 final Long seed; 863 if (randomSeed.isPresent()) 864 { 865 seed = Long.valueOf(randomSeed.getValue()); 866 } 867 else 868 { 869 seed = null; 870 } 871 872 // Create the value patterns for the target entry DN and proxied 873 // authorization identities. 874 final ValuePattern dnPattern; 875 try 876 { 877 dnPattern = new ValuePattern(entryDN.getValue(), seed); 878 } 879 catch (final ParseException pe) 880 { 881 debugException(pe); 882 err("Unable to parse the entry DN value pattern: ", pe.getMessage()); 883 return ResultCode.PARAM_ERROR; 884 } 885 886 final ValuePattern authzIDPattern; 887 if (proxyAs.isPresent()) 888 { 889 try 890 { 891 authzIDPattern = new ValuePattern(proxyAs.getValue(), seed); 892 } 893 catch (final ParseException pe) 894 { 895 debugException(pe); 896 err("Unable to parse the proxied authorization pattern: ", 897 pe.getMessage()); 898 return ResultCode.PARAM_ERROR; 899 } 900 } 901 else 902 { 903 authzIDPattern = null; 904 } 905 906 907 // Get the set of controls to include in modify requests. 908 final ArrayList<Control> controlList = new ArrayList<Control>(5); 909 if (assertionFilter.isPresent()) 910 { 911 controlList.add(new AssertionRequestControl(assertionFilter.getValue())); 912 } 913 914 if (permissiveModify.isPresent()) 915 { 916 controlList.add(new PermissiveModifyRequestControl()); 917 } 918 919 if (preReadAttribute.isPresent()) 920 { 921 final List<String> attrList = preReadAttribute.getValues(); 922 final String[] attrArray = new String[attrList.size()]; 923 attrList.toArray(attrArray); 924 controlList.add(new PreReadRequestControl(attrArray)); 925 } 926 927 if (postReadAttribute.isPresent()) 928 { 929 final List<String> attrList = postReadAttribute.getValues(); 930 final String[] attrArray = new String[attrList.size()]; 931 attrList.toArray(attrArray); 932 controlList.add(new PostReadRequestControl(attrArray)); 933 } 934 935 if (control.isPresent()) 936 { 937 controlList.addAll(control.getValues()); 938 } 939 940 final Control[] controlArray = new Control[controlList.size()]; 941 controlList.toArray(controlArray); 942 943 944 // Get the names of the attributes to modify. 945 final String[] attrs = new String[attribute.getValues().size()]; 946 attribute.getValues().toArray(attrs); 947 948 949 // Get the character set as a byte array. 950 final byte[] charSet = getBytes(characterSet.getValue()); 951 952 953 // If the --ratePerSecond option was specified, then limit the rate 954 // accordingly. 955 FixedRateBarrier fixedRateBarrier = null; 956 if (ratePerSecond.isPresent() || variableRateData.isPresent()) 957 { 958 // We might not have a rate per second if --variableRateData is specified. 959 // The rate typically doesn't matter except when we have warm-up 960 // intervals. In this case, we'll run at the max rate. 961 final int intervalSeconds = collectionInterval.getValue(); 962 final int ratePerInterval = 963 (ratePerSecond.getValue() == null) 964 ? Integer.MAX_VALUE 965 : ratePerSecond.getValue() * intervalSeconds; 966 fixedRateBarrier = 967 new FixedRateBarrier(1000L * intervalSeconds, ratePerInterval); 968 } 969 970 971 // If --variableRateData was specified, then initialize a RateAdjustor. 972 RateAdjustor rateAdjustor = null; 973 if (variableRateData.isPresent()) 974 { 975 try 976 { 977 rateAdjustor = RateAdjustor.newInstance(fixedRateBarrier, 978 ratePerSecond.getValue(), variableRateData.getValue()); 979 } 980 catch (final IOException e) 981 { 982 debugException(e); 983 err("Initializing the variable rates failed: " + e.getMessage()); 984 return ResultCode.PARAM_ERROR; 985 } 986 catch (final IllegalArgumentException e) 987 { 988 debugException(e); 989 err("Initializing the variable rates failed: " + e.getMessage()); 990 return ResultCode.PARAM_ERROR; 991 } 992 } 993 994 995 // Determine whether to include timestamps in the output and if so what 996 // format should be used for them. 997 final boolean includeTimestamp; 998 final String timeFormat; 999 if (timestampFormat.getValue().equalsIgnoreCase("with-date")) 1000 { 1001 includeTimestamp = true; 1002 timeFormat = "dd/MM/yyyy HH:mm:ss"; 1003 } 1004 else if (timestampFormat.getValue().equalsIgnoreCase("without-date")) 1005 { 1006 includeTimestamp = true; 1007 timeFormat = "HH:mm:ss"; 1008 } 1009 else 1010 { 1011 includeTimestamp = false; 1012 timeFormat = null; 1013 } 1014 1015 1016 // Determine whether any warm-up intervals should be run. 1017 final long totalIntervals; 1018 final boolean warmUp; 1019 int remainingWarmUpIntervals = warmUpIntervals.getValue(); 1020 if (remainingWarmUpIntervals > 0) 1021 { 1022 warmUp = true; 1023 totalIntervals = 0L + numIntervals.getValue() + remainingWarmUpIntervals; 1024 } 1025 else 1026 { 1027 warmUp = true; 1028 totalIntervals = 0L + numIntervals.getValue(); 1029 } 1030 1031 1032 // Create the table that will be used to format the output. 1033 final OutputFormat outputFormat; 1034 if (csvFormat.isPresent()) 1035 { 1036 outputFormat = OutputFormat.CSV; 1037 } 1038 else 1039 { 1040 outputFormat = OutputFormat.COLUMNS; 1041 } 1042 1043 final ColumnFormatter formatter = new ColumnFormatter(includeTimestamp, 1044 timeFormat, outputFormat, " ", 1045 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", 1046 "Mods/Sec"), 1047 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", 1048 "Avg Dur ms"), 1049 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", 1050 "Errors/Sec"), 1051 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall", 1052 "Mods/Sec"), 1053 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall", 1054 "Avg Dur ms")); 1055 1056 1057 // Create values to use for statistics collection. 1058 final AtomicLong modCounter = new AtomicLong(0L); 1059 final AtomicLong errorCounter = new AtomicLong(0L); 1060 final AtomicLong modDurations = new AtomicLong(0L); 1061 final ResultCodeCounter rcCounter = new ResultCodeCounter(); 1062 1063 1064 // Determine the length of each interval in milliseconds. 1065 final long intervalMillis = 1000L * collectionInterval.getValue(); 1066 1067 1068 // Create a random number generator to use for seeding the per-thread 1069 // generators. 1070 final Random random = new Random(); 1071 1072 1073 // Create the threads to use for the modifications. 1074 final CyclicBarrier barrier = new CyclicBarrier(numThreads.getValue() + 1); 1075 final ModRateThread[] threads = new ModRateThread[numThreads.getValue()]; 1076 for (int i=0; i < threads.length; i++) 1077 { 1078 final LDAPConnection connection; 1079 try 1080 { 1081 connection = getConnection(); 1082 } 1083 catch (final LDAPException le) 1084 { 1085 debugException(le); 1086 err("Unable to connect to the directory server: ", 1087 getExceptionMessage(le)); 1088 return le.getResultCode(); 1089 } 1090 1091 threads[i] = new ModRateThread(this, i, connection, dnPattern, attrs, 1092 charSet, valueLength.getValue(), valueCount.getValue(), 1093 increment.isPresent(), incrementAmount.getValue(), controlArray, 1094 authzIDPattern, random.nextLong(), 1095 iterationsBeforeReconnect.getValue(), barrier, modCounter, 1096 modDurations, errorCounter, rcCounter, fixedRateBarrier); 1097 threads[i].start(); 1098 } 1099 1100 1101 // Display the table header. 1102 for (final String headerLine : formatter.getHeaderLines(true)) 1103 { 1104 out(headerLine); 1105 } 1106 1107 1108 // Start the RateAdjustor before the threads so that the initial value is 1109 // in place before any load is generated unless we're doing a warm-up in 1110 // which case, we'll start it after the warm-up is complete. 1111 if ((rateAdjustor != null) && (remainingWarmUpIntervals <= 0)) 1112 { 1113 rateAdjustor.start(); 1114 } 1115 1116 1117 // Indicate that the threads can start running. 1118 try 1119 { 1120 barrier.await(); 1121 } 1122 catch (final Exception e) 1123 { 1124 debugException(e); 1125 } 1126 1127 long overallStartTime = System.nanoTime(); 1128 long nextIntervalStartTime = System.currentTimeMillis() + intervalMillis; 1129 1130 1131 boolean setOverallStartTime = false; 1132 long lastDuration = 0L; 1133 long lastNumErrors = 0L; 1134 long lastNumMods = 0L; 1135 long lastEndTime = System.nanoTime(); 1136 for (long i=0; i < totalIntervals; i++) 1137 { 1138 if (rateAdjustor != null) 1139 { 1140 if (! rateAdjustor.isAlive()) 1141 { 1142 out("All of the rates in " + variableRateData.getValue().getName() + 1143 " have been completed."); 1144 break; 1145 } 1146 } 1147 1148 final long startTimeMillis = System.currentTimeMillis(); 1149 final long sleepTimeMillis = nextIntervalStartTime - startTimeMillis; 1150 nextIntervalStartTime += intervalMillis; 1151 if (sleepTimeMillis > 0) 1152 { 1153 sleeper.sleep(sleepTimeMillis); 1154 } 1155 1156 if (stopRequested.get()) 1157 { 1158 break; 1159 } 1160 1161 final long endTime = System.nanoTime(); 1162 final long intervalDuration = endTime - lastEndTime; 1163 1164 final long numMods; 1165 final long numErrors; 1166 final long totalDuration; 1167 if (warmUp && (remainingWarmUpIntervals > 0)) 1168 { 1169 numMods = modCounter.getAndSet(0L); 1170 numErrors = errorCounter.getAndSet(0L); 1171 totalDuration = modDurations.getAndSet(0L); 1172 } 1173 else 1174 { 1175 numMods = modCounter.get(); 1176 numErrors = errorCounter.get(); 1177 totalDuration = modDurations.get(); 1178 } 1179 1180 final long recentNumMods = numMods - lastNumMods; 1181 final long recentNumErrors = numErrors - lastNumErrors; 1182 final long recentDuration = totalDuration - lastDuration; 1183 1184 final double numSeconds = intervalDuration / 1000000000.0d; 1185 final double recentModRate = recentNumMods / numSeconds; 1186 final double recentErrorRate = recentNumErrors / numSeconds; 1187 1188 final double recentAvgDuration; 1189 if (recentNumMods > 0L) 1190 { 1191 recentAvgDuration = 1.0d * recentDuration / recentNumMods / 1000000; 1192 } 1193 else 1194 { 1195 recentAvgDuration = 0.0d; 1196 } 1197 1198 if (warmUp && (remainingWarmUpIntervals > 0)) 1199 { 1200 out(formatter.formatRow(recentModRate, recentAvgDuration, 1201 recentErrorRate, "warming up", "warming up")); 1202 1203 remainingWarmUpIntervals--; 1204 if (remainingWarmUpIntervals == 0) 1205 { 1206 out("Warm-up completed. Beginning overall statistics collection."); 1207 setOverallStartTime = true; 1208 if (rateAdjustor != null) 1209 { 1210 rateAdjustor.start(); 1211 } 1212 } 1213 } 1214 else 1215 { 1216 if (setOverallStartTime) 1217 { 1218 overallStartTime = lastEndTime; 1219 setOverallStartTime = false; 1220 } 1221 1222 final double numOverallSeconds = 1223 (endTime - overallStartTime) / 1000000000.0d; 1224 final double overallAuthRate = numMods / numOverallSeconds; 1225 1226 final double overallAvgDuration; 1227 if (numMods > 0L) 1228 { 1229 overallAvgDuration = 1.0d * totalDuration / numMods / 1000000; 1230 } 1231 else 1232 { 1233 overallAvgDuration = 0.0d; 1234 } 1235 1236 out(formatter.formatRow(recentModRate, recentAvgDuration, 1237 recentErrorRate, overallAuthRate, overallAvgDuration)); 1238 1239 lastNumMods = numMods; 1240 lastNumErrors = numErrors; 1241 lastDuration = totalDuration; 1242 } 1243 1244 final List<ObjectPair<ResultCode,Long>> rcCounts = 1245 rcCounter.getCounts(true); 1246 if ((! suppressErrorsArgument.isPresent()) && (! rcCounts.isEmpty())) 1247 { 1248 err("\tError Results:"); 1249 for (final ObjectPair<ResultCode,Long> p : rcCounts) 1250 { 1251 err("\t", p.getFirst().getName(), ": ", p.getSecond()); 1252 } 1253 } 1254 1255 lastEndTime = endTime; 1256 } 1257 1258 // Shut down the RateAdjustor if we have one. 1259 if (rateAdjustor != null) 1260 { 1261 rateAdjustor.shutDown(); 1262 } 1263 1264 // Stop all of the threads. 1265 ResultCode resultCode = ResultCode.SUCCESS; 1266 for (final ModRateThread t : threads) 1267 { 1268 final ResultCode r = t.stopRunning(); 1269 if (resultCode == ResultCode.SUCCESS) 1270 { 1271 resultCode = r; 1272 } 1273 } 1274 1275 return resultCode; 1276 } 1277 1278 1279 1280 /** 1281 * Requests that this tool stop running. This method will attempt to wait 1282 * for all threads to complete before returning control to the caller. 1283 */ 1284 public void stopRunning() 1285 { 1286 stopRequested.set(true); 1287 sleeper.wakeup(); 1288 1289 final Thread t = runningThread; 1290 if (t != null) 1291 { 1292 try 1293 { 1294 t.join(); 1295 } 1296 catch (final Exception e) 1297 { 1298 debugException(e); 1299 1300 if (e instanceof InterruptedException) 1301 { 1302 Thread.currentThread().interrupt(); 1303 } 1304 } 1305 } 1306 } 1307 1308 1309 1310 /** 1311 * {@inheritDoc} 1312 */ 1313 @Override() 1314 public LinkedHashMap<String[],String> getExampleUsages() 1315 { 1316 final LinkedHashMap<String[],String> examples = 1317 new LinkedHashMap<String[],String>(2); 1318 1319 String[] args = 1320 { 1321 "--hostname", "server.example.com", 1322 "--port", "389", 1323 "--bindDN", "uid=admin,dc=example,dc=com", 1324 "--bindPassword", "password", 1325 "--entryDN", "uid=user.[1-1000000],ou=People,dc=example,dc=com", 1326 "--attribute", "description", 1327 "--valueLength", "12", 1328 "--numThreads", "10" 1329 }; 1330 String description = 1331 "Test modify performance by randomly selecting entries across a set " + 1332 "of one million users located below 'ou=People,dc=example,dc=com' " + 1333 "with ten concurrent threads and replacing the values for the " + 1334 "description attribute with a string of 12 randomly-selected " + 1335 "lowercase alphabetic characters."; 1336 examples.put(args, description); 1337 1338 args = new String[] 1339 { 1340 "--generateSampleRateFile", "variable-rate-data.txt" 1341 }; 1342 description = 1343 "Generate a sample variable rate definition file that may be used " + 1344 "in conjunction with the --variableRateData argument. The sample " + 1345 "file will include comments that describe the format for data to be " + 1346 "included in this file."; 1347 examples.put(args, description); 1348 1349 return examples; 1350 } 1351}