001/*
002 * The contents of this file are subject to the terms of the Common Development and
003 * Distribution License (the License). You may not use this file except in compliance with the
004 * License.
005 *
006 * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
007 * specific language governing permission and limitations under the License.
008 *
009 * When distributing Covered Software, include this CDDL Header Notice in each file and include
010 * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
011 * Header, with the fields enclosed by brackets [] replaced by your own identifying
012 * information: "Portions Copyright [year] [name of copyright owner]".
013 *
014 * Copyright 2006-2010 Sun Microsystems, Inc.
015 * Portions Copyright 2011-2016 ForgeRock AS.
016 */
017package org.opends.server.tools;
018
019import static org.opends.messages.ToolMessages.*;
020import static org.opends.server.config.ConfigConstants.*;
021import static org.opends.server.util.StaticUtils.*;
022import static com.forgerock.opendj.cli.CommonArguments.*;
023
024import static com.forgerock.opendj.cli.ArgumentConstants.*;
025import static com.forgerock.opendj.cli.Utils.*;
026
027import java.io.File;
028import java.io.OutputStream;
029import java.io.PrintStream;
030import java.util.ArrayList;
031import java.util.HashSet;
032import java.util.List;
033import java.util.Random;
034import java.util.Set;
035
036import org.forgerock.i18n.LocalizableMessage;
037import org.forgerock.i18n.slf4j.LocalizedLogger;
038import org.forgerock.opendj.config.server.ConfigException;
039import org.forgerock.opendj.ldap.ResultCode;
040import org.opends.server.admin.std.server.BackendCfg;
041import org.opends.server.api.Backend;
042import org.opends.server.api.Backend.BackendOperation;
043import org.opends.server.api.plugin.PluginType;
044import org.opends.server.core.CoreConfigManager;
045import org.opends.server.core.DirectoryServer;
046import org.opends.server.core.LockFileManager;
047import org.opends.server.core.PluginConfigManager;
048import org.opends.server.extensions.ConfigFileHandler;
049import org.opends.server.loggers.ErrorLogPublisher;
050import org.opends.server.loggers.ErrorLogger;
051import org.opends.server.loggers.JDKLogging;
052import org.opends.server.loggers.TextErrorLogPublisher;
053import org.opends.server.loggers.TextWriter;
054import org.opends.server.protocols.ldap.LDAPAttribute;
055import org.opends.server.tasks.ImportTask;
056import org.opends.server.tools.makeldif.TemplateFile;
057import org.opends.server.tools.tasks.TaskTool;
058import org.forgerock.opendj.ldap.schema.AttributeType;
059import org.forgerock.opendj.ldap.DN;
060import org.opends.server.types.DirectoryException;
061import org.opends.server.types.ExistingFileBehavior;
062import org.opends.server.types.InitializationException;
063import org.opends.server.types.LDIFImportConfig;
064import org.opends.server.types.LDIFImportResult;
065import org.opends.server.types.NullOutputStream;
066import org.opends.server.types.RawAttribute;
067import org.opends.server.types.SearchFilter;
068import org.opends.server.util.cli.LDAPConnectionArgumentParser;
069
070import com.forgerock.opendj.cli.Argument;
071import com.forgerock.opendj.cli.ArgumentException;
072import com.forgerock.opendj.cli.BooleanArgument;
073import com.forgerock.opendj.cli.ClientException;
074import com.forgerock.opendj.cli.IntegerArgument;
075import com.forgerock.opendj.cli.StringArgument;
076
077/**
078 * This program provides a utility that may be used to import the contents of an
079 * LDIF file into a Directory Server backend.  This will be a process that is
080 * intended to run separate from Directory Server and not internally within the
081 * server process (e.g., via the tasks interface).
082 */
083public class ImportLDIF extends TaskTool {
084
085  private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
086
087  /**
088   * The buffer size that should be used when reading data from LDIF.
089   */
090  public static final int LDIF_BUFFER_SIZE = 1048576;
091
092
093  /**
094   * The main method for ImportLDIF tool.
095   *
096   * @param  args  The command-line arguments provided to this program.
097   */
098  public static void main(String[] args)
099  {
100    int retCode = mainImportLDIF(args, true, System.out, System.err);
101
102    if(retCode != 0)
103    {
104      System.exit(filterExitCode(retCode));
105    }
106  }
107
108  /**
109   * Processes the command-line arguments and invokes the import process.
110   *
111   * @param  args  The command-line arguments provided to this program.
112   * @return The error code.
113   */
114  public static int mainImportLDIF(String[] args)
115  {
116    return mainImportLDIF(args, true, System.out, System.err);
117  }
118
119  /**
120   * Processes the command-line arguments and invokes the import process.
121   *
122   * @param  args              The command-line arguments provided to this
123   *                           program.
124   * @param  initializeServer  Indicates whether to initialize the server.
125   * @param  outStream         The output stream to use for standard output, or
126   *                           {@code null} if standard output is not needed.
127   * @param  errStream         The output stream to use for standard error, or
128   *                           {@code null} if standard error is not needed.
129   *
130   * @return The error code.
131   */
132  public static int mainImportLDIF(String[] args, boolean initializeServer,
133                                   OutputStream outStream,
134                                   OutputStream errStream)
135  {
136    ImportLDIF tool = new ImportLDIF();
137    return tool.process(args, initializeServer, outStream, errStream);
138  }
139
140  /** Define the command-line arguments that may be used with this program. */
141  private BooleanArgument countRejects;
142  private BooleanArgument isCompressed;
143  private BooleanArgument isEncrypted;
144  private BooleanArgument overwrite;
145  private BooleanArgument quietMode;
146  private BooleanArgument skipSchemaValidation;
147  private BooleanArgument clearBackend;
148  private IntegerArgument randomSeed;
149  private StringArgument  backendID;
150  private StringArgument  configClass;
151  private StringArgument  configFile;
152  private StringArgument  excludeAttributeStrings;
153  private StringArgument  excludeBranchStrings;
154  private StringArgument  excludeFilterStrings;
155  private StringArgument  includeAttributeStrings;
156  private StringArgument  includeBranchStrings;
157  private StringArgument  includeFilterStrings;
158  private StringArgument  ldifFiles;
159  private StringArgument  rejectFile;
160  private StringArgument  skipFile;
161  private StringArgument  templateFile;
162  private BooleanArgument skipDNValidation;
163  private IntegerArgument threadCount;
164  private IntegerArgument offHeapSize;
165  private StringArgument  tmpDirectory;
166
167  private int process(String[] args, boolean initializeServer,
168                      OutputStream outStream, OutputStream errStream) {
169
170    PrintStream out = NullOutputStream.wrapOrNullStream(outStream);
171    PrintStream err = NullOutputStream.wrapOrNullStream(errStream);
172    JDKLogging.disableLogging();
173
174    // FIXME -- Need to add a mechanism for verifying the file signature.
175
176
177    // Create the command-line argument parser for use with this program.
178    LDAPConnectionArgumentParser argParser =
179            createArgParser("org.opends.server.tools.ImportLDIF", INFO_LDIFIMPORT_TOOL_DESCRIPTION.get());
180    argParser.setShortToolDescription(REF_SHORT_DESC_IMPORT_LDIF.get());
181
182    // Initialize all the command-line argument types and register them with the
183    // parser.
184    try
185    {
186      createArguments(argParser);
187    }
188    catch (ArgumentException ae)
189    {
190      printWrappedText(err, ERR_CANNOT_INITIALIZE_ARGS.get(ae.getMessage()));
191      return 1;
192    }
193
194    // Init the default values so that they can appear also on the usage.
195    argParser.getArguments().initArgumentsWithConfiguration(argParser);
196
197    // Parse the command-line arguments provided to this program.
198    try
199    {
200      argParser.parseArguments(args);
201      validateTaskArgs();
202    }
203    catch (ArgumentException ae)
204    {
205      argParser.displayMessageAndUsageReference(err, ERR_ERROR_PARSING_ARGS.get(ae.getMessage()));
206      return 1;
207    }
208    catch (ClientException ce)
209    {
210      // No need to display the usage since the problem comes with a provided value.
211      printWrappedText(err, ce.getMessageObject());
212      return 1;
213    }
214
215
216    if (argParser.usageOrVersionDisplayed())
217    {
218      return 0;
219    }
220
221
222    // Make sure that either the "ldifFile" argument or the "templateFile"
223    // argument was provided, but not both.
224    if (ldifFiles.isPresent())
225    {
226      if (templateFile.isPresent())
227      {
228        printWrappedText(err, conflictingArgsErrorMessage(ldifFiles, templateFile));
229        return 1;
230      }
231    }
232    else if (! templateFile.isPresent())
233    {
234      argParser.displayMessageAndUsageReference(err, ERR_LDIFIMPORT_MISSING_REQUIRED_ARGUMENT.get(
235          ldifFiles.getLongIdentifier(), templateFile.getLongIdentifier()));
236      return 1;
237    }
238
239    // Make sure that either the "includeBranchStrings" argument or the
240    // "backendID" argument was provided.
241    if(!includeBranchStrings.isPresent() && !backendID.isPresent())
242    {
243      argParser.displayMessageAndUsageReference(err, ERR_LDIFIMPORT_MISSING_BACKEND_ARGUMENT.get(
244          includeBranchStrings.getLongIdentifier(), backendID.getLongIdentifier()));
245      return 1;
246    }
247
248    // Count rejects option requires the ability to return result codes
249    // which are potentially greater than 1. This is not supported by
250    // the task framework.
251    if (countRejects.isPresent() && argParser.connectionArgumentsPresent())
252    {
253      argParser.displayMessageAndUsageReference(err, ERR_LDIFIMPORT_COUNT_REJECTS_REQUIRES_OFFLINE.get(
254          countRejects.getLongIdentifier()));
255      return 1;
256    }
257
258    // Don't write non-error messages to console if quite
259    if (quietMode.isPresent()) {
260      out = new PrintStream(NullOutputStream.instance());
261    }
262
263    // Checks the version - if upgrade required, the tool is unusable
264    try
265    {
266      checkVersion();
267    }
268    catch (InitializationException e)
269    {
270      printWrappedText(err, e.getMessage());
271      return 1;
272    }
273
274    return process(argParser, initializeServer, out, err);
275  }
276
277  private void createArguments(LDAPConnectionArgumentParser argParser) throws ArgumentException
278  {
279      configClass =
280              StringArgument.builder(OPTION_LONG_CONFIG_CLASS)
281                      .shortIdentifier(OPTION_SHORT_CONFIG_CLASS)
282                      .description(INFO_DESCRIPTION_CONFIG_CLASS.get())
283                      .hidden()
284                      .required()
285                      .defaultValue(ConfigFileHandler.class.getName())
286                      .valuePlaceholder(INFO_CONFIGCLASS_PLACEHOLDER.get())
287                      .buildAndAddToParser(argParser);
288      configFile =
289              StringArgument.builder("configFile")
290                      .shortIdentifier('f')
291                      .description(INFO_DESCRIPTION_CONFIG_FILE.get())
292                      .hidden()
293                      .required()
294                      .valuePlaceholder(INFO_CONFIGFILE_PLACEHOLDER.get())
295                      .buildAndAddToParser(argParser);
296      ldifFiles =
297              StringArgument.builder(OPTION_LONG_LDIF_FILE)
298                      .shortIdentifier(OPTION_SHORT_LDIF_FILE)
299                      .description(INFO_LDIFIMPORT_DESCRIPTION_LDIF_FILE.get())
300                      .multiValued()
301                      .valuePlaceholder(INFO_LDIFFILE_PLACEHOLDER.get())
302                      .buildAndAddToParser(argParser);
303      templateFile =
304              StringArgument.builder("templateFile")
305                      .shortIdentifier('A')
306                      .description(INFO_LDIFIMPORT_DESCRIPTION_TEMPLATE_FILE.get())
307                      .valuePlaceholder(INFO_TEMPLATE_FILE_PLACEHOLDER.get())
308                      .buildAndAddToParser(argParser);
309      backendID =
310              StringArgument.builder("backendID")
311                      .shortIdentifier('n')
312                      .description(INFO_LDIFIMPORT_DESCRIPTION_BACKEND_ID.get())
313                      .valuePlaceholder(INFO_BACKENDNAME_PLACEHOLDER.get())
314                      .buildAndAddToParser(argParser);
315      clearBackend =
316              BooleanArgument.builder("clearBackend")
317                      .shortIdentifier('F')
318                      .description(INFO_LDIFIMPORT_DESCRIPTION_CLEAR_BACKEND.get())
319                      .buildAndAddToParser(argParser);
320      includeBranchStrings =
321              StringArgument.builder("includeBranch")
322                      .shortIdentifier('b')
323                      .description(INFO_LDIFIMPORT_DESCRIPTION_INCLUDE_BRANCH.get())
324                      .multiValued()
325                      .valuePlaceholder(INFO_BRANCH_DN_PLACEHOLDER.get())
326                      .buildAndAddToParser(argParser);
327      excludeBranchStrings =
328              StringArgument.builder("excludeBranch")
329                      .shortIdentifier('B')
330                      .description(INFO_LDIFIMPORT_DESCRIPTION_EXCLUDE_BRANCH.get())
331                      .multiValued()
332                      .valuePlaceholder(INFO_BRANCH_DN_PLACEHOLDER.get())
333                      .buildAndAddToParser(argParser);
334      includeAttributeStrings =
335              StringArgument.builder("includeAttribute")
336                      .shortIdentifier('i')
337                      .description(INFO_LDIFIMPORT_DESCRIPTION_INCLUDE_ATTRIBUTE.get())
338                      .multiValued()
339                      .valuePlaceholder(INFO_ATTRIBUTE_PLACEHOLDER.get())
340                      .buildAndAddToParser(argParser);
341      excludeAttributeStrings =
342              StringArgument.builder("excludeAttribute")
343                      .shortIdentifier('e')
344                      .description(INFO_LDIFIMPORT_DESCRIPTION_EXCLUDE_ATTRIBUTE.get())
345                      .multiValued()
346                      .valuePlaceholder(INFO_ATTRIBUTE_PLACEHOLDER.get())
347                      .buildAndAddToParser(argParser);
348      includeFilterStrings =
349              StringArgument.builder("includeFilter")
350                      .shortIdentifier('I')
351                      .description(INFO_LDIFIMPORT_DESCRIPTION_INCLUDE_FILTER.get())
352                      .multiValued()
353                      .valuePlaceholder(INFO_FILTER_PLACEHOLDER.get())
354                      .buildAndAddToParser(argParser);
355      excludeFilterStrings =
356              StringArgument.builder("excludeFilter")
357                      .shortIdentifier('E')
358                      .description(INFO_LDIFIMPORT_DESCRIPTION_EXCLUDE_FILTER.get())
359                      .multiValued()
360                      .valuePlaceholder(INFO_FILTER_PLACEHOLDER.get())
361                      .buildAndAddToParser(argParser);
362      rejectFile =
363              StringArgument.builder("rejectFile")
364                      .shortIdentifier('R')
365                      .description(INFO_LDIFIMPORT_DESCRIPTION_REJECT_FILE.get())
366                      .valuePlaceholder(INFO_REJECT_FILE_PLACEHOLDER.get())
367                      .buildAndAddToParser(argParser);
368      skipFile =
369              StringArgument.builder("skipFile")
370                      .description(INFO_LDIFIMPORT_DESCRIPTION_SKIP_FILE.get())
371                      .valuePlaceholder(INFO_SKIP_FILE_PLACEHOLDER.get())
372                      .buildAndAddToParser(argParser);
373      overwrite =
374              BooleanArgument.builder("overwrite")
375                      .shortIdentifier('O')
376                      .description(INFO_LDIFIMPORT_DESCRIPTION_OVERWRITE.get())
377                      .buildAndAddToParser(argParser);
378      randomSeed =
379              IntegerArgument.builder(OPTION_LONG_RANDOM_SEED)
380                      .shortIdentifier(OPTION_SHORT_RANDOM_SEED)
381                      .description(INFO_LDIFIMPORT_DESCRIPTION_RANDOM_SEED.get())
382                      .defaultValue(0)
383                      .valuePlaceholder(INFO_SEED_PLACEHOLDER.get())
384                      .buildAndAddToParser(argParser);
385      skipSchemaValidation =
386              BooleanArgument.builder("skipSchemaValidation")
387                      .shortIdentifier('S')
388                      .description(INFO_LDIFIMPORT_DESCRIPTION_SKIP_SCHEMA_VALIDATION.get())
389                      .buildAndAddToParser(argParser);
390      skipDNValidation =
391              BooleanArgument.builder("skipDNValidation")
392                      .description(INFO_LDIFIMPORT_DESCRIPTION_DN_VALIDATION.get())
393                      .buildAndAddToParser(argParser);
394      threadCount =
395              IntegerArgument.builder("threadCount")
396                      .description(INFO_LDIFIMPORT_DESCRIPTION_THREAD_COUNT.get())
397                      .lowerBound(1)
398                      .defaultValue(0)
399                      .valuePlaceholder(INFO_LDIFIMPORT_THREAD_COUNT_PLACEHOLDER.get())
400                      .buildAndAddToParser(argParser);
401      offHeapSize =
402              IntegerArgument.builder("offHeapSize")
403                      .description(INFO_LDIFIMPORT_DESCRIPTION_OFFHEAP_SIZE.get())
404                      .lowerBound(0)
405                      .defaultValue(700)
406                      .valuePlaceholder(INFO_LDIFIMPORT_OFFHEAP_SIZE_PLACEHOLDER.get())
407                      .buildAndAddToParser(argParser);
408      tmpDirectory =
409              StringArgument.builder("tmpdirectory")
410                      .description(INFO_LDIFIMPORT_DESCRIPTION_TEMP_DIRECTORY.get())
411                      .defaultValue("import-tmp")
412                      .valuePlaceholder(INFO_LDIFIMPORT_TEMP_DIR_PLACEHOLDER.get())
413                      .buildAndAddToParser(argParser);
414      countRejects =
415              BooleanArgument.builder("countRejects")
416                      .description(INFO_LDIFIMPORT_DESCRIPTION_COUNT_REJECTS.get())
417                      .buildAndAddToParser(argParser);
418      isCompressed =
419              BooleanArgument.builder("isCompressed")
420                      .shortIdentifier('c')
421                      .description(INFO_LDIFIMPORT_DESCRIPTION_IS_COMPRESSED.get())
422                      .buildAndAddToParser(argParser);
423      isEncrypted =
424              BooleanArgument.builder("isEncrypted")
425                      .shortIdentifier('y')
426                      .description(INFO_LDIFIMPORT_DESCRIPTION_IS_ENCRYPTED.get())
427                      .hidden() //See issue #27
428                      .buildAndAddToParser(argParser);
429      quietMode =
430              BooleanArgument.builder(OPTION_LONG_QUIET)
431                      .shortIdentifier(OPTION_SHORT_QUIET)
432                      .description(INFO_LDIFIMPORT_DESCRIPTION_QUIET.get())
433                      .buildAndAddToParser(argParser);
434
435    final BooleanArgument displayUsage = showUsageArgument();
436      argParser.addArgument(displayUsage);
437      argParser.setUsageArgument(displayUsage);
438  }
439
440  @Override
441  public void addTaskAttributes(List<RawAttribute> attributes)
442  {
443    // Required attributes
444    addAttribute(attributes, ATTR_IMPORT_LDIF_FILE, ldifFiles.getValues());
445    addAttribute(attributes, ATTR_IMPORT_TEMPLATE_FILE, templateFile.getValue());
446    addAttribute(attributes, ATTR_IMPORT_RANDOM_SEED, randomSeed.getValue());
447    addAttribute(attributes, ATTR_IMPORT_THREAD_COUNT, threadCount.getValue());
448    addAttribute(attributes, ATTR_IMPORT_OFFHEAP_SIZE, offHeapSize.getValue());
449
450    // Optional attributes
451    addAttribute2(attributes, ATTR_IMPORT_BACKEND_ID, backendID);
452    addAttribute(attributes, ATTR_IMPORT_INCLUDE_ATTRIBUTE, includeAttributeStrings.getValues());
453    addAttribute(attributes, ATTR_IMPORT_EXCLUDE_ATTRIBUTE, excludeAttributeStrings.getValues());
454    addAttribute(attributes, ATTR_IMPORT_INCLUDE_FILTER, includeFilterStrings.getValues());
455    addAttribute(attributes, ATTR_IMPORT_EXCLUDE_FILTER, excludeFilterStrings.getValues());
456    addAttribute(attributes, ATTR_IMPORT_INCLUDE_BRANCH, includeBranchStrings.getValues());
457    addAttribute(attributes, ATTR_IMPORT_EXCLUDE_BRANCH, excludeBranchStrings.getValues());
458    addAttribute2(attributes, ATTR_IMPORT_REJECT_FILE, rejectFile);
459    addAttribute2(attributes, ATTR_IMPORT_SKIP_FILE, skipFile);
460    addAttribute2(attributes, ATTR_IMPORT_OVERWRITE, overwrite);
461    addAttribute2(attributes, ATTR_IMPORT_SKIP_SCHEMA_VALIDATION, skipSchemaValidation);
462    addAttribute2(attributes, ATTR_IMPORT_TMP_DIRECTORY, tmpDirectory);
463    addAttribute2(attributes, ATTR_IMPORT_SKIP_DN_VALIDATION, skipDNValidation);
464    addAttribute2(attributes, ATTR_IMPORT_IS_COMPRESSED, isCompressed);
465    addAttribute2(attributes, ATTR_IMPORT_IS_ENCRYPTED, isEncrypted);
466    addAttribute2(attributes, ATTR_IMPORT_CLEAR_BACKEND, clearBackend);
467  }
468
469  private void addAttribute(List<RawAttribute> attributes, String attrName, String value)
470  {
471    if (value != null)
472    {
473      attributes.add(new LDAPAttribute(attrName, value));
474    }
475  }
476
477  private void addAttribute2(List<RawAttribute> attributes, String attrName, Argument arg)
478  {
479    final String value = arg.getValue();
480    if (value != null && !value.equals(arg.getDefaultValue()))
481    {
482      attributes.add(new LDAPAttribute(attrName, value));
483    }
484  }
485
486  private void addAttribute(List<RawAttribute> attributes, String attrName, List<String> attrValues)
487  {
488    if (attrValues != null && !attrValues.isEmpty())
489    {
490      attributes.add(new LDAPAttribute(attrName, attrValues));
491    }
492  }
493
494  @Override
495  public String getTaskObjectclass() {
496    return "ds-task-import";
497  }
498
499  @Override
500  public Class<?> getTaskClass() {
501    return ImportTask.class;
502  }
503
504  @Override
505  protected int processLocal(boolean initializeServer,
506                           PrintStream out,
507                           PrintStream err) {
508
509
510    // Perform the initial bootstrap of the Directory Server and process the configuration.
511    DirectoryServer directoryServer = DirectoryServer.getInstance();
512    if (initializeServer)
513    {
514      try
515      {
516        DirectoryServer.bootstrapClient();
517        DirectoryServer.initializeJMX();
518      }
519      catch (Exception e)
520      {
521        printWrappedText(err, ERR_SERVER_BOOTSTRAP_ERROR.get(getExceptionMessage(e)));
522        return 1;
523      }
524
525      try
526      {
527        directoryServer.initializeConfiguration(configClass.getValue(),
528                                                configFile.getValue());
529      }
530      catch (InitializationException ie)
531      {
532        printWrappedText(err, ERR_CANNOT_LOAD_CONFIG.get(ie.getMessage()));
533        return 1;
534      }
535      catch (Exception e)
536      {
537        printWrappedText(err, ERR_CANNOT_LOAD_CONFIG.get(getExceptionMessage(e)));
538        return 1;
539      }
540
541
542
543      // Initialize the Directory Server schema elements.
544      try
545      {
546        directoryServer.initializeSchema();
547      }
548      catch (Exception e)
549      {
550        printWrappedText(err, ERR_CANNOT_LOAD_SCHEMA.get(getMessage(e)));
551        return 1;
552      }
553
554
555      // Initialize the Directory Server core configuration.
556      try
557      {
558        CoreConfigManager coreConfigManager = new CoreConfigManager(directoryServer.getServerContext());
559        coreConfigManager.initializeCoreConfig();
560      }
561      catch (Exception e)
562      {
563        printWrappedText(err, ERR_CANNOT_INITIALIZE_CORE_CONFIG.get(getMessage(e)));
564        return 1;
565      }
566
567
568      // Initialize the Directory Server crypto manager.
569      try
570      {
571        directoryServer.initializeCryptoManager();
572      }
573      catch (Exception e)
574      {
575        printWrappedText(err, ERR_CANNOT_INITIALIZE_CRYPTO_MANAGER.get(getMessage(e)));
576        return 1;
577      }
578
579
580      if (! quietMode.isPresent())
581      {
582        try
583        {
584          ErrorLogPublisher errorLogPublisher =
585              TextErrorLogPublisher.getToolStartupTextErrorPublisher(
586                  new TextWriter.STREAM(out));
587          ErrorLogger.getInstance().addLogPublisher(errorLogPublisher);
588        }
589        catch(Exception e)
590        {
591          err.println("Error installing the custom error logger: " +
592              stackTraceToSingleLineString(e));
593        }
594      }
595
596      // Initialize the root DNs.
597      try
598      {
599        directoryServer.initializeRootDNConfigManager();
600      }
601      catch (Exception e)
602      {
603        printWrappedText(err, ERR_CANNOT_INITIALIZE_ROOTDN_MANAGER.get(getMessage(e)));
604        return 1;
605      }
606
607      // Initialize the plugin manager.
608      try
609      {
610        HashSet<PluginType> pluginTypes = new HashSet<>(1);
611        directoryServer.initializePlugins(pluginTypes);
612      }
613      catch (Exception e)
614      {
615        printWrappedText(err, ERR_LDIFIMPORT_CANNOT_INITIALIZE_PLUGINS.get(getMessage(e)));
616        return 1;
617      }
618
619      // Initialize the subentry manager.
620      try
621      {
622        directoryServer.initializeSubentryManager();
623      }
624      catch (InitializationException ie)
625      {
626        printWrappedText(err, ERR_CANNOT_INITIALIZE_SUBENTRY_MANAGER.get(ie.getMessage()));
627        return 1;
628      }
629
630      // Initialize all the password policy information.
631      try
632      {
633        directoryServer.initializeAuthenticationPolicyComponents();
634      }
635      catch (Exception e)
636      {
637        printWrappedText(err, ERR_LDIFIMPORT_CANNOT_INITIALIZE_PWPOLICY.get(getMessage(e)));
638        return 1;
639      }
640    }
641
642    // Make sure that the plugin initialization is performed.
643    try
644    {
645      HashSet<PluginType> pluginTypes = new HashSet<>(1);
646      pluginTypes.add(PluginType.LDIF_IMPORT);
647      PluginConfigManager pluginConfigManager =
648              DirectoryServer.getPluginConfigManager();
649      pluginConfigManager.initializeUserPlugins(pluginTypes);
650    }
651    catch (Exception e)
652    {
653      printWrappedText(err, ERR_LDIFIMPORT_CANNOT_INITIALIZE_PLUGINS.get(getMessage(e)));
654      return 1;
655    }
656
657    // See if there were any user-defined sets of include/exclude attributes or
658    // filters.  If so, then process them.
659    HashSet<AttributeType> excludeAttributes;
660    boolean excludeAllUserAttributes = false;
661    boolean excludeAllOperationalAttributes = false;
662    if (excludeAttributeStrings == null)
663    {
664      excludeAttributes = null;
665    }
666    else
667    {
668      excludeAttributes = new HashSet<>();
669      for (String attrName : excludeAttributeStrings.getValues())
670      {
671        String lowerName = attrName.toLowerCase();
672        if (lowerName.equals("*"))
673        {
674          excludeAllUserAttributes = true;
675        }
676        else if (lowerName.equals("+"))
677        {
678          excludeAllOperationalAttributes = true;
679        }
680        else
681        {
682          excludeAttributes.add(DirectoryServer.getAttributeType(attrName));
683        }
684      }
685    }
686
687    HashSet<AttributeType> includeAttributes;
688    boolean includeAllUserAttributes = false;
689    boolean includeAllOperationalAttributes = false;
690    if (includeAttributeStrings == null)
691    {
692      includeAttributes = null;
693    }
694    else
695    {
696      includeAttributes = new HashSet<>();
697      for (String attrName : includeAttributeStrings.getValues())
698      {
699        String lowerName = attrName.toLowerCase();
700        if (lowerName.equals("*"))
701        {
702          includeAllUserAttributes = true;
703        }
704        else if (lowerName.equals("+"))
705        {
706          includeAllOperationalAttributes = true;
707        }
708        else
709        {
710          includeAttributes.add(DirectoryServer.getAttributeType(attrName));
711        }
712      }
713    }
714
715    ArrayList<SearchFilter> excludeFilters;
716    if (excludeFilterStrings == null)
717    {
718      excludeFilters = null;
719    }
720    else
721    {
722      excludeFilters = new ArrayList<>();
723      for (String filterString : excludeFilterStrings.getValues())
724      {
725        try
726        {
727          excludeFilters.add(SearchFilter.createFilterFromString(filterString));
728        }
729        catch (DirectoryException de)
730        {
731          logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_EXCLUDE_FILTER, filterString, de.getMessageObject());
732          return 1;
733        }
734        catch (Exception e)
735        {
736          logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_EXCLUDE_FILTER, filterString, getExceptionMessage(e));
737          return 1;
738        }
739      }
740    }
741
742    ArrayList<SearchFilter> includeFilters;
743    if (includeFilterStrings == null)
744    {
745      includeFilters = null;
746    }
747    else
748    {
749      includeFilters = new ArrayList<>();
750      for (String filterString : includeFilterStrings.getValues())
751      {
752        try
753        {
754          includeFilters.add(SearchFilter.createFilterFromString(filterString));
755        }
756        catch (DirectoryException de)
757        {
758          logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_INCLUDE_FILTER, filterString, de.getMessageObject());
759          return 1;
760        }
761        catch (Exception e)
762        {
763          logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_INCLUDE_FILTER, filterString, getExceptionMessage(e));
764          return 1;
765        }
766      }
767    }
768
769
770    // Get information about the backends defined in the server.  Iterate
771    // through them, finding the one backend into which the LDIF should be
772    // imported and finding backends with subordinate base DNs that should be
773    // excluded from the import.
774    Backend<?> backend = null;
775    Set<DN> defaultIncludeBranches = null;
776    Set<DN> excludeBranches = new HashSet<>();
777    Set<DN> includeBranches = new HashSet<>();
778
779    if (includeBranchStrings.isPresent())
780    {
781      for (String s : includeBranchStrings.getValues())
782      {
783        DN includeBranch;
784        try
785        {
786          includeBranch = DN.valueOf(s);
787        }
788        catch (Exception e)
789        {
790          logger.error(ERR_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE, s, getExceptionMessage(e));
791          return 1;
792        }
793
794        includeBranches.add(includeBranch);
795      }
796    }
797
798    ArrayList<Backend>     backendList = new ArrayList<>();
799    ArrayList<BackendCfg>  entryList   = new ArrayList<>();
800    ArrayList<List<DN>> dnList = new ArrayList<>();
801    int code = BackendToolUtils.getBackends(backendList, entryList, dnList);
802    if (code != 0)
803    {
804      return code;
805    }
806
807    int numBackends = backendList.size();
808    for (int i=0; i < numBackends; i++)
809    {
810      Backend<?> b = backendList.get(i);
811
812      if(backendID.isPresent())
813      {
814        if (! backendID.getValue().equals(b.getBackendID()))
815        {
816          continue;
817        }
818      }
819      else
820      {
821        if (!useBackend(includeBranches, dnList.get(i)))
822        {
823          continue;
824        }
825      }
826
827      if (backend == null)
828      {
829        backend                = b;
830        defaultIncludeBranches = new HashSet<>(dnList.get(i));
831      }
832      else
833      {
834        logger.error(ERR_LDIFIMPORT_MULTIPLE_BACKENDS_FOR_ID);
835        return 1;
836      }
837    }
838
839    if (backend == null)
840    {
841      logger.error(ERR_LDIFIMPORT_NO_BACKENDS_FOR_ID);
842      return 1;
843    }
844    else if (!backend.supports(BackendOperation.LDIF_IMPORT))
845    {
846      logger.error(ERR_LDIFIMPORT_CANNOT_IMPORT, backendID.getValue());
847      return 1;
848    }
849
850    for (List<DN> baseList : dnList)
851    {
852      for (DN baseDN : baseList)
853      {
854        for (DN importBase : defaultIncludeBranches)
855        {
856          if (!baseDN.equals(importBase) && baseDN.isSubordinateOrEqualTo(importBase))
857          {
858            excludeBranches.add(baseDN);
859            break;
860          }
861        }
862      }
863    }
864
865    for (String s : excludeBranchStrings.getValues())
866    {
867      DN excludeBranch;
868      try
869      {
870        excludeBranch = DN.valueOf(s);
871      }
872      catch (Exception e)
873      {
874        logger.error(ERR_LDIFIMPORT_CANNOT_DECODE_EXCLUDE_BASE, s, getExceptionMessage(e));
875        return 1;
876      }
877
878      excludeBranches.add(excludeBranch);
879    }
880
881    if (! includeBranchStrings.isPresent())
882    {
883      includeBranches = defaultIncludeBranches;
884    }
885    else
886    {
887      // Make sure the selected backend will handle all the include branches
888      for(DN includeBranch : includeBranches)
889      {
890        if (! Backend.handlesEntry(includeBranch, defaultIncludeBranches,
891                                   excludeBranches))
892        {
893          logger.error(ERR_LDIFIMPORT_INVALID_INCLUDE_BASE, includeBranch, backendID.getValue());
894          return 1;
895        }
896      }
897    }
898
899
900    // See if the data should be read from LDIF files or generated via MakeLDIF.
901    LDIFImportConfig importConfig;
902    if (ldifFiles.isPresent())
903    {
904      ArrayList<String> fileList = new ArrayList<>(ldifFiles.getValues());
905      int badFileCount = 0;
906      for (String pathname : fileList)
907      {
908        File f = new File(pathname);
909        if (!f.canRead())
910        {
911          logger.error(ERR_LDIFIMPORT_CANNOT_READ_FILE, pathname);
912          badFileCount++;
913        }
914      }
915      if (badFileCount > 0)
916      {
917        return 1;
918      }
919      importConfig = new LDIFImportConfig(fileList);
920    }
921    else
922    {
923      Random random = newRandom();
924
925      String resourcePath = DirectoryServer.getInstanceRoot() + File.separator +
926                            PATH_MAKELDIF_RESOURCE_DIR;
927      TemplateFile tf = new TemplateFile(resourcePath, random);
928
929      ArrayList<LocalizableMessage> warnings = new ArrayList<>();
930      try
931      {
932        tf.parse(templateFile.getValue(), warnings);
933      }
934      catch (Exception e)
935      {
936        logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_TEMPLATE_FILE, templateFile.getValue(), e.getMessage());
937        return 1;
938      }
939
940      importConfig = new LDIFImportConfig(tf);
941    }
942
943
944    // Create the LDIF import configuration to use when reading the LDIF.
945    importConfig.setCompressed(isCompressed.isPresent());
946    importConfig.setClearBackend(clearBackend.isPresent());
947    importConfig.setEncrypted(isEncrypted.isPresent());
948    importConfig.setExcludeAttributes(excludeAttributes);
949    importConfig.setExcludeBranches(excludeBranches);
950    importConfig.setExcludeFilters(excludeFilters);
951    importConfig.setIncludeAttributes(includeAttributes);
952    importConfig.setIncludeBranches(includeBranches);
953    importConfig.setIncludeFilters(includeFilters);
954    importConfig.setValidateSchema(!skipSchemaValidation.isPresent());
955    importConfig.setSkipDNValidation(skipDNValidation.isPresent());
956    importConfig.setTmpDirectory(tmpDirectory.getValue());
957
958    try
959    {
960        importConfig.setThreadCount(threadCount.getIntValue());
961    }
962    catch(Exception e)
963    {
964        logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_THREAD_COUNT,
965                threadCount.getValue(), e.getMessage());
966        return 1;
967    }
968
969    try
970    {
971      importConfig.setOffHeapSize(offHeapSize.getIntValue());
972    }
973    catch (Exception e)
974    {
975      logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_OFFHEAP_SIZE, offHeapSize.getValue(), e.getMessage());
976      return 1;
977    }
978
979    importConfig.setBufferSize(LDIF_BUFFER_SIZE);
980    importConfig.setExcludeAllUserAttributes(excludeAllUserAttributes);
981    importConfig.setExcludeAllOperationalAttributes(excludeAllOperationalAttributes);
982    importConfig.setIncludeAllOpAttributes(includeAllOperationalAttributes);
983    importConfig.setIncludeAllUserAttributes(includeAllUserAttributes);
984
985    // FIXME -- Should this be conditional?
986    importConfig.setInvokeImportPlugins(true);
987
988    if (rejectFile != null)
989    {
990      try
991      {
992        ExistingFileBehavior existingBehavior = overwrite.isPresent()
993            ? ExistingFileBehavior.OVERWRITE
994            : ExistingFileBehavior.APPEND;
995
996        importConfig.writeRejectedEntries(rejectFile.getValue(),
997                                          existingBehavior);
998      }
999      catch (Exception e)
1000      {
1001        logger.error(ERR_LDIFIMPORT_CANNOT_OPEN_REJECTS_FILE, rejectFile.getValue(), getExceptionMessage(e));
1002        return 1;
1003      }
1004    }
1005
1006    if (skipFile != null)
1007    {
1008      try
1009      {
1010        ExistingFileBehavior existingBehavior = overwrite.isPresent()
1011            ? ExistingFileBehavior.OVERWRITE
1012            : ExistingFileBehavior.APPEND;
1013
1014        importConfig.writeSkippedEntries(skipFile.getValue(),
1015                                          existingBehavior);
1016      }
1017      catch (Exception e)
1018      {
1019        logger.error(ERR_LDIFIMPORT_CANNOT_OPEN_SKIP_FILE, skipFile.getValue(), getExceptionMessage(e));
1020        return 1;
1021      }
1022    }
1023
1024    // Get the set of base DNs for the backend as an array.
1025    DN[] baseDNs = new DN[defaultIncludeBranches.size()];
1026    defaultIncludeBranches.toArray(baseDNs);
1027
1028
1029    // Acquire an exclusive lock for the backend.
1030    try
1031    {
1032      String lockFile = LockFileManager.getBackendLockFileName(backend);
1033      StringBuilder failureReason = new StringBuilder();
1034      if (! LockFileManager.acquireExclusiveLock(lockFile, failureReason))
1035      {
1036        logger.error(ERR_LDIFIMPORT_CANNOT_LOCK_BACKEND, backend.getBackendID(), failureReason);
1037        return 1;
1038      }
1039    }
1040    catch (Exception e)
1041    {
1042      logger.error(ERR_LDIFIMPORT_CANNOT_LOCK_BACKEND, backend.getBackendID(), getExceptionMessage(e));
1043      return 1;
1044    }
1045
1046
1047    // Launch the import.
1048    int retCode = 0;
1049    try
1050    {
1051      LDIFImportResult importResult =
1052          backend.importLDIF(importConfig, DirectoryServer.getInstance().getServerContext());
1053      if (countRejects.isPresent())
1054      {
1055        if (importResult.getEntriesRejected() > Integer.MAX_VALUE)
1056        {
1057          retCode = Integer.MAX_VALUE;
1058        }
1059        else
1060        {
1061          retCode = (int) importResult.getEntriesRejected();
1062        }
1063      }
1064    }
1065    catch (DirectoryException de)
1066    {
1067      LocalizableMessage msg;
1068      if (de.getResultCode() == ResultCode.CONSTRAINT_VIOLATION)
1069      {
1070        msg = ERR_LDIFIMPORT_ERROR_CONSTRAINT_VIOLATION.get();
1071      }
1072      else
1073      {
1074        msg = de.getMessageObject();
1075      }
1076      logger.error(ERR_LDIFIMPORT_ERROR_DURING_IMPORT.get(msg));
1077      retCode = 1;
1078    }
1079    catch (Exception e)
1080    {
1081      logger.error(ERR_LDIFIMPORT_ERROR_DURING_IMPORT, getExceptionMessage(e));
1082      retCode = 1;
1083    }
1084
1085
1086    // Release the exclusive lock on the backend.
1087    try
1088    {
1089      String lockFile = LockFileManager.getBackendLockFileName(backend);
1090      StringBuilder failureReason = new StringBuilder();
1091      if (! LockFileManager.releaseLock(lockFile, failureReason))
1092      {
1093        logger.warn(WARN_LDIFIMPORT_CANNOT_UNLOCK_BACKEND, backend.getBackendID(), failureReason);
1094        retCode = 1;
1095      }
1096    }
1097    catch (Exception e)
1098    {
1099      logger.warn(WARN_LDIFIMPORT_CANNOT_UNLOCK_BACKEND, backend.getBackendID(), getExceptionMessage(e));
1100      retCode = 1;
1101    }
1102
1103
1104    // Clean up after the import by closing the import config.
1105    importConfig.close();
1106    return retCode;
1107  }
1108
1109  private Object getMessage(Exception e)
1110  {
1111    try
1112    {
1113      throw e;
1114    }
1115    catch (ConfigException | InitializationException e2)
1116    {
1117      return e2.getMessage();
1118    }
1119    catch (Exception e2)
1120    {
1121      return getExceptionMessage(e2);
1122    }
1123  }
1124
1125  private boolean useBackend(Set<DN> includeBranches, List<DN> dnlist)
1126  {
1127    for (DN baseDN : dnlist)
1128    {
1129      for (DN includeDN : includeBranches)
1130      {
1131        if (baseDN.isSuperiorOrEqualTo(includeDN))
1132        {
1133          return true;
1134        }
1135      }
1136    }
1137    return false;
1138  }
1139
1140  private Random newRandom()
1141  {
1142    if (randomSeed.isPresent())
1143    {
1144      try
1145      {
1146        return new Random(randomSeed.getIntValue());
1147      }
1148      catch (Exception ignored)
1149      {
1150        // ignore
1151      }
1152    }
1153    return new Random();
1154  }
1155
1156  @Override
1157  public String getTaskId() {
1158    // NYI.
1159    return null;
1160  }
1161}