public final class CheckIndex
extends java.lang.Object
implements java.io.Closeable
As this tool checks every byte in the index, on a large index it can take quite a long time to run.
Modifier and Type | Class and Description |
---|---|
private static class |
CheckIndex.ConstantRelationIntersectVisitor |
private static interface |
CheckIndex.DocValuesIteratorSupplier |
static class |
CheckIndex.Options
Run-time configuration options for CheckIndex commands.
|
static class |
CheckIndex.Status
Returned from
checkIndex() detailing the health and status of the index. |
static class |
CheckIndex.VerifyPointsVisitor
Walks the entire N-dimensional points space, verifying that all points fall within the last cell's boundaries.
|
Modifier and Type | Field and Description |
---|---|
private static boolean |
assertsOn |
private boolean |
checksumsOnly |
private boolean |
closed |
private Directory |
dir |
private boolean |
doSlowChecks |
private boolean |
failFast |
private java.io.PrintStream |
infoStream |
private boolean |
verbose |
private Lock |
writeLock |
Constructor and Description |
---|
CheckIndex(Directory dir)
Create a new CheckIndex on the directory.
|
CheckIndex(Directory dir,
Lock writeLock)
Expert: create a directory with the specified lock.
|
Modifier and Type | Method and Description |
---|---|
static boolean |
assertsOn()
Check whether asserts are enabled or not.
|
private static void |
checkBinaryDocValues(java.lang.String fieldName,
int maxDoc,
BinaryDocValues bdv,
BinaryDocValues bdv2) |
private static void |
checkDocValues(FieldInfo fi,
DocValuesProducer dvReader,
int maxDoc,
java.io.PrintStream infoStream,
CheckIndex.Status.DocValuesStatus status) |
private static void |
checkDVIterator(FieldInfo fi,
int maxDoc,
CheckIndex.DocValuesIteratorSupplier producer) |
private static CheckIndex.Status.TermIndexStatus |
checkFields(Fields fields,
Bits liveDocs,
int maxDoc,
FieldInfos fieldInfos,
NormsProducer normsProducer,
boolean doPrint,
boolean isVectors,
java.io.PrintStream infoStream,
boolean verbose,
boolean doSlowChecks)
checks Fields api is consistent with itself.
|
(package private) static void |
checkImpacts(Impacts impacts,
int lastTarget) |
CheckIndex.Status |
checkIndex()
Returns a
CheckIndex.Status instance detailing
the state of the index. |
CheckIndex.Status |
checkIndex(java.util.List<java.lang.String> onlySegments)
Returns a
CheckIndex.Status instance detailing
the state of the index. |
private static void |
checkNumericDocValues(java.lang.String fieldName,
NumericDocValues ndv,
NumericDocValues ndv2) |
private static boolean |
checkSingleTermRange(java.lang.String field,
int maxDoc,
Terms terms,
BytesRef minTerm,
BytesRef maxTerm,
FixedBitSet normalDocs,
FixedBitSet intersectDocs)
Test Terms.intersect on this range, and validates that it returns the same doc ids as using non-intersect TermsEnum.
|
private static void |
checkSoftDeletes(java.lang.String softDeletesField,
SegmentCommitInfo info,
SegmentReader reader,
java.io.PrintStream infoStream,
boolean failFast) |
private static void |
checkSortedDocValues(java.lang.String fieldName,
int maxDoc,
SortedDocValues dv,
SortedDocValues dv2) |
private static void |
checkSortedNumericDocValues(java.lang.String fieldName,
int maxDoc,
SortedNumericDocValues ndv,
SortedNumericDocValues ndv2) |
private static void |
checkSortedSetDocValues(java.lang.String fieldName,
int maxDoc,
SortedSetDocValues dv,
SortedSetDocValues dv2) |
void |
close() |
int |
doCheck(CheckIndex.Options opts)
Actually perform the index check
|
private static int |
doMain(java.lang.String[] args) |
boolean |
doSlowChecks()
|
private void |
ensureOpen() |
void |
exorciseIndex(CheckIndex.Status result)
Repairs the index using previously returned result
from
checkIndex() . |
boolean |
getChecksumsOnly()
See
getChecksumsOnly() . |
private static long |
getDocsFromTermRange(java.lang.String field,
int maxDoc,
TermsEnum termsEnum,
FixedBitSet docsSeen,
BytesRef minTerm,
BytesRef maxTerm,
boolean isIntersect)
Visits all terms in the range minTerm (inclusive) to maxTerm (exclusive), marking all doc IDs encountered into allDocsSeen, and
returning the total number of terms visited.
|
boolean |
getFailFast()
See
setFailFast(boolean) . |
static void |
main(java.lang.String[] args)
Command-line interface to check and exorcise corrupt segments from an index.
|
private static void |
msg(java.io.PrintStream out,
java.lang.String msg) |
private static double |
nsToSec(long ns) |
static CheckIndex.Options |
parseOptions(java.lang.String[] args)
Parse command line args into fields
|
void |
setChecksumsOnly(boolean v)
If true, only validate physical integrity for all files.
|
void |
setDoSlowChecks(boolean v)
If true, additional slow checks are performed.
|
void |
setFailFast(boolean v)
If true, just throw the original exception immediately when
corruption is detected, rather than continuing to iterate to other
segments looking for more corruption.
|
void |
setInfoStream(java.io.PrintStream out)
Set infoStream where messages should go.
|
void |
setInfoStream(java.io.PrintStream out,
boolean verbose)
Set infoStream where messages should go.
|
private static boolean |
testAsserts() |
static CheckIndex.Status.DocValuesStatus |
testDocValues(CodecReader reader,
java.io.PrintStream infoStream,
boolean failFast)
Test docvalues.
|
static CheckIndex.Status.FieldInfoStatus |
testFieldInfos(CodecReader reader,
java.io.PrintStream infoStream,
boolean failFast)
Test field infos.
|
static CheckIndex.Status.FieldNormStatus |
testFieldNorms(CodecReader reader,
java.io.PrintStream infoStream,
boolean failFast)
Test field norms.
|
static CheckIndex.Status.LiveDocStatus |
testLiveDocs(CodecReader reader,
java.io.PrintStream infoStream,
boolean failFast)
Test live docs.
|
static CheckIndex.Status.PointsStatus |
testPoints(CodecReader reader,
java.io.PrintStream infoStream,
boolean failFast)
Test the points index
|
static CheckIndex.Status.TermIndexStatus |
testPostings(CodecReader reader,
java.io.PrintStream infoStream)
Test the term index.
|
static CheckIndex.Status.TermIndexStatus |
testPostings(CodecReader reader,
java.io.PrintStream infoStream,
boolean verbose,
boolean doSlowChecks,
boolean failFast)
Test the term index.
|
static CheckIndex.Status.IndexSortStatus |
testSort(CodecReader reader,
Sort sort,
java.io.PrintStream infoStream,
boolean failFast)
Tests index sort order.
|
static CheckIndex.Status.StoredFieldStatus |
testStoredFields(CodecReader reader,
java.io.PrintStream infoStream,
boolean failFast)
Test stored fields.
|
static CheckIndex.Status.TermVectorStatus |
testTermVectors(CodecReader reader,
java.io.PrintStream infoStream)
Test term vectors.
|
static CheckIndex.Status.TermVectorStatus |
testTermVectors(CodecReader reader,
java.io.PrintStream infoStream,
boolean verbose,
boolean doSlowChecks,
boolean failFast)
Test term vectors.
|
private java.io.PrintStream infoStream
private Directory dir
private Lock writeLock
private volatile boolean closed
private boolean doSlowChecks
private boolean failFast
private boolean verbose
private boolean checksumsOnly
private static boolean assertsOn
public CheckIndex(Directory dir) throws java.io.IOException
java.io.IOException
public CheckIndex(Directory dir, Lock writeLock)
private void ensureOpen()
public void close() throws java.io.IOException
close
in interface java.io.Closeable
close
in interface java.lang.AutoCloseable
java.io.IOException
public void setDoSlowChecks(boolean v)
public boolean doSlowChecks()
public void setFailFast(boolean v)
public boolean getFailFast()
setFailFast(boolean)
.public boolean getChecksumsOnly()
getChecksumsOnly()
.public void setChecksumsOnly(boolean v)
public void setInfoStream(java.io.PrintStream out, boolean verbose)
public void setInfoStream(java.io.PrintStream out)
setInfoStream(PrintStream,boolean)
.private static void msg(java.io.PrintStream out, java.lang.String msg)
public CheckIndex.Status checkIndex() throws java.io.IOException
CheckIndex.Status
instance detailing
the state of the index.
As this method checks every byte in the index, on a large index it can take quite a long time to run.
WARNING: make sure you only call this when the index is not opened by any writer.
java.io.IOException
public CheckIndex.Status checkIndex(java.util.List<java.lang.String> onlySegments) throws java.io.IOException
CheckIndex.Status
instance detailing
the state of the index.onlySegments
- list of specific segment names to check
As this method checks every byte in the specified segments, on a large index it can take quite a long time to run.
java.io.IOException
public static CheckIndex.Status.IndexSortStatus testSort(CodecReader reader, Sort sort, java.io.PrintStream infoStream, boolean failFast) throws java.io.IOException
java.io.IOException
public static CheckIndex.Status.LiveDocStatus testLiveDocs(CodecReader reader, java.io.PrintStream infoStream, boolean failFast) throws java.io.IOException
java.io.IOException
public static CheckIndex.Status.FieldInfoStatus testFieldInfos(CodecReader reader, java.io.PrintStream infoStream, boolean failFast) throws java.io.IOException
java.io.IOException
public static CheckIndex.Status.FieldNormStatus testFieldNorms(CodecReader reader, java.io.PrintStream infoStream, boolean failFast) throws java.io.IOException
java.io.IOException
private static long getDocsFromTermRange(java.lang.String field, int maxDoc, TermsEnum termsEnum, FixedBitSet docsSeen, BytesRef minTerm, BytesRef maxTerm, boolean isIntersect) throws java.io.IOException
java.io.IOException
private static boolean checkSingleTermRange(java.lang.String field, int maxDoc, Terms terms, BytesRef minTerm, BytesRef maxTerm, FixedBitSet normalDocs, FixedBitSet intersectDocs) throws java.io.IOException
java.io.IOException
private static CheckIndex.Status.TermIndexStatus checkFields(Fields fields, Bits liveDocs, int maxDoc, FieldInfos fieldInfos, NormsProducer normsProducer, boolean doPrint, boolean isVectors, java.io.PrintStream infoStream, boolean verbose, boolean doSlowChecks) throws java.io.IOException
java.io.IOException
static void checkImpacts(Impacts impacts, int lastTarget)
public static CheckIndex.Status.TermIndexStatus testPostings(CodecReader reader, java.io.PrintStream infoStream) throws java.io.IOException
java.io.IOException
public static CheckIndex.Status.TermIndexStatus testPostings(CodecReader reader, java.io.PrintStream infoStream, boolean verbose, boolean doSlowChecks, boolean failFast) throws java.io.IOException
java.io.IOException
public static CheckIndex.Status.PointsStatus testPoints(CodecReader reader, java.io.PrintStream infoStream, boolean failFast) throws java.io.IOException
java.io.IOException
public static CheckIndex.Status.StoredFieldStatus testStoredFields(CodecReader reader, java.io.PrintStream infoStream, boolean failFast) throws java.io.IOException
java.io.IOException
public static CheckIndex.Status.DocValuesStatus testDocValues(CodecReader reader, java.io.PrintStream infoStream, boolean failFast) throws java.io.IOException
java.io.IOException
private static void checkDVIterator(FieldInfo fi, int maxDoc, CheckIndex.DocValuesIteratorSupplier producer) throws java.io.IOException
java.io.IOException
private static void checkBinaryDocValues(java.lang.String fieldName, int maxDoc, BinaryDocValues bdv, BinaryDocValues bdv2) throws java.io.IOException
java.io.IOException
private static void checkSortedDocValues(java.lang.String fieldName, int maxDoc, SortedDocValues dv, SortedDocValues dv2) throws java.io.IOException
java.io.IOException
private static void checkSortedSetDocValues(java.lang.String fieldName, int maxDoc, SortedSetDocValues dv, SortedSetDocValues dv2) throws java.io.IOException
java.io.IOException
private static void checkSortedNumericDocValues(java.lang.String fieldName, int maxDoc, SortedNumericDocValues ndv, SortedNumericDocValues ndv2) throws java.io.IOException
java.io.IOException
private static void checkNumericDocValues(java.lang.String fieldName, NumericDocValues ndv, NumericDocValues ndv2) throws java.io.IOException
java.io.IOException
private static void checkDocValues(FieldInfo fi, DocValuesProducer dvReader, int maxDoc, java.io.PrintStream infoStream, CheckIndex.Status.DocValuesStatus status) throws java.lang.Exception
java.lang.Exception
public static CheckIndex.Status.TermVectorStatus testTermVectors(CodecReader reader, java.io.PrintStream infoStream) throws java.io.IOException
java.io.IOException
public static CheckIndex.Status.TermVectorStatus testTermVectors(CodecReader reader, java.io.PrintStream infoStream, boolean verbose, boolean doSlowChecks, boolean failFast) throws java.io.IOException
java.io.IOException
public void exorciseIndex(CheckIndex.Status result) throws java.io.IOException
checkIndex()
. Note that this does not
remove any of the unreferenced files after it's done;
you must separately open an IndexWriter
, which
deletes unreferenced files when it's created.
WARNING: this writes a new segments file into the index, effectively removing all documents in broken segments from the index. BE CAREFUL.
java.io.IOException
private static boolean testAsserts()
public static boolean assertsOn()
public static void main(java.lang.String[] args) throws java.io.IOException, java.lang.InterruptedException
Run it like this:
java -ea:org.apache.lucene... org.apache.lucene.index.CheckIndex pathToIndex [-exorcise] [-verbose] [-segment X] [-segment Y]
-exorcise
: actually write a new segments_N file, removing any problematic segments. *LOSES DATA*
-segment X
: only check the specified
segment(s). This can be specified multiple times,
to check more than one segment, eg -segment _2
-segment _a
. You can't use this with the -exorcise
option.
WARNING: -exorcise
should only be used on an emergency basis as it will cause
documents (perhaps many) to be permanently removed from the index. Always make
a backup copy of your index before running this! Do not run this tool on an index
that is actively being written to. You have been warned!
Run without -exorcise, this tool will open the index, report version information and report any exceptions it hits and what action it would take if -exorcise were specified. With -exorcise, this tool will remove any segments that have issues and write a new segments_N file. This means all documents contained in the affected segments will be removed.
This tool exits with exit code 1 if the index cannot be opened or has any corruption, else 0.
java.io.IOException
java.lang.InterruptedException
private static int doMain(java.lang.String[] args) throws java.io.IOException, java.lang.InterruptedException
java.io.IOException
java.lang.InterruptedException
public static CheckIndex.Options parseOptions(java.lang.String[] args)
args
- The command line argumentsjava.lang.IllegalArgumentException
- if any of the CLI args are invalidpublic int doCheck(CheckIndex.Options opts) throws java.io.IOException, java.lang.InterruptedException
opts
- The options to use for this checkjava.io.IOException
java.lang.InterruptedException
private static void checkSoftDeletes(java.lang.String softDeletesField, SegmentCommitInfo info, SegmentReader reader, java.io.PrintStream infoStream, boolean failFast) throws java.io.IOException
java.io.IOException
private static double nsToSec(long ns)