public final class CompressingStoredFieldsWriter extends StoredFieldsWriter
Modifier and Type | Class and Description |
---|---|
private static class |
CompressingStoredFieldsWriter.CompressingStoredFieldsMergeSub |
StoredFieldsWriter.MergeVisitor
Modifier and Type | Field and Description |
---|---|
private GrowableByteArrayDataOutput |
bufferedDocs |
(package private) static boolean |
BULK_MERGE_ENABLED |
(package private) static java.lang.String |
BULK_MERGE_ENABLED_SYSPROP |
(package private) static int |
BYTE_ARR |
private int |
chunkSize |
(package private) static java.lang.String |
CODEC_SFX_DAT |
(package private) static java.lang.String |
CODEC_SFX_IDX |
private CompressionMode |
compressionMode |
private Compressor |
compressor |
(package private) static long |
DAY |
(package private) static int |
DAY_ENCODING |
private int |
docBase |
private int[] |
endOffsets |
static java.lang.String |
FIELDS_EXTENSION
Extension of stored fields file
|
static java.lang.String |
FIELDS_INDEX_EXTENSION
Extension of stored fields index file
|
private IndexOutput |
fieldsStream |
(package private) static long |
HOUR |
(package private) static int |
HOUR_ENCODING |
private CompressingStoredFieldsIndexWriter |
indexWriter |
private int |
maxDocsPerChunk |
(package private) static long |
NEGATIVE_ZERO_DOUBLE |
(package private) static int |
NEGATIVE_ZERO_FLOAT |
private int |
numBufferedDocs |
private long |
numChunks |
private long |
numDirtyChunks |
(package private) static int |
NUMERIC_DOUBLE |
(package private) static int |
NUMERIC_FLOAT |
(package private) static int |
NUMERIC_INT |
(package private) static int |
NUMERIC_LONG |
private int[] |
numStoredFields |
private int |
numStoredFieldsInDoc |
(package private) static long |
SECOND |
(package private) static int |
SECOND_ENCODING |
private java.lang.String |
segment |
(package private) static int |
STRING |
(package private) static int |
TYPE_BITS |
(package private) static int |
TYPE_MASK |
(package private) static int |
VERSION_CURRENT |
(package private) static int |
VERSION_START |
Constructor and Description |
---|
CompressingStoredFieldsWriter(Directory directory,
SegmentInfo si,
java.lang.String segmentSuffix,
IOContext context,
java.lang.String formatName,
CompressionMode compressionMode,
int chunkSize,
int maxDocsPerChunk,
int blockSize)
Sole constructor.
|
Modifier and Type | Method and Description |
---|---|
void |
close() |
void |
finish(FieldInfos fis,
int numDocs)
Called before
StoredFieldsWriter.close() , passing in the number
of documents that were written. |
void |
finishDocument()
Called when a document and all its fields have been added.
|
private void |
flush() |
int |
merge(MergeState mergeState)
Merges in the stored fields from the readers in
mergeState . |
private static void |
saveInts(int[] values,
int length,
DataOutput out) |
void |
startDocument()
Called before writing the stored fields of the document.
|
(package private) boolean |
tooDirty(CompressingStoredFieldsReader candidate)
Returns true if we should recompress this reader, even though we could bulk merge compressed data
|
private boolean |
triggerFlush() |
void |
writeField(FieldInfo info,
IndexableField field)
Writes a single stored field.
|
private void |
writeHeader(int docBase,
int numBufferedDocs,
int[] numStoredFields,
int[] lengths,
boolean sliced) |
(package private) static void |
writeTLong(DataOutput out,
long l)
Writes a long in a variable-length format.
|
(package private) static void |
writeZDouble(DataOutput out,
double d)
Writes a float in a variable-length format.
|
(package private) static void |
writeZFloat(DataOutput out,
float f)
Writes a float in a variable-length format.
|
public static final java.lang.String FIELDS_EXTENSION
public static final java.lang.String FIELDS_INDEX_EXTENSION
static final int STRING
static final int BYTE_ARR
static final int NUMERIC_INT
static final int NUMERIC_FLOAT
static final int NUMERIC_LONG
static final int NUMERIC_DOUBLE
static final int TYPE_BITS
static final int TYPE_MASK
static final java.lang.String CODEC_SFX_IDX
static final java.lang.String CODEC_SFX_DAT
static final int VERSION_START
static final int VERSION_CURRENT
private final java.lang.String segment
private CompressingStoredFieldsIndexWriter indexWriter
private IndexOutput fieldsStream
private Compressor compressor
private final CompressionMode compressionMode
private final int chunkSize
private final int maxDocsPerChunk
private final GrowableByteArrayDataOutput bufferedDocs
private int[] numStoredFields
private int[] endOffsets
private int docBase
private int numBufferedDocs
private long numChunks
private long numDirtyChunks
private int numStoredFieldsInDoc
static final int NEGATIVE_ZERO_FLOAT
static final long NEGATIVE_ZERO_DOUBLE
static final long SECOND
static final long HOUR
static final long DAY
static final int SECOND_ENCODING
static final int HOUR_ENCODING
static final int DAY_ENCODING
static final java.lang.String BULK_MERGE_ENABLED_SYSPROP
static final boolean BULK_MERGE_ENABLED
public CompressingStoredFieldsWriter(Directory directory, SegmentInfo si, java.lang.String segmentSuffix, IOContext context, java.lang.String formatName, CompressionMode compressionMode, int chunkSize, int maxDocsPerChunk, int blockSize) throws java.io.IOException
java.io.IOException
public void close() throws java.io.IOException
close
in interface java.io.Closeable
close
in interface java.lang.AutoCloseable
close
in class StoredFieldsWriter
java.io.IOException
public void startDocument() throws java.io.IOException
StoredFieldsWriter
StoredFieldsWriter.writeField(FieldInfo, IndexableField)
will be called
for each stored field. Note that this is
called even if the document has no stored fields.startDocument
in class StoredFieldsWriter
java.io.IOException
public void finishDocument() throws java.io.IOException
StoredFieldsWriter
finishDocument
in class StoredFieldsWriter
java.io.IOException
private static void saveInts(int[] values, int length, DataOutput out) throws java.io.IOException
java.io.IOException
private void writeHeader(int docBase, int numBufferedDocs, int[] numStoredFields, int[] lengths, boolean sliced) throws java.io.IOException
java.io.IOException
private boolean triggerFlush()
private void flush() throws java.io.IOException
java.io.IOException
public void writeField(FieldInfo info, IndexableField field) throws java.io.IOException
StoredFieldsWriter
writeField
in class StoredFieldsWriter
java.io.IOException
static void writeZFloat(DataOutput out, float f) throws java.io.IOException
ZFloat --> Header, Bytes*?
Uint8
. When it is
equal to 0xFF then the value is negative and stored in the next
4 bytes. Otherwise if the first bit is set then the other bits
in the header encode the value plus one and no other
bytes are read. Otherwise, the value is a positive float value
whose first byte is the header, and 3 bytes need to be read to
complete it.
java.io.IOException
static void writeZDouble(DataOutput out, double d) throws java.io.IOException
ZFloat --> Header, Bytes*?
Uint8
. When it is
equal to 0xFF then the value is negative and stored in the next
8 bytes. When it is equal to 0xFE then the value is stored as a
float in the next 4 bytes. Otherwise if the first bit is set
then the other bits in the header encode the value plus one and
no other bytes are read. Otherwise, the value is a positive float
value whose first byte is the header, and 7 bytes need to be read
to complete it.
java.io.IOException
static void writeTLong(DataOutput out, long l) throws java.io.IOException
ZLong --> Header, Bytes*?
zigzag-decode
it and finally
multiply by the multiple corresponding to the compression scheme.
java.io.IOException
public void finish(FieldInfos fis, int numDocs) throws java.io.IOException
StoredFieldsWriter
StoredFieldsWriter.close()
, passing in the number
of documents that were written. Note that this is
intentionally redundant (equivalent to the number of
calls to StoredFieldsWriter.startDocument()
, but a Codec should
check that this is the case to detect the JRE bug described
in LUCENE-1282.finish
in class StoredFieldsWriter
java.io.IOException
public int merge(MergeState mergeState) throws java.io.IOException
StoredFieldsWriter
mergeState
. The default implementation skips
over deleted documents, and uses StoredFieldsWriter.startDocument()
,
StoredFieldsWriter.writeField(FieldInfo, IndexableField)
, and StoredFieldsWriter.finish(FieldInfos, int)
,
returning the number of documents that were written.
Implementations can override this method for more sophisticated
merging (bulk-byte copying, etc).merge
in class StoredFieldsWriter
java.io.IOException
boolean tooDirty(CompressingStoredFieldsReader candidate)
The last chunk written for a segment is typically incomplete, so without recompressing, in some worst-case situations (e.g. frequent reopen with tiny flushes), over time the compression ratio can degrade. This is a safety switch.