final class DocumentsWriterFlushControl extends java.lang.Object implements Accountable
DocumentsWriterPerThread
flushing during
indexing. It tracks the memory consumption per
DocumentsWriterPerThread
and uses a configured FlushPolicy
to
decide if a DocumentsWriterPerThread
must flush.
In addition to the FlushPolicy
the flush control might set certain
DocumentsWriterPerThread
as flush pending iff a
DocumentsWriterPerThread
exceeds the
IndexWriterConfig.getRAMPerThreadHardLimitMB()
to prevent address
space exhaustion.
Modifier and Type | Class and Description |
---|---|
private static class |
DocumentsWriterFlushControl.BlockedFlush |
Modifier and Type | Field and Description |
---|---|
private long |
activeBytes |
private java.util.Queue<DocumentsWriterFlushControl.BlockedFlush> |
blockedFlushes |
private boolean |
closed |
private LiveIndexWriterConfig |
config |
private DocumentsWriter |
documentsWriter |
(package private) boolean |
flushByRAMWasDisabled |
private long |
flushBytes |
(package private) java.util.concurrent.atomic.AtomicBoolean |
flushDeletes |
private java.util.IdentityHashMap<DocumentsWriterPerThread,java.lang.Long> |
flushingWriters |
private FlushPolicy |
flushPolicy |
private java.util.Queue<DocumentsWriterPerThread> |
flushQueue |
private boolean |
fullFlush |
private java.util.List<DocumentsWriterPerThread> |
fullFlushBuffer |
private long |
hardMaxBytesPerDWPT |
private InfoStream |
infoStream |
(package private) double |
maxConfiguredRamBuffer |
private int |
numDocsSinceStalled |
private int |
numPending |
(package private) long |
peakActiveBytes |
(package private) long |
peakDelta |
(package private) long |
peakFlushBytes |
(package private) long |
peakNetBytes |
private DocumentsWriterPerThreadPool |
perThreadPool |
(package private) DocumentsWriterStallControl |
stallControl |
private long |
stallStartNS |
Constructor and Description |
---|
DocumentsWriterFlushControl(DocumentsWriter documentsWriter,
LiveIndexWriterConfig config) |
Modifier and Type | Method and Description |
---|---|
(package private) void |
abortFullFlushes() |
(package private) void |
abortPendingFlushes() |
long |
activeBytes() |
(package private) void |
addFlushableState(DocumentsWriterPerThreadPool.ThreadState perThread) |
java.util.Iterator<DocumentsWriterPerThreadPool.ThreadState> |
allActiveThreadStates()
Returns an iterator that provides access to all currently active
DocumentsWriterPerThreadPool.ThreadState s |
(package private) boolean |
anyStalledThreads()
Returns
true iff stalled |
private boolean |
assertActiveDeleteQueue(DocumentsWriterDeleteQueue queue) |
(package private) boolean |
assertBlockedFlushes(DocumentsWriterDeleteQueue flushingQueue) |
private boolean |
assertMemory() |
private boolean |
assertNumDocsSinceStalled(boolean stalled) |
private DocumentsWriterPerThread |
checkout(DocumentsWriterPerThreadPool.ThreadState perThread,
boolean markPending) |
private void |
checkoutAndBlock(DocumentsWriterPerThreadPool.ThreadState perThread) |
(package private) DocumentsWriterPerThread |
checkoutLargestNonPendingWriter()
Returns the largest non-pending flushable DWPT or
null if there is none. |
private void |
commitPerThreadBytes(DocumentsWriterPerThreadPool.ThreadState perThread) |
(package private) DocumentsWriterPerThread |
doAfterDocument(DocumentsWriterPerThreadPool.ThreadState perThread,
boolean isUpdate) |
(package private) void |
doAfterFlush(DocumentsWriterPerThread dwpt) |
(package private) void |
doOnAbort(DocumentsWriterPerThreadPool.ThreadState state) |
(package private) void |
doOnDelete() |
(package private) DocumentsWriterPerThreadPool.ThreadState |
findLargestNonPendingWriter() |
(package private) void |
finishFullFlush() |
boolean |
getAndResetApplyAllDeletes() |
long |
getDeleteBytesUsed()
Returns heap bytes currently consumed by buffered deletes/updates that would be
freed if we pushed all deletes.
|
long |
getFlushingBytes() |
InfoStream |
getInfoStream()
Returns the
IndexWriter InfoStream |
private java.util.Iterator<DocumentsWriterPerThreadPool.ThreadState> |
getPerThreadsIterator(int upto) |
private DocumentsWriterPerThread |
internalTryCheckOutForFlush(DocumentsWriterPerThreadPool.ThreadState perThread) |
(package private) boolean |
isFullFlush()
Returns
true if a full flush is currently running |
(package private) long |
markForFullFlush() |
long |
netBytes() |
(package private) DocumentsWriterPerThread |
nextPendingFlush() |
(package private) int |
numBlockedFlushes()
Returns the number of flushes that are checked out but not yet available
for flushing.
|
(package private) int |
numFlushingDWPT() |
(package private) int |
numQueuedFlushes()
Returns the number of flushes that are already checked out but not yet
actively flushing
|
(package private) DocumentsWriterPerThreadPool.ThreadState |
obtainAndLock() |
private void |
pruneBlockedQueue(DocumentsWriterDeleteQueue flushingQueue)
Prunes the blockedQueue by removing all DWPT that are associated with the given flush queue.
|
long |
ramBytesUsed()
Return the memory usage of this object in bytes.
|
void |
setApplyAllDeletes() |
(package private) void |
setClosed() |
void |
setFlushPending(DocumentsWriterPerThreadPool.ThreadState perThread)
Sets flush pending state on the given
DocumentsWriterPerThreadPool.ThreadState . |
private long |
stallLimitBytes() |
java.lang.String |
toString() |
(package private) DocumentsWriterPerThread |
tryCheckoutForFlush(DocumentsWriterPerThreadPool.ThreadState perThread) |
private boolean |
updatePeaks(long delta) |
private boolean |
updateStallState() |
void |
waitForFlush() |
(package private) void |
waitIfStalled()
This method will block if too many DWPT are currently flushing and no
checked out DWPT are available
|
clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait
getChildResources
private final long hardMaxBytesPerDWPT
private long activeBytes
private volatile long flushBytes
private volatile int numPending
private int numDocsSinceStalled
final java.util.concurrent.atomic.AtomicBoolean flushDeletes
private boolean fullFlush
private final java.util.Queue<DocumentsWriterPerThread> flushQueue
private final java.util.Queue<DocumentsWriterFlushControl.BlockedFlush> blockedFlushes
private final java.util.IdentityHashMap<DocumentsWriterPerThread,java.lang.Long> flushingWriters
double maxConfiguredRamBuffer
long peakActiveBytes
long peakFlushBytes
long peakNetBytes
long peakDelta
boolean flushByRAMWasDisabled
final DocumentsWriterStallControl stallControl
private final DocumentsWriterPerThreadPool perThreadPool
private final FlushPolicy flushPolicy
private boolean closed
private final DocumentsWriter documentsWriter
private final LiveIndexWriterConfig config
private final InfoStream infoStream
private long stallStartNS
private final java.util.List<DocumentsWriterPerThread> fullFlushBuffer
DocumentsWriterFlushControl(DocumentsWriter documentsWriter, LiveIndexWriterConfig config)
public long activeBytes()
public long getFlushingBytes()
public long netBytes()
private long stallLimitBytes()
private boolean assertMemory()
private void commitPerThreadBytes(DocumentsWriterPerThreadPool.ThreadState perThread)
private boolean updatePeaks(long delta)
DocumentsWriterPerThread doAfterDocument(DocumentsWriterPerThreadPool.ThreadState perThread, boolean isUpdate)
private DocumentsWriterPerThread checkout(DocumentsWriterPerThreadPool.ThreadState perThread, boolean markPending)
private boolean assertNumDocsSinceStalled(boolean stalled)
void doAfterFlush(DocumentsWriterPerThread dwpt)
private boolean updateStallState()
public void waitForFlush()
public void setFlushPending(DocumentsWriterPerThreadPool.ThreadState perThread)
DocumentsWriterPerThreadPool.ThreadState
. The
DocumentsWriterPerThreadPool.ThreadState
must have indexed at least on Document and must not be
already pending.void doOnAbort(DocumentsWriterPerThreadPool.ThreadState state)
DocumentsWriterPerThread tryCheckoutForFlush(DocumentsWriterPerThreadPool.ThreadState perThread)
private void checkoutAndBlock(DocumentsWriterPerThreadPool.ThreadState perThread)
private DocumentsWriterPerThread internalTryCheckOutForFlush(DocumentsWriterPerThreadPool.ThreadState perThread)
public java.lang.String toString()
toString
in class java.lang.Object
DocumentsWriterPerThread nextPendingFlush()
void setClosed()
public java.util.Iterator<DocumentsWriterPerThreadPool.ThreadState> allActiveThreadStates()
DocumentsWriterPerThreadPool.ThreadState
sprivate java.util.Iterator<DocumentsWriterPerThreadPool.ThreadState> getPerThreadsIterator(int upto)
void doOnDelete()
public long getDeleteBytesUsed()
public long ramBytesUsed()
Accountable
ramBytesUsed
in interface Accountable
int numFlushingDWPT()
public boolean getAndResetApplyAllDeletes()
public void setApplyAllDeletes()
DocumentsWriterPerThreadPool.ThreadState obtainAndLock()
long markForFullFlush()
private boolean assertActiveDeleteQueue(DocumentsWriterDeleteQueue queue)
void addFlushableState(DocumentsWriterPerThreadPool.ThreadState perThread)
private void pruneBlockedQueue(DocumentsWriterDeleteQueue flushingQueue)
void finishFullFlush()
boolean assertBlockedFlushes(DocumentsWriterDeleteQueue flushingQueue)
void abortFullFlushes()
void abortPendingFlushes()
boolean isFullFlush()
true
if a full flush is currently runningint numQueuedFlushes()
int numBlockedFlushes()
void waitIfStalled()
boolean anyStalledThreads()
true
iff stalledpublic InfoStream getInfoStream()
IndexWriter
InfoStream
DocumentsWriterPerThreadPool.ThreadState findLargestNonPendingWriter()
final DocumentsWriterPerThread checkoutLargestNonPendingWriter()
null
if there is none.