@InterfaceAudience.Private public class BlockManager extends Object
Modifier and Type | Field and Description |
---|---|
static org.apache.commons.logging.Log |
blockLog |
static float |
DEFAULT_MAP_LOAD_FACTOR
Default load factor of map
|
int |
defaultReplication
Default number of replicas
|
Map<String,LightWeightLinkedSet<Block>> |
excessReplicateMap
Maps a StorageID to the set of blocks that are "extra" for this
DataNode.
|
short |
maxReplication
The maximum number of replicas allowed for a block
|
short |
minReplication
Minimum copies needed or else write is disallowed
|
org.apache.hadoop.hdfs.server.blockmanagement.UnderReplicatedBlocks |
neededReplications
Store set of Blocks that need to be replicated 1 or more times.
|
Constructor and Description |
---|
BlockManager(Namesystem namesystem,
FSClusterStats stats,
Configuration conf) |
Modifier and Type | Method and Description |
---|---|
void |
activate(Configuration conf) |
BlockInfo |
addBlockCollection(BlockInfo block,
BlockCollection bc) |
short |
adjustReplication(short replication)
Clamp the specified replication between the minimum and the maximum
replication levels.
|
boolean |
checkMinReplication(Block block) |
void |
checkReplication(BlockCollection bc)
Check replication of the blocks in the collection.
|
DatanodeDescriptor[] |
chooseTarget(String src,
int numOfReplicas,
DatanodeDescriptor client,
HashMap<Node,Node> excludedNodes,
long blocksize,
List<String> favoredNodes)
Choose target datanodes according to the replication policy.
|
void |
clearQueues()
Clear all queues that hold decisions previously made by
this NameNode.
|
void |
close() |
boolean |
commitOrCompleteLastBlock(MutableBlockCollection bc,
Block commitBlock)
Commit the last block of the file and mark it as complete if it has
meets the minimum replication requirement
|
LocatedBlock |
convertLastBlockToUnderConstruction(MutableBlockCollection bc)
Convert the last block of the file to an under construction block.
|
NumberReplicas |
countNodes(Block b)
Return the number of nodes hosting a given block, grouped
by the state of those replicas.
|
LocatedBlocks |
createLocatedBlocks(BlockInfo[] blocks,
long fileSizeExcludeBlocksUnderConstruction,
boolean isFileUnderConstruction,
long offset,
long length,
boolean needBlockToken,
boolean inSnapshot)
Create a LocatedBlocks.
|
Iterator<DatanodeDescriptor> |
datanodeIterator(Block block) |
void |
findAndMarkBlockAsCorrupt(ExtendedBlock blk,
DatanodeInfo dn,
String reason)
Mark the block belonging to datanode as corrupt
|
BlockInfo |
forceCompleteBlock(MutableBlockCollection bc,
BlockInfoUnderConstruction block)
Force the given block in the given file to be marked as complete,
regardless of whether enough replicas are present.
|
DataEncryptionKey |
generateDataEncryptionKey() |
int |
getActiveBlockCount() |
BlockCollection |
getBlockCollection(Block b) |
ExportedBlockKeys |
getBlockKeys() |
BlockPlacementPolicy |
getBlockPlacementPolicy() |
BlocksWithLocations |
getBlocks(DatanodeID datanode,
long size)
return a list of blocks & their locations on
datanode whose
total size is size |
BlockTokenSecretManager |
getBlockTokenSecretManager()
get the BlockTokenSecretManager
|
int |
getCapacity() |
long[] |
getCorruptReplicaBlockIds(int numExpectedBlocks,
Long startingBlockId)
Return a range of corrupt replica block ids.
|
Iterator<Block> |
getCorruptReplicaBlockIterator()
Return an iterator over the set of blocks for which there are no replicas.
|
long |
getCorruptReplicaBlocksCount()
Used by metrics
|
DatanodeManager |
getDatanodeManager() |
long |
getExcessBlocksCount()
Used by metrics
|
int |
getMaxReplicationStreams() |
long |
getMissingBlocksCount() |
DatanodeDescriptor[] |
getNodes(BlockInfo block) |
int |
getPendingDataNodeMessageCount()
Used by metrics
|
long |
getPendingDeletionBlocksCount()
Used by metrics
|
long |
getPendingReplicationBlocksCount()
Used by metrics
|
long |
getPostponedMisreplicatedBlocksCount()
Used by metrics
|
long |
getScheduledReplicationBlocksCount()
Used by metrics
|
BlockInfo |
getStoredBlock(Block block) |
int |
getTotalBlocks() |
long |
getUnderReplicatedBlocksCount()
Used by metrics
|
int |
getUnderReplicatedNotMissingBlocks()
Return number of under-replicated but not missing blocks
|
void |
metaSave(PrintWriter out)
Dump meta data to out.
|
int |
numCorruptReplicas(Block block) |
int |
numOfUnderReplicatedBlocks() |
void |
processAllPendingDNMessages()
Process any remaining queued datanode messages after entering
active state.
|
void |
processIncrementalBlockReport(DatanodeID nodeID,
String poolId,
ReceivedDeletedBlockInfo[] blockInfos)
The given node is reporting incremental information about some blocks.
|
void |
processMisReplicatedBlocks()
For each block in the name-node verify whether it belongs to any file,
over or under replicated.
|
void |
processQueuedMessagesForBlock(Block b)
Try to process any messages that were previously queued for the given
block.
|
void |
processReport(DatanodeID nodeID,
String poolId,
BlockListAsLongs newReport)
The given datanode is reporting all its blocks.
|
void |
removeBlock(Block block) |
void |
removeBlockFromMap(Block block) |
void |
removeStoredBlock(Block block,
DatanodeDescriptor node)
Modify (block-->datanode) map.
|
void |
setBlockPlacementPolicy(BlockPlacementPolicy newpolicy)
Set BlockPlacementPolicy
|
void |
setBlockPoolId(String blockPoolId) |
void |
setBlockToken(LocatedBlock b,
BlockTokenSecretManager.AccessMode mode)
Generate a block token for the located block.
|
void |
setPostponeBlocksFromFuture(boolean postpone) |
void |
setReplication(short oldRepl,
short newRepl,
String src,
Block... blocks)
Set replication for the blocks.
|
void |
shutdown() |
void |
verifyReplication(String src,
short replication,
String clientName)
Check whether the replication parameter is within the range
determined by system configuration.
|
public static final org.apache.commons.logging.Log blockLog
public static final float DEFAULT_MAP_LOAD_FACTOR
public final Map<String,LightWeightLinkedSet<Block>> excessReplicateMap
public final org.apache.hadoop.hdfs.server.blockmanagement.UnderReplicatedBlocks neededReplications
public final short maxReplication
public final short minReplication
public final int defaultReplication
public BlockManager(Namesystem namesystem, FSClusterStats stats, Configuration conf) throws IOException
IOException
public long getPendingReplicationBlocksCount()
public long getUnderReplicatedBlocksCount()
public long getCorruptReplicaBlocksCount()
public long getScheduledReplicationBlocksCount()
public long getPendingDeletionBlocksCount()
public long getExcessBlocksCount()
public long getPostponedMisreplicatedBlocksCount()
public int getPendingDataNodeMessageCount()
public void setBlockPoolId(String blockPoolId)
public BlockTokenSecretManager getBlockTokenSecretManager()
public void activate(Configuration conf)
public void close()
public DatanodeManager getDatanodeManager()
public BlockPlacementPolicy getBlockPlacementPolicy()
public void setBlockPlacementPolicy(BlockPlacementPolicy newpolicy)
public void metaSave(PrintWriter out)
public int getMaxReplicationStreams()
public boolean checkMinReplication(Block block)
block
- public boolean commitOrCompleteLastBlock(MutableBlockCollection bc, Block commitBlock) throws IOException
bc
- block collectioncommitBlock
- - contains client reported block length and generationIOException
- if the block does not have at least a minimal number
of replicas reported from data-nodes.public BlockInfo forceCompleteBlock(MutableBlockCollection bc, BlockInfoUnderConstruction block) throws IOException
IOException
public LocatedBlock convertLastBlockToUnderConstruction(MutableBlockCollection bc) throws IOException
The block is converted only if the file has blocks and the last one
is a partial block (its size is less than the preferred block size).
The converted block is returned to the client.
The client uses the returned block locations to form the data pipeline
for this block.
The methods returns null if there is no partial block at the end.
The client is supposed to allocate a new block with the next call.
bc
- fileIOException
public LocatedBlocks createLocatedBlocks(BlockInfo[] blocks, long fileSizeExcludeBlocksUnderConstruction, boolean isFileUnderConstruction, long offset, long length, boolean needBlockToken, boolean inSnapshot) throws IOException
IOException
public ExportedBlockKeys getBlockKeys()
public void setBlockToken(LocatedBlock b, BlockTokenSecretManager.AccessMode mode) throws IOException
IOException
public DataEncryptionKey generateDataEncryptionKey()
public short adjustReplication(short replication)
public void verifyReplication(String src, short replication, String clientName) throws IOException
IOException
public BlocksWithLocations getBlocks(DatanodeID datanode, long size) throws IOException
datanode
whose
total size is size
datanode
- on which blocks are locatedsize
- total size of blocksIOException
public void findAndMarkBlockAsCorrupt(ExtendedBlock blk, DatanodeInfo dn, String reason) throws IOException
blk
- Block to be marked as corruptdn
- Datanode which holds the corrupt replicareason
- a textual reason why the block should be marked corrupt,
for logging purposesIOException
public void setPostponeBlocksFromFuture(boolean postpone)
public int getUnderReplicatedNotMissingBlocks()
public DatanodeDescriptor[] chooseTarget(String src, int numOfReplicas, DatanodeDescriptor client, HashMap<Node,Node> excludedNodes, long blocksize, List<String> favoredNodes) throws IOException
IOException
- if the number of targets < minimum replication.BlockPlacementPolicy.chooseTarget(String, int, DatanodeDescriptor,
List, boolean, HashMap, long)
public void processReport(DatanodeID nodeID, String poolId, BlockListAsLongs newReport) throws IOException
IOException
public void processQueuedMessagesForBlock(Block b) throws IOException
IOException
public void processAllPendingDNMessages() throws IOException
IOException
public void processMisReplicatedBlocks()
public void setReplication(short oldRepl, short newRepl, String src, Block... blocks)
public void removeStoredBlock(Block block, DatanodeDescriptor node)
public void processIncrementalBlockReport(DatanodeID nodeID, String poolId, ReceivedDeletedBlockInfo[] blockInfos) throws IOException
IOException
public NumberReplicas countNodes(Block b)
public int getActiveBlockCount()
public DatanodeDescriptor[] getNodes(BlockInfo block)
public int getTotalBlocks()
public void removeBlock(Block block)
public void checkReplication(BlockCollection bc)
public long getMissingBlocksCount()
public BlockInfo addBlockCollection(BlockInfo block, BlockCollection bc)
public BlockCollection getBlockCollection(Block b)
public Iterator<DatanodeDescriptor> datanodeIterator(Block block)
public int numCorruptReplicas(Block block)
public void removeBlockFromMap(Block block)
public int getCapacity()
public long[] getCorruptReplicaBlockIds(int numExpectedBlocks, Long startingBlockId)
numExpectedBlocks
- Number of block ids to return.
0 <= numExpectedBlocks <= 100startingBlockId
- Block id from which to start. If null, start at
beginning.public Iterator<Block> getCorruptReplicaBlockIterator()
public int numOfUnderReplicatedBlocks()
public void clearQueues()
public void shutdown()
Copyright © 2013 Apache Software Foundation. All rights reserved.