博客
关于我
kafka日志存储(五):LogSegment
阅读量:250 次
发布时间:2019-03-01

本文共 3218 字,大约阅读时间需要 10 分钟。

为了防止Log文件过大,Log文件会被切分成多个日志文件,每个日志文件对应一个LogSegment。在LogSegment中,封装了FileMessageSet和OffsetIndex对象。LogSegment类的实现如下:

LogSegment类结构

class LogSegment(val log: FileMessageSet,
val index: OffsetIndex,
val baseOffset: Long,
val indexIntervalBytes: Int,
val rollJitterMs: Long,
time: Time) extends Logging {
private var bytesSinceLastIndexEntry = 0
var created: Long = time.milliseconds
}

append方法

def append(offset: Long, messages: ByteBufferMessageSet): Unit = {
if (messages.sizeInBytes > 0) {
trace("Inserting %d bytes at offset %d at position %d".format(
messages.sizeInBytes, offset, log.sizeInBytes()))
if (bytesSinceLastIndexEntry > indexIntervalBytes) {
index.append(offset, log.sizeInBytes())
this.bytesSinceLastIndexEntry = 0
}
log.append(messages)
this.bytesSinceLastIndexEntry += messages.sizeInBytes
}
}

read方法

def read(
startOffset: Long,
maxOffset: Option[Long],
maxSize: Int,
maxPosition: Long = size
): FetchDataInfo = {
if (maxSize < 0) {
throw new IllegalArgumentException("Invalid max size for log read (%d)".format(maxSize))
}
val logSize = log.sizeInBytes
val startPosition = translateOffset(startOffset)
if (startPosition == null) {
return null
}
val offsetMetadata = new LogOffsetMetadata(
startOffset,
this.baseOffset,
startPosition.position
)
if (maxSize == 0) {
return FetchDataInfo(offsetMetadata, MessageSet.Empty)
}
val length = maxOffset match {
case None =>
min((maxPosition - startPosition.position).toInt, maxSize)
case Some(offset) =>
if (offset < startOffset) {
return FetchDataInfo(offsetMetadata, MessageSet.Empty)
}
val mapping = translateOffset(offset, startPosition.position)
val endPosition = if (mapping == null) {
logSize
} else {
mapping.position
}
min(min(maxPosition, endPosition) - startPosition.position, maxSize).toInt
}
FetchDataInfo(offsetMetadata, log.read(startPosition.position, length))
}

recover方法

def recover(maxMessageSize: Int): Int = {
index.truncate()
index.resize(index.maxIndexSize)
var validBytes = 0
var lastIndexEntry = 0
val iter = log.iterator(maxMessageSize)
try {
while (iter.hasNext) {
val entry = iter.next
entry.message.ensureValid()
if (validBytes - lastIndexEntry > indexIntervalBytes) {
val startOffset = entry.message.compressionCodec match {
case NoCompressionCodec =>
entry.offset
case _ =>
ByteBufferMessageSet.deepIterator(entry).next().offset
}
index.append(startOffset, validBytes)
lastIndexEntry = validBytes
}
validBytes += MessageSet.entrySize(entry.message)
}
} catch {
case e: CorruptRecordException =>
logger.warn("Found invalid messages in log segment %s at byte offset %d: %s.".format(
log.file.getAbsolutePath, validBytes, e.getMessage))
}
val truncated = log.sizeInBytes - validBytes
log.truncateTo(validBytes)
index.trimToValidSize()
truncated
}

转载地址:http://gjxx.baihongyu.com/

你可能感兴趣的文章
No Datastore Session bound to thread, and configuration does not allow creation of non-transactional
查看>>
No fallbackFactory instance of type class com.ruoyi---SpringCloud Alibaba_若依微服务框架改造---工作笔记005
查看>>
No Feign Client for loadBalancing defined. Did you forget to include spring-cloud-starter-loadbalanc
查看>>
No mapping found for HTTP request with URI [/...] in DispatcherServlet with name ...的解决方法
查看>>
No mapping found for HTTP request with URI [/logout.do] in DispatcherServlet with name 'springmvc'
查看>>
No module named 'crispy_forms'等使用pycharm开发
查看>>
No module named cv2
查看>>
No module named tensorboard.main在安装tensorboardX的时候遇到的问题
查看>>
No module named ‘MySQLdb‘错误解决No module named ‘MySQLdb‘错误解决
查看>>
No new migrations found. Your system is up-to-date.
查看>>
No qualifying bean of type XXX found for dependency XXX.
查看>>
No qualifying bean of type ‘com.netflix.discovery.AbstractDiscoveryClientOptionalArgs<?>‘ available
查看>>
No resource identifier found for attribute 'srcCompat' in package的解决办法
查看>>
no session found for current thread
查看>>
No toolchains found in the NDK toolchains folder for ABI with prefix: mips64el-linux-android
查看>>
NO.23 ZenTaoPHP目录结构
查看>>
no1
查看>>
NO32 网络层次及OSI7层模型--TCP三次握手四次断开--子网划分
查看>>
NOAA(美国海洋和大气管理局)气象数据获取与POI点数据获取
查看>>
NoClassDefFoundError: org/springframework/boot/context/properties/ConfigurationBeanFactoryMetadata
查看>>