Din Djarin, also known as “the Mandalorian” or simply “Mando,” was a human male Mandalorian who worked as a famous bounty hunter during the New Republic Era.
// New 初始化 block 这里会将第一个原始时间戳写入到 block 中 funcNew(t0 uint32) *Series { s := Series{ T0: t0, leading: ^uint8(0), }
s.bw.writeBits(uint64(t0), 32) return &s }
// Push 负责写入时序数据 func(s *Series)Push(t uint32, v float64) { // .... // 如果是第一个数据点的话写入原始数据后直接返回 if s.t == 0 { s.t = t s.val = v s.tDelta = t - s.T0 // 实际上这里为 0
// The block header stores the starting time stamp, t-1（前一个时间戳）, // which is aligned to a two hour window; the first time // stamp, t0, in the block is stored as a delta from t−1 in 14 bits. // 用 14 个 bit 写入时间戳差值 s.bw.writeBits(uint64(s.tDelta), 14) // 原始数据点完整写入 s.bw.writeBits(math.Float64bits(v), 64) return }
tDelta := t - s.t dod := int32(tDelta - s.tDelta) // 计算差值的差值 Detla of Delta
// 下面开始就处理非第一个数据点的情况了 switch { // If D is zero, then store a single ‘0’ bit // 如果是零的话 那直接用 '0' 一个字节就可以直接表示 case dod == 0: s.bw.writeBit(zero)
// If D is between [-63, 64], store ‘10’ followed by the value (7 bits) case-63 <= dod && dod <= 64: s.bw.writeBits(0x02, 2) // 控制位 '10' s.bw.writeBits(uint64(dod), 7) // 7bits 可以表示 [-63, 64] 的范围
// If D is between [-255, 256], store ‘110’ followed by the value (9 bits) case-255 <= dod && dod <= 256: s.bw.writeBits(0x06, 3) // 控制位 '110' s.bw.writeBits(uint64(dod), 9)
// if D is between [-2047, 2048], store ‘1110’ followed by the value (12 bits) case-2047 <= dod && dod <= 2048: s.bw.writeBits(0x0e, 4) // 控制位 '1110' s.bw.writeBits(uint64(dod), 12)
// Otherwise store ‘1111’ followed by D using 32 bits default: s.bw.writeBits(0x0f, 4) // 其余情况控制位均用 '1111' s.bw.writeBits(uint64(dod), 32) }
// clamp number of leading zeros to avoid overflow when encoding if leading >= 32 { leading = 31 }
// (Control bit ‘0’) If the block of meaningful bits // falls within the block of previous meaningful bits, // i.e., there are at least as many leading zeros and // as many trailing zeros as with the previous value, // use that information for the block position and // just store the meaningful XORed value.
// (Control bit ‘1’) Store the length of the number // of leading zeros in the next 5 bits, then store the // length of the meaningful XORed value in the next // 6 bits. Finally store the meaningful bits of the XORed value. s.leading, s.trailing = leading, trailing
从 Linux 角度来看，操作系统的内存空间被分为「内核空间」和「用户空间」两大部分，其中内核空间和用户空间的空间大小、操作权限以及核心功能都不相同。这里的内核空间是指操作系统本身使用的内存空间，而用户空间则是提供给各个进程使用的内存空间。由于用户进程不具有访问内核资源的权限，例如访问硬件资源，因此当一个用户进程需要使用内核资源的时候，就需要通过 系统调用 来完成。
// optimizeConcatRegex returns literal prefix/suffix text that can be safely // checked against the label value before running the regexp matcher. funcoptimizeConcatRegex(r *syntax.Regexp)(prefix, suffix, contains string) { sub := r.Sub
// We can safely remove begin and end text matchers respectively // at the beginning and end of the regexp. iflen(sub) > 0 && sub[0].Op == syntax.OpBeginText { sub = sub[1:] } iflen(sub) > 0 && sub[len(sub)-1].Op == syntax.OpEndText { sub = sub[:len(sub)-1] }
iflen(sub) == 0 { return }
// Given Prometheus regex matchers are always anchored to the begin/end // of the text, if the first/last operations are literals, we can safely // treat them as prefix/suffix. if sub[0].Op == syntax.OpLiteral && (sub[0].Flags&syntax.FoldCase) == 0 { prefix = string(sub[0].Rune) } if last := len(sub) - 1; sub[last].Op == syntax.OpLiteral && (sub[last].Flags&syntax.FoldCase) == 0 { suffix = string(sub[last].Rune) }
// If contains any literal which is not a prefix/suffix, we keep the // 1st one. We do not keep the whole list of literals to simplify the // fast path. for i := 1; i < len(sub)-1; i++ { if sub[i].Op == syntax.OpLiteral && (sub[i].Flags&syntax.FoldCase) == 0 { contains = string(sub[i].Rune) break } }
funcmain() { store := mandodb.OpenTSDB(mandodb.WithMetaBytesCompressorType(mandodb.ZstdBytesCompressor)) defer store.Close() // ... }
// 压缩效果 28M -> 25M ❯ 🐶 ll seg-1627711905-1627719165 Permissions Size User Date Modified Name .rwxr-xr-x 25M chenjiandongx 1 Aug 00:13 data .rwxr-xr-x 110 chenjiandongx 1 Aug 00:13 meta.json
SnappyBytesCompressor
funcmain() { store := mandodb.OpenTSDB(mandodb.WithMetaBytesCompressorType(mandodb.SnappyBytesCompressor)) defer store.Close() // ... }
// 压缩效果 28M -> 26M ❯ 🐶 ll seg-1627763918-1627771178 Permissions Size User Date Modified Name .rwxr-xr-x 26M chenjiandongx 1 Aug 14:39 data .rwxr-xr-x 110 chenjiandongx 1 Aug 14:39 meta.json
多多少少还是有点效果的 🤪…
压缩是有成本的，压缩体积的同时会增大 CPU 开销（mbp 可以煎鸡蛋了），减缓写入速率。
敲黑板，接下来就要来好好讲讲 data 文件到底写了什么东西。data 存储布局如下。
Figure: Segment Stroage
TOC 描述了 Data Block 和 Meta Block（Series Block + Labels Block）的体积，用于后面对 data 进行解析读取。Data Block 存储了每条时间线具体的数据点，时间线之间数据紧挨存储。DataContent 就是使用 Gorilla 差值算法压缩的 block。
Figure: Data Block
Labels Block 记录了具体的 Label 值以及对应 Label 与哪些 Series 相关联。
Figure: Labels Block
Series Block 记录了每条时间线的元数据，字段解释如下。
SidLength: Sid 的长度。
Sid: 时间线的唯一标识。
StartOffset: 时间线数据块在 Data Block 中的起始偏移。
EndOffset: 时间线数据块在 Data Block 中的终止偏移。
LabelCount: 时间线包含的 Label 数量。
Labels: 标签在 Labels Block 中的序号（仅记录序号，不记录具体值）。
Figure: Series Block
了解完设计，再看看 Meta Block 编码和解编码的代码实现，binaryMetaSerializer 实现了 MetaSerializer 接口。
// series block for idx, series := range meta.Series { encf.MarshalUint16(uint16(len(series.Sid))) encf.MarshalString(series.Sid) encf.MarshalUint64(series.StartOffset, series.EndOffset)
