astelm/minio
1
0
Fork 0
mirror of https://github.com/pgsty/minio.git synced 2026-07-22 13:40:22 +03:00
Code Issues Packages Projects Releases Wiki Activity

Store object EC in metadata header (#19534)

Browse Source 
Keep the EC in header, so it can be retrieved easily for dynamic quorum calculations.

To not force a full metadata decode on every read the value will be 0/0 for data written in previous versions.

Size is expected to increase by 2 bytes per version, since all valid values can be represented with 1 byte each.

Example:
```
λ xl-meta xl.meta
{
  "Versions": [
    {
      "Header": {
        "EcM": 4,
        "EcN": 8,
        "Flags": 6,
        "ModTime": "2024-04-17T11:46:25.325613+02:00",
        "Signature": "0a409875",
        "Type": 1,
        "VersionID": "8e03504e11234957b2727bc53eda0d55"
      },
...
```

Not used for operations yet.
This commit is contained in:
Klaus Post
2024-04-19 09:43:43 -07:00
committed by GitHub
parent 2ca9befd2a
commit 5f774951b1
5 changed files with 230 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/metacache-entries.go
+10
View File
@@ -120,6 +120,16 @@ func (e *metaCacheEntry) matches(other *metaCacheEntry, strict bool) (prefer *me
for i, eVer := range eVers.versions {
oVer := oVers.versions[i]
if eVer.header != oVer.header {
if eVer.header.hasEC() != oVer.header.hasEC() {
// One version has EC and the other doesn't - may have been written later.
// Compare without considering EC.
a, b := eVer.header, oVer.header
a.EcN, a.EcM = 0, 0
b.EcN, b.EcM = 0, 0
if a == b {
continue
}
}
if !strict && eVer.header.matchesNotStrict(oVer.header) {
if prefer == nil {
if eVer.header.sortsBefore(oVer.header) {
cmd/xl-storage-format-v2-legacy.go
+107
View File
@@ -29,6 +29,9 @@ func (x *xlMetaV2VersionHeader) unmarshalV(v uint8, bts []byte) (o []byte, err e
switch v {
case 1:
return x.unmarshalV1(bts)
case 2:
x2 := xlMetaV2VersionHeaderV2{xlMetaV2VersionHeader: x}
return x2.UnmarshalMsg(bts)
case xlHeaderVersion:
return x.UnmarshalMsg(bts)
}
@@ -123,3 +126,107 @@ func (j *xlMetaV2Version) unmarshalV(v uint8, bts []byte) (o []byte, err error)
}
return o, err
}
// xlMetaV2VersionHeaderV2 is a version 2 of xlMetaV2VersionHeader before EcN and EcM were added.
type xlMetaV2VersionHeaderV2 struct {
*xlMetaV2VersionHeader
}
// UnmarshalMsg implements msgp.Unmarshaler
func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte) (o []byte, err error) {
z.EcN, z.EcN = 0, 0
var zb0001 uint32
zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
if zb0001 != 5 {
err = msgp.ArrayError{Wanted: 5, Got: zb0001}
return
}
bts, err = msgp.ReadExactBytes(bts, (z.VersionID)[:])
if err != nil {
err = msgp.WrapError(err, "VersionID")
return
}
z.ModTime, bts, err = msgp.ReadInt64Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "ModTime")
return
}
bts, err = msgp.ReadExactBytes(bts, (z.Signature)[:])
if err != nil {
err = msgp.WrapError(err, "Signature")
return
}
{
var zb0002 uint8
zb0002, bts, err = msgp.ReadUint8Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "Type")
return
}
z.Type = VersionType(zb0002)
}
{
var zb0003 uint8
zb0003, bts, err = msgp.ReadUint8Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "Flags")
return
}
z.Flags = xlFlags(zb0003)
}
o = bts
return
}
// DecodeMsg implements msgp.Decodable
func (z *xlMetaV2VersionHeaderV2) DecodeMsg(dc *msgp.Reader) (err error) {
z.EcN, z.EcN = 0, 0
var zb0001 uint32
zb0001, err = dc.ReadArrayHeader()
if err != nil {
err = msgp.WrapError(err)
return
}
if zb0001 != 5 {
err = msgp.ArrayError{Wanted: 5, Got: zb0001}
return
}
err = dc.ReadExactBytes((z.VersionID)[:])
if err != nil {
err = msgp.WrapError(err, "VersionID")
return
}
z.ModTime, err = dc.ReadInt64()
if err != nil {
err = msgp.WrapError(err, "ModTime")
return
}
err = dc.ReadExactBytes((z.Signature)[:])
if err != nil {
err = msgp.WrapError(err, "Signature")
return
}
{
var zb0002 uint8
zb0002, err = dc.ReadUint8()
if err != nil {
err = msgp.WrapError(err, "Type")
return
}
z.Type = VersionType(zb0002)
}
{
var zb0003 uint8
zb0003, err = dc.ReadUint8()
if err != nil {
err = msgp.WrapError(err, "Flags")
return
}
z.Flags = xlFlags(zb0003)
}
return
}
cmd/xl-storage-format-v2.go
+15 -2
View File
@@ -250,15 +250,17 @@ type xlMetaV2VersionHeader struct {
Signature [4]byte
Type VersionType
Flags xlFlags
EcM, EcN uint8 // Note that these will be 0/0 for non-v2 objects and older xl.meta
}
func (x xlMetaV2VersionHeader) String() string {
return fmt.Sprintf("Type: %s, VersionID: %s, Signature: %s, ModTime: %s, Flags: %s",
return fmt.Sprintf("Type: %s, VersionID: %s, Signature: %s, ModTime: %s, Flags: %s, N: %d, M: %d",
x.Type.String(),
hex.EncodeToString(x.VersionID[:]),
hex.EncodeToString(x.Signature[:]),
time.Unix(0, x.ModTime),
x.Flags.String(),
x.EcN, x.EcM,
)
}
@@ -274,6 +276,11 @@ func (x xlMetaV2VersionHeader) matchesNotStrict(o xlMetaV2VersionHeader) bool {
x.Type == o.Type
}
// hasEC will return true if the version has erasure coding information.
func (x xlMetaV2VersionHeader) hasEC() bool {
return x.EcM > 0 && x.EcN > 0
}
// sortsBefore can be used as a tiebreaker for stable sorting/selecting.
// Returns false on ties.
func (x xlMetaV2VersionHeader) sortsBefore(o xlMetaV2VersionHeader) bool {
@@ -351,12 +358,18 @@ func (j *xlMetaV2Version) header() xlMetaV2VersionHeader {
if j.Type == ObjectType && j.ObjectV2.InlineData() {
flags |= xlFlagInlineData
}
var ecM, ecN uint8
if j.Type == ObjectType && j.ObjectV2 != nil {
ecM, ecN = uint8(j.ObjectV2.ErasureM), uint8(j.ObjectV2.ErasureN)
}
return xlMetaV2VersionHeader{
VersionID: j.getVersionID(),
ModTime: j.getModTime().UnixNano(),
Signature: j.getSignature(),
Type: j.Type,
Flags: flags,
EcN: ecM,
EcM: ecN,
}
}
@@ -440,7 +453,7 @@ func (j *xlMetaV2Version) ToFileInfo(volume, path string, allParts bool) (fi Fil
}
const (
xlHeaderVersion = 2
xlHeaderVersion = 3
xlMetaVersion = 2
)
cmd/xl-storage-format-v2_gen.go
+41 -9
View File
@@ -2198,8 +2198,8 @@ func (z *xlMetaV2VersionHeader) DecodeMsg(dc *msgp.Reader) (err error) {
err = msgp.WrapError(err)
return
}
if zb0001 != 5 {
err = msgp.ArrayError{Wanted: 5, Got: zb0001}
if zb0001 != 7 {
err = msgp.ArrayError{Wanted: 7, Got: zb0001}
return
}
err = dc.ReadExactBytes((z.VersionID)[:])
@@ -2235,13 +2235,23 @@ func (z *xlMetaV2VersionHeader) DecodeMsg(dc *msgp.Reader) (err error) {
}
z.Flags = xlFlags(zb0003)
}
z.EcM, err = dc.ReadUint8()
if err != nil {
err = msgp.WrapError(err, "EcM")
return
}
z.EcN, err = dc.ReadUint8()
if err != nil {
err = msgp.WrapError(err, "EcN")
return
}
return
}
// EncodeMsg implements msgp.Encodable
func (z *xlMetaV2VersionHeader) EncodeMsg(en *msgp.Writer) (err error) {
// array header, size 5
err = en.Append(0x95)
// array header, size 7
err = en.Append(0x97)
if err != nil {
return
}
@@ -2270,19 +2280,31 @@ func (z *xlMetaV2VersionHeader) EncodeMsg(en *msgp.Writer) (err error) {
err = msgp.WrapError(err, "Flags")
return
}
err = en.WriteUint8(z.EcM)
if err != nil {
err = msgp.WrapError(err, "EcM")
return
}
err = en.WriteUint8(z.EcN)
if err != nil {
err = msgp.WrapError(err, "EcN")
return
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z *xlMetaV2VersionHeader) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// array header, size 5
o = append(o, 0x95)
// array header, size 7
o = append(o, 0x97)
o = msgp.AppendBytes(o, (z.VersionID)[:])
o = msgp.AppendInt64(o, z.ModTime)
o = msgp.AppendBytes(o, (z.Signature)[:])
o = msgp.AppendUint8(o, uint8(z.Type))
o = msgp.AppendUint8(o, uint8(z.Flags))
o = msgp.AppendUint8(o, z.EcM)
o = msgp.AppendUint8(o, z.EcN)
return
}
@@ -2294,8 +2316,8 @@ func (z *xlMetaV2VersionHeader) UnmarshalMsg(bts []byte) (o []byte, err error) {
err = msgp.WrapError(err)
return
}
if zb0001 != 5 {
err = msgp.ArrayError{Wanted: 5, Got: zb0001}
if zb0001 != 7 {
err = msgp.ArrayError{Wanted: 7, Got: zb0001}
return
}
bts, err = msgp.ReadExactBytes(bts, (z.VersionID)[:])
@@ -2331,12 +2353,22 @@ func (z *xlMetaV2VersionHeader) UnmarshalMsg(bts []byte) (o []byte, err error) {
}
z.Flags = xlFlags(zb0003)
}
z.EcM, bts, err = msgp.ReadUint8Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "EcM")
return
}
z.EcN, bts, err = msgp.ReadUint8Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "EcN")
return
}
o = bts
return
}
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *xlMetaV2VersionHeader) Msgsize() (s int) {
s = 1 + msgp.ArrayHeaderSize + (16 * (msgp.ByteSize)) + msgp.Int64Size + msgp.ArrayHeaderSize + (4 * (msgp.ByteSize)) + msgp.Uint8Size + msgp.Uint8Size
s = 1 + msgp.ArrayHeaderSize + (16 * (msgp.ByteSize)) + msgp.Int64Size + msgp.ArrayHeaderSize + (4 * (msgp.ByteSize)) + msgp.Uint8Size + msgp.Uint8Size + msgp.Uint8Size + msgp.Uint8Size
return
}