Skip to content

data.encoder #

V Binary Encoder/Decoder

A high-performance binary encoder/decoder module for V that provides efficient serialization and deserialization of data structures. The encoder supports automatic encoding/decoding of structs using V's compile-time reflection capabilities.

Features

  • Automatic struct encoding/decoding using compile-time reflection
  • Support for primitive types, arrays, maps, and nested structs
  • Compact binary format with length prefixing
  • Size limits to prevent memory issues (64KB for strings/lists)
  • Comprehensive error handling
  • Built-in versioning support

Format

The binary format starts with a version byte (currently v1), followed by the encoded data:

[version_byte][encoded_data...]

Supported Types

Primitive Types

  • string
  • int (32-bit)
  • u8
  • u16
  • u32
  • u64
  • time.Time

Arrays

  • []string
  • []int
  • []u8
  • []u16
  • []u32
  • []u64

Maps

  • map[string]string
  • map[string][]u8

Structs

  • Nested struct support with automatic encoding/decoding

Usage

Basic Encoding

import freeflowuniverse.crystallib.data.encoder

// Create a new encoder
mut e := encoder.new()

// Add primitive values
e.add_string('hello')
e.add_int(42)
e.add_u8(255)
e.add_u16(65535)
e.add_u32(4294967295)
e.add_u64(18446744073709551615)

// Add arrays
e.add_list_string(['one', 'two', 'three'])
e.add_list_int([1, 2, 3])

// Add maps
e.add_map_string({
    'key1': 'value1'
    'key2': 'value2'
})

// Get encoded bytes
encoded := e.data

Basic Decoding

// Create decoder from bytes
mut d := encoder.decoder_new(encoded)

// Read values in same order as encoded
str := d.get_string()
num := d.get_int()
byte := d.get_u8()
u16_val := d.get_u16()
u32_val := d.get_u32()
u64_val := d.get_u64()

// Read arrays
strings := d.get_list_string()
ints := d.get_list_int()

// Read maps
str_map := d.get_map_string()

Automatic Struct Encoding/Decoding

struct Person {
    name string
    age  int
    tags []string
    meta map[string]string
}

// Create struct instance
person := Person{
    name: 'John'
    age: 30
    tags: ['developer', 'v']
    meta: {
        'location': 'NYC'
        'role': 'engineer'
    }
}

// Encode struct
encoded := encoder.encode(person)!

// Decode back to struct
decoded := encoder.decode[Person](encoded)!

Example

Here's a complete example showing how to encode nested structs:

import freeflowuniverse.crystallib.data.encoder

// Define some nested structs
struct Address {
    street string
    number int
    country string
}

struct Person {
    name string
    age int
    addresses []Address    // nested array of structs
    metadata map[string]string
}

// Example usage
fn main() {
    // Create test data
    mut person := Person{
        name: 'John Doe'
        age: 30
        addresses: [
            Address{
                street: 'Main St'
                number: 123
                country: 'USA'
            },
            Address{
                street: 'Side St'
                number: 456
                country: 'Canada'
            }
        ]
        metadata: {
            'id': 'abc123'
            'type': 'customer'
        }
    }

    // Encode the data
    mut e := encoder.new()

    // Add version byte (v1)
    e.add_u8(1)

    // Encode the Person struct
    e.add_string(person.name)
    e.add_int(person.age)

    // Encode the addresses array
    e.add_u16(u16(person.addresses.len))  // number of addresses
    for addr in person.addresses {
        e.add_string(addr.street)
        e.add_int(addr.number)
        e.add_string(addr.country)
    }

    // Encode the metadata map
    e.add_map_string(person.metadata)

    // The binary data is now in e.data
    encoded := e.data

    // Later, when decoding, first byte tells us the version
    version := encoded[0]
    assert version == 1
}

Binary Format Details

For the example above, the binary layout would be:

[1]                     // version byte (v1)
[len][John Doe]         // name (u16 length + bytes)
[30]                    // age (int/u32)
[2]                     // number of addresses (u16)
  [len][Main St]        // address 1 street
  [123]                 // address 1 number
  [len][USA]           // address 1 country
  [len][Side St]       // address 2 street
  [456]                // address 2 number
  [len][Canada]        // address 2 country
[2]                     // number of metadata entries (u16)
  [len][id]            // key 1
  [len][abc123]        // value 1
  [len][type]          // key 2
  [len][customer]      // value 2

Implementation Details

Binary Format

The encoded data follows this format:

  1. For strings:
  • u16 length prefix
  • raw string bytes
  1. For arrays:
  • u16 length prefix
  • encoded elements
  1. For maps:
  • u16 count of entries
  • encoded key-value pairs

fn decode #

fn decode[T](data []u8) !T

fn decoder_new #

fn decoder_new(data []u8) Decoder

fn encode #

fn encode[T](obj T) ![]u8

example see https://github.com/vlang/v/blob/master/examples/compiletime/reflection.v

fn new #

fn new() Encoder

enum DataType{ string int bytes u8 u16 u32 u64 time list_string list_int list_u8 list_u16 list_u32 list_u64 map_string map_bytes }

struct Decoder #

struct Decoder {
pub mut:
	version u8 = 1 // is important
	data    []u8
}

fn (Decoder) get_string #

fn (mut d Decoder) get_string() string

fn (Decoder) get_int #

fn (mut d Decoder) get_int() int

fn (Decoder) get_bytes #

fn (mut d Decoder) get_bytes() []u8

fn (Decoder) get_u8 #

fn (mut d Decoder) get_u8() u8

adds u16 length of string in bytes + the bytes

fn (Decoder) get_u16 #

fn (mut d Decoder) get_u16() u16

fn (Decoder) get_u32 #

fn (mut d Decoder) get_u32() u32

fn (Decoder) get_u64 #

fn (mut d Decoder) get_u64() u64

fn (Decoder) get_i64 #

fn (mut d Decoder) get_i64() i64

fn (Decoder) get_time #

fn (mut d Decoder) get_time() time.Time

fn (Decoder) get_ourtime #

fn (mut d Decoder) get_ourtime() ourtime.OurTime

fn (Decoder) get_list_string #

fn (mut d Decoder) get_list_string() []string

fn (Decoder) get_list_int #

fn (mut d Decoder) get_list_int() []int

fn (Decoder) get_list_u8 #

fn (mut d Decoder) get_list_u8() []u8

fn (Decoder) get_list_u16 #

fn (mut d Decoder) get_list_u16() []u16

fn (Decoder) get_list_u32 #

fn (mut d Decoder) get_list_u32() []u32

fn (Decoder) get_list_u64 #

fn (mut d Decoder) get_list_u64() []u64

fn (Decoder) get_map_string #

fn (mut d Decoder) get_map_string() map[string]string

fn (Decoder) get_map_bytes #

fn (mut d Decoder) get_map_bytes() map[string][]u8

struct Encoder #

struct Encoder {
pub mut:
	data []u8
	// datatypes []DataType
}

fn (Encoder) add_string #

fn (mut b Encoder) add_string(data string)

adds u16 length of string in bytes + the bytes

fn (Encoder) add_int #

fn (mut b Encoder) add_int(data int)

Please note that unlike C and Go, int is always a 32 bit integer. We borrow the add_u32() function to handle the encoding of a 32 bit type

fn (Encoder) add_bytes #

fn (mut b Encoder) add_bytes(data []u8)

add bytes or bytestring

fn (Encoder) add_u8 #

fn (mut b Encoder) add_u8(data u8)

fn (Encoder) add_u16 #

fn (mut b Encoder) add_u16(data u16)

fn (Encoder) add_u32 #

fn (mut b Encoder) add_u32(data u32)

fn (Encoder) add_u64 #

fn (mut b Encoder) add_u64(data u64)

fn (Encoder) add_i64 #

fn (mut b Encoder) add_i64(data i64)

fn (Encoder) add_time #

fn (mut b Encoder) add_time(data time.Time)

fn (Encoder) add_ourtime #

fn (mut b Encoder) add_ourtime(data ourtime.OurTime)

fn (Encoder) add_list_string #

fn (mut b Encoder) add_list_string(data []string)

fn (Encoder) add_list_int #

fn (mut b Encoder) add_list_int(data []int)

fn (Encoder) add_list_u8 #

fn (mut b Encoder) add_list_u8(data []u8)

fn (Encoder) add_list_u16 #

fn (mut b Encoder) add_list_u16(data []u16)

fn (Encoder) add_list_u32 #

fn (mut b Encoder) add_list_u32(data []u32)

fn (Encoder) add_list_u64 #

fn (mut b Encoder) add_list_u64(data []u64)

fn (Encoder) add_map_string #

fn (mut b Encoder) add_map_string(data map[string]string)

when complicated hash e.g. map of other object need to serialize each sub object

fn (Encoder) add_map_bytes #

fn (mut b Encoder) add_map_bytes(data map[string][]u8)

when complicated hash e.g. map of other object need to serialize each sub object