data.vstor #

compression / encryption

• for now only as supported natively in vlang
• encryption is done by the blake192 hash
• encryption after compression

fn cache_get(zdb u32) !ZDBSTAT

fn new() !VSTOR

fn slice_decode(zo Slice) ![]u8

fetch the object from ZDB's decode the data back to binary stream

fn slice_new(data []u8, parts u8, parity u8) !Slice

encode data to ZStor Objecta parts is the relevant parts of data e.g. 16 parity is the overhead e.g. 4

struct FileMeta {
mut:
	slices []SliceMeta
pub mut:
	name             string
	hash             string // blake 192
	compression_type CompressionType
	encryption_type  EncryptionType
}

the metadata as required for retrieving info from set of ZDB's this is the metadata which needs to be kept, otherwise data cannot be retrieved

struct Location {
pub mut:
	farmid      string // as in TFChain
	name        string
	description string
}

Todo: define what is good way to specify a location, what do we have in TFChain

struct Slice {
pub mut:
	data   [][]u8
	parity u8 // nr of parity parts
	crc32  []u32
}

the base object which is encoded and stored on set of ZDB is the data part, part of a file

fn (mut zo Slice) check(crc bool) !

verify that the dataobject has properly constructuted if crc set, then will recalculate crc to verify

struct SliceMeta {
pub mut:
	zdbids      []u32 // list of ZDB's used = the position in VSTOR.zdbs
	zdbposition []u32 // the position in the ZDB (the key as u32)
	crc32       []u32 // crc32 of each part in the zdb
}

the metadata as required for retrieving info from set of ZDB's

struct VSTOR {
pub mut:
	zdbs           []ZDB
	current_spread ZDBCurrentSpread
}

fn (mut vstor VSTOR) location_new(args Location) !Location

fn (mut vstor VSTOR) monitor() !

start a thread which will every hour do a check of all the known ZDBs info will be cached in redis eventually we will monitor other things too, so we know when system is no longer good

fn (mut vstor VSTOR) spread_get(zdbid u32) !ZDBCurrentSpread

walk over the zdb stats, figure out which zdb's are best return those

fn (mut vstor VSTOR) stor(mut path pathlib.Path) !

cut file in slices depending the current_spread encode & store the parts in the ZStor

fn (mut vstor VSTOR) zdb_new(args ZDB) !ZDB

fn (mut vstor VSTOR) zdbstats_check() !

fn (mut vstor VSTOR) zdbstats_get(zdbid u32) !ZDBSTAT

return ZDBSTAT object

struct ZDB {
pub mut:
	location  Location
	address   string
	port      u32
	secret    string
	namespace string
	stats     []ZDBSTAT
}

import freeflowuniverse.crystallib.core.pathlib import time

struct ZDBCurrentSpread {
pub mut:
	spread           []u32 // zdb's we are using
	parts            u8    // nr of parts data needs to be cut in e.g. 16
	parity           u8    // size of parity e.g. 4
	last_check       time.Time
	slice_size_kb    u32 // default 4 MB
	compression_type CompressionType
	encryption_type  EncryptionType
}

is the current spread as used when storing new data to ZDB's

struct ZDBMeasurement {
pub mut:
	status              ZDBSTATUS
	latencies           u32 // in millisecs
	storage_capacity_gb u32 // amount of data stored in GB
	storage_used_gb     u32 // amount of data used in the zdb
	time                time.Time
}

struct ZDBSTAT {
pub mut:
	id           u32 // links to id of ZDB in VSTOR.zdbs
	measurements []ZDBMeasurement
}

fn (mut stat ZDBSTAT) check() !

execute on a check, remote poll the ZDB which returns the info only keep 12 checks

fn (mut stat ZDBSTAT) latency() u32

return latency averaged

fn (mut stat ZDBSTAT) storage_capacity() u32

fn (mut stat ZDBSTAT) storage_used() u32

fn (mut stat ZDBSTAT) percent_free() u8

return 100 for full, 0 for empty, 10 for 10%

fn (mut stat ZDBSTAT) cache_set() !

fn (mut stat ZDBSTAT) cache_get() !