Tools to work with docker

How to install a docker machine

TODO: make examples how to deploy a docker machine

TODO: make examples how to use docker compose, make sure we have nice lib for it

Builders

see https://github.com/threefoldtech/vbuilders/tree/development/builders for examples

How to get started see

https://github.com/threefoldtech/vbuilders/blob/development/docsrc/src/gettingstarted/ubuntu/ubuntu.md

requirements

can only use a node.executor to execute what is required

fn new(args DockerEngineArgs) !DockerEngine

if sshkeys_allowed empty array will check the local machine for loaded sshkeys

enum BuildPlatformType {
	linux_arm64
	linux_amd64
}

enum PlatformType {
	alpine
	ubuntu
}

only 2 supported for now

struct AddFileEmbeddedArgs {
pub mut:
	source          string // is the filename, needs to be embedded
	dest            string // in the container we're building
	make_executable bool
}

struct AddFileEmbeddedItem {
pub mut:
	source          string
	dest            string // in the container we're building
	recipe          &DockerBuilderRecipe @[str: skip]
	make_executable bool
	check_embed     bool = true
}

fn (mut i AddFileEmbeddedItem) check() !

fn (mut i AddFileEmbeddedItem) render() !string

struct BuildArgs {
pub mut:
	reset  bool // resets the docker state, will build again
	strict bool // question: ?
	engine &DockerEngine
	args   Params
}

struct CmdArgs {
pub mut:
	cmd string
}

struct CmdItem {
pub mut:
	cmd    string
	recipe &DockerBuilderRecipe @[str: skip]
}

fn (mut i CmdItem) check() !

fn (mut i CmdItem) render() !string

struct CodeGetArgs {
pub mut:
	url string // e.g.  https://github.com/vlang/v
	// other example url := 'https://github.com/threefoldfoundation/www_examplesite/tree/development/manual'
	pull  bool
	reset bool
	name  string
	dest  string // where does the directory need to be checked out to
}

struct ComposeArgs {
pub mut:
	name        string @[required]
	params      string
	composepath string // can be empty, if empty will be buildpath/compose
}

struct ComposeService {
pub mut:
	name    string
	content string // optional
	params  Params
	files   []embed_file.EmbedFileData
	recipe  &DockerComposeRecipe @[str: skip]
	render  bool = true
	env     map[string]string
	ports   []PortMap
	volumes []VolumeMap
	restart bool
	image   string
}

fn (mut cs ComposeService) env_add(key string, val string)

add an environment variable to the service

fn (mut cs ComposeService) restart_set()

make sure the service always restarts

fn (mut cs ComposeService) port_expose(indocker int, host int) !

make sure the service always restarts

fn (mut cs ComposeService) volume_add(hostpath string, containerpath string) !

make sure the service always restarts

struct ComposeServiceArgs {
pub:
	name    string @[required]
	image   string @[required]
	params  string
	content string // optional, if this is set then will not render
}

struct ContainerGetArgs {
pub mut:
	name     string
	id       string
	image_id string
	// tag    string
	// digest string
}

EXISTS, GET

struct ContainerGetError {
	Error
pub:
	args     ContainerGetArgs
	notfound bool
	toomany  bool
}

fn (err ContainerGetError) msg() string

fn (err ContainerGetError) code() int

struct CopyArgs {
pub mut:
	from            string
	source          string
	dest            string
	make_executable bool // if set will make the file copied executable
}

struct CopyItem {
pub mut:
	from            string
	source          string
	dest            string
	recipe          &DockerBuilderRecipe @[str: skip]
	make_executable bool // if set will make the file copied executable
	// check_embed bool = true
}

fn (mut i CopyItem) check() !

fn (mut i CopyItem) render() !string

struct DockerBuilderRecipe {
pub mut:
	name     string
	prefix   string
	tag      string
	params   Params
	files    []embed_file.EmbedFileData
	engine   &DockerEngine @[str: skip]
	items    []RecipeItem
	platform PlatformType
	zinit    bool
}

fn (mut b DockerBuilderRecipe) add_cmd(args CmdArgs) !

add run command to docker, is the cmd which is run when docker get's built

fn (mut r DockerBuilderRecipe) add_codeget(args_ CodeGetArgs) !

checkout a code repository on right location

fn (mut b DockerBuilderRecipe) add_copy(args CopyArgs) !

to do something like: 'Add alpine:latest'

fn (mut r DockerBuilderRecipe) add_download(args_ DownloadArgs) !

checkout a code repository on right location

fn (mut b DockerBuilderRecipe) add_entrypoint(args EntryPointArgs) !

fn (mut b DockerBuilderRecipe) add_env(name string, val string) !

fn (mut b DockerBuilderRecipe) add_expose(args ExposeARgs) !

to do something like: 'Expose 8080/udp'

fn (mut b DockerBuilderRecipe) add_file_embedded(args AddFileEmbeddedArgs) !

to do something like: 'Add alpine:latest'

fn (mut b DockerBuilderRecipe) add_from(args FromArgs) !

to do something like: 'FROM alpine:latest'

fn (mut r DockerBuilderRecipe) add_go_package(args GoPackageArgs) !

install go components

fn (mut r DockerBuilderRecipe) add_gobuild_from_code(args GoBuildArgs) !

do a build of a go package . will get the code, pull and/or reset . will the pull all dependencies . will do the build DEBUG TRICK: put debug flag on, which will not execute the build cmd . you can go to /tmp/build/buildname and do shell.sh to debug

fn (mut r DockerBuilderRecipe) add_gobuilder() !

fn (mut r DockerBuilderRecipe) add_nodejsbuilder() !

fn (mut b DockerBuilderRecipe) add_package(args PackageArgs) !

add one of more name (alpine packages), no need to do update, upgrade first,

fn (mut b DockerBuilderRecipe) add_run(args RunArgs) !

to do something like: 'FROM alpine:latest'

fn (mut r DockerBuilderRecipe) add_rust_package(args RustPackageArgs) !

use cargo install to install rust components

fn (mut r DockerBuilderRecipe) add_rustbuild_from_code(args RustBuildArgs) !

do a build of a rust package . will get the code, pull and/or reset . will the pull all dependencies . will do the build DEBUG TRICK: put debug flag on, which will not execute the build cmd . you can go to /tmp/build/buildname and do shell.sh to debug

fn (mut r DockerBuilderRecipe) add_sshserver() !

add ssh server and init scripts (note: zinit needs to be installed)

fn (mut r DockerBuilderRecipe) add_vbuilder() !

fn (mut b DockerBuilderRecipe) add_volume(args VolumeArgs) !

to do something like: 'Volume /data'

fn (mut b DockerBuilderRecipe) add_workdir(args WorkDirArgs) !

to do something like: 'FROM alpine:latest'

fn (mut r DockerBuilderRecipe) add_zinit() !

import freeflowuniverse.crystallib.develop.gittools import freeflowuniverse.crystallib.core.pathlib

fn (mut b DockerBuilderRecipe) add_zinit_cmd(args ZinitAddArgs) !

add a zinit to the docker container each init will launch a process in the container the init files are rendered as yaml files in the build directory and added to the docker at the end how to use zinit see: https://github.com/threefoldtech/zinit/tree/master/docs

fn (mut b DockerBuilderRecipe) build(reset bool) !

fn (mut b DockerBuilderRecipe) delete() !

delete the working directory

fn (mut r DockerBuilderRecipe) execute(args ExecuteArgs) !

execute the file as embedded

fn (mut b DockerBuilderRecipe) path() string

fn (mut b DockerBuilderRecipe) render() !string

fn (mut r DockerBuilderRecipe) write_file(args WriteFileArgs) !

the content will be written to dest all trailing spaces will be removed (dedent)

struct DockerComposeRecipe {
pub mut:
	name    string
	content string
	params  Params
	engine  &DockerEngine @[str: skip]
	items   []&ComposeService
	path    string
}

fn (mut b DockerComposeRecipe) delete() !

fn (mut recipe DockerComposeRecipe) service_new(args ComposeServiceArgs) !&ComposeService

fn (mut b DockerComposeRecipe) start() !

fn (mut b DockerComposeRecipe) stop() !

struct DockerContainer {
pub mut:
	id              string
	name            string
	created         time.Time
	ssh_enabled     bool // if yes make sure ssh is enabled to the container
	ipaddr          IPAddress
	forwarded_ports []string
	mounts          []utils.ContainerVolume
	ssh_port        int // ssh port on node that is used to get ssh
	ports           []string
	networks        []string
	labels          map[string]string @[str: skip]
	image           &DockerImage      @[str: skip]
	engine          &DockerEngine     @[str: skip]
	status          utils.ContainerStatus
	memsize         int // in MB
	command         string
}

need to fill in what is relevant

fn (mut container DockerContainer) start() !

create/start container (first need to get a dockercontainer before we can start)

fn (mut container DockerContainer) halt() !

delete docker container

fn (mut container DockerContainer) delete() !

delete docker container

fn (mut container DockerContainer) save2image(image_repo string, image_tag string) !string

save the docker container to image

fn (mut container DockerContainer) export(path string) !

export docker to tgz

fn (mut container DockerContainer) shell(args DockerShellArgs) !

open shell to the container using docker, is interactive, cannot use in script

fn (mut container DockerContainer) execute(cmd_ string, silent bool) !

struct DockerContainerCreateArgs {
	name             string
	hostname         string
	forwarded_ports  []string          // ["80:9000/tcp", "1000, 10000/udp"]
	mounted_volumes  []string          // ["/root:/root", ]
	env              map[string]string // map of environment variables that will be passed to the container
	privileged       bool
	remove_when_done bool = true // remove the container when it shuts down
pub mut:
	image_repo string
	image_tag  string
	command    string = '/bin/bash'
}

struct DockerEngine {
	name string
pub mut:
	sshkeys_allowed []string // all keys here have access over ssh into the machine, when ssh enabled
	images          []DockerImage
	containers      []DockerContainer
	buildpath       string
	localonly       bool
	cache           bool = true
	push            bool
	platform        []BuildPlatformType // used to build
	registries      []DockerRegistry    // one or more supported DockerRegistries
	prefix          string
}

https://docs.docker.com/reference/

fn (mut e DockerEngine) compose_new(args_ ComposeArgs) DockerComposeRecipe

fn (mut e DockerEngine) container_create(args DockerContainerCreateArgs) !&DockerContainer

fn (mut e DockerEngine) container_delete(args ContainerGetArgs) !

fn (mut e DockerEngine) container_exists(args ContainerGetArgs) !bool

fn (mut e DockerEngine) container_get(args_ ContainerGetArgs) !&DockerContainer

get container from memory, can use match_glob see https://modules.vlang.io/index.html#string.match_glob

fn (mut e DockerEngine) container_import(path string, mut args DockerContainerCreateArgs) !&DockerContainer

import a container into an image, run docker container with it image_repo examples ['myimage', 'myimage:latest'] if DockerContainerCreateArgs contains a name, container will be created and restarted

fn (mut e DockerEngine) containers_delete(args ContainerGetArgs) !

remove one or more container

fn (mut e DockerEngine) containers_get(args_ ContainerGetArgs) ![]&DockerContainer

get containers from memory params: name string (can also be a glob e.g. use *,? and []) id string image_id string

fn (mut e DockerEngine) containers_load() !

load all containers, they can be consulted in e.containers see obj: DockerContainer as result in e.containers

fn (mut e DockerEngine) get_free_port() ?int

Get free port

fn (mut e DockerEngine) image_exists(args ImageGetArgs) !bool

fn (mut e DockerEngine) image_get(args ImageGetArgs) !&DockerImage

find image based on repo and optional tag args: repo string tag string digest string id string

fn (mut e DockerEngine) init() !

check docker has been installed & enabled on node

fn (mut e DockerEngine) load() !

reload the state from system

fn (mut e DockerEngine) recipe_new(args RecipeArgs) DockerBuilderRecipe

params

name     string
prefix 	 string (e.g. despiegk/ or myimage registry-host:5000/despiegk/) is added to the name when pushing
                    also exists on docker engine level, if not used here will come from the docker engine level
tag		 string (default is 'latest' but can be a version nr)
platform PlatformType
zinit    bool = true

fn (mut e DockerEngine) registry_add(args DockerRegistryArgs) !

check docker has been installed & enabled on node

fn (mut e DockerEngine) reset_all() !

reset all images & containers, CAREFUL!

struct DockerEngineArgs {
pub mut:
	sshkeys_allowed []string // ssh keys which will be inserted when a docker gets deployed (is not implemented yet)
	name            string = 'default'
	localonly       bool   // do you build for local utilization only
	prefix          string // e.g. despiegk/ or myimage registry-host:5000/despiegk/) is added to the name when pushing	
}

struct DockerImage {
pub mut:
	repo    string
	id      string
	tag     string
	digest  string
	size    int // size in MB
	created time.Time
	engine  &DockerEngine @[str: skip]
}

fn (mut image DockerImage) delete(force bool) !

delete docker image

fn (mut image DockerImage) export(path string) !string

export docker image to tar.gz

fn (mut image DockerImage) load(path string) !

import docker image back into the local env

struct DockerRegistry {
pub mut:
	name     string = 'default'
	datapath string
	ssl      bool
}

struct DockerRegistryArgs {
pub mut:
	name      string = 'default' @[required]
	datapath  string
	ssl       bool
	reset     bool // if reset will reset existing running one
	reset_ssl bool // if reset will reset the sslkey
	secret    string = '1234' @[required]
}

struct DockerShellArgs {
pub mut:
	cmd string
}

// open ssh shell to the cobtainer pub fn (mut container DockerContainer) ssh_shell(cmd string) ! { container.engine.node.shell(cmd)! }

struct DownloadArgs {
pub mut:
	name       string // unique name which will be used in the cache
	url        string
	reset      bool // will remove
	minsize_kb u32 = 10 // is always in kb
	maxsize_kb u32
	hash       string // if hash is known, will verify what hash is
	dest       string // if specified will copy to that destination	
	timeout    int = 180
	retry      int = 3
}

struct EntryPointArgs {
pub mut:
	cmd string
}

import freeflowuniverse.crystallib.builder

struct EntryPointItem {
pub mut:
	cmd    string
	recipe &DockerBuilderRecipe @[str: skip]
}

fn (mut i EntryPointItem) check() !

fn (mut i EntryPointItem) render() !string

struct EnvItem {
pub mut:
	name  string
	value string
}

fn (mut i EnvItem) check() !

fn (mut i EnvItem) render() !string

struct ExecuteArgs {
pub mut:
	source string // is the filename, needs to be embedded
	debug  bool
}

struct ExposeARgs {
pub mut:
	ports []string
}

struct ExposeItem {
pub mut:
	ports  []string
	recipe &DockerBuilderRecipe @[str: skip]
}

fn (mut i ExposeItem) check() !

fn (mut i ExposeItem) render() !string

struct FromArgs {
pub mut:
	image string
	tag   string
	alias string
}

struct FromItem {
pub mut:
	image  string
	tag    string
	recipe &DockerBuilderRecipe @[str: skip]
	alias  string
}

fn (mut i FromItem) check() !

fn (mut i FromItem) render() !string

struct GoBuildArgs {
pub mut:
	url      string // e.g.  https://github.com/valeriansaliou/sonic
	pull     bool
	reset    bool
	buildcmd string = 'go run build.go '
	copycmd  string
	name     string
}

struct GoPackageArgs {
pub mut:
	name    string // can be comma separated, can also be url e.g. github.com/caddyserver/xcaddy/cmd/xcaddy@latest
	postcmd string // normally empty
}

struct ImageGetArgs {
pub:
	repo   string
	tag    string
	digest string
	id     string
}

struct ImageGetError {
	Error
pub:
	args     ImageGetArgs
	notfound bool
	toomany  bool
}

fn (err ImageGetError) msg() string

fn (err ImageGetError) code() int

struct PackageArgs {
pub mut:
	name  string
	names []string
}

struct PackageItem {
pub mut:
	names    []string
	recipe   &DockerBuilderRecipe @[str: skip]
	platform PlatformType
}

fn (mut i PackageItem) check() !

fn (mut i PackageItem) render() !string

struct PortMap {
pub:
	indocker int
	host     int
}

struct RecipeArgs {
pub:
	name   string
	prefix string // string (e.g. despiegk/ or myimage registry-host:5000/despiegk/) is added to the name when pushing
	// also exists on docker engine level, if not used here will come from the docker engine level
	tag      string //(default is 'latest' but can be a version nr)
	platform PlatformType
	zinit    bool = true
}

struct RunArgs {
pub mut:
	cmd string
}

struct RunItem {
pub mut:
	cmd    string
	recipe &DockerBuilderRecipe @[str: skip]
}

fn (mut i RunItem) check() !

fn (mut i RunItem) render() !string

struct RustBuildArgs {
pub mut:
	url      string // e.g.  https://github.com/valeriansaliou/sonic
	pull     bool
	reset    bool
	buildcmd string = 'cargo build --release'
	copycmd  string
	debug    bool // to be able to easily debug the intermediate step
	name     string
}

import freeflowuniverse.crystallib.develop.gittools import freeflowuniverse.crystallib.core.pathlib

struct RustPackageArgs {
pub mut:
	name    string // can be comma separated
	copycmd string // normally empty
}

struct VolumeArgs {
pub mut:
	mount_points []string
}

struct VolumeItem {
pub mut:
	mount_points []string
	recipe       &DockerBuilderRecipe @[str: skip]
}

fn (mut i VolumeItem) check() !

fn (mut i VolumeItem) render() !string

struct VolumeMap {
pub:
	hostpath      string
	containerpath string
}

fn (mut vm VolumeMap) check() !

struct WorkDirArgs {
pub mut:
	workdir string
}

struct WorkDirItem {
pub mut:
	workdir string
	recipe  &DockerBuilderRecipe @[str: skip]
}

fn (mut i WorkDirItem) check() !

fn (mut i WorkDirItem) render() !string

struct WriteFileArgs {
pub mut:
	name            string // such a file needs to have a name
	content         string
	dest            string // where the content will be saved
	make_executable bool   // if true then will be made executable
}

struct ZinitAddArgs {
pub mut:
	// more info see https://github.com/threefoldtech/zinit/tree/master/docs
	name    string
	exec    string // e.g. redis-server --port 7777
	test    string // e.g. redis-cli -p 7777 PING
	after   string // e.g. 'copy,tmuxp'  these are the names of the ones we are depending on, is comma separated
	log     ZinitLogType
	signal  string // could be 'SIGKILL' as well, if not specified then 'SIGTERM'
	oneshot bool   // is the command supposed to stop and only be used one, so not as daemon
}

struct ZinitItem {
pub mut:
	name    string
	exec    string   // e.g. redis-server --port 7777
	test    string   // e.g. redis-cli -p 7777 PING
	after   []string // e.g. copy,tmuxp  these are the names of the ones we are depending on
	log     ZinitLogType
	signal  string // could be 'SIGKILL' as well, if not specified then 'SIGTERM'
	oneshot bool   // is the command supposed to stop and only be used one, so not as daemon	
}

fn (mut i ZinitItem) check() !

fn (mut i ZinitItem) render() !string