Skip to content
Snippets Groups Projects
Unverified Commit 8cb3fded authored by Nic Jackson's avatar Nic Jackson
Browse files

Update tests and images

parent 1f106420
No related branches found
No related tags found
No related merge requests found
Showing with 614 additions and 28 deletions
......@@ -70,3 +70,12 @@ jobs:
KUBERNETES_VERSION=${{steps.collect.outputs.version}}
push: true
tags: ghcr.io/jumppad-labs/kubernetes:${{steps.collect.outputs.version}}
- name: Test built image
if: steps.collect.outputs.version != steps.collect.outputs.current
run: |-
# Install Jumppad
curl https://jumppad.dev/install | bash
# Test the image
jumppad test --tags @Single --var version=${{steps.collect.outputs.version}} ./kubernetes/tests
\ No newline at end of file
......@@ -75,3 +75,12 @@ jobs:
NOMAD_VERSION=${{steps.version.outputs.value}}
push: true
tags: ghcr.io/jumppad-labs/nomad:${{steps.collect.outputs.version}}
- name: Test built image
if: steps.collect.outputs.version != steps.collect.outputs.current
run: |-
# Install Jumppad
curl https://jumppad.dev/install | bash
# Test the image
jumppad test --tags @Single --var version=${{steps.collect.outputs.version}} ./nomad/tests
\ No newline at end of file
......@@ -7,20 +7,13 @@ ENV container=docker LANG=C.UTF-8
RUN apt-get update && \
apt-get install -y -o APT::Install-Suggests=0 -o APT::Install-Recommends=0 \
software-properties-common supervisor rsyslog iproute2 python3 python3-apt sudo bash ca-certificates \
curl net-tools jq unzip vim dnsutils iputils-ping socat wget
# Setup bash
RUN sed -i 's/\/bin\/ash/\/bin\/bash/g' /etc/passwd
# Setup supervisor
COPY files/supervisord.conf /etc/supervisor/supervisord.conf
ca-certificates curl gnupg lsb-release
# Setup docker
RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg && \
RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg && \
echo \
"deb [arch=${TARGETARCH} signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
$(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null && \
$(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null && \
apt-get update && \
apt-get install -y -o APT::Install-Suggests=0 -o APT::Install-Recommends=0 docker-ce docker-ce-cli containerd.io && \
apt-get clean && rm -rf /var/lib/apt/lists/*
......@@ -29,13 +22,11 @@ RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor
ENV DIND_COMMIT d58df1fc6c866447ce2cd129af10e5b507705624
RUN set -eux; \
wget -O /usr/local/bin/dind "https://raw.githubusercontent.com/docker/docker/${DIND_COMMIT}/hack/dind"; \
curl -o /usr/local/bin/dind "https://raw.githubusercontent.com/docker/docker/${DIND_COMMIT}/hack/dind"; \
chmod +x /usr/local/bin/dind
COPY --chmod=0755 files/dockerd.sh /usr/local/bin/dockerd.sh
COPY files/docker.conf /etc/supervisor/conf.d/docker.conf
VOLUME /var/lib/docker
ENTRYPOINT ["/usr/bin/supervisord"]
CMD ["-c", "/etc/supervisor/supervisord.conf"]
\ No newline at end of file
ENTRYPOINT ["/usr/local/bin/dockerd.sh"]
\ No newline at end of file
server:
dataStorage:
size: "128Mb"
dev:
enabled: true
standalone:
enabled: true
authDelegator:
enabled: true
variable "version" {
default = "v1.29.1"
}
resource "network" "cloud" {
subnet = "10.5.0.0/16"
}
resource "k8s_cluster" "k3s" {
image {
name = "ghcr.io/jumppad-labs/kubernetes:${variable.version}"
}
network {
id = resource.network.cloud.meta.id
}
}
resource "helm" "vault" {
cluster = resource.k8s_cluster.k3s
repository {
name = "hashicorp"
url = "https://helm.releases.hashicorp.com"
}
chart = "hashicorp/vault"
version = "v0.18.0"
values = "./files/vault-values.yaml"
health_check {
timeout = "240s"
pods = ["app.kubernetes.io/name=vault"]
}
}
resource "ingress" "vault_http" {
port = 18200
target {
resource = resource.k8s_cluster.k3s
port = 8200
config = {
service = "vault"
namespace = "default"
}
}
}
output "KUBECONFIG" {
value = resource.k8s_cluster.k3s.kube_config.path
}
\ No newline at end of file
Feature: Kubernetes Cluster
In order to test Kubernetes clusters
I should apply a blueprint
And test the output
@Single
Scenario Outline: K3s Cluster
Given I have a running blueprint
Then the following resources should be running
| name |
| resource.network.cloud |
| resource.k8s_cluster.k3s |
And a HTTP call to "http://localhost:18200" should result in status 200
@All
Scenario Outline: K3s Cluster
Given the jumppad variable "version" has a value "<version>"
And I have a running blueprint
Then the following resources should be running
| name |
| resource.network.cloud |
| resource.k8s_cluster.k3s |
And a HTTP call to "http://localhost:18200" should result in status 200
Examples:
| version |
| v1.27.8 |
| v1.29.0 |
| v1.29.1 |
| v1.31.0 |
| v1.31.1 |
main.hcl 0 → 100644
resource "network" "main" {
subnet = "10.5.0.0/16"
}
resource "container" "postgres" {
network {
id = resource.network.main.meta.id
}
image {
name = "postgres:13"
}
environment = {
POSTGRES_PASSWORD = "temporal"
POSTGRES_USER = "temporal"
}
health_check {
timeout = "30s"
exec {
script = "pg_isready"
}
}
port {
local = 5432
host = 5432
}
}
resource "template" "temporal_config" {
source = <<-EOF
limit.maxIDLength:
- value: 255
constraints: {}
system.forceSearchAttributesCacheRefreshOnRead:
- value: true # Dev setup only. Please don't turn this on in production.
constraints: {}
EOF
destination = "${data("temporal")}/config.yaml"
}
resource "container" "temporal" {
network {
id = resource.network.main.meta.id
}
image {
name = "temporalio/auto-setup:1.24.2"
}
environment = {
DB = "postgres12"
ID = resource.container.postgres.meta.id
DB_PORT_3 = "${resource.container.postgres.port.0.local}"
POSTGRES_SEEDS = "postgres"
POSTGRES_USER = "temporal"
POSTGRES_PWD = "temporal"
}
volume {
source = data("temporal")
destination = "/etc/temporal/config/dynamicconfig"
}
port {
local = 7233
host = 7233
}
}
\ No newline at end of file
FROM ghcr.io/jumppad-labs/dind:v1.0.0
FROM ghcr.io/jumppad-labs/dind:v1.1.2
ARG NOMAD_VERSION
ARG TARGETARCH
ENV CNI_PLUGINS=1.3.0
ENV CNI_PLUGINS=1.5.1
RUN apt-get update && \
apt-get install -y -o APT::Install-Suggests=0 -o APT::Install-Recommends=0 \
unzip iproute2 \
&& apt-get clean && rm -rf /var/lib/apt/lists/*
RUN curl -L https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_linux_${TARGETARCH}.zip -o /tmp/nomad.zip && \
cd /tmp && \
......@@ -78,4 +83,4 @@ RUN mkdir -p /var/lib/nomad
COPY files/config.hcl /etc/nomad.d/config.hcl
COPY --chmod=0755 files/nomad.sh /usr/local/bin/nomad.sh
COPY files/nomad.conf /etc/supervisor/conf.d/nomad.conf
\ No newline at end of file
ENTRYPOINT ["/usr/local/bin/nomad.sh"]
\ No newline at end of file
[program:nomad]
command=/usr/local/bin/nomad.sh
autorestart=true
priority=300
stopsignal=SIGTERM
stdout_logfile=/dev/stdout
stderr_logfile=/dev/stderr
stdout_logfile_maxbytes=0
stderr_logfile_maxbytes=0
environment=HTTP_PROXY="",HTTPS_PROXY="",NO_PROXY=""
\ No newline at end of file
......@@ -10,4 +10,12 @@ else
update-ca-certificates
fi
exec /usr/bin/nomad agent -config /etc/nomad.d -log-level=DEBUG
\ No newline at end of file
# Start the docker daemon
/usr/local/bin/dockerd.sh &
# Start the nomad agent
/usr/bin/nomad agent -config /etc/nomad.d -log-level=DEBUG &
wait -n
exit $?
\ No newline at end of file
job "example_1" {
datacenters = ["dc1"]
type = "service"
update {
max_parallel = 1
min_healthy_time = "10s"
healthy_deadline = "3m"
progress_deadline = "10m"
auto_revert = false
canary = 0
}
migrate {
max_parallel = 1
health_check = "checks"
min_healthy_time = "10s"
healthy_deadline = "5m"
}
group "fake_service" {
count = 1
network {
port "http" {
to = 19090
static = 19090
}
}
restart {
# The number of attempts to run the job within the specified interval.
attempts = 2
interval = "30m"
delay = "15s"
mode = "fail"
}
ephemeral_disk {
size = 30
}
task "fake_service" {
# The "driver" parameter specifies the task driver that should be used to
# run the task.
driver = "docker"
logs {
max_files = 2
max_file_size = 10
}
env {
LISTEN_ADDR = ":19090"
NAME = "Example1"
}
config {
image = "nicholasjackson/fake-service:v0.18.1"
ports = ["http"]
}
resources {
cpu = 500 # 500 MHz
memory = 256 # 256MB
}
}
}
}
variable "version" {
default = "v1.8.0"
}
resource "network" "cloud" {
subnet = "10.5.0.0/16"
}
resource "nomad_cluster" "dev" {
client_nodes = 3
network {
id = resource.network.cloud.meta.id
}
}
resource "nomad_job" "example_1" {
cluster = resource.nomad_cluster.dev
paths = ["./files/example.nomad"]
health_check {
timeout = "60s"
jobs = ["example_1"]
}
}
resource "ingress" "fake_service_1" {
port = 19090
target {
resource = resource.nomad_cluster.dev
named_port = "http"
config = {
job = "example_1"
group = "fake_service"
task = "fake_service"
}
}
}
\ No newline at end of file
Feature: Nomad Cluster
In order to test Nomad clusters
I should apply a blueprint
And test the output
@Single
Scenario Outline: Nomad Cluster
Given I have a running blueprint
Then the following resources should be running
| name |
| resource.network.cloud |
| resource.nomad_cluster.dev |
And a HTTP call to "http://localhost:19090" should result in status 200
@All
Scenario Outline: Nomad Cluster
Given the jumppad variable "version" has a value "<version>"
And I have a running blueprint
Then the following resources should be running
| name |
| resource.network.cloud |
| resource.nomad_cluster.dev |
And a HTTP call to "http://localhost:19090" should result in status 200
Examples:
| version |
| v1.8.0 |
| v1.8.1 |
| v1.8.2 |
| v1.8.3 |
| v1.8.4 |
FROM ubuntu:22.04
ARG TARGETARCH
ENV DEBIAN_FRONTEND=noninteractive
ENV container=docker LANG=C.UTF-8
RUN apt-get update && \
apt-get install -y -o APT::Install-Suggests=0 -o APT::Install-Recommends=0 \
software-properties-common supervisor rsyslog iproute2 python3 python3-apt sudo bash ca-certificates \
curl net-tools jq unzip vim dnsutils iputils-ping socat wget
# Setup bash
RUN sed -i 's/\/bin\/ash/\/bin\/bash/g' /etc/passwd
# Setup supervisor
COPY files/supervisord.conf /etc/supervisor/supervisord.conf
# Setup docker
RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg && \
echo \
"deb [arch=${TARGETARCH} signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
$(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null && \
apt-get update && \
apt-get install -y -o APT::Install-Suggests=0 -o APT::Install-Recommends=0 docker-ce docker-ce-cli containerd.io && \
apt-get clean && rm -rf /var/lib/apt/lists/*
# https://github.com/docker/docker/tree/master/hack/dind
ENV DIND_COMMIT d58df1fc6c866447ce2cd129af10e5b507705624
RUN set -eux; \
wget -O /usr/local/bin/dind "https://raw.githubusercontent.com/docker/docker/${DIND_COMMIT}/hack/dind"; \
chmod +x /usr/local/bin/dind
COPY --chmod=0755 files/dockerd.sh /usr/local/bin/dockerd.sh
COPY files/docker.conf /etc/supervisor/conf.d/docker.conf
VOLUME /var/lib/docker
ENTRYPOINT ["/usr/bin/supervisord"]
CMD ["-c", "/etc/supervisor/supervisord.conf"]
\ No newline at end of file
[program:docker]
command=/usr/local/bin/dockerd.sh
autorestart=true
priority=100
#stdout_logfile=/dev/stdout
#stderr_logfile=/dev/stderr
#stdout_logfile_maxbytes=0
#stderr_logfile_maxbytes=0
\ No newline at end of file
#!/bin/sh
set -eu
_tls_ensure_private() {
local f="$1"; shift
[ -s "$f" ] || openssl genrsa -out "$f" 4096
}
_tls_san() {
{
ip -oneline address | awk '{ gsub(/\/.+$/, "", $4); print "IP:" $4 }'
{
cat /etc/hostname
echo 'docker'
echo 'localhost'
hostname -f
hostname -s
} | sed 's/^/DNS:/'
[ -z "${DOCKER_TLS_SAN:-}" ] || echo "$DOCKER_TLS_SAN"
} | sort -u | xargs printf '%s,' | sed "s/,\$//"
}
_tls_generate_certs() {
local dir="$1"; shift
# if ca/key.pem || !ca/cert.pem, generate CA public if necessary
# if ca/key.pem, generate server public
# if ca/key.pem, generate client public
# (regenerating public certs every startup to account for SAN/IP changes and/or expiration)
# https://github.com/FiloSottile/mkcert/issues/174
local certValidDays='825'
if [ -s "$dir/ca/key.pem" ] || [ ! -s "$dir/ca/cert.pem" ]; then
# if we either have a CA private key or do *not* have a CA public key, then we should create/manage the CA
mkdir -p "$dir/ca"
_tls_ensure_private "$dir/ca/key.pem"
openssl req -new -key "$dir/ca/key.pem" \
-out "$dir/ca/cert.pem" \
-subj '/CN=docker:dind CA' -x509 -days "$certValidDays"
fi
if [ -s "$dir/ca/key.pem" ]; then
# if we have a CA private key, we should create/manage a server key
mkdir -p "$dir/server"
_tls_ensure_private "$dir/server/key.pem"
openssl req -new -key "$dir/server/key.pem" \
-out "$dir/server/csr.pem" \
-subj '/CN=docker:dind server'
cat > "$dir/server/openssl.cnf" <<-EOF
[ x509_exts ]
subjectAltName = $(_tls_san)
EOF
openssl x509 -req \
-in "$dir/server/csr.pem" \
-CA "$dir/ca/cert.pem" \
-CAkey "$dir/ca/key.pem" \
-CAcreateserial \
-out "$dir/server/cert.pem" \
-days "$certValidDays" \
-extfile "$dir/server/openssl.cnf" \
-extensions x509_exts
cp "$dir/ca/cert.pem" "$dir/server/ca.pem"
openssl verify -CAfile "$dir/server/ca.pem" "$dir/server/cert.pem"
fi
if [ -s "$dir/ca/key.pem" ]; then
# if we have a CA private key, we should create/manage a client key
mkdir -p "$dir/client"
_tls_ensure_private "$dir/client/key.pem"
chmod 0644 "$dir/client/key.pem" # openssl defaults to 0600 for the private key, but this one needs to be shared with arbitrary client contexts
openssl req -new \
-key "$dir/client/key.pem" \
-out "$dir/client/csr.pem" \
-subj '/CN=docker:dind client'
cat > "$dir/client/openssl.cnf" <<-'EOF'
[ x509_exts ]
extendedKeyUsage = clientAuth
EOF
openssl x509 -req \
-in "$dir/client/csr.pem" \
-CA "$dir/ca/cert.pem" \
-CAkey "$dir/ca/key.pem" \
-CAcreateserial \
-out "$dir/client/cert.pem" \
-days "$certValidDays" \
-extfile "$dir/client/openssl.cnf" \
-extensions x509_exts
cp "$dir/ca/cert.pem" "$dir/client/ca.pem"
openssl verify -CAfile "$dir/client/ca.pem" "$dir/client/cert.pem"
fi
}
# no arguments passed
# or first arg is `-f` or `--some-option`
if [ "$#" -eq 0 ] || [ "${1#-}" != "$1" ]; then
# set "dockerSocket" to the default "--host" *unix socket* value (for both standard or rootless)
uid="$(id -u)"
if [ "$uid" = '0' ]; then
dockerSocket='unix:///var/run/docker.sock'
else
# if we're not root, we must be trying to run rootless
: "${XDG_RUNTIME_DIR:=/run/user/$uid}"
dockerSocket="unix://$XDG_RUNTIME_DIR/docker.sock"
fi
case "${DOCKER_HOST:-}" in
unix://*)
dockerSocket="$DOCKER_HOST"
;;
esac
# add our default arguments
if [ -n "${DOCKER_TLS_CERTDIR:-}" ] \
&& _tls_generate_certs "$DOCKER_TLS_CERTDIR" \
&& [ -s "$DOCKER_TLS_CERTDIR/server/ca.pem" ] \
&& [ -s "$DOCKER_TLS_CERTDIR/server/cert.pem" ] \
&& [ -s "$DOCKER_TLS_CERTDIR/server/key.pem" ] \
; then
# generate certs and use TLS if requested/possible (default in 19.03+)
set -- dockerd \
--host="$dockerSocket" \
--host=tcp://0.0.0.0:2376 \
--tlsverify \
--tlscacert "$DOCKER_TLS_CERTDIR/server/ca.pem" \
--tlscert "$DOCKER_TLS_CERTDIR/server/cert.pem" \
--tlskey "$DOCKER_TLS_CERTDIR/server/key.pem" \
"$@"
DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS="${DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS:-} -p 0.0.0.0:2376:2376/tcp"
else
# TLS disabled (-e DOCKER_TLS_CERTDIR='') or missing certs
set -- dockerd \
--host="$dockerSocket" \
--host=tcp://0.0.0.0:2375 \
"$@"
DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS="${DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS:-} -p 0.0.0.0:2375:2375/tcp"
fi
fi
if [ "$1" = 'dockerd' ]; then
# explicitly remove Docker's default PID file to ensure that it can start properly if it was stopped uncleanly (and thus didn't clean up the PID file)
find /run /var/run -iname 'docker*.pid' -delete || :
if dockerd --version | grep -qF ' 20.10.'; then
# XXX inject "docker-init" (tini) as pid1 to workaround https://github.com/docker-library/docker/issues/318 (zombie container-shim processes)
set -- docker-init -- "$@"
fi
uid="$(id -u)"
if [ "$uid" != '0' ]; then
# if we're not root, we must be trying to run rootless
if ! command -v rootlesskit > /dev/null; then
echo >&2 "error: attempting to run rootless dockerd but missing 'rootlesskit' (perhaps the 'docker:dind-rootless' image variant is intended?)"
exit 1
fi
user="$(id -un 2>/dev/null || :)"
if ! grep -qE "^($uid${user:+|$user}):" /etc/subuid || ! grep -qE "^($uid${user:+|$user}):" /etc/subgid; then
echo >&2 "error: attempting to run rootless dockerd but missing necessary entries in /etc/subuid and/or /etc/subgid for $uid"
exit 1
fi
: "${XDG_RUNTIME_DIR:=/run/user/$uid}"
export XDG_RUNTIME_DIR
if ! mkdir -p "$XDG_RUNTIME_DIR" || [ ! -w "$XDG_RUNTIME_DIR" ] || ! mkdir -p "$HOME/.local/share/docker" || [ ! -w "$HOME/.local/share/docker" ]; then
echo >&2 "error: attempting to run rootless dockerd but need writable HOME ($HOME) and XDG_RUNTIME_DIR ($XDG_RUNTIME_DIR) for user $uid"
exit 1
fi
if [ -f /proc/sys/kernel/unprivileged_userns_clone ] && unprivClone="$(cat /proc/sys/kernel/unprivileged_userns_clone)" && [ "$unprivClone" != '1' ]; then
echo >&2 "error: attempting to run rootless dockerd but need 'kernel.unprivileged_userns_clone' (/proc/sys/kernel/unprivileged_userns_clone) set to 1"
exit 1
fi
if [ -f /proc/sys/user/max_user_namespaces ] && maxUserns="$(cat /proc/sys/user/max_user_namespaces)" && [ "$maxUserns" = '0' ]; then
echo >&2 "error: attempting to run rootless dockerd but need 'user.max_user_namespaces' (/proc/sys/user/max_user_namespaces) set to a sufficiently large value"
exit 1
fi
# TODO overlay support detection?
exec rootlesskit \
--net="${DOCKERD_ROOTLESS_ROOTLESSKIT_NET:-vpnkit}" \
--mtu="${DOCKERD_ROOTLESS_ROOTLESSKIT_MTU:-1500}" \
--disable-host-loopback \
--port-driver=builtin \
--copy-up=/etc \
--copy-up=/run \
${DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS:-} \
"$@"
elif [ -x '/usr/local/bin/dind' ]; then
# if we have the (mostly defunct now) Docker-in-Docker wrapper script, use it
set -- '/usr/local/bin/dind' "$@"
fi
else
# if it isn't `dockerd` we're trying to run, pass it through `docker-entrypoint.sh` so it gets `DOCKER_HOST` set appropriately too
set -- docker-entrypoint.sh "$@"
fi
exec "$@"
\ No newline at end of file
[include]
files = /etc/supervisor/conf.d/*.conf
[unix_http_server]
file=/var/run/supervisor.sock
chmod=0700
username=username
password=password
[supervisord]
user=root
nodaemon=true
logfile=/var/log/supervisor/supervisord.log
pidfile=/var/run/supervisord.pid
childlogdir=/var/log/supervisor
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
[supervisorctl]
serverurl=unix:///var/run/supervisor.sock
username=username
password=password
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment