From aff20f82f59e452cb1bcf942ffc7d2ef38a28529 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 20 Dec 2023 17:52:09 -0300 Subject: [PATCH 01/86] Transforma VM em k8s --- README.md | 212 ++++++++++++++- flake.nix | 245 +++++++++++------- kubernetes-examples/appvia/deployment.yaml | 21 ++ kubernetes-examples/appvia/exec-liveness.yaml | 21 ++ kubernetes-examples/appvia/ingress.yaml | 18 ++ kubernetes-examples/appvia/notes.md | 21 ++ kubernetes-examples/appvia/service.yaml | 11 + .../minimal-pod-with-busybox-example.yaml | 9 + 8 files changed, 467 insertions(+), 91 deletions(-) create mode 100644 kubernetes-examples/appvia/deployment.yaml create mode 100644 kubernetes-examples/appvia/exec-liveness.yaml create mode 100644 kubernetes-examples/appvia/ingress.yaml create mode 100644 kubernetes-examples/appvia/notes.md create mode 100644 kubernetes-examples/appvia/service.yaml create mode 100644 kubernetes-examples/minimal-pod-with-busybox-example/minimal-pod-with-busybox-example.yaml diff --git a/README.md b/README.md index 9fadeb9..81ddb70 100644 --- a/README.md +++ b/README.md @@ -42,4 +42,214 @@ source .env ```bash source .env RUNNER_SCOPE="org" SCOPE_TARGET="imobanco" bash ./ops/bash/entrypoint.sh -``` \ No newline at end of file +``` + + +# k8s in NixOS VM + + +Gerar token: +- onde gerar: https://github.com/settings/tokens +- com os seguintes checks: https://github.com/myoung34/docker-github-actions-runner/wiki/Usage#token-scope + + +```bash +rm -fv nixos.qcow2; nix run --impure --refresh --verbose .#vm +``` + + +Copie e cole no terminal da VM e edite com seu PAT gerado no pasos anterior: +```bash +GITHUB_TOKEN=ghp_yyyyyyyyyyyyyyy +``` + + + +```bash +# Bem haky, só bypassei o problema que deixava como pending +kubectl label nodes nixos size=linux +kubectl get nodes nixos --show-labels + +NAME_SPACE_RUNNER='actions-runner-system' +kubectl create ns "$NAME_SPACE_RUNNER" + +mkdir -pv ~/k8s-bootstrap-runner \ +&& ~/k8s-bootstrap-runner + +cat > script.sh <<-'EOF' +rm -rf *.pem *.csr *.srl || true + +# Step 1: CA +# ---------- + +# create CA, it is secret, keep it safe +openssl genrsa -out ca.private.pem 2048 + +# create public CA, give it to everyone so they can add it to trusted root +openssl req -x509 -new -key ca.private.pem -out ca.public.pem -days 10000 -subj "/C=UA/L=Kiev" + +# Step 2: Certificate +# ------------------- + +# create certificate, it is secret, keep it safe +openssl genrsa -out cert.private.pem 2048 + +# create "certificate signing request" (csr) +openssl req -new -key cert.private.pem -out cert.csr -subj "/CN=actions-runner-controller-webhook.actions-runner-system.svc" + +# config +cat <> cert.conf +[SAN] +subjectAltName = @alt_names +[alt_names] +DNS.1 = actions-runner-controller-webhook.actions-runner-system.svc +DNS.2 = actions-runner-controller-webhook.actions-runner-system.svc.cluster.local +EOT + +# sign it with our CA +openssl x509 -req -in cert.csr -CA ca.public.pem -CAkey ca.private.pem -CAcreateserial -out cert.public.pem -days 10000 -extensions SAN -extfile cert.conf + +# clean +rm -rf *.csr *.srl cert.conf || true + +# check +openssl x509 -in cert.public.pem -text -noout | grep DNS +EOF + +chmod 0755 script.sh + +./script.sh + +# smoke tests, better than nothing?! +test -f cert.public.pem || exit 1 +test -f cert.private.pem || exit 1 +test -f ca.public.pem || exit 1 + + +kubectl create secret tls actions-runner-controller-serving-cert \ +--namespace="$NAME_SPACE_RUNNER" \ +--cert=cert.public.pem \ +--key=cert.private.pem + + +# + +cat > values.yml <<-'EOF' +authSecret: + create: true + github_token: ghp_xxxxxxxxxxxxxxx + +# POI: disable cert manager +certManagerEnabled: false + +admissionWebHooks: + # POI: cat ca.public.pem | base64 + caBundle: xxxxxxxxxxxxxxxxxxxxxxxxxxxx= + +nodeSelector: +size: linux + +podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/path: /metrics + prometheus.io/port: "8080" +EOF + + +sed -i 's/xxxxxxxxxxxxxxxxxxxxxxxxxxxx=/'"$(cat ca.public.pem | base64 -w 0)"'/g' values.yml +sed -i 's/ghp_xxxxxxxxxxxxxxx/'"$GITHUB_TOKEN"'/g' values.yml + +helm upgrade actions-runner-controller actions-runner-controller \ +--install \ +--namespace "$NAME_SPACE_RUNNER" \ +--repo https://actions-runner-controller.github.io/actions-runner-controller \ +-f values.yml + + +cat > runner.yml <<-'EOF' +apiVersion: actions.summerwind.dev/v1alpha1 +kind: RunnerDeployment +metadata: + name: gha + namespace: actions-runner-system +spec: + replicas: 1 + template: + spec: + # https://github.com/actions-runner-controller/actions-runner-controller/blob/master/docs/detailed-docs.md#runner-with-dind + dockerdWithinRunnerContainer: true + organization: Imobanco + labels: + - 'gha' + - 'gha-dev' + nodeSelector: + size: linux + + resources: + limits: + cpu: "6.0" + memory: "5Gi" + requests: + cpu: "3.0" + memory: "4Gi" +EOF + +kubectl --namespace="$NAME_SPACE_RUNNER" apply -f runner.yml + +export POD_NAME=$(kubectl get pods --namespace "$NAME_SPACE_RUNNER" -l "app.kubernetes.io/name=actions-runner-controller,app.kubernetes.io/instance=actions-runner-controller" -o jsonpath="{.items[0].metadata.name}") + +export CONTAINER_PORT=$(kubectl get pod --namespace "$NAME_SPACE_RUNNER" $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + + +echo "$POD_NAME" +echo "$CONTAINER_PORT" + +``` +Refs.: +- https://mac-blog.org.ua/github-actions-kubernetes-runner-without-certmanager/ +- https://serverfault.com/questions/1099167/node-pool-selection +- https://superuser.com/a/1225139 + + +```bash +kubectl describe -n "$NAME_SPACE_RUNNER" pod "$POD_NAME" +``` + + +```bash +kubectl describe pod "$POD_NAME" -n "$NAME_SPACE_RUNNER" +``` + + +TODO: +Pq esse comando não termina? +O que esse comando faz exatamente? +```bash +kubectl \ +--namespace "$NAME_SPACE_RUNNER" \ +port-forward $POD_NAME 8080:$CONTAINER_PORT +``` + + + + +```bash +kubectl -n actions-runner-system get pods +kubectl -n actions-runner-system get runners +``` +Refs.: +- https://mac-blog.org.ua/github-actions-kubernetes-runner-without-certmanager/ + + +```bash +while ! false; do + kubectl get pod --all-namespaces -o wide \ + && echo \ + && kubectl get services --all-namespaces -o wide \ + && echo \ + && kubectl get nodes --all-namespaces -o wide; + sleep 2; + clear; +done +``` + diff --git a/flake.nix b/flake.nix index a3479d1..50f3b7b 100644 --- a/flake.nix +++ b/flake.nix @@ -117,7 +117,9 @@ system = builtins.currentSystem; modules = [ - + # export QEMU_NET_OPTS="hostfwd=tcp::2200-:10022" && nix run .#vm + # Then connect with ssh -p 2200 nixuser@localhost + # ps -p $(pgrep -f qemu-kvm) -o args | tr ' ' '\n' ({ config, nixpkgs, pkgs, lib, modulesPath, ... }: let nixuserKeys = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIExR+PSB/jBwJYKfpLN+MMXs3miRn70oELTV3sXdgzpr"; @@ -136,14 +138,17 @@ fileSystems."/" = { device = "/dev/hda1"; }; virtualisation.vmVariant = { - virtualisation.useNixStoreImage = false; # TODO: hardening + + virtualisation.useNixStoreImage = false; virtualisation.writableStore = true; # TODO: hardening + virtualisation.docker.enable = true; + programs.dconf.enable = true; # security.polkit.enable = true; # TODO: hardening? - virtualisation.memorySize = 1024 * 2; # Use MiB memory. - virtualisation.diskSize = 1024 * 16; # Use MiB memory. + virtualisation.memorySize = 1024 * 10; # Use MiB memory. + virtualisation.diskSize = 1024 * 50; # Use MiB memory. virtualisation.cores = 8; # Number of cores. virtualisation.graphics = true; @@ -160,6 +165,9 @@ "-chardev qemu-vdagent,id=ch1,name=vdagent,clipboard=on" "-device virtio-serial-pci" "-device virtserialport,chardev=ch1,id=ch1,name=com.redhat.spice.0" + + # https://serverfault.com/a/1119403 + # "-device intel-iommu,intremap=on" ]; }; @@ -196,44 +204,22 @@ ]; packages = with pkgs; [ awscli + bashInteractive btop coreutils direnv file + firefox git gnumake + kubernetes-helm + nix-info openssh + openssl starship which - sops - - github-runner - curl - jq - httpie - - ( - writeScriptBin "start-github-runner-with-pat" '' - #! ${pkgs.runtimeShell} -e - - PAT="$1" - - # https://github.com/actions/runner/issues/323#issuecomment-583640437 - # https://github.com/actions/runner/issues/2538#issuecomment-1517988404 - config.sh \ - --ephemeral \ - --pat "$PAT" \ - --replace \ - --runasservice \ - --unattended \ - --url https://github.com/imobanco/github-ci-runner \ - && run.sh - '' - ) - ]; - shell = pkgs.zsh; uid = 1234; autoSubUidGidRange = true; @@ -247,49 +233,15 @@ ]; }; - # imports = [ allAttrs.sops-nix.nixosModules.sops ]; - /* - https://github.com/NixOS/nixpkgs/issues/169812 - https://github.com/actions/runner/issues/1882#issuecomment-1427930611 - - nix shell nixpkgs#github-runner --command \ - sh \ - -c \ - 'config.sh --url https://github.com/imobanco/github-ci-runner --pat "$PAT" --ephemeral && run.sh' - - config.sh --url https://github.com/imobanco/github-ci-runner --pat "$PAT" --ephemeral && run.sh - - TODO: https://www.youtube.com/watch?v=G5f6GC7SnhU - */ - # services.github-runner.enable = true; - # services.github-runner.url = "https://github.com/imobanco"; - # services.github-runner.tokenFile = config.sops.secrets."github-runner/token".path; - # services.github-runner.extraPackages = with pkgs; [ config.virtualisation.docker.package ]; - # virtualisation.docker.enable = true; - # systemd.services.github-runner.serviceConfig.SupplementaryGroups = [ "docker" ]; - - virtualisation.docker.enable = true; - /* - https://github.com/vimjoyer/sops-nix-video/tree/25e5698044e60841a14dcd64955da0b1b66957a2 - https://github.com/Mic92/sops-nix/issues/65#issuecomment-929082304 - https://discourse.nixos.org/t/qmenu-secrets-sops-and-nixos/13621/8 - https://www.youtube.com/watch?v=1BquzE3Yb4I - https://github.com/FiloSottile/age#encrypting-to-a-github-user - https://devops.datenkollektiv.de/using-sops-with-age-and-git-like-a-pro.html - - sudo cat /run/secrets/example-key - */ - /* - sops.defaultSopsFile = ./secrets/secrets.yaml.encrypted; - sops.defaultSopsFormat = "yaml"; - sops.gnupg.sshKeyPaths = []; - sops.age.sshKeyPaths = []; - sops.age.keyFile = ./secrets/keys.txt; - sops.secrets.example-key = { }; - */ - + # https://github.com/NixOS/nixpkgs/blob/3a44e0112836b777b176870bb44155a2c1dbc226/nixos/modules/programs/zsh/oh-my-zsh.nix#L119 + # https://discourse.nixos.org/t/nix-completions-for-zsh/5532 + # https://github.com/NixOS/nixpkgs/blob/09aa1b23bb5f04dfc0ac306a379a464584fc8de7/nixos/modules/programs/zsh/zsh.nix#L230-L231 programs.zsh = { enable = true; + shellAliases = { + vim = "nvim"; + k = "kubectl"; + }; enableCompletion = true; autosuggestions.enable = true; syntaxHighlighting.enable = true; @@ -332,6 +284,31 @@ enableGhostscriptFonts = true; }; + # Hack to fix annoying zsh warning, too overkill probably + # https://www.reddit.com/r/NixOS/comments/cg102t/how_to_run_a_shell_command_upon_startup/eudvtz1/?utm_source=reddit&utm_medium=web2x&context=3 + # https://stackoverflow.com/questions/638975/how-wdo-i-tell-if-a-regular-file-does-not-exist-in-bash#comment25226870_638985 + systemd.user.services.fix-zsh-warning = { + script = '' + test -f /home/nixuser/.zshrc || touch /home/nixuser/.zshrc && chown nixuser: -Rv /home/nixuser + ''; + wantedBy = [ "default.target" ]; + }; + + # journalctl -u fix-k8s.service -b -f + systemd.services.fix-k8s = { + script = '' + echo "Fixing k8s" + + CLUSTER_ADMIN_KEY_PATH=/var/lib/kubernetes/secrets/cluster-admin-key.pem + + while ! test -f "$CLUSTER_ADMIN_KEY_PATH"; do echo $(date +'%d/%m/%Y %H:%M:%S:%3N'); sleep 0.5; done + + chmod 0660 -v "$CLUSTER_ADMIN_KEY_PATH" + chown root:kubernetes -v "$CLUSTER_ADMIN_KEY_PATH" + ''; + wantedBy = [ "multi-user.target" ]; + }; + # Enable ssh services.sshd.enable = true; @@ -349,21 +326,12 @@ ]; }; - # Hack to fix annoying zsh warning, too overkill probably - # https://www.reddit.com/r/NixOS/comments/cg102t/how_to_run_a_shell_command_upon_startup/eudvtz1/?utm_source=reddit&utm_medium=web2x&context=3 - systemd.user.services.fix-zsh-warning = { - script = '' - echo "Fixing a zsh warning" - # https://stackoverflow.com/questions/638975/how-wdo-i-tell-if-a-regular-file-does-not-exist-in-bash#comment25226870_638985 - test -f /home/nixuser/.zshrc || touch /home/nixuser/.zshrc && chown nixuser: -Rv /home/nixuser - ''; - wantedBy = [ "default.target" ]; - }; - # https://nixos.wiki/wiki/Libvirt # https://discourse.nixos.org/t/set-up-vagrant-with-libvirt-qemu-kvm-on-nixos/14653 boot.extraModprobeConfig = "options kvm_intel nested=1"; + services.qemuGuest.enable = true; + # X configuration services.xserver.enable = true; services.xserver.layout = "br"; @@ -380,19 +348,16 @@ services.spice-vdagentd.enable = true; nixpkgs.config.allowUnfree = true; + nix = { extraOptions = "experimental-features = nix-command flakes"; package = pkgs.nixVersions.nix_2_10; readOnlyStore = true; registry.nixpkgs.flake = nixpkgs; # https://bou.ke/blog/nix-tips/ - - nixPath = [ - "nixpkgs=/etc/channels/nixpkgs" - "nixos-config=/etc/nixos/configuration.nix" - ]; + nixPath = ["nixpkgs=${pkgs.path}"]; }; - environment.etc."channels/nixpkgs".source = nixpkgs.outPath; + environment.etc."channels/nixpkgs".source = "${pkgs.path}"; environment.systemPackages = with pkgs; [ bashInteractive @@ -408,6 +373,107 @@ zsh-autosuggestions zsh-completions + # Looks like kubernetes needs at least all this + kubectl + kubernetes + # + cni + cni-plugins + conntrack-tools + cri-o + cri-tools + ebtables + ethtool + flannel + iptables + socat + + ( + writeScriptBin "fix-k8s-cluster-admin-key" '' + #! ${pkgs.runtimeShell} -e + sudo chmod 0660 -v /var/lib/kubernetes/secrets/cluster-admin-key.pem + sudo chown root:kubernetes -v /var/lib/kubernetes/secrets/cluster-admin-key.pem + '' + ) + ]; + + # Is this a must to kubernetes? + swapDevices = pkgs.lib.mkForce [ ]; + + # Is it a must for k8s? + # Take a look into: + # https://github.com/NixOS/nixpkgs/blob/9559834db0df7bb274062121cf5696b46e31bc8c/nixos/modules/services/cluster/kubernetes/kubelet.nix#L255-L259 + boot.kernel.sysctl = { + # If it is enabled it conflicts with what kubelet is doing + # "net.bridge.bridge-nf-call-ip6tables" = 1; + # "net.bridge.bridge-nf-call-iptables" = 1; + + # https://docs.projectcalico.org/v3.9/getting-started/kubernetes/installation/migration-from-flannel + # https://access.redhat.com/solutions/53031 + "net.ipv4.conf.all.rp_filter" = 1; + # https://www.tenable.com/audits/items/CIS_Debian_Linux_8_Server_v2.0.2_L1.audit:bb0f399418f537997c2b44741f2cd634 + # "net.ipv4.conf.default.rp_filter" = 1; + "vm.swappiness" = 0; + }; + + environment.variables.KUBECONFIG = "/etc/kubernetes/cluster-admin.kubeconfig"; + + services.kubernetes.roles = [ "master" "node" ]; + services.kubernetes.masterAddress = "nixos"; + services.kubernetes = { + flannel.enable = true; + }; + + environment.etc."kubernets/kubernetes-examples/appvia/deployment.yaml" = { + mode = "0644"; + text = "${builtins.readFile ./kubernetes-examples/appvia/deployment.yaml}"; + }; + + environment.etc."kubernets/kubernetes-examples/appvia/service.yaml" = { + mode = "0644"; + text = "${builtins.readFile ./kubernetes-examples/appvia/service.yaml}"; + }; + + environment.etc."kubernets/kubernetes-examples/appvia/ingress.yaml" = { + mode = "0644"; + text = "${builtins.readFile ./kubernetes-examples/appvia/ingress.yaml}"; + }; + + environment.etc."kubernets/kubernetes-examples/appvia/notes.md" = { + mode = "0644"; + text = "${builtins.readFile ./kubernetes-examples/appvia/notes.md}"; + }; + + # journalctl -u move-kubernetes-examples.service -b + systemd.services.move-kubernetes-examples = { + script = '' + echo "Started move-kubernets-examples" + + # cp -rv ''\${./kubernetes-examples} /home/nixuser/ + cp -Rv /etc/kubernets/kubernetes-examples/ /home/nixuser/ + + chown -Rv nixuser:nixgroup /home/nixuser/kubernetes-examples + + kubectl \ + apply \ + --file /home/nixuser/kubernetes-examples/deployment.yaml \ + --file /home/nixuser/kubernetes-examples/service.yaml \ + --file /home/nixuser/kubernetes-examples/ingress.yaml + ''; + wantedBy = [ "multi-user.target" ]; + }; + + # https://discourse.nixos.org/t/nixos-firewall-with-kubernetes/23673/2 + # networking.firewall.trustedInterfaces ?? + networking.firewall.allowedTCPPorts = [ 8000 8080 8443 9000 9443 ]; + + boot.kernelParams = [ + "swapaccount=0" + "systemd.unified_cgroup_hierarchy=0" + "group_enable=memory" + "cgroup_enable=cpuset" + "cgroup_memory=1" + "cgroup_enable=memory" ]; system.stateVersion = "22.11"; @@ -415,7 +481,6 @@ ]; specialArgs = { inherit nixpkgs allAttrs; }; - }; }; } diff --git a/kubernetes-examples/appvia/deployment.yaml b/kubernetes-examples/appvia/deployment.yaml new file mode 100644 index 0000000..655c181 --- /dev/null +++ b/kubernetes-examples/appvia/deployment.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: podinfo + labels: + app: podinfo +spec: + selector: + matchLabels: + app: podinfo + template: + metadata: + labels: + app: podinfo + spec: + containers: + - name: podinfo-container + image: stefanprodan/podinfo + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9898 \ No newline at end of file diff --git a/kubernetes-examples/appvia/exec-liveness.yaml b/kubernetes-examples/appvia/exec-liveness.yaml new file mode 100644 index 0000000..7d6ca96 --- /dev/null +++ b/kubernetes-examples/appvia/exec-liveness.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + test: liveness + name: liveness-exec +spec: + containers: + - name: liveness + image: registry.k8s.io/busybox + args: + - /bin/sh + - -c + - touch /tmp/healthy; sleep 30; rm -f /tmp/healthy; sleep 600 + livenessProbe: + exec: + command: + - cat + - /tmp/healthy + initialDelaySeconds: 5 + periodSeconds: 5 diff --git a/kubernetes-examples/appvia/ingress.yaml b/kubernetes-examples/appvia/ingress.yaml new file mode 100644 index 0000000..7ca267c --- /dev/null +++ b/kubernetes-examples/appvia/ingress.yaml @@ -0,0 +1,18 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: podinfo + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / +spec: + ingressClassName: nginx-example + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: podinfo + port: + number: 3000 diff --git a/kubernetes-examples/appvia/notes.md b/kubernetes-examples/appvia/notes.md new file mode 100644 index 0000000..99c54e7 --- /dev/null +++ b/kubernetes-examples/appvia/notes.md @@ -0,0 +1,21 @@ + + +```bash +cd ~/kubernetes-examples/appvia \ +&& kubectl apply \ +-f deployment.yaml \ +-f service.yaml \ +-f ingress.yaml + + +while true; do + kubectl get pod --all-namespaces -o wide \ + && echo \ + && kubectl get services --all-namespaces -o wide \ + && echo \ + && kubectl get nodes --all-namespaces -o wide; + sleep 2; + clear; +done +``` + diff --git a/kubernetes-examples/appvia/service.yaml b/kubernetes-examples/appvia/service.yaml new file mode 100644 index 0000000..e38a178 --- /dev/null +++ b/kubernetes-examples/appvia/service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: my-service +spec: + selector: + app.kubernetes.io/name: my-podinfo + ports: + - protocol: TCP + port: 3000 + targetPort: 9898 diff --git a/kubernetes-examples/minimal-pod-with-busybox-example/minimal-pod-with-busybox-example.yaml b/kubernetes-examples/minimal-pod-with-busybox-example/minimal-pod-with-busybox-example.yaml new file mode 100644 index 0000000..4bc3ee4 --- /dev/null +++ b/kubernetes-examples/minimal-pod-with-busybox-example/minimal-pod-with-busybox-example.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test-pod +spec: + containers: + - name: test-pod + image: busybox + command: ['sh', '-c', "while ! false; do echo $(date +'%d/%m/%Y %H:%M:%S:%3N'); sleep 1; done"] From 49b81d1308eaca9a64bec5f41d81e9ca704b9f93 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 20 Dec 2023 17:59:37 -0300 Subject: [PATCH 02/86] ajusts --- flake.nix | 3 ++- kubernetes-examples/appvia/deployment.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 50f3b7b..fdee7d8 100644 --- a/flake.nix +++ b/flake.nix @@ -354,7 +354,7 @@ package = pkgs.nixVersions.nix_2_10; readOnlyStore = true; registry.nixpkgs.flake = nixpkgs; # https://bou.ke/blog/nix-tips/ - nixPath = ["nixpkgs=${pkgs.path}"]; + nixPath = [ "nixpkgs=${pkgs.path}" ]; }; environment.etc."channels/nixpkgs".source = "${pkgs.path}"; @@ -481,6 +481,7 @@ ]; specialArgs = { inherit nixpkgs allAttrs; }; + }; }; } diff --git a/kubernetes-examples/appvia/deployment.yaml b/kubernetes-examples/appvia/deployment.yaml index 655c181..c85e7bb 100644 --- a/kubernetes-examples/appvia/deployment.yaml +++ b/kubernetes-examples/appvia/deployment.yaml @@ -18,4 +18,4 @@ spec: image: stefanprodan/podinfo imagePullPolicy: IfNotPresent ports: - - containerPort: 9898 \ No newline at end of file + - containerPort: 9898 From 41f2c70e6f594d73bdaab21f230859b70d831e3b Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 20 Dec 2023 18:55:09 -0300 Subject: [PATCH 03/86] Adds alias --- flake.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index fdee7d8..197b750 100644 --- a/flake.nix +++ b/flake.nix @@ -139,7 +139,7 @@ virtualisation.vmVariant = { - virtualisation.useNixStoreImage = false; + virtualisation.useNixStoreImage = false; # TODO: hardening virtualisation.writableStore = true; # TODO: hardening virtualisation.docker.enable = true; @@ -241,6 +241,7 @@ shellAliases = { vim = "nvim"; k = "kubectl"; + kaf = "kubectl apply -f"; }; enableCompletion = true; autosuggestions.enable = true; From d101c11ac8371061f7820bb998e73f205a23742e Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 20 Dec 2023 19:08:31 -0300 Subject: [PATCH 04/86] Corrige eexemplo mais simples --- flake.nix | 11 +++++++++++ .../minimal-pod-with-busybox-example/notes.md | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 kubernetes-examples/minimal-pod-with-busybox-example/notes.md diff --git a/flake.nix b/flake.nix index 197b750..0bfcf11 100644 --- a/flake.nix +++ b/flake.nix @@ -425,6 +425,17 @@ flannel.enable = true; }; + # TODO: refatorar, talvez usar self? + environment.etc."kubernets/kubernetes-examples/minimal-pod-with-busybox-example/minimal-pod-with-busybox-example.yaml" = { + mode = "0644"; + text = "${builtins.readFile ./kubernetes-examples/minimal-pod-with-busybox-example/minimal-pod-with-busybox-example.yaml}"; + }; + + environment.etc."kubernets/kubernetes-examples/minimal-pod-with-busybox-example/notes.md" = { + mode = "0644"; + text = "${builtins.readFile ./kubernetes-examples/minimal-pod-with-busybox-example/notes.md}"; + }; + environment.etc."kubernets/kubernetes-examples/appvia/deployment.yaml" = { mode = "0644"; text = "${builtins.readFile ./kubernetes-examples/appvia/deployment.yaml}"; diff --git a/kubernetes-examples/minimal-pod-with-busybox-example/notes.md b/kubernetes-examples/minimal-pod-with-busybox-example/notes.md new file mode 100644 index 0000000..180a64f --- /dev/null +++ b/kubernetes-examples/minimal-pod-with-busybox-example/notes.md @@ -0,0 +1,19 @@ + + +```bash +cd ~/kubernetes-examples/minimal-pod-with-busybox-example \ +&& kubectl apply \ +-f minimal-pod-with-busybox-example.yaml + + +while true; do + kubectl get pod --all-namespaces -o wide \ + && echo \ + && kubectl get services --all-namespaces -o wide \ + && echo \ + && kubectl get nodes --all-namespaces -o wide; + sleep 2; + clear; +done +``` + From 4d53bb48656f74a39db4e1b6623e1e8c908148e8 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 20 Dec 2023 22:17:40 -0300 Subject: [PATCH 05/86] networking.firewall.enable = false; --- flake.nix | 15 +++++++- kubernetes-examples/appvia/notes.md | 2 +- .../official-examples/notes.md | 37 +++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 kubernetes-examples/official-examples/notes.md diff --git a/flake.nix b/flake.nix index 0bfcf11..22e6e34 100644 --- a/flake.nix +++ b/flake.nix @@ -169,6 +169,10 @@ # https://serverfault.com/a/1119403 # "-device intel-iommu,intremap=on" ]; + + virtualisation.forwardPorts = [ + { from = "host"; host.port = 8080; guest.port = 8080; } + ]; }; users.users.root = { @@ -477,7 +481,16 @@ # https://discourse.nixos.org/t/nixos-firewall-with-kubernetes/23673/2 # networking.firewall.trustedInterfaces ?? - networking.firewall.allowedTCPPorts = [ 8000 8080 8443 9000 9443 ]; + # networking.firewall.allowedTCPPorts = [ 80 8000 8080 8443 9000 9443 ]; + networking.firewall.enable = false; + + environment.etc."containers/registries.conf" = { + mode = "0644"; + text = '' + [registries.search] + registries = ['docker.io', 'localhost', 'us-docker.pkg.dev', 'gcr.io'] + ''; + }; boot.kernelParams = [ "swapaccount=0" diff --git a/kubernetes-examples/appvia/notes.md b/kubernetes-examples/appvia/notes.md index 99c54e7..5d1ce06 100644 --- a/kubernetes-examples/appvia/notes.md +++ b/kubernetes-examples/appvia/notes.md @@ -14,7 +14,7 @@ while true; do && kubectl get services --all-namespaces -o wide \ && echo \ && kubectl get nodes --all-namespaces -o wide; - sleep 2; + sleep 1; clear; done ``` diff --git a/kubernetes-examples/official-examples/notes.md b/kubernetes-examples/official-examples/notes.md new file mode 100644 index 0000000..5dcc2e2 --- /dev/null +++ b/kubernetes-examples/official-examples/notes.md @@ -0,0 +1,37 @@ + + + + +https://cloud.google.com/kubernetes-engine/docs/tutorials/guestbook + + +```bash +git clone https://github.com/GoogleCloudPlatform/kubernetes-engine-samples \ +&& cd kubernetes-engine-samples/quickstarts/guestbook \ +&& BASE_URL='https://k8s.io/examples/application/guestbook' +kubectl apply \ +-f redis-leader-deployment.yaml \ +-f redis-leader-service.yaml \ +-f redis-follower-deployment.yaml \ +-f redis-follower-service.yaml \ +-f frontend-service.yaml +``` + +```bash +kubectl get service frontend +``` + +```bash +while true; do + kubectl get pod --all-namespaces -o wide \ + && echo \ + && kubectl get services --all-namespaces -o wide \ + && echo \ + && kubectl get deployments.apps --all-namespaces -o wide \ + && echo \ + && kubectl get nodes --all-namespaces -o wide; + sleep 2; + clear; +done +``` + From b5c1a57a0dee8ec71c4dea28c8f8b2c9f0b18d63 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 20 Dec 2023 22:49:06 -0300 Subject: [PATCH 06/86] runs-on: arc-runner-set --- .github/workflows/tests.yml | 2 +- README.md | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ba6e418..a189062 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,7 @@ on: jobs: test: - runs-on: self-hosted + runs-on: arc-runner-set name: Testes steps: - name: checkout PR merge commit diff --git a/README.md b/README.md index 81ddb70..1d1c424 100644 --- a/README.md +++ b/README.md @@ -253,3 +253,27 @@ while ! false; do done ``` + + +https://docs.github.com/en/enterprise-server@3.11/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/quickstart-for-actions-runner-controller + +```bash +NAMESPACE="arc-systems" +helm install arc \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller +``` + +```bash +INSTALLATION_NAME="arc-runner-set" +NAMESPACE="arc-runners" +GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" +GITHUB_PAT="ghp_xxxxxxxxxxxxxxx" +helm install "${INSTALLATION_NAME}" \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ + --set githubConfigSecret.github_token="${GITHUB_PAT}" \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set +``` \ No newline at end of file From ffae54145e679b0d191f433c528aacf7c2c5c821 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 20 Dec 2023 22:50:42 -0300 Subject: [PATCH 07/86] bump --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1d1c424..48229f6 100644 --- a/README.md +++ b/README.md @@ -276,4 +276,5 @@ helm install "${INSTALLATION_NAME}" \ --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ --set githubConfigSecret.github_token="${GITHUB_PAT}" \ oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set -``` \ No newline at end of file +``` + From c506c35782ad665316d45609e0a5ebe442e439fc Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Thu, 21 Dec 2023 16:23:31 -0300 Subject: [PATCH 08/86] Multiplos ajustes, e workflow_dispatch --- .github/workflows/tests.yml | 19 +-- README.md | 111 +++++++++---- flake.nix | 14 +- kubernetes-examples/appvia/notes.md | 15 +- kubernetes-examples/flask-gunicorn/notes.md | 148 ++++++++++++++++++ .../minimal-pod-with-busybox-example/notes.md | 3 +- kubernetes-examples/nodejs-node-port/notes.md | 92 +++++++++++ .../official-examples/notes.md | 13 ++ 8 files changed, 367 insertions(+), 48 deletions(-) create mode 100644 kubernetes-examples/flask-gunicorn/notes.md create mode 100644 kubernetes-examples/nodejs-node-port/notes.md diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a189062..178c732 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,14 +1,15 @@ name: tests -on: - push: - branches: - - main - pull_request: - branchs: - - 'feature/**' - - 'fix/**' - - 'refactor/**' +on: workflow_dispatch +# on: +# push: +# branches: +# - main +# pull_request: +# branchs: +# - 'feature/**' +# - 'fix/**' +# - 'refactor/**' jobs: diff --git a/README.md b/README.md index 48229f6..111c4c7 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,50 @@ GITHUB_TOKEN=ghp_yyyyyyyyyyyyyyy ``` +### + +https://docs.github.com/en/enterprise-server@3.11/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/quickstart-for-actions-runner-controller + +Substitua pelo seu PAT: +```bash +GITHUB_PAT=ghp_yyyyyyyyyyyyyyy + +GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" +INSTALLATION_NAME="arc-runner-set" +NAMESPACE="arc-runners" + +helm install arc \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller \ +&& helm install "${INSTALLATION_NAME}" \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ + --set githubConfigSecret.github_token="${GITHUB_PAT}" \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set +``` + + + +```bash +while true; do + kubectl get pod --all-namespaces -o wide \ + && echo \ + && kubectl get services --all-namespaces -o wide \ + && echo \ + && kubectl get deployments.apps --all-namespaces -o wide \ + && echo \ + && kubectl get nodes --all-namespaces -o wide; + sleep 2; + clear; +done +``` + + + +### + ```bash # Bem haky, só bypassei o problema que deixava como pending @@ -223,7 +267,8 @@ kubectl describe pod "$POD_NAME" -n "$NAME_SPACE_RUNNER" TODO: Pq esse comando não termina? -O que esse comando faz exatamente? +O que esse comando faz exatamente? Explicação: +https://kubernetes.io/docs/tutorials/stateless-application/guestbook/#viewing-the-frontend-service-via-kubectl-port-forward ```bash kubectl \ --namespace "$NAME_SPACE_RUNNER" \ @@ -231,8 +276,6 @@ port-forward $POD_NAME 8080:$CONTAINER_PORT ``` - - ```bash kubectl -n actions-runner-system get pods kubectl -n actions-runner-system get runners @@ -241,40 +284,46 @@ Refs.: - https://mac-blog.org.ua/github-actions-kubernetes-runner-without-certmanager/ -```bash -while ! false; do - kubectl get pod --all-namespaces -o wide \ - && echo \ - && kubectl get services --all-namespaces -o wide \ - && echo \ - && kubectl get nodes --all-namespaces -o wide; - sleep 2; - clear; -done -``` +### -https://docs.github.com/en/enterprise-server@3.11/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/quickstart-for-actions-runner-controller - ```bash -NAMESPACE="arc-systems" -helm install arc \ - --namespace "${NAMESPACE}" \ - --create-namespace \ - oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller +GITHUB_TOKEN=ghp_yyyyyyyyyyyyyyy ``` + ```bash -INSTALLATION_NAME="arc-runner-set" -NAMESPACE="arc-runners" -GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" -GITHUB_PAT="ghp_xxxxxxxxxxxxxxx" -helm install "${INSTALLATION_NAME}" \ - --namespace "${NAMESPACE}" \ - --create-namespace \ - --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ - --set githubConfigSecret.github_token="${GITHUB_PAT}" \ - oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set +NAME_SPACE_RUNNER='actions-runner-system' + + +helm repo add jetstack https://charts.jetstack.io + +helm install \ + cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --create-namespace \ + --version v1.12.4 \ + --set installCRDs=true + +curl -L https://github.com/summerwind/actions-runner-controller/releases/latest/download/actions-runner-controller.yaml > actions-runner-controller.yaml + +kubectl apply -f actions-runner-controller.yaml + +kubectl create secret generic controller-manager \ +--from-literal=github_token=${GITHUB_TOKEN} \ +-n actions-runner-system + +cat > runner.yaml <<-'EOF' +apiVersion: actions.summerwind.dev/v1alpha1 +kind: Runner +metadata: + name: example-runner +spec: + repository: summerwind/actions-runner-controller + env: [] +EOF + +kubectl apply -f runner.yaml -n "$NAME_SPACE_RUNNER" ``` diff --git a/flake.nix b/flake.nix index 22e6e34..f58fbd7 100644 --- a/flake.nix +++ b/flake.nix @@ -168,10 +168,19 @@ # https://serverfault.com/a/1119403 # "-device intel-iommu,intremap=on" + + # "-net user,hostfwd=tcp::8090-::8080" ]; + # https://discourse.nixos.org/t/nixpkgs-support-for-linux-builders-running-on-macos/24313/2 virtualisation.forwardPorts = [ - { from = "host"; host.port = 8080; guest.port = 8080; } + { + from = "host"; + # host.address = "127.0.0.1"; + host.port = 8090; + # guest.address = "34.74.203.201"; + guest.port = 30163; + } ]; }; @@ -369,9 +378,10 @@ openssh direnv - nix-direnv fzf + jq neovim + nix-direnv nixos-option oh-my-zsh zsh diff --git a/kubernetes-examples/appvia/notes.md b/kubernetes-examples/appvia/notes.md index 5d1ce06..f08db0a 100644 --- a/kubernetes-examples/appvia/notes.md +++ b/kubernetes-examples/appvia/notes.md @@ -1,21 +1,26 @@ +TODO: qual a fonte desse exemplo? Perdi e não encontrei novamente. + ```bash cd ~/kubernetes-examples/appvia \ && kubectl apply \ --f deployment.yaml \ --f service.yaml \ --f ingress.yaml + -f deployment.yaml \ + -f service.yaml \ + -f ingress.yaml +``` +```bash while true; do kubectl get pod --all-namespaces -o wide \ && echo \ && kubectl get services --all-namespaces -o wide \ && echo \ + && kubectl get deployments.apps --all-namespaces -o wide \ + && echo \ && kubectl get nodes --all-namespaces -o wide; - sleep 1; + sleep 2; clear; done ``` - diff --git a/kubernetes-examples/flask-gunicorn/notes.md b/kubernetes-examples/flask-gunicorn/notes.md new file mode 100644 index 0000000..f9b49cb --- /dev/null +++ b/kubernetes-examples/flask-gunicorn/notes.md @@ -0,0 +1,148 @@ + + + + +Adaptado para usar `nodePort`: +```bash +cat > flask-node-port.yml <<-'EOF' +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: hello-service + name: hello-service +spec: + replicas: 3 + selector: + matchLabels: + app: hello-service + template: + metadata: + labels: + app: hello-service + spec: + containers: + - name: hello-service + image: quay.io/lordofthejars/hello-flask:1.0.0 + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: hello-service + labels: + app: hello-service +spec: + # Expose the service on a static port on each node + # so that we can access the service from outside the cluster + type: NodePort + + # When the node receives a request on the static port (30163) + # "select pods with the label 'app' set to 'echo-hostname'" + # and forward the request to one of them + selector: + app: hello-service + + ports: + # Three types of ports for a service + # nodePort - a static port assigned on each the node + # port - port exposed internally in the cluster + # targetPort - the container port to send requests to + - nodePort: 30163 + port: 8080 + targetPort: 8080 +EOF + +kubectl apply -f flask-node-port.yml +``` +Refs.: +- [How to deploy a Flask application in Python with Gunicorn](https://developers.redhat.com/articles/2023/08/17/how-deploy-flask-application-python-gunicorn#) +- https://matthewpalmer.net/kubernetes-app-developer/articles/service-kubernetes-example-tutorial.html +- https://medium.com/google-cloud/kubernetes-nodeport-vs-loadbalancer-vs-ingress-when-should-i-use-what-922f010849e0 + + + +```bash +while true; do + kubectl get pod --all-namespaces -o wide \ + && echo \ + && kubectl get services --all-namespaces -o wide \ + && echo \ + && kubectl get deployments.apps --all-namespaces -o wide \ + && echo \ + && kubectl get nodes --all-namespaces -o wide; + sleep 2; + clear; +done +``` + +Em um terminal na VM NixOS: +```bash +curl localhost:30163/ +``` + + +Em um terminal no host: +```bash +curl localhost:8090/ +``` + + +## Usando LoadBalancer e ip hardcoded + +```bash +cat > flask-load-balancer.yml <<-'EOF' +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: hello-service + name: hello-service +spec: + replicas: 1 + selector: + matchLabels: + app: hello-service + template: + metadata: + labels: + app: hello-service + spec: + containers: + - name: hello-service + image: quay.io/lordofthejars/hello-flask:1.0.0 + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: hello-service + labels: + app: hello-service +spec: + ports: + - name: http + port: 8080 + selector: + app: hello-service + type: LoadBalancer + externalIPs: + - "34.74.203.201" +EOF + + +kubectl apply -f flask-load-balancer.yml + +``` +Refs.: +- https://paul-boone.medium.com/kubernetes-loadbalancer-ip-stuck-in-pending-6ddea72b8ff5 + + +No terminal da VM NixOS: +```bash +curl 34.74.203.201:8080/ +``` + +Pelo que entendi não funciona externamente. diff --git a/kubernetes-examples/minimal-pod-with-busybox-example/notes.md b/kubernetes-examples/minimal-pod-with-busybox-example/notes.md index 180a64f..72ae7e0 100644 --- a/kubernetes-examples/minimal-pod-with-busybox-example/notes.md +++ b/kubernetes-examples/minimal-pod-with-busybox-example/notes.md @@ -11,9 +11,10 @@ while true; do && echo \ && kubectl get services --all-namespaces -o wide \ && echo \ + && kubectl get deployments.apps --all-namespaces -o wide \ + && echo \ && kubectl get nodes --all-namespaces -o wide; sleep 2; clear; done ``` - diff --git a/kubernetes-examples/nodejs-node-port/notes.md b/kubernetes-examples/nodejs-node-port/notes.md new file mode 100644 index 0000000..4a3e3b5 --- /dev/null +++ b/kubernetes-examples/nodejs-node-port/notes.md @@ -0,0 +1,92 @@ + + +Atualmente quebrado! + +https://kubernetes.io/docs/tutorials/stateless-application/expose-external-ip-address/#creating-a-service-for-an-application-running-in-five-pods + +Adaptado para usar `nodePort`: +```bash +cat > nodejs-node-port.yml <<-'EOF' +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: load-balancer-example + name: hello-world +spec: + replicas: 5 + selector: + matchLabels: + app.kubernetes.io/name: load-balancer-example + template: + metadata: + labels: + app.kubernetes.io/name: load-balancer-example + spec: + containers: + - image: gcr.io/google-samples/node-hello:1.0 + name: hello-world + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: hello-world + labels: + app: hello-world +spec: + # Expose the service on a static port on each node + # so that we can access the service from outside the cluster + type: NodePort + + # When the node receives a request on the static port (30163) + # "select pods with the label 'app' set to 'echo-hostname'" + # and forward the request to one of them + selector: + app: hello-world + + ports: + # Three types of ports for a service + # nodePort - a static port assigned on each the node + # port - port exposed internally in the cluster + # targetPort - the container port to send requests to + - nodePort: 30163 + port: 8080 + targetPort: 8080 +EOF + +kubectl apply -f nodejs-node-port.yml +``` +Refs.: +- [How to deploy a Flask application in Python with Gunicorn](https://developers.redhat.com/articles/2023/08/17/how-deploy-flask-application-python-gunicorn#) +- https://matthewpalmer.net/kubernetes-app-developer/articles/service-kubernetes-example-tutorial.html +- https://medium.com/google-cloud/kubernetes-nodeport-vs-loadbalancer-vs-ingress-when-should-i-use-what-922f010849e0 + + + +```bash +while true; do + kubectl get pod --all-namespaces -o wide \ + && echo \ + && kubectl get services --all-namespaces -o wide \ + && echo \ + && kubectl get deployments.apps --all-namespaces -o wide \ + && echo \ + && kubectl get nodes --all-namespaces -o wide; + sleep 2; + clear; +done +``` + +Em um terminal na VM NixOS: +```bash +curl localhost:30163/ +``` + + +Em um terminal no host: +```bash +curl localhost:8090/ +``` + diff --git a/kubernetes-examples/official-examples/notes.md b/kubernetes-examples/official-examples/notes.md index 5dcc2e2..76cc121 100644 --- a/kubernetes-examples/official-examples/notes.md +++ b/kubernetes-examples/official-examples/notes.md @@ -1,5 +1,17 @@ +Originalmente encontrado aqui: +https://kubernetes.io/docs/tutorials/stateless-application/guestbook + +Está quebrado! Pelo menos uma das imagens usadas não existem mais! +```bash +docker pull gcr.io/google_samples/gb-frontend:v5 +``` +Refs.: +- https://kubernetes.io/docs/tutorials/stateless-application/guestbook/#creating-the-guestbook-frontend-deployment + + +Encontrei o repo original, mas também não consegui fazer funcionar 100%. https://cloud.google.com/kubernetes-engine/docs/tutorials/guestbook @@ -21,6 +33,7 @@ kubectl apply \ kubectl get service frontend ``` + ```bash while true; do kubectl get pod --all-namespaces -o wide \ From 7e8c21a3a6c5d9773d1c19baac9e6404b9a4b052 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Thu, 21 Dec 2023 17:13:49 -0300 Subject: [PATCH 09/86] Simplifica testes --- .github/workflows/tests.yml | 10 +++++----- README.md | 4 ---- flake.nix | 2 ++ 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 178c732..29f8924 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,13 +26,13 @@ jobs: - name: Executa docker do host run: | # sudo apt-get install -y podman - docker pull alpine - docker images - docker info + # docker pull alpine + # docker images + # docker info - name: Run alpine OCI image with docker and metadatas run: | - docker run alpine cat /etc/os*release + # docker run alpine cat /etc/os*release cat /etc/os*release uname -a - nix flake metadata nixpkgs + # nix flake metadata nixpkgs diff --git a/README.md b/README.md index 111c4c7..d8dd6c2 100644 --- a/README.md +++ b/README.md @@ -86,11 +86,7 @@ helm install arc \ --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ --set githubConfigSecret.github_token="${GITHUB_PAT}" \ oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set -``` - - -```bash while true; do kubectl get pod --all-namespaces -o wide \ && echo \ diff --git a/flake.nix b/flake.nix index f58fbd7..faa0e07 100644 --- a/flake.nix +++ b/flake.nix @@ -68,6 +68,7 @@ coreutils curl gettext + gh gnumake hack httpie @@ -223,6 +224,7 @@ direnv file firefox + gh git gnumake kubernetes-helm From 57c57a97d553bf1c59211c6100ac04bc3b66cf8e Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Thu, 21 Dec 2023 17:17:01 -0300 Subject: [PATCH 10/86] Testa instalar nix --- .github/workflows/tests.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 29f8924..d5d3f81 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,3 +36,25 @@ jobs: cat /etc/os*release uname -a # nix flake metadata nixpkgs + + - name: Instala nix single user + run: | + BASE_URL='https://raw.githubusercontent.com/ES-Nix/get-nix/' \ + && SHA256=87fa0f1dbfdd28a1f99b39d5bd4dcc39de97bc64 \ + && NIX_RELEASE_VERSION='2.10.2' \ + && curl -fsSL "${BASE_URL}""$SHA256"/get-nix.sh | sh -s -- ${NIX_RELEASE_VERSION} \ + && . "$HOME"/.nix-profile/etc/profile.d/nix.sh \ + && . ~/."$(basename $SHELL)"rc \ + && export TMPDIR=/tmp \ + && nix flake --version \ + && nix registry pin nixpkgs github:NixOS/nixpkgs/ea4c80b39be4c09702b0cb3b42eab59e2ba4f24b \ + && curl -fsSL https://raw.githubusercontent.com/ES-Nix/get-nix/"$SHA256"/install_direnv_and_nix_direnv.sh | sh \ + && . ~/."$(basename $SHELL)"rc \ + && direnv --version + echo "$HOME"/.nix-profile/bin >> $GITHUB_PATH + + - name: Testa nix + run: | + cat /etc/os*release + uname -a + nix flake metadata nixpkgs \ No newline at end of file From 6ba80ffbfed62f3895417eecb41c2ffcfea4619d Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Thu, 21 Dec 2023 17:22:43 -0300 Subject: [PATCH 11/86] Testa instalar curl para instalar nix --- .github/workflows/tests.yml | 17 ++++++----------- README.md | 12 ++++++++++++ 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d5d3f81..f689d79 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,21 +21,16 @@ jobs: uses: actions/checkout@v3 - name: Rodar um echo - run: echo 'teste' + run: echo 'Olá mundo!' - - name: Executa docker do host - run: | - # sudo apt-get install -y podman - # docker pull alpine - # docker images - # docker info - - - name: Run alpine OCI image with docker and metadatas + - name: Metadatas run: | - # docker run alpine cat /etc/os*release cat /etc/os*release uname -a - # nix flake metadata nixpkgs + + - name: Executa docker do host + run: | + apt-get update && apt-get install -y curl - name: Instala nix single user run: | diff --git a/README.md b/README.md index d8dd6c2..e339de9 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,18 @@ while true; do done ``` +Verifique que o runner aparece no link: +https://github.com/imobanco/github-ci-runner/actions/runners?tab=self-hosted + + +```bash +export GH_TOKEN=ghp_yyyyyyyyyyyyyyy +``` + + +```bash +gh workflow run tests.yml --ref feature/k8s +``` ### From d7e8c2a2a6f2d1a3efb95b418fdb37e70813a5e9 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Thu, 21 Dec 2023 17:24:26 -0300 Subject: [PATCH 12/86] Debugando o ambiente --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f689d79..82ccc22 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,7 +26,9 @@ jobs: - name: Metadatas run: | cat /etc/os*release + cat /etc/groups uname -a + id - name: Executa docker do host run: | From 7ddd270618241fa2eaadb6ccb296d63ee9b3fc74 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Thu, 21 Dec 2023 17:25:16 -0300 Subject: [PATCH 13/86] Debugando o ambiente --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 82ccc22..028a22c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,7 +26,7 @@ jobs: - name: Metadatas run: | cat /etc/os*release - cat /etc/groups + cat /etc/group uname -a id From 0a0f94953385aac18fb938c4c84980618cd930fb Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Thu, 21 Dec 2023 17:26:59 -0300 Subject: [PATCH 14/86] Debugando o ambiente, 3 --- .github/workflows/tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 028a22c..0bdaf7f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -29,10 +29,11 @@ jobs: cat /etc/group uname -a id + sudo id - name: Executa docker do host run: | - apt-get update && apt-get install -y curl + sudo apt-get update && sudo apt-get install -y curl - name: Instala nix single user run: | From 682a1e6bb6850b1ab5a187ae32ab8e7cf5986bcf Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Thu, 21 Dec 2023 17:29:09 -0300 Subject: [PATCH 15/86] Debugando o ambiente, 4 --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0bdaf7f..6218dfa 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,7 +33,7 @@ jobs: - name: Executa docker do host run: | - sudo apt-get update && sudo apt-get install -y curl + sudo apt-get update && sudo apt-get install -y curl xz - name: Instala nix single user run: | From 12eab64121bd90e8b6daba887609a239330c6f98 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Thu, 21 Dec 2023 17:30:50 -0300 Subject: [PATCH 16/86] Debugando o ambiente, 5 --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6218dfa..001672d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,7 +33,7 @@ jobs: - name: Executa docker do host run: | - sudo apt-get update && sudo apt-get install -y curl xz + sudo apt-get update && sudo apt-get install -y curl xz-utils - name: Instala nix single user run: | From 41864ad7717b56c5f3076645097d908b41bd1637 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Thu, 21 Dec 2023 17:44:41 -0300 Subject: [PATCH 17/86] Debugando o ambiente, 6 --- .github/workflows/tests.yml | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 001672d..af1d1c8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,25 +34,3 @@ jobs: - name: Executa docker do host run: | sudo apt-get update && sudo apt-get install -y curl xz-utils - - - name: Instala nix single user - run: | - BASE_URL='https://raw.githubusercontent.com/ES-Nix/get-nix/' \ - && SHA256=87fa0f1dbfdd28a1f99b39d5bd4dcc39de97bc64 \ - && NIX_RELEASE_VERSION='2.10.2' \ - && curl -fsSL "${BASE_URL}""$SHA256"/get-nix.sh | sh -s -- ${NIX_RELEASE_VERSION} \ - && . "$HOME"/.nix-profile/etc/profile.d/nix.sh \ - && . ~/."$(basename $SHELL)"rc \ - && export TMPDIR=/tmp \ - && nix flake --version \ - && nix registry pin nixpkgs github:NixOS/nixpkgs/ea4c80b39be4c09702b0cb3b42eab59e2ba4f24b \ - && curl -fsSL https://raw.githubusercontent.com/ES-Nix/get-nix/"$SHA256"/install_direnv_and_nix_direnv.sh | sh \ - && . ~/."$(basename $SHELL)"rc \ - && direnv --version - echo "$HOME"/.nix-profile/bin >> $GITHUB_PATH - - - name: Testa nix - run: | - cat /etc/os*release - uname -a - nix flake metadata nixpkgs \ No newline at end of file From ff6d19b8e593cb7479b0fc150d900673448eb01a Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Thu, 21 Dec 2023 17:55:07 -0300 Subject: [PATCH 18/86] =?UTF-8?q?Simplifica=20instru=C3=A7=C3=B5es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/tests.yml | 3 +- README.md | 237 ++---------------------------------- 2 files changed, 10 insertions(+), 230 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index af1d1c8..cc1f71d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,4 +33,5 @@ jobs: - name: Executa docker do host run: | - sudo apt-get update && sudo apt-get install -y curl xz-utils + sudo apt-get update && sudo apt-get install -y curl hello xz-utils + hello \ No newline at end of file diff --git a/README.md b/README.md index e339de9..87d031b 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,8 @@ RUNNER_SCOPE="org" SCOPE_TARGET="imobanco" bash ./ops/bash/entrypoint.sh # k8s in NixOS VM + + Gerar token: - onde gerar: https://github.com/settings/tokens - com os seguintes checks: https://github.com/myoung34/docker-github-actions-runner/wiki/Usage#token-scope @@ -60,16 +62,6 @@ rm -fv nixos.qcow2; nix run --impure --refresh --verbose .#vm Copie e cole no terminal da VM e edite com seu PAT gerado no pasos anterior: ```bash -GITHUB_TOKEN=ghp_yyyyyyyyyyyyyyy -``` - - -### - -https://docs.github.com/en/enterprise-server@3.11/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/quickstart-for-actions-runner-controller - -Substitua pelo seu PAT: -```bash GITHUB_PAT=ghp_yyyyyyyyyyyyyyy GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" @@ -103,7 +95,7 @@ done Verifique que o runner aparece no link: https://github.com/imobanco/github-ci-runner/actions/runners?tab=self-hosted - +No terminal do clone local (apenas para testes manuais) do repositório: ```bash export GH_TOKEN=ghp_yyyyyyyyyyyyyyy ``` @@ -113,225 +105,12 @@ export GH_TOKEN=ghp_yyyyyyyyyyyyyyy gh workflow run tests.yml --ref feature/k8s ``` +Pelo navegador: +https://github.com/imobanco/github-ci-runner/actions -### - - -```bash -# Bem haky, só bypassei o problema que deixava como pending -kubectl label nodes nixos size=linux -kubectl get nodes nixos --show-labels - -NAME_SPACE_RUNNER='actions-runner-system' -kubectl create ns "$NAME_SPACE_RUNNER" - -mkdir -pv ~/k8s-bootstrap-runner \ -&& ~/k8s-bootstrap-runner - -cat > script.sh <<-'EOF' -rm -rf *.pem *.csr *.srl || true - -# Step 1: CA -# ---------- - -# create CA, it is secret, keep it safe -openssl genrsa -out ca.private.pem 2048 - -# create public CA, give it to everyone so they can add it to trusted root -openssl req -x509 -new -key ca.private.pem -out ca.public.pem -days 10000 -subj "/C=UA/L=Kiev" - -# Step 2: Certificate -# ------------------- - -# create certificate, it is secret, keep it safe -openssl genrsa -out cert.private.pem 2048 - -# create "certificate signing request" (csr) -openssl req -new -key cert.private.pem -out cert.csr -subj "/CN=actions-runner-controller-webhook.actions-runner-system.svc" - -# config -cat <> cert.conf -[SAN] -subjectAltName = @alt_names -[alt_names] -DNS.1 = actions-runner-controller-webhook.actions-runner-system.svc -DNS.2 = actions-runner-controller-webhook.actions-runner-system.svc.cluster.local -EOT - -# sign it with our CA -openssl x509 -req -in cert.csr -CA ca.public.pem -CAkey ca.private.pem -CAcreateserial -out cert.public.pem -days 10000 -extensions SAN -extfile cert.conf - -# clean -rm -rf *.csr *.srl cert.conf || true - -# check -openssl x509 -in cert.public.pem -text -noout | grep DNS -EOF - -chmod 0755 script.sh - -./script.sh - -# smoke tests, better than nothing?! -test -f cert.public.pem || exit 1 -test -f cert.private.pem || exit 1 -test -f ca.public.pem || exit 1 - - -kubectl create secret tls actions-runner-controller-serving-cert \ ---namespace="$NAME_SPACE_RUNNER" \ ---cert=cert.public.pem \ ---key=cert.private.pem - - -# - -cat > values.yml <<-'EOF' -authSecret: - create: true - github_token: ghp_xxxxxxxxxxxxxxx - -# POI: disable cert manager -certManagerEnabled: false - -admissionWebHooks: - # POI: cat ca.public.pem | base64 - caBundle: xxxxxxxxxxxxxxxxxxxxxxxxxxxx= - -nodeSelector: -size: linux - -podAnnotations: - prometheus.io/scrape: "true" - prometheus.io/path: /metrics - prometheus.io/port: "8080" -EOF - - -sed -i 's/xxxxxxxxxxxxxxxxxxxxxxxxxxxx=/'"$(cat ca.public.pem | base64 -w 0)"'/g' values.yml -sed -i 's/ghp_xxxxxxxxxxxxxxx/'"$GITHUB_TOKEN"'/g' values.yml - -helm upgrade actions-runner-controller actions-runner-controller \ ---install \ ---namespace "$NAME_SPACE_RUNNER" \ ---repo https://actions-runner-controller.github.io/actions-runner-controller \ --f values.yml - - -cat > runner.yml <<-'EOF' -apiVersion: actions.summerwind.dev/v1alpha1 -kind: RunnerDeployment -metadata: - name: gha - namespace: actions-runner-system -spec: - replicas: 1 - template: - spec: - # https://github.com/actions-runner-controller/actions-runner-controller/blob/master/docs/detailed-docs.md#runner-with-dind - dockerdWithinRunnerContainer: true - organization: Imobanco - labels: - - 'gha' - - 'gha-dev' - nodeSelector: - size: linux - resources: - limits: - cpu: "6.0" - memory: "5Gi" - requests: - cpu: "3.0" - memory: "4Gi" -EOF +Links: +- https://docs.github.com/en/enterprise-server@3.11/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/quickstart-for-actions-runner-controller +- -kubectl --namespace="$NAME_SPACE_RUNNER" apply -f runner.yml - -export POD_NAME=$(kubectl get pods --namespace "$NAME_SPACE_RUNNER" -l "app.kubernetes.io/name=actions-runner-controller,app.kubernetes.io/instance=actions-runner-controller" -o jsonpath="{.items[0].metadata.name}") - -export CONTAINER_PORT=$(kubectl get pod --namespace "$NAME_SPACE_RUNNER" $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - - -echo "$POD_NAME" -echo "$CONTAINER_PORT" - -``` -Refs.: -- https://mac-blog.org.ua/github-actions-kubernetes-runner-without-certmanager/ -- https://serverfault.com/questions/1099167/node-pool-selection -- https://superuser.com/a/1225139 - - -```bash -kubectl describe -n "$NAME_SPACE_RUNNER" pod "$POD_NAME" -``` - - -```bash -kubectl describe pod "$POD_NAME" -n "$NAME_SPACE_RUNNER" -``` - - -TODO: -Pq esse comando não termina? -O que esse comando faz exatamente? Explicação: -https://kubernetes.io/docs/tutorials/stateless-application/guestbook/#viewing-the-frontend-service-via-kubectl-port-forward -```bash -kubectl \ ---namespace "$NAME_SPACE_RUNNER" \ -port-forward $POD_NAME 8080:$CONTAINER_PORT -``` - - -```bash -kubectl -n actions-runner-system get pods -kubectl -n actions-runner-system get runners -``` -Refs.: -- https://mac-blog.org.ua/github-actions-kubernetes-runner-without-certmanager/ - - - -### - - -```bash -GITHUB_TOKEN=ghp_yyyyyyyyyyyyyyy -``` - - -```bash -NAME_SPACE_RUNNER='actions-runner-system' - - -helm repo add jetstack https://charts.jetstack.io - -helm install \ - cert-manager jetstack/cert-manager \ - --namespace cert-manager \ - --create-namespace \ - --version v1.12.4 \ - --set installCRDs=true - -curl -L https://github.com/summerwind/actions-runner-controller/releases/latest/download/actions-runner-controller.yaml > actions-runner-controller.yaml - -kubectl apply -f actions-runner-controller.yaml - -kubectl create secret generic controller-manager \ ---from-literal=github_token=${GITHUB_TOKEN} \ --n actions-runner-system - -cat > runner.yaml <<-'EOF' -apiVersion: actions.summerwind.dev/v1alpha1 -kind: Runner -metadata: - name: example-runner -spec: - repository: summerwind/actions-runner-controller - env: [] -EOF - -kubectl apply -f runner.yaml -n "$NAME_SPACE_RUNNER" -``` From d29ed06afc01fde096d6d47c3822bb2797132b99 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Thu, 21 Dec 2023 17:58:10 -0300 Subject: [PATCH 19/86] Ajusta texto --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cc1f71d..3c090bd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,7 +31,7 @@ jobs: id sudo id - - name: Executa docker do host + - name: Executa instalação via apt para teste mínimo run: | - sudo apt-get update && sudo apt-get install -y curl hello xz-utils + sudo apt-get update && sudo apt-get install -y hello hello \ No newline at end of file From 1fff74335c2040046e1c1f6150a7342c671bfc84 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Thu, 21 Dec 2023 19:57:48 -0300 Subject: [PATCH 20/86] Ajustes, pricipalmente time.timeZone = "America/Recife"; --- README.md | 7 +++---- flake.nix | 4 ++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 87d031b..2bcce59 100644 --- a/README.md +++ b/README.md @@ -45,10 +45,7 @@ RUNNER_SCOPE="org" SCOPE_TARGET="imobanco" bash ./ops/bash/entrypoint.sh ``` -# k8s in NixOS VM - - - +# github self-hosted runner em k8s em uma VM NixOS Gerar token: - onde gerar: https://github.com/settings/tokens @@ -104,6 +101,8 @@ export GH_TOKEN=ghp_yyyyyyyyyyyyyyy ```bash gh workflow run tests.yml --ref feature/k8s ``` +Refs.: +- https://docs.github.com/en/enterprise-server@3.11/actions/using-workflows/manually-running-a-workflow?tool=cli#running-a-workflow Pelo navegador: https://github.com/imobanco/github-ci-runner/actions diff --git a/flake.nix b/flake.nix index faa0e07..f076698 100644 --- a/flake.nix +++ b/flake.nix @@ -131,6 +131,9 @@ # i18n.defaultLocale = "pt_BR.UTF-8"; console.keyMap = "br-abnt2"; + # Set your time zone. + time.timeZone = "America/Recife"; + # Why # nix flake show --impure .# # break if it does not exists? @@ -435,6 +438,7 @@ environment.variables.KUBECONFIG = "/etc/kubernetes/cluster-admin.kubeconfig"; + # services.kubernetes.kubelet.seedDockerImages = []; services.kubernetes.roles = [ "master" "node" ]; services.kubernetes.masterAddress = "nixos"; services.kubernetes = { From 113b773968598c4f4556c8bc2194feb0ea97c950 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Fri, 22 Dec 2023 14:02:50 -0300 Subject: [PATCH 21/86] Ajusts --- README.md | 5 ++++- kubernetes-examples/official-examples/notes.md | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2bcce59..a62d031 100644 --- a/README.md +++ b/README.md @@ -57,10 +57,13 @@ rm -fv nixos.qcow2; nix run --impure --refresh --verbose .#vm ``` -Copie e cole no terminal da VM e edite com seu PAT gerado no pasos anterior: +Copie e cole no terminal da VM e EDITE com seu PAT gerado no pasos anterior: ```bash GITHUB_PAT=ghp_yyyyyyyyyyyyyyy +``` + +```bash GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" INSTALLATION_NAME="arc-runner-set" NAMESPACE="arc-runners" diff --git a/kubernetes-examples/official-examples/notes.md b/kubernetes-examples/official-examples/notes.md index 76cc121..07ea4cb 100644 --- a/kubernetes-examples/official-examples/notes.md +++ b/kubernetes-examples/official-examples/notes.md @@ -10,6 +10,10 @@ docker pull gcr.io/google_samples/gb-frontend:v5 Refs.: - https://kubernetes.io/docs/tutorials/stateless-application/guestbook/#creating-the-guestbook-frontend-deployment +```bash +journalctl --unit docker +``` + Encontrei o repo original, mas também não consegui fazer funcionar 100%. From e356c862146a78a9cee9de812d6494dde24755aa Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Fri, 22 Dec 2023 14:11:33 -0300 Subject: [PATCH 22/86] Reduz RAM para 8Gigas --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index f076698..3b6425e 100644 --- a/flake.nix +++ b/flake.nix @@ -151,7 +151,7 @@ programs.dconf.enable = true; # security.polkit.enable = true; # TODO: hardening? - virtualisation.memorySize = 1024 * 10; # Use MiB memory. + virtualisation.memorySize = 1024 * 8; # Use MiB memory. virtualisation.diskSize = 1024 * 50; # Use MiB memory. virtualisation.cores = 8; # Number of cores. virtualisation.graphics = true; From bc968d8917c79a8041e4fc2d5d8f5e51c5fd5f0d Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Fri, 22 Dec 2023 18:39:34 -0300 Subject: [PATCH 23/86] Debugando o ambiente, DinD tests, 0 --- README.md | 148 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 147 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a62d031..c03724f 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ rm -fv nixos.qcow2; nix run --impure --refresh --verbose .#vm ``` -Copie e cole no terminal da VM e EDITE com seu PAT gerado no pasos anterior: +Copie e cole no terminal da VM e EDITE com seu PAT gerado no passo anterior: ```bash GITHUB_PAT=ghp_yyyyyyyyyyyyyyy ``` @@ -116,3 +116,149 @@ Links: - + +```bash +cd "$HOME" \ +&& git clone https://github.com/actions/actions-runner-controller.git \ +&& cd actions-runner-controller + +mkdir -pv ~/arc-configuration/{controller,runner-scale-set-1,runner-scale-set-2} \ +&& cd ~/arc-configuration + + +cd ~/actions-runner-controller/charts \ +&& cp -v actions-runner-controller/values.yaml ~/arc-configuration/controller \ +&& cp -v gha-runner-scale-set/values.yaml ~/arc-configuration/runner-scale-set-1 \ +&& cp -v gha-runner-scale-set/values.yaml ~/arc-configuration/runner-scale-set-2 + +``` + + +```bash +helm install arc \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + --set image.tag="0.4.0" \ + -f ~/arc-configuration/controller/values.yaml \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller \ + --version "0.4.0" +``` + + + +## DinD + + +Copie e cole no terminal da VM e EDITE com seu PAT gerado no passo anterior: +```bash +GITHUB_PAT=ghp_yyyyyyyyyyyyyyy +``` + + +```bash +GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" +INSTALLATION_NAME="arc-runner-set" +NAMESPACE="arc-runners" + +helm install arc \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller \ +&& helm install "${INSTALLATION_NAME}" \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ + --set githubConfigSecret.github_token="${GITHUB_PAT}" \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set + +while true; do + kubectl get pod --all-namespaces -o wide \ + && echo \ + && kubectl get services --all-namespaces -o wide \ + && echo \ + && kubectl get deployments.apps --all-namespaces -o wide \ + && echo \ + && kubectl get nodes --all-namespaces -o wide; + sleep 2; + clear; +done +``` + + + + +```bash +cd "$HOME" \ +&& git clone https://github.com/actions/actions-runner-controller.git \ +&& cd actions-runner-controller + + +cat << 'EOF' > enables-dind.patch +diff --git a/charts/gha-runner-scale-set/values.yaml b/charts/gha-runner-scale-set/values.yaml +index 021fecb..b474e88 100644 +--- a/charts/gha-runner-scale-set/values.yaml ++++ b/charts/gha-runner-scale-set/values.yaml +@@ -75,8 +75,8 @@ githubConfigSecret: + ## + ## If any customization is required for dind or kubernetes mode, containerMode should remain + ## empty, and configuration should be applied to the template. +-# containerMode: +-# type: "dind" ## type can be set to dind or kubernetes ++containerMode: ++ type: "dind" ## type can be set to dind or kubernetes + # ## the following is required when containerMode.type=kubernetes + # kubernetesModeWorkVolumeClaim: + # accessModes: ["ReadWriteOnce"] +@@ -199,6 +199,6 @@ template: + ## In case the helm chart can't find the right service account, you can explicitly pass in the following value + ## to help it finish RoleBinding with the right service account. + ## Note: if your controller is installed to only watch a single namespace, you have to pass these values explicitly. +-# controllerServiceAccount: +-# namespace: arc-system +-# name: test-arc-gha-runner-scale-set-controller ++controllerServiceAccount: ++ namespace: arc-system +EOF + +git apply enables-dind.patch +``` + + +```bash +helm install arc-runner-set \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + --set githubConfigSecret.github_token="${GITHUB_PAT}" \ + --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ + --set image.tag="0.4.0" \ + -f ~/actions-runner-controller/charts/gha-runner-scale-set/values.yaml \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \ + --version "0.4.0" +``` + + + + + + + +```bash +GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" +INSTALLATION_NAME="arc-runner-set" +NAMESPACE="arc-runners" + +helm install arc \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller \ +&& helm install arc-runner-set \ + --create-namespace \ + --namespace "${NAMESPACE}" \ + --set githubConfigSecret.github_token="${GITHUB_PAT}" \ + --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ + --set image.tag="0.4.0" \ + --version "0.4.0" \ + -f ~/actions-runner-controller/charts/gha-runner-scale-set/values.yaml \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set +``` + From dd3b62b27967f3a9a85bd5bb9ee44c7ca056acd1 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Fri, 22 Dec 2023 18:44:58 -0300 Subject: [PATCH 24/86] Debugando o ambiente, DinD tests, 1 --- .github/workflows/tests.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3c090bd..8ba6fc9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,4 +34,20 @@ jobs: - name: Executa instalação via apt para teste mínimo run: | sudo apt-get update && sudo apt-get install -y hello - hello \ No newline at end of file + hello + + - name: Instala nix single user + run: | + BASE_URL='https://raw.githubusercontent.com/ES-Nix/get-nix/' \ + && SHA256=87fa0f1dbfdd28a1f99b39d5bd4dcc39de97bc64 \ + && NIX_RELEASE_VERSION='2.10.2' \ + && curl -fsSL "${BASE_URL}""$SHA256"/get-nix.sh | sh -s -- ${NIX_RELEASE_VERSION} \ + && . "$HOME"/.nix-profile/etc/profile.d/nix.sh \ + && . ~/."$(basename $SHELL)"rc \ + && export TMPDIR=/tmp \ + && nix flake --version \ + && nix registry pin nixpkgs github:NixOS/nixpkgs/ea4c80b39be4c09702b0cb3b42eab59e2ba4f24b \ + && curl -fsSL https://raw.githubusercontent.com/ES-Nix/get-nix/"$SHA256"/install_direnv_and_nix_direnv.sh | sh \ + && . ~/."$(basename $SHELL)"rc \ + && direnv --version + echo "$HOME"/.nix-profile/bin >> $GITHUB_PATH From 7504015490a10e4f6f5858c3f7b86336468426a7 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Fri, 22 Dec 2023 18:46:24 -0300 Subject: [PATCH 25/86] Debugando o ambiente, DinD tests, 2 --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8ba6fc9..9f572e9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,7 +33,7 @@ jobs: - name: Executa instalação via apt para teste mínimo run: | - sudo apt-get update && sudo apt-get install -y hello + sudo apt-get update && sudo apt-get install -y hello curl xz-utils hello - name: Instala nix single user From a50873abeaf1a8daadbd708b8f49dd0fb753c28a Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Fri, 22 Dec 2023 18:48:32 -0300 Subject: [PATCH 26/86] Debugando o ambiente, DinD tests, 3 --- .github/workflows/tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9f572e9..e5436bf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,6 +36,10 @@ jobs: sudo apt-get update && sudo apt-get install -y hello curl xz-utils hello + - name: Instala nix single user + run: | + docker images + - name: Instala nix single user run: | BASE_URL='https://raw.githubusercontent.com/ES-Nix/get-nix/' \ From 8d1224157654b97326137c57733a383d2cb0023b Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Fri, 22 Dec 2023 19:03:27 -0300 Subject: [PATCH 27/86] Debugando o ambiente, DinD tests, 4 --- .github/workflows/tests.yml | 19 ++--------- README.md | 66 ++++++++++++++++++++----------------- 2 files changed, 38 insertions(+), 47 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e5436bf..fc4c976 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,7 +26,7 @@ jobs: - name: Metadatas run: | cat /etc/os*release - cat /etc/group + # cat /etc/group uname -a id sudo id @@ -38,20 +38,5 @@ jobs: - name: Instala nix single user run: | + docker info docker images - - - name: Instala nix single user - run: | - BASE_URL='https://raw.githubusercontent.com/ES-Nix/get-nix/' \ - && SHA256=87fa0f1dbfdd28a1f99b39d5bd4dcc39de97bc64 \ - && NIX_RELEASE_VERSION='2.10.2' \ - && curl -fsSL "${BASE_URL}""$SHA256"/get-nix.sh | sh -s -- ${NIX_RELEASE_VERSION} \ - && . "$HOME"/.nix-profile/etc/profile.d/nix.sh \ - && . ~/."$(basename $SHELL)"rc \ - && export TMPDIR=/tmp \ - && nix flake --version \ - && nix registry pin nixpkgs github:NixOS/nixpkgs/ea4c80b39be4c09702b0cb3b42eab59e2ba4f24b \ - && curl -fsSL https://raw.githubusercontent.com/ES-Nix/get-nix/"$SHA256"/install_direnv_and_nix_direnv.sh | sh \ - && . ~/."$(basename $SHELL)"rc \ - && direnv --version - echo "$HOME"/.nix-profile/bin >> $GITHUB_PATH diff --git a/README.md b/README.md index c03724f..e12f67d 100644 --- a/README.md +++ b/README.md @@ -155,42 +155,14 @@ GITHUB_PAT=ghp_yyyyyyyyyyyyyyy ``` -```bash -GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" -INSTALLATION_NAME="arc-runner-set" -NAMESPACE="arc-runners" - -helm install arc \ - --namespace "${NAMESPACE}" \ - --create-namespace \ - oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller \ -&& helm install "${INSTALLATION_NAME}" \ - --namespace "${NAMESPACE}" \ - --create-namespace \ - --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ - --set githubConfigSecret.github_token="${GITHUB_PAT}" \ - oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set - -while true; do - kubectl get pod --all-namespaces -o wide \ - && echo \ - && kubectl get services --all-namespaces -o wide \ - && echo \ - && kubectl get deployments.apps --all-namespaces -o wide \ - && echo \ - && kubectl get nodes --all-namespaces -o wide; - sleep 2; - clear; -done -``` - ```bash cd "$HOME" \ && git clone https://github.com/actions/actions-runner-controller.git \ -&& cd actions-runner-controller +&& cd actions-runner-controller \ +&& git checkout 1f9b7541e6545a9d5ffa052481a84aad7ba4aa4d cat << 'EOF' > enables-dind.patch @@ -218,12 +190,46 @@ index 021fecb..b474e88 100644 -# name: test-arc-gha-runner-scale-set-controller +controllerServiceAccount: + namespace: arc-system ++ name: test-arc-gha-runner-scale-set-controller EOF git apply enables-dind.patch ``` +```bash +GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" +INSTALLATION_NAME="arc-runner-set" +NAMESPACE="arc-runners" + +helm install arc \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller \ +&& helm install arc-runner-set \ + --create-namespace \ + --namespace "${NAMESPACE}" \ + --set githubConfigSecret.github_token="${GITHUB_PAT}" \ + --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ + --set image.tag="0.4.0" \ + --version "0.4.0" \ + -f ~/actions-runner-controller/charts/gha-runner-scale-set/values.yaml \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set + +while true; do + kubectl get pod --all-namespaces -o wide \ + && echo \ + && kubectl get services --all-namespaces -o wide \ + && echo \ + && kubectl get deployments.apps --all-namespaces -o wide \ + && echo \ + && kubectl get nodes --all-namespaces -o wide; + sleep 2; + clear; +done +``` + + ```bash helm install arc-runner-set \ --namespace "${NAMESPACE}" \ From 0415e2130494215335d35c1046baf54f759777a0 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Fri, 22 Dec 2023 23:48:49 -0300 Subject: [PATCH 28/86] WIP --- .github/workflows/tests.yml | 2 +- README.md | 201 +++++++++++++++++++----------------- flake.nix | 17 +++ 3 files changed, 127 insertions(+), 93 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fc4c976..1e6e607 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,7 +14,7 @@ on: workflow_dispatch jobs: test: - runs-on: arc-runner-set + runs-on: arc-runner-set-dind name: Testes steps: - name: checkout PR merge commit diff --git a/README.md b/README.md index e12f67d..28c367e 100644 --- a/README.md +++ b/README.md @@ -64,32 +64,26 @@ GITHUB_PAT=ghp_yyyyyyyyyyyyyyy ```bash -GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" -INSTALLATION_NAME="arc-runner-set" -NAMESPACE="arc-runners" +NAMESPACE="arc-systems" helm install arc \ --namespace "${NAMESPACE}" \ --create-namespace \ - oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller \ -&& helm install "${INSTALLATION_NAME}" \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller + + +INSTALLATION_NAME="arc-runner-set" +NAMESPACE="arc-runners" +GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" + +helm install "${INSTALLATION_NAME}" \ --namespace "${NAMESPACE}" \ --create-namespace \ --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ --set githubConfigSecret.github_token="${GITHUB_PAT}" \ oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set -while true; do - kubectl get pod --all-namespaces -o wide \ - && echo \ - && kubectl get services --all-namespaces -o wide \ - && echo \ - && kubectl get deployments.apps --all-namespaces -o wide \ - && echo \ - && kubectl get nodes --all-namespaces -o wide; - sleep 2; - clear; -done +wk8s ``` Verifique que o runner aparece no link: @@ -117,35 +111,6 @@ Links: -```bash -cd "$HOME" \ -&& git clone https://github.com/actions/actions-runner-controller.git \ -&& cd actions-runner-controller - -mkdir -pv ~/arc-configuration/{controller,runner-scale-set-1,runner-scale-set-2} \ -&& cd ~/arc-configuration - - -cd ~/actions-runner-controller/charts \ -&& cp -v actions-runner-controller/values.yaml ~/arc-configuration/controller \ -&& cp -v gha-runner-scale-set/values.yaml ~/arc-configuration/runner-scale-set-1 \ -&& cp -v gha-runner-scale-set/values.yaml ~/arc-configuration/runner-scale-set-2 - -``` - - -```bash -helm install arc \ - --namespace "${NAMESPACE}" \ - --create-namespace \ - --set image.tag="0.4.0" \ - -f ~/arc-configuration/controller/values.yaml \ - oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller \ - --version "0.4.0" -``` - - - ## DinD @@ -155,9 +120,6 @@ GITHUB_PAT=ghp_yyyyyyyyyyyyyyy ``` - - - ```bash cd "$HOME" \ && git clone https://github.com/actions/actions-runner-controller.git \ @@ -167,9 +129,23 @@ cd "$HOME" \ cat << 'EOF' > enables-dind.patch diff --git a/charts/gha-runner-scale-set/values.yaml b/charts/gha-runner-scale-set/values.yaml -index 021fecb..b474e88 100644 +index 021fecb..4395555 100644 --- a/charts/gha-runner-scale-set/values.yaml +++ b/charts/gha-runner-scale-set/values.yaml +@@ -37,11 +37,11 @@ githubConfigSecret: + # - example.org + + ## maxRunners is the max number of runners the autoscaling runner set will scale up to. +-# maxRunners: 5 ++maxRunners: 3 + + ## minRunners is the min number of idle runners. The target number of runners created will be + ## calculated as a sum of minRunners and the number of jobs assigned to the scale set. +-# minRunners: 0 ++minRunners: 1 + + # runnerGroup: "default" + @@ -75,8 +75,8 @@ githubConfigSecret: ## ## If any customization is required for dind or kubernetes mode, containerMode should remain @@ -194,77 +170,118 @@ index 021fecb..b474e88 100644 EOF git apply enables-dind.patch -``` +rm -v enables-dind.patch -```bash -GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" -INSTALLATION_NAME="arc-runner-set" -NAMESPACE="arc-runners" + +NAMESPACE="arc-systems" helm install arc \ --namespace "${NAMESPACE}" \ --create-namespace \ - oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller \ -&& helm install arc-runner-set \ - --create-namespace \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller + + +INSTALLATION_NAME="arc-runner-set1" +NAMESPACE="arc-runners" +GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" + +helm install "${INSTALLATION_NAME}" \ --namespace "${NAMESPACE}" \ - --set githubConfigSecret.github_token="${GITHUB_PAT}" \ + --create-namespace \ --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ - --set image.tag="0.4.0" \ - --version "0.4.0" \ - -f ~/actions-runner-controller/charts/gha-runner-scale-set/values.yaml \ + --set githubConfigSecret.github_token="${GITHUB_PAT}" \ oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set -while true; do - kubectl get pod --all-namespaces -o wide \ - && echo \ - && kubectl get services --all-namespaces -o wide \ - && echo \ - && kubectl get deployments.apps --all-namespaces -o wide \ - && echo \ - && kubectl get nodes --all-namespaces -o wide; - sleep 2; - clear; -done -``` +INSTALLATION_NAME="arc-runner-set-dind" +# NAMESPACE="arc-runners" +GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" -```bash -helm install arc-runner-set \ +helm install "${INSTALLATION_NAME}" \ --namespace "${NAMESPACE}" \ --create-namespace \ - --set githubConfigSecret.github_token="${GITHUB_PAT}" \ --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ - --set image.tag="0.4.0" \ - -f ~/actions-runner-controller/charts/gha-runner-scale-set/values.yaml \ - oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \ - --version "0.4.0" + --set githubConfigSecret.github_token="${GITHUB_PAT}" \ + --values ~/actions-runner-controller/charts/gha-runner-scale-set/values.yaml \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set + +wk8s ``` +```bash +kubectl get pods -n arc-runners +``` +### + ```bash -GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" +NAMESPACE="arc-systems" + +helm \ +install \ +--dry-run \ +arc \ +--namespace "${NAMESPACE}" \ +--create-namespace \ +oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller \ +-o yaml > gha-runner-scale-set-controller.yml + + INSTALLATION_NAME="arc-runner-set" NAMESPACE="arc-runners" +GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" + + +helm \ +install \ +--dry-run \ +"${INSTALLATION_NAME}" \ +--namespace "${NAMESPACE}" \ +--create-namespace \ +--set githubConfigUrl="${GITHUB_CONFIG_URL}" \ +--set githubConfigSecret.github_token="${GITHUB_PAT}" \ +oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \ +-o yaml > gha-runner-scale-set.yml + +wk8s +``` + + + +```bash +helm install --dry-run --debug + +#kubectl create namespace arc-runners +#kubectl create secret generic pre-defined-secret \ +#--namespace=arc-runners \ +#--from-literal=github_token="${GITHUB_PAT}" + -helm install arc \ - --namespace "${NAMESPACE}" \ - --create-namespace \ - oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller \ -&& helm install arc-runner-set \ - --create-namespace \ - --namespace "${NAMESPACE}" \ - --set githubConfigSecret.github_token="${GITHUB_PAT}" \ - --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ --set image.tag="0.4.0" \ --version "0.4.0" \ - -f ~/actions-runner-controller/charts/gha-runner-scale-set/values.yaml \ - oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set ``` + +```bash +cd "$HOME" \ +&& git clone https://github.com/actions/actions-runner-controller.git \ +&& cd actions-runner-controller \ +&& git checkout 1f9b7541e6545a9d5ffa052481a84aad7ba4aa4d + + +mkdir -pv ~/arc-configuration/{controller,runner-scale-set-1,runner-scale-set-2} \ +&& cd ~/arc-configuration + + +cd ~/actions-runner-controller/charts \ +&& cp -v actions-runner-controller/values.yaml ~/arc-configuration/controller \ +&& cp -v gha-runner-scale-set/values.yaml ~/arc-configuration/runner-scale-set-1 \ +&& cp -v gha-runner-scale-set/values.yaml ~/arc-configuration/runner-scale-set-2 + +``` diff --git a/flake.nix b/flake.nix index 3b6425e..050edf9 100644 --- a/flake.nix +++ b/flake.nix @@ -415,6 +415,23 @@ sudo chown root:kubernetes -v /var/lib/kubernetes/secrets/cluster-admin-key.pem '' ) + + ( + writeScriptBin "wk8s" '' + #! ${pkgs.runtimeShell} -e + while true; do + kubectl get pod --all-namespaces -o wide \ + && echo \ + && kubectl get services --all-namespaces -o wide \ + && echo \ + && kubectl get deployments.apps --all-namespaces -o wide \ + && echo \ + && kubectl get nodes --all-namespaces -o wide; + sleep 1; + clear; + done + '' + ) ]; # Is this a must to kubernetes? From d927884289789d669710ac9d0f80bf4fd97d82a5 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Mon, 25 Dec 2023 23:50:05 -0300 Subject: [PATCH 29/86] WIP, testanod dind --- .github/workflows/tests.yml | 2 +- README.md | 222 ++++++++++++++++-- flake.nix | 1 + kubernetes-examples/appvia/notes.md | 12 +- kubernetes-examples/flask-gunicorn/notes.md | 12 +- .../minimal-pod-with-busybox-example/notes.md | 16 +- kubernetes-examples/nodejs-node-port/notes.md | 12 +- .../official-examples/notes.md | 59 ++++- 8 files changed, 264 insertions(+), 72 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1e6e607..fc4c976 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,7 +14,7 @@ on: workflow_dispatch jobs: test: - runs-on: arc-runner-set-dind + runs-on: arc-runner-set name: Testes steps: - name: checkout PR merge commit diff --git a/README.md b/README.md index 28c367e..c318c4f 100644 --- a/README.md +++ b/README.md @@ -69,8 +69,9 @@ NAMESPACE="arc-systems" helm install arc \ --namespace "${NAMESPACE}" \ --create-namespace \ - oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller - + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller \ + --set image.tag="0.8.1" \ + --version "0.8.1" INSTALLATION_NAME="arc-runner-set" NAMESPACE="arc-runners" @@ -81,11 +82,15 @@ helm install "${INSTALLATION_NAME}" \ --create-namespace \ --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ --set githubConfigSecret.github_token="${GITHUB_PAT}" \ - oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set - + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \ + --set image.tag="0.8.1" \ + --version "0.8.1" wk8s ``` + + + Verifique que o runner aparece no link: https://github.com/imobanco/github-ci-runner/actions/runners?tab=self-hosted @@ -173,16 +178,25 @@ git apply enables-dind.patch rm -v enables-dind.patch - NAMESPACE="arc-systems" - -helm install arc \ +helm install arc --wait \ --namespace "${NAMESPACE}" \ --create-namespace \ oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller +#INSTALLATION_NAME="arc-runner-set1" +#NAMESPACE="arc-runners" +#GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" +# +#helm install "${INSTALLATION_NAME}" --wait \ +# --namespace "${NAMESPACE}" \ +# --create-namespace \ +# --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ +# --set githubConfigSecret.github_token="${GITHUB_PAT}" \ +# oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set -INSTALLATION_NAME="arc-runner-set1" + +INSTALLATION_NAME="arc-runner-set-dind" NAMESPACE="arc-runners" GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" @@ -191,11 +205,79 @@ helm install "${INSTALLATION_NAME}" \ --create-namespace \ --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ --set githubConfigSecret.github_token="${GITHUB_PAT}" \ - oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set + --values ~/actions-runner-controller/charts/gha-runner-scale-set/values.yaml \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \ + --set image.tag="0.4.0" \ + --version "0.4.0" +wk8s +``` -INSTALLATION_NAME="arc-runner-set-dind" -# NAMESPACE="arc-runners" +```bash +kubectl get pods -n arc-systems +kubectl get pods -n arc-runners +``` + + +### Tentativa + + +```bash +cd "$HOME" \ +&& git clone https://github.com/actions/actions-runner-controller.git \ +&& cd actions-runner-controller \ +&& git checkout 1f9b7541e6545a9d5ffa052481a84aad7ba4aa4d + + +cat << 'EOF' > pin-tags-and-dind.patch +diff --git a/charts/gha-runner-scale-set/values.yaml b/charts/gha-runner-scale-set/values.yaml +index 021fecb..6bc29fe 100644 +--- a/charts/gha-runner-scale-set/values.yaml ++++ b/charts/gha-runner-scale-set/values.yaml +@@ -75,8 +75,8 @@ githubConfigSecret: + ## + ## If any customization is required for dind or kubernetes mode, containerMode should remain + ## empty, and configuration should be applied to the template. +-# containerMode: +-# type: "dind" ## type can be set to dind or kubernetes ++containerMode: ++ type: "dind" ## type can be set to dind or kubernetes + # ## the following is required when containerMode.type=kubernetes + # kubernetesModeWorkVolumeClaim: + # accessModes: ["ReadWriteOnce"] +@@ -133,7 +133,7 @@ template: + ## mountPath: /run/docker + ## readOnly: true + ## - name: dind +- ## image: docker:dind ++ ## image: docker:24.0.7-dind-alpine3.18 + ## args: + ## - dockerd + ## - --host=unix:///run/docker/docker.sock +@@ -190,7 +190,7 @@ template: + spec: + containers: + - name: runner +- image: ghcr.io/actions/actions-runner:latest ++ image: ghcr.io/actions/actions-runner:2.306.0 + command: ["/home/runner/run.sh"] + + ## Optional controller service account that needs to have required Role and RoleBinding +EOF + +git apply pin-tags-and-dind.patch +rm -v pin-tags-and-dind.patch + + +NAMESPACE="arc-systems" +helm install arc-1 \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller + + +INSTALLATION_NAME="arc-runner-set" +NAMESPACE="arc-runners" GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" helm install "${INSTALLATION_NAME}" \ @@ -204,17 +286,129 @@ helm install "${INSTALLATION_NAME}" \ --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ --set githubConfigSecret.github_token="${GITHUB_PAT}" \ --values ~/actions-runner-controller/charts/gha-runner-scale-set/values.yaml \ - oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set - + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \ + --set image.tag="0.8.1" \ + --version "0.8.1" wk8s ``` + + +#### Kaniko + + +https://some-natalie.dev/blog/kaniko-in-arc/ + ```bash -kubectl get pods -n arc-runners + +NAMESPACE="arc-systems" + +helm install arc \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller + + +INSTALLATION_NAME="arc-runner-set" +NAMESPACE="arc-runners" +GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" + +helm install "${INSTALLATION_NAME}" \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ + --set githubConfigSecret.github_token="${GITHUB_PAT}" \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set + + +cat > k8s-storage.yml << 'EOF' +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: k8s-mode + namespace: test-runners # just showing the test namespace +provisioner: file.csi.azure.com # change this to your provisioner +allowVolumeExpansion: true # probably not strictly necessary +reclaimPolicy: Delete +mountOptions: + - dir_mode=0777 # this mounts at a directory needing this + - file_mode=0777 + - uid=1000 # match your pod's user id, this is for actions/actions-runner + - gid=1000 + - mfsymlinks + - cache=strict + - actimeo=30 +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: test-k8s-cache-pvc + namespace: test-runners +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: k8s-mode # we'll need this in the runner Helm chart +EOF + + +kubectl create namespace test-runners +kubectl apply -f k8s-storage.yml ``` +```bash +cat > helm-kaniko.yml << 'EOF' +template: + spec: + initContainers: # needed to set permissions to use the PVC + - name: kube-init + image: ghcr.io/actions/actions-runner:latest + command: ["sudo", "chown", "-R", "runner:runner", "/home/runner/_work"] + volumeMounts: + - name: work + mountPath: /home/runner/_work + containers: + - name: runner + image: ghcr.io/actions/actions-runner:latest + command: ["/home/runner/run.sh"] + env: + - name: ACTIONS_RUNNER_CONTAINER_HOOKS + value: /home/runner/k8s/index.js + - name: ACTIONS_RUNNER_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: ACTIONS_RUNNER_REQUIRE_JOB_CONTAINER + value: "false" # allow non-container steps, makes life easier + volumeMounts: + - name: work + mountPath: /home/runner/_work + +containerMode: + type: "kubernetes" + kubernetesModeWorkVolumeClaim: + accessModes: ["ReadWriteOnce"] + storageClassName: "k8s-mode" + resources: + requests: + storage: 1Gi +EOF + +helm install kaniko-worker \ + --namespace "test-runners" \ + --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ + --set githubConfigSecret.github_token="${GITHUB_PAT}" \ + -f helm-kaniko.yml \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \ + --version 0.8.1 + +kubectl get pods -n "test-runners" +``` + ### diff --git a/flake.nix b/flake.nix index 050edf9..42e63cf 100644 --- a/flake.nix +++ b/flake.nix @@ -456,6 +456,7 @@ environment.variables.KUBECONFIG = "/etc/kubernetes/cluster-admin.kubeconfig"; # services.kubernetes.kubelet.seedDockerImages = []; + # dockerTools.examples.redis services.kubernetes.roles = [ "master" "node" ]; services.kubernetes.masterAddress = "nixos"; services.kubernetes = { diff --git a/kubernetes-examples/appvia/notes.md b/kubernetes-examples/appvia/notes.md index f08db0a..2865eba 100644 --- a/kubernetes-examples/appvia/notes.md +++ b/kubernetes-examples/appvia/notes.md @@ -12,15 +12,5 @@ cd ~/kubernetes-examples/appvia \ ```bash -while true; do - kubectl get pod --all-namespaces -o wide \ - && echo \ - && kubectl get services --all-namespaces -o wide \ - && echo \ - && kubectl get deployments.apps --all-namespaces -o wide \ - && echo \ - && kubectl get nodes --all-namespaces -o wide; - sleep 2; - clear; -done +wk8s ``` diff --git a/kubernetes-examples/flask-gunicorn/notes.md b/kubernetes-examples/flask-gunicorn/notes.md index f9b49cb..2eb88c4 100644 --- a/kubernetes-examples/flask-gunicorn/notes.md +++ b/kubernetes-examples/flask-gunicorn/notes.md @@ -64,17 +64,7 @@ Refs.: ```bash -while true; do - kubectl get pod --all-namespaces -o wide \ - && echo \ - && kubectl get services --all-namespaces -o wide \ - && echo \ - && kubectl get deployments.apps --all-namespaces -o wide \ - && echo \ - && kubectl get nodes --all-namespaces -o wide; - sleep 2; - clear; -done +wk8s ``` Em um terminal na VM NixOS: diff --git a/kubernetes-examples/minimal-pod-with-busybox-example/notes.md b/kubernetes-examples/minimal-pod-with-busybox-example/notes.md index 72ae7e0..bbb6a9e 100644 --- a/kubernetes-examples/minimal-pod-with-busybox-example/notes.md +++ b/kubernetes-examples/minimal-pod-with-busybox-example/notes.md @@ -4,17 +4,9 @@ cd ~/kubernetes-examples/minimal-pod-with-busybox-example \ && kubectl apply \ -f minimal-pod-with-busybox-example.yaml +``` - -while true; do - kubectl get pod --all-namespaces -o wide \ - && echo \ - && kubectl get services --all-namespaces -o wide \ - && echo \ - && kubectl get deployments.apps --all-namespaces -o wide \ - && echo \ - && kubectl get nodes --all-namespaces -o wide; - sleep 2; - clear; -done +```bash +wk8s ``` + diff --git a/kubernetes-examples/nodejs-node-port/notes.md b/kubernetes-examples/nodejs-node-port/notes.md index 4a3e3b5..e53bfd9 100644 --- a/kubernetes-examples/nodejs-node-port/notes.md +++ b/kubernetes-examples/nodejs-node-port/notes.md @@ -66,17 +66,7 @@ Refs.: ```bash -while true; do - kubectl get pod --all-namespaces -o wide \ - && echo \ - && kubectl get services --all-namespaces -o wide \ - && echo \ - && kubectl get deployments.apps --all-namespaces -o wide \ - && echo \ - && kubectl get nodes --all-namespaces -o wide; - sleep 2; - clear; -done +wk8s ``` Em um terminal na VM NixOS: diff --git a/kubernetes-examples/official-examples/notes.md b/kubernetes-examples/official-examples/notes.md index 07ea4cb..2a45701 100644 --- a/kubernetes-examples/official-examples/notes.md +++ b/kubernetes-examples/official-examples/notes.md @@ -37,18 +37,53 @@ kubectl apply \ kubectl get service frontend ``` - ```bash -while true; do - kubectl get pod --all-namespaces -o wide \ - && echo \ - && kubectl get services --all-namespaces -o wide \ - && echo \ - && kubectl get deployments.apps --all-namespaces -o wide \ - && echo \ - && kubectl get nodes --all-namespaces -o wide; - sleep 2; - clear; -done +wk8s ``` + + + +```bash +cat > static-docker-example.yml <<-'EOF' +--- +apiVersion: v1 +kind: Pod +metadata: + name: static-docker-example +spec: + volumes: + - name: dockersocket + emptyDir: {} + + containers: + + # This is going to be our docker service container. + - name: docker-service + image: docker:dind-rootless + + # IMPORTANT! This is security related. + # Read up about running privileged containers + securityContext: + privileged: true + + volumeMounts: + - name: dockersocket + mountPath: /run/user/1000/ + + # We will run commands in this one. + - name: docker-commander + image: docker:dind-rootless + # Just keep the container running + command: [ "/bin/sh", "-c", "sleep 86000s" ] + volumeMounts: + - name: dockersocket + mountPath: /var/run +EOF + + +kubectl apply -f static-docker-example.yml + +``` +Refs.: +- From 3e338b17eb1f49166ac43ceb1aea44a0402cd52b Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Tue, 26 Dec 2023 01:21:47 -0300 Subject: [PATCH 30/86] WIP, testanod dind --- .github/workflows/tests.yml | 12 +- README.md | 107 +++++++++++++----- .../official-examples/notes.md | 3 +- 3 files changed, 89 insertions(+), 33 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fc4c976..0792263 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,6 +23,13 @@ jobs: - name: Rodar um echo run: echo 'Olá mundo!' + - name: Instala nix single user + run: | + env | sort + which docker + docker info + docker images + - name: Metadatas run: | cat /etc/os*release @@ -35,8 +42,3 @@ jobs: run: | sudo apt-get update && sudo apt-get install -y hello curl xz-utils hello - - - name: Instala nix single user - run: | - docker info - docker images diff --git a/README.md b/README.md index c318c4f..9796408 100644 --- a/README.md +++ b/README.md @@ -223,21 +223,65 @@ kubectl get pods -n arc-runners ```bash +#cd "$HOME" \ +#&& git clone https://github.com/actions/actions-runner-controller.git \ +#&& cd actions-runner-controller \ +#&& git checkout 1f9b7541e6545a9d5ffa052481a84aad7ba4aa4d +# +#cat << 'EOF' > pin-tags-and-dind.patch +#diff --git a/charts/gha-runner-scale-set/values.yaml b/charts/gha-runner-scale-set/values.yaml +#index 021fecb..6bc29fe 100644 +#--- a/charts/gha-runner-scale-set/values.yaml +#+++ b/charts/gha-runner-scale-set/values.yaml +#@@ -75,8 +75,8 @@ githubConfigSecret: +# ## +# ## If any customization is required for dind or kubernetes mode, containerMode should remain +# ## empty, and configuration should be applied to the template. +#-# containerMode: +#-# type: "dind" ## type can be set to dind or kubernetes +#+containerMode: +#+ type: "dind" ## type can be set to dind or kubernetes +# # ## the following is required when containerMode.type=kubernetes +# # kubernetesModeWorkVolumeClaim: +# # accessModes: ["ReadWriteOnce"] +#@@ -133,7 +133,7 @@ template: +# ## mountPath: /run/docker +# ## readOnly: true +# ## - name: dind +#- ## image: docker:dind +#+ ## image: docker:24.0.7-dind-alpine3.18 +# ## args: +# ## - dockerd +# ## - --host=unix:///run/docker/docker.sock +#@@ -190,7 +190,7 @@ template: +# spec: +# containers: +# - name: runner +#- image: ghcr.io/actions/actions-runner:latest +#+ image: ghcr.io/actions/actions-runner:2.306.0 +# command: ["/home/runner/run.sh"] +# +# ## Optional controller service account that needs to have required Role and RoleBinding +#EOF +# +#git apply pin-tags-and-dind.patch +#rm -v pin-tags-and-dind.patch + + cd "$HOME" \ && git clone https://github.com/actions/actions-runner-controller.git \ && cd actions-runner-controller \ -&& git checkout 1f9b7541e6545a9d5ffa052481a84aad7ba4aa4d - +&& git checkout e0a7e142e0fcd446c58e7875d4d44a7eea6e72f2 cat << 'EOF' > pin-tags-and-dind.patch diff --git a/charts/gha-runner-scale-set/values.yaml b/charts/gha-runner-scale-set/values.yaml -index 021fecb..6bc29fe 100644 +index bbd58ac..8c2db90 100644 --- a/charts/gha-runner-scale-set/values.yaml +++ b/charts/gha-runner-scale-set/values.yaml -@@ -75,8 +75,8 @@ githubConfigSecret: - ## - ## If any customization is required for dind or kubernetes mode, containerMode should remain - ## empty, and configuration should be applied to the template. +@@ -68,8 +68,8 @@ githubConfigSecret: + # key: ca.crt + # runnerMountPath: /usr/local/share/ca-certificates/ + -# containerMode: -# type: "dind" ## type can be set to dind or kubernetes +containerMode: @@ -245,41 +289,47 @@ index 021fecb..6bc29fe 100644 # ## the following is required when containerMode.type=kubernetes # kubernetesModeWorkVolumeClaim: # accessModes: ["ReadWriteOnce"] -@@ -133,7 +133,7 @@ template: - ## mountPath: /run/docker - ## readOnly: true - ## - name: dind -- ## image: docker:dind -+ ## image: docker:24.0.7-dind-alpine3.18 - ## args: - ## - dockerd - ## - --host=unix:///run/docker/docker.sock -@@ -190,7 +190,7 @@ template: +@@ -158,7 +158,7 @@ template: spec: containers: - - name: runner -- image: ghcr.io/actions/actions-runner:latest -+ image: ghcr.io/actions/actions-runner:2.306.0 - command: ["/home/runner/run.sh"] + - name: runner +- image: ghcr.io/actions/actions-runner:latest ++ image: ghcr.io/actions/actions-runner:2.306.0 + command: ["/home/runner/run.sh"] ## Optional controller service account that needs to have required Role and RoleBinding EOF -git apply pin-tags-and-dind.patch +git pin-tags-and-dind.patch rm -v pin-tags-and-dind.patch NAMESPACE="arc-systems" -helm install arc-1 \ + +helm install arc \ --namespace "${NAMESPACE}" \ --create-namespace \ - oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller - + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller \ + --set image.tag="0.4.0" \ + --version "0.4.0" INSTALLATION_NAME="arc-runner-set" NAMESPACE="arc-runners" GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" +helm install "${INSTALLATION_NAME}" \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ + --set githubConfigSecret.github_token="${GITHUB_PAT}" \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \ + --set image.tag="0.4.0" \ + --version "0.4.0" + +INSTALLATION_NAME="arc-runner-set-dind" +NAMESPACE="arc-runners" +GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" + helm install "${INSTALLATION_NAME}" \ --namespace "${NAMESPACE}" \ --create-namespace \ @@ -287,8 +337,9 @@ helm install "${INSTALLATION_NAME}" \ --set githubConfigSecret.github_token="${GITHUB_PAT}" \ --values ~/actions-runner-controller/charts/gha-runner-scale-set/values.yaml \ oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \ - --set image.tag="0.8.1" \ - --version "0.8.1" + --set image.tag="0.4.0" \ + --version "0.4.0" + wk8s ``` @@ -299,6 +350,8 @@ wk8s https://some-natalie.dev/blog/kaniko-in-arc/ +https://snyk.io/blog/building-docker-images-kubernetes/ + ```bash diff --git a/kubernetes-examples/official-examples/notes.md b/kubernetes-examples/official-examples/notes.md index 2a45701..f7a4030 100644 --- a/kubernetes-examples/official-examples/notes.md +++ b/kubernetes-examples/official-examples/notes.md @@ -86,4 +86,5 @@ kubectl apply -f static-docker-example.yml ``` Refs.: -- +- https://discuss.kubernetes.io/t/can-k8s-or-k8s-api-build-image-with-dockerfile/16059/2 + From 34ca7e2060033e8bd089adb2769706a8e7653dad Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Tue, 26 Dec 2023 01:27:57 -0300 Subject: [PATCH 31/86] WIP, testanod dind --- .github/workflows/tests.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0792263..e3c576f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,13 +23,6 @@ jobs: - name: Rodar um echo run: echo 'Olá mundo!' - - name: Instala nix single user - run: | - env | sort - which docker - docker info - docker images - - name: Metadatas run: | cat /etc/os*release @@ -38,7 +31,14 @@ jobs: id sudo id + - name: Instala nix single user + run: | + env | sort + which docker + docker info + docker images + - name: Executa instalação via apt para teste mínimo run: | - sudo apt-get update && sudo apt-get install -y hello curl xz-utils + sudo apt-get update && sudo apt-get install -y curl nix-bin hello xz-utils hello From 574f26d8325ecbf8a13d62ae8c5625960a0cfabb Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Tue, 26 Dec 2023 01:29:58 -0300 Subject: [PATCH 32/86] WIP, testanod dind --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e3c576f..6c85963 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,8 +35,8 @@ jobs: run: | env | sort which docker - docker info - docker images + # docker info + # docker images - name: Executa instalação via apt para teste mínimo run: | From 410a0ac6b6f9f65829656d7e9834250f7f1db392 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Tue, 26 Dec 2023 01:47:42 -0300 Subject: [PATCH 33/86] WIP, testanod dind --- .github/workflows/tests.yml | 2 +- README.md | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6c85963..5f747a3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,7 +14,7 @@ on: workflow_dispatch jobs: test: - runs-on: arc-runner-set + runs-on: arc-runner-set-dind name: Testes steps: - name: checkout PR merge commit diff --git a/README.md b/README.md index 9796408..3920b57 100644 --- a/README.md +++ b/README.md @@ -344,7 +344,13 @@ wk8s ``` +```bash +INSTALLATION_NAME="arc-runner-set" +NAMESPACE="arc-runners" +helm uninstall "${INSTALLATION_NAME}" \ + --namespace "${NAMESPACE}" +``` #### Kaniko From ab096f39c8ef605ea6bab3468dd5025d64bfc99b Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Tue, 26 Dec 2023 01:48:46 -0300 Subject: [PATCH 34/86] WIP, testanod dind --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5f747a3..5846967 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,8 +35,8 @@ jobs: run: | env | sort which docker - # docker info - # docker images + docker info + docker images - name: Executa instalação via apt para teste mínimo run: | From d3d3cde976c08954b45a0c7346f43da354bd8fa7 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Tue, 26 Dec 2023 01:51:08 -0300 Subject: [PATCH 35/86] WIP, testanod dind --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3920b57..3e3b08e 100644 --- a/README.md +++ b/README.md @@ -348,7 +348,7 @@ wk8s INSTALLATION_NAME="arc-runner-set" NAMESPACE="arc-runners" -helm uninstall "${INSTALLATION_NAME}" \ +helm uninstall "${INSTALLATION_NAME}" \ --namespace "${NAMESPACE}" ``` From 7ba7a153df944a983a9c26bc3feb4590da147ed3 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Tue, 26 Dec 2023 01:54:08 -0300 Subject: [PATCH 36/86] WIP, testanod dind --- .github/workflows/tests.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5846967..589bbf5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,17 +25,19 @@ jobs: - name: Metadatas run: | - cat /etc/os*release + # cat /etc/os*release # cat /etc/group uname -a id sudo id - - name: Instala nix single user + - name: Testes sobre docker run: | - env | sort + # env | sort which docker + docker --version docker info + sudo docker info docker images - name: Executa instalação via apt para teste mínimo From c6ede82218322f61b66bd627f8b7e5f33a74c3d2 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Tue, 26 Dec 2023 13:18:27 -0300 Subject: [PATCH 37/86] =?UTF-8?q?WIP,=20refatorando,=20v=C3=A1rios=20patch?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/tests.yml | 4 +- README.md | 299 ++++++++++++++++++++++++++++++------ flake.nix | 5 + 3 files changed, 263 insertions(+), 45 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 589bbf5..d10d205 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,7 +14,7 @@ on: workflow_dispatch jobs: test: - runs-on: arc-runner-set-dind + runs-on: arc-runner-set name: Testes steps: - name: checkout PR merge commit @@ -36,8 +36,8 @@ jobs: # env | sort which docker docker --version - docker info sudo docker info + docker info docker images - name: Executa instalação via apt para teste mínimo diff --git a/README.md b/README.md index 3e3b08e..984f46c 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,8 @@ RUNNER_SCOPE="org" SCOPE_TARGET="imobanco" bash ./ops/bash/entrypoint.sh ``` -# github self-hosted runner em k8s em uma VM NixOS +# github self-hosted runner em kubernetes em uma máquina virtual NixOS + Gerar token: - onde gerar: https://github.com/settings/tokens @@ -99,7 +100,8 @@ No terminal do clone local (apenas para testes manuais) do repositório: export GH_TOKEN=ghp_yyyyyyyyyyyyyyy ``` - +Note: o remoto tenta iniciar a execução com o código que está no REMOTO, ou seja, +modificações apenas locais não são executadas. ```bash gh workflow run tests.yml --ref feature/k8s ``` @@ -119,38 +121,84 @@ Links: ## DinD -Copie e cole no terminal da VM e EDITE com seu PAT gerado no passo anterior: +Copie e cole no terminal da VM e EDITE com seu PAT: ```bash GITHUB_PAT=ghp_yyyyyyyyyyyyyyy ``` +### O mais simples e que deveria funcionar + + + ```bash cd "$HOME" \ && git clone https://github.com/actions/actions-runner-controller.git \ && cd actions-runner-controller \ && git checkout 1f9b7541e6545a9d5ffa052481a84aad7ba4aa4d +cat << 'EOF' > enables-dind.patch +diff --git a/charts/gha-runner-scale-set/values.yaml b/charts/gha-runner-scale-set/values.yaml +index 021fecb..ca32c0e 100644 +--- a/charts/gha-runner-scale-set/values.yaml ++++ b/charts/gha-runner-scale-set/values.yaml +@@ -75,8 +75,8 @@ githubConfigSecret: + ## + ## If any customization is required for dind or kubernetes mode, containerMode should remain + ## empty, and configuration should be applied to the template. +-# containerMode: +-# type: "dind" ## type can be set to dind or kubernetes ++containerMode: ++ type: "dind" ## type can be set to dind or kubernetes + # ## the following is required when containerMode.type=kubernetes + # kubernetesModeWorkVolumeClaim: + # accessModes: ["ReadWriteOnce"] +EOF + +git apply enables-dind.patch +rm -v enables-dind.patch + + +NAMESPACE="arc-systems" +helm install arc \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller + +INSTALLATION_NAME="arc-runner-set" +NAMESPACE="arc-runners" +GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" + +helm install "${INSTALLATION_NAME}" \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ + --set githubConfigSecret.github_token="${GITHUB_PAT}" \ + --values ~/actions-runner-controller/charts/gha-runner-scale-set/values.yaml \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \ + --set image.tag="0.8.1" \ + --version "0.8.1" + +wk8s +``` + +### Pinando versões de várias coisas + + +TODO: falta docker:24.0.7-dind-alpine3.18 +https://github.com/actions/actions-runner-controller/issues/3159#issuecomment-1864952928 + +```bash +cd "$HOME" \ +&& git clone https://github.com/actions/actions-runner-controller.git \ +&& cd actions-runner-controller \ +&& git checkout 1f9b7541e6545a9d5ffa052481a84aad7ba4aa4d cat << 'EOF' > enables-dind.patch diff --git a/charts/gha-runner-scale-set/values.yaml b/charts/gha-runner-scale-set/values.yaml -index 021fecb..4395555 100644 +index 021fecb..56fc9f1 100644 --- a/charts/gha-runner-scale-set/values.yaml +++ b/charts/gha-runner-scale-set/values.yaml -@@ -37,11 +37,11 @@ githubConfigSecret: - # - example.org - - ## maxRunners is the max number of runners the autoscaling runner set will scale up to. --# maxRunners: 5 -+maxRunners: 3 - - ## minRunners is the min number of idle runners. The target number of runners created will be - ## calculated as a sum of minRunners and the number of jobs assigned to the scale set. --# minRunners: 0 -+minRunners: 1 - - # runnerGroup: "default" - @@ -75,8 +75,8 @@ githubConfigSecret: ## ## If any customization is required for dind or kubernetes mode, containerMode should remain @@ -162,16 +210,16 @@ index 021fecb..4395555 100644 # ## the following is required when containerMode.type=kubernetes # kubernetesModeWorkVolumeClaim: # accessModes: ["ReadWriteOnce"] -@@ -199,6 +199,6 @@ template: - ## In case the helm chart can't find the right service account, you can explicitly pass in the following value - ## to help it finish RoleBinding with the right service account. - ## Note: if your controller is installed to only watch a single namespace, you have to pass these values explicitly. --# controllerServiceAccount: --# namespace: arc-system --# name: test-arc-gha-runner-scale-set-controller -+controllerServiceAccount: -+ namespace: arc-system -+ name: test-arc-gha-runner-scale-set-controller +@@ -190,7 +190,7 @@ template: + spec: + containers: + - name: runner +- image: ghcr.io/actions/actions-runner:latest ++ image: ghcr.io/actions/actions-runner:2.311.0 + command: ["/home/runner/run.sh"] + + ## Optional controller service account that needs to have required Role and RoleBinding + EOF git apply enables-dind.patch @@ -179,24 +227,185 @@ rm -v enables-dind.patch NAMESPACE="arc-systems" -helm install arc --wait \ +helm install arc \ --namespace "${NAMESPACE}" \ --create-namespace \ oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller -#INSTALLATION_NAME="arc-runner-set1" -#NAMESPACE="arc-runners" -#GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" -# -#helm install "${INSTALLATION_NAME}" --wait \ -# --namespace "${NAMESPACE}" \ -# --create-namespace \ -# --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ -# --set githubConfigSecret.github_token="${GITHUB_PAT}" \ -# oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set +INSTALLATION_NAME="arc-runner-set" +NAMESPACE="arc-runners" +GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" + +helm install "${INSTALLATION_NAME}" \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ + --set githubConfigSecret.github_token="${GITHUB_PAT}" \ + --values ~/actions-runner-controller/charts/gha-runner-scale-set/values.yaml \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \ + --set image.tag="0.8.1" \ + --version "0.8.1" + +wk8s +``` -INSTALLATION_NAME="arc-runner-set-dind" +```bash +kubectl get pods -n arc-systems +kubectl get pods -n arc-runners +``` + + + +#### Expação do template manualmente + + +TODO: WIP +- https://github.com/actions/actions-runner-controller/issues/2967#issuecomment-1790955550 +- https://github.com/actions/actions-runner-controller/issues/2967#issuecomment-1846987624 + + +Copie e cole no terminal da VM e EDITE com seu PAT: +```bash +GITHUB_PAT=ghp_yyyyyyyyyyyyyyy +``` + + +```bash +cd "$HOME" \ +&& git clone https://github.com/actions/actions-runner-controller.git \ +&& cd actions-runner-controller \ +&& git checkout 1f9b7541e6545a9d5ffa052481a84aad7ba4aa4d + + +cat << 'EOF' > enables-dind.patch +diff --git a/charts/gha-runner-scale-set/values.yaml b/charts/gha-runner-scale-set/values.yaml +index 021fecb..2177920 100644 +--- a/charts/gha-runner-scale-set/values.yaml ++++ b/charts/gha-runner-scale-set/values.yaml +@@ -110,53 +110,53 @@ githubConfigSecret: + template: + ## template.spec will be modified if you change the container mode + ## with containerMode.type=dind, we will populate the template.spec with following pod spec +- ## template: +- ## spec: +- ## initContainers: +- ## - name: init-dind-externals +- ## image: ghcr.io/actions/actions-runner:latest +- ## command: ["cp", "-r", "-v", "/home/runner/externals/.", "/home/runner/tmpDir/"] +- ## volumeMounts: +- ## - name: dind-externals +- ## mountPath: /home/runner/tmpDir +- ## containers: +- ## - name: runner +- ## image: ghcr.io/actions/actions-runner:latest +- ## command: ["/home/runner/run.sh"] +- ## env: +- ## - name: DOCKER_HOST +- ## value: unix:///run/docker/docker.sock +- ## volumeMounts: +- ## - name: work +- ## mountPath: /home/runner/_work +- ## - name: dind-sock +- ## mountPath: /run/docker +- ## readOnly: true +- ## - name: dind +- ## image: docker:dind +- ## args: +- ## - dockerd +- ## - --host=unix:///run/docker/docker.sock +- ## - --group=$(DOCKER_GROUP_GID) +- ## env: +- ## - name: DOCKER_GROUP_GID +- ## value: "123" +- ## securityContext: +- ## privileged: true +- ## volumeMounts: +- ## - name: work +- ## mountPath: /home/runner/_work +- ## - name: dind-sock +- ## mountPath: /run/docker +- ## - name: dind-externals +- ## mountPath: /home/runner/externals +- ## volumes: +- ## - name: work +- ## emptyDir: {} +- ## - name: dind-sock +- ## emptyDir: {} +- ## - name: dind-externals +- ## emptyDir: {} ++ template: ++ spec: ++ initContainers: ++ - name: init-dind-externals ++ image: ghcr.io/actions/actions-runner:latest ++ command: ["cp", "-r", "-v", "/home/runner/externals/.", "/home/runner/tmpDir/"] ++ volumeMounts: ++ - name: dind-externals ++ mountPath: /home/runner/tmpDir ++ containers: ++ - name: runner ++ image: ghcr.io/actions/actions-runner:latest ++ command: ["/home/runner/run.sh"] ++ env: ++ - name: DOCKER_HOST ++ value: unix:///run/docker/docker.sock ++ volumeMounts: ++ - name: work ++ mountPath: /home/runner/_work ++ - name: dind-sock ++ mountPath: /run/docker ++ readOnly: true ++ - name: dind ++ image: docker:dind ++ args: ++ - dockerd ++ - --host=unix:///run/docker/docker.sock ++ - --group=$(DOCKER_GROUP_GID) ++ env: ++ - name: DOCKER_GROUP_GID ++ value: "123" ++ securityContext: ++ privileged: true ++ volumeMounts: ++ - name: work ++ mountPath: /home/runner/_work ++ - name: dind-sock ++ mountPath: /run/docker ++ - name: dind-externals ++ mountPath: /home/runner/externals ++ volumes: ++ - name: work ++ emptyDir: {} ++ - name: dind-sock ++ emptyDir: {} ++ - name: dind-externals ++ emptyDir: {} + ###################################################################################################### + ## with containerMode.type=kubernetes, we will populate the template.spec with following pod spec + ## template: +@@ -190,7 +190,7 @@ template: + spec: + containers: + - name: runner +- image: ghcr.io/actions/actions-runner:latest ++ image: ghcr.io/actions/actions-runner:2.311.0 + command: ["/home/runner/run.sh"] + + ## Optional controller service account that needs to have required Role and RoleBinding +EOF + +git apply enables-dind.patch +rm -v enables-dind.patch + + +NAMESPACE="arc-systems" +helm install arc \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller + +INSTALLATION_NAME="arc-runner-set" NAMESPACE="arc-runners" GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" @@ -207,8 +416,9 @@ helm install "${INSTALLATION_NAME}" \ --set githubConfigSecret.github_token="${GITHUB_PAT}" \ --values ~/actions-runner-controller/charts/gha-runner-scale-set/values.yaml \ oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \ - --set image.tag="0.4.0" \ - --version "0.4.0" + --set image.tag="0.8.1" \ + --version "0.8.1" + wk8s ``` @@ -219,6 +429,7 @@ kubectl get pods -n arc-runners ``` + ### Tentativa @@ -326,7 +537,7 @@ helm install "${INSTALLATION_NAME}" \ --set image.tag="0.4.0" \ --version "0.4.0" -INSTALLATION_NAME="arc-runner-set-dind" +INSTALLATION_NAME="arc-runner-set" NAMESPACE="arc-runners" GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" @@ -352,6 +563,8 @@ helm uninstall "${INSTALLATION_NAME}" \ --namespace "${NAMESPACE}" ``` + + #### Kaniko diff --git a/flake.nix b/flake.nix index 42e63cf..4f464ca 100644 --- a/flake.nix +++ b/flake.nix @@ -356,6 +356,10 @@ services.xserver.layout = "br"; services.xserver.displayManager.autoLogin.user = "nixuser"; + services.xserver.displayManager.sessionCommands = '' + exo-open --launch TerminalEmulator --zoom=-3 + export ABC_XYZ=42 + ''; # https://nixos.org/manual/nixos/stable/#sec-xfce services.xserver.desktopManager.xfce.enable = true; @@ -456,6 +460,7 @@ environment.variables.KUBECONFIG = "/etc/kubernetes/cluster-admin.kubeconfig"; # services.kubernetes.kubelet.seedDockerImages = []; + # ghcr.io/actions/actions-runner:2.311.0 # dockerTools.examples.redis services.kubernetes.roles = [ "master" "node" ]; services.kubernetes.masterAddress = "nixos"; From a1ac2e06c9d365566b9f17d72b60c944900c5ed5 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Tue, 26 Dec 2023 21:49:22 -0300 Subject: [PATCH 38/86] WIP, dind ainda quebrado --- .github/workflows/tests.yml | 1 + README.md | 407 ++++++++++++++++++------------------ flake.nix | 16 +- 3 files changed, 219 insertions(+), 205 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d10d205..528514e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,6 +34,7 @@ jobs: - name: Testes sobre docker run: | # env | sort + touch /dev/kvm which docker docker --version sudo docker info diff --git a/README.md b/README.md index 984f46c..d1ead90 100644 --- a/README.md +++ b/README.md @@ -114,8 +114,11 @@ https://github.com/imobanco/github-ci-runner/actions Links: - https://docs.github.com/en/enterprise-server@3.11/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/quickstart-for-actions-runner-controller -- - +- https://github.com/actions/actions-runner-controller/discussions/2775 +- https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/about-actions-runner-controller#about-actions-runner-controller +- https://docs.github.com/en/enterprise-server@3.11/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/using-actions-runner-controller-runners-in-a-workflow#about-using-arc-runners-in-a-workflow-file +- https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/about-actions-runner-controller#assets-and-releases +- https://github.com/actions/runner/blob/9e3e57ff90c089641a3a5833c2211841da1a37f8/images/Dockerfile ## DinD @@ -127,8 +130,7 @@ GITHUB_PAT=ghp_yyyyyyyyyyyyyyy ``` -### O mais simples e que deveria funcionar - +### O mais simples (que deveria funcionar!) ```bash @@ -139,7 +141,7 @@ cd "$HOME" \ cat << 'EOF' > enables-dind.patch diff --git a/charts/gha-runner-scale-set/values.yaml b/charts/gha-runner-scale-set/values.yaml -index 021fecb..ca32c0e 100644 +index 021fecb..b474e88 100644 --- a/charts/gha-runner-scale-set/values.yaml +++ b/charts/gha-runner-scale-set/values.yaml @@ -75,8 +75,8 @@ githubConfigSecret: @@ -153,6 +155,7 @@ index 021fecb..ca32c0e 100644 # ## the following is required when containerMode.type=kubernetes # kubernetesModeWorkVolumeClaim: # accessModes: ["ReadWriteOnce"] + EOF git apply enables-dind.patch @@ -182,10 +185,14 @@ helm install "${INSTALLATION_NAME}" \ wk8s ``` -### Pinando versões de várias coisas + +### Pina versões -TODO: falta docker:24.0.7-dind-alpine3.18 +docker:dind -> docker:24.0.7-dind-alpine3.18 +actions-runner:latest -> actions-runner:2.311.0 + + https://github.com/actions/actions-runner-controller/issues/3159#issuecomment-1864952928 ```bash @@ -196,7 +203,7 @@ cd "$HOME" \ cat << 'EOF' > enables-dind.patch diff --git a/charts/gha-runner-scale-set/values.yaml b/charts/gha-runner-scale-set/values.yaml -index 021fecb..56fc9f1 100644 +index 021fecb..3cbb9f6 100644 --- a/charts/gha-runner-scale-set/values.yaml +++ b/charts/gha-runner-scale-set/values.yaml @@ -75,8 +75,8 @@ githubConfigSecret: @@ -210,6 +217,15 @@ index 021fecb..56fc9f1 100644 # ## the following is required when containerMode.type=kubernetes # kubernetesModeWorkVolumeClaim: # accessModes: ["ReadWriteOnce"] +@@ -133,7 +133,7 @@ template: + ## mountPath: /run/docker + ## readOnly: true + ## - name: dind +- ## image: docker:dind ++ ## image: docker:24.0.7-dind-alpine3.18 + ## args: + ## - dockerd + ## - --host=unix:///run/docker/docker.sock @@ -190,7 +190,7 @@ template: spec: containers: @@ -230,7 +246,9 @@ NAMESPACE="arc-systems" helm install arc \ --namespace "${NAMESPACE}" \ --create-namespace \ - oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller \ + --set image.tag="0.8.1" \ + --version "0.8.1" INSTALLATION_NAME="arc-runner-set" NAMESPACE="arc-runners" @@ -255,135 +273,44 @@ kubectl get pods -n arc-systems kubectl get pods -n arc-runners ``` - - -#### Expação do template manualmente - - -TODO: WIP -- https://github.com/actions/actions-runner-controller/issues/2967#issuecomment-1790955550 -- https://github.com/actions/actions-runner-controller/issues/2967#issuecomment-1846987624 - - -Copie e cole no terminal da VM e EDITE com seu PAT: ```bash -GITHUB_PAT=ghp_yyyyyyyyyyyyyyy +kubectl -n arc-systems get pod arc-runner-set-754b578d-listener -o yaml +kubectl -n arc-systems get pod arc-gha-rs-controller-58d944bbdb-rkdzh -o yaml ``` +Muda de `+ type: "dind"` para `+ type: ""` ```bash cd "$HOME" \ && git clone https://github.com/actions/actions-runner-controller.git \ && cd actions-runner-controller \ && git checkout 1f9b7541e6545a9d5ffa052481a84aad7ba4aa4d - cat << 'EOF' > enables-dind.patch diff --git a/charts/gha-runner-scale-set/values.yaml b/charts/gha-runner-scale-set/values.yaml -index 021fecb..2177920 100644 +index 021fecb..3cbb9f6 100644 --- a/charts/gha-runner-scale-set/values.yaml +++ b/charts/gha-runner-scale-set/values.yaml -@@ -110,53 +110,53 @@ githubConfigSecret: - template: - ## template.spec will be modified if you change the container mode - ## with containerMode.type=dind, we will populate the template.spec with following pod spec -- ## template: -- ## spec: -- ## initContainers: -- ## - name: init-dind-externals -- ## image: ghcr.io/actions/actions-runner:latest -- ## command: ["cp", "-r", "-v", "/home/runner/externals/.", "/home/runner/tmpDir/"] -- ## volumeMounts: -- ## - name: dind-externals -- ## mountPath: /home/runner/tmpDir -- ## containers: -- ## - name: runner -- ## image: ghcr.io/actions/actions-runner:latest -- ## command: ["/home/runner/run.sh"] -- ## env: -- ## - name: DOCKER_HOST -- ## value: unix:///run/docker/docker.sock -- ## volumeMounts: -- ## - name: work -- ## mountPath: /home/runner/_work -- ## - name: dind-sock -- ## mountPath: /run/docker -- ## readOnly: true -- ## - name: dind +@@ -75,8 +75,8 @@ githubConfigSecret: + ## + ## If any customization is required for dind or kubernetes mode, containerMode should remain + ## empty, and configuration should be applied to the template. +-# containerMode: +-# type: "dind" ## type can be set to dind or kubernetes ++containerMode: ++ type: "" ## type can be set to dind or kubernetes + # ## the following is required when containerMode.type=kubernetes + # kubernetesModeWorkVolumeClaim: + # accessModes: ["ReadWriteOnce"] +@@ -133,7 +133,7 @@ template: + ## mountPath: /run/docker + ## readOnly: true + ## - name: dind - ## image: docker:dind -- ## args: -- ## - dockerd -- ## - --host=unix:///run/docker/docker.sock -- ## - --group=$(DOCKER_GROUP_GID) -- ## env: -- ## - name: DOCKER_GROUP_GID -- ## value: "123" -- ## securityContext: -- ## privileged: true -- ## volumeMounts: -- ## - name: work -- ## mountPath: /home/runner/_work -- ## - name: dind-sock -- ## mountPath: /run/docker -- ## - name: dind-externals -- ## mountPath: /home/runner/externals -- ## volumes: -- ## - name: work -- ## emptyDir: {} -- ## - name: dind-sock -- ## emptyDir: {} -- ## - name: dind-externals -- ## emptyDir: {} -+ template: -+ spec: -+ initContainers: -+ - name: init-dind-externals -+ image: ghcr.io/actions/actions-runner:latest -+ command: ["cp", "-r", "-v", "/home/runner/externals/.", "/home/runner/tmpDir/"] -+ volumeMounts: -+ - name: dind-externals -+ mountPath: /home/runner/tmpDir -+ containers: -+ - name: runner -+ image: ghcr.io/actions/actions-runner:latest -+ command: ["/home/runner/run.sh"] -+ env: -+ - name: DOCKER_HOST -+ value: unix:///run/docker/docker.sock -+ volumeMounts: -+ - name: work -+ mountPath: /home/runner/_work -+ - name: dind-sock -+ mountPath: /run/docker -+ readOnly: true -+ - name: dind -+ image: docker:dind -+ args: -+ - dockerd -+ - --host=unix:///run/docker/docker.sock -+ - --group=$(DOCKER_GROUP_GID) -+ env: -+ - name: DOCKER_GROUP_GID -+ value: "123" -+ securityContext: -+ privileged: true -+ volumeMounts: -+ - name: work -+ mountPath: /home/runner/_work -+ - name: dind-sock -+ mountPath: /run/docker -+ - name: dind-externals -+ mountPath: /home/runner/externals -+ volumes: -+ - name: work -+ emptyDir: {} -+ - name: dind-sock -+ emptyDir: {} -+ - name: dind-externals -+ emptyDir: {} - ###################################################################################################### - ## with containerMode.type=kubernetes, we will populate the template.spec with following pod spec - ## template: ++ ## image: docker:24.0.7-dind-alpine3.18 + ## args: + ## - dockerd + ## - --host=unix:///run/docker/docker.sock @@ -190,7 +190,7 @@ template: spec: containers: @@ -393,6 +320,7 @@ index 021fecb..2177920 100644 command: ["/home/runner/run.sh"] ## Optional controller service account that needs to have required Role and RoleBinding + EOF git apply enables-dind.patch @@ -403,7 +331,9 @@ NAMESPACE="arc-systems" helm install arc \ --namespace "${NAMESPACE}" \ --create-namespace \ - oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller \ + --set image.tag="0.8.1" \ + --version "0.8.1" INSTALLATION_NAME="arc-runner-set" NAMESPACE="arc-runners" @@ -423,6 +353,122 @@ wk8s ``` + +#### Expação do template manualmente + + +TODO: WIP +- https://github.com/actions/actions-runner-controller/issues/2967#issuecomment-1790955550 +- https://github.com/actions/actions-runner-controller/issues/2967#issuecomment-1846987624 +- https://helm.sh/docs/chart_template_guide/debugging/ + + +Copie e cole no terminal da VM e EDITE com seu PAT: +```bash +GITHUB_PAT=ghp_yyyyyyyyyyyyyyy +``` + + +```bash +cd "$HOME" \ +&& git clone https://github.com/actions/actions-runner-controller.git \ +&& cd actions-runner-controller \ +&& git checkout 1f9b7541e6545a9d5ffa052481a84aad7ba4aa4d + +cat << 'EOF' > ~/actions-runner-controller/charts/gha-runner-scale-set/values.yaml +## githubConfigUrl is the GitHub url for where you want to configure runners +## ex: https://github.com/myorg/myrepo or https://github.com/myorg +githubConfigUrl: "" + +## githubConfigSecret is the k8s secrets to use when auth with GitHub API. +## You can choose to use GitHub App or a PAT token +githubConfigSecret: + ### GitHub PAT Configuration + github_token: "" + +## template is the PodSpec for each runner Pod +## For reference: https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec +template: + spec: + initContainers: + - name: init-dind-externals + image: ghcr.io/actions/actions-runner:2.311.0 + command: ["cp", "-r", "-v", "/home/runner/externals/.", "/home/runner/tmpDir/"] + volumeMounts: + - name: dind-externals + mountPath: /home/runner/tmpDir + containers: + - name: runner + image: ghcr.io/actions/actions-runner:2.311.0 + command: ["/home/runner/run.sh"] + env: + - name: DOCKER_HOST + value: unix:///var/run/docker.sock + volumeMounts: + - name: work + mountPath: /home/runner/_work + - name: dind-sock + mountPath: /var/run + readOnly: true + - name: dind + image: docker:24.0.7-dind-alpine3.18 + args: + - dockerd + - --host=unix:///var/run/docker.sock + - --group=$(DOCKER_GROUP_GID) + env: + - name: DOCKER_GROUP_GID + value: "123" + securityContext: + privileged: true + volumeMounts: + - name: work + mountPath: /home/runner/_work + - name: dind-sock + mountPath: /var/run + - name: dind-externals + mountPath: /home/runner/externals + volumes: + - name: work + emptyDir: {} + - name: dind-sock + emptyDir: {} + - name: dind-externals + emptyDir: {} + #controllerServiceAccount: + # namespace: arc-systems + # name: arc-gha-rs-controller +EOF + + +NAMESPACE="arc-systems" +helm install arc \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller \ + --set image.tag="0.8.1" \ + --version "0.8.1" + +INSTALLATION_NAME="arc-runner-set" +NAMESPACE="arc-runners" +GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" + +helm install "${INSTALLATION_NAME}" \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ + --set githubConfigSecret.github_token="${GITHUB_PAT}" \ + --values ~/actions-runner-controller/charts/gha-runner-scale-set/values.yaml \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \ + --set image.tag="0.8.1" \ + --version "0.8.1" + +wk8s +``` +Refs.: +- https://docs.github.com/en/enterprise-server@3.11/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/deploying-runner-scale-sets-with-actions-runner-controller#using-docker-in-docker-mode +- + ```bash kubectl get pods -n arc-systems kubectl get pods -n arc-runners @@ -430,55 +476,29 @@ kubectl get pods -n arc-runners + ### Tentativa ```bash -#cd "$HOME" \ -#&& git clone https://github.com/actions/actions-runner-controller.git \ -#&& cd actions-runner-controller \ -#&& git checkout 1f9b7541e6545a9d5ffa052481a84aad7ba4aa4d -# -#cat << 'EOF' > pin-tags-and-dind.patch -#diff --git a/charts/gha-runner-scale-set/values.yaml b/charts/gha-runner-scale-set/values.yaml -#index 021fecb..6bc29fe 100644 -#--- a/charts/gha-runner-scale-set/values.yaml -#+++ b/charts/gha-runner-scale-set/values.yaml -#@@ -75,8 +75,8 @@ githubConfigSecret: -# ## -# ## If any customization is required for dind or kubernetes mode, containerMode should remain -# ## empty, and configuration should be applied to the template. -#-# containerMode: -#-# type: "dind" ## type can be set to dind or kubernetes -#+containerMode: -#+ type: "dind" ## type can be set to dind or kubernetes -# # ## the following is required when containerMode.type=kubernetes -# # kubernetesModeWorkVolumeClaim: -# # accessModes: ["ReadWriteOnce"] -#@@ -133,7 +133,7 @@ template: -# ## mountPath: /run/docker -# ## readOnly: true -# ## - name: dind -#- ## image: docker:dind -#+ ## image: docker:24.0.7-dind-alpine3.18 -# ## args: -# ## - dockerd -# ## - --host=unix:///run/docker/docker.sock -#@@ -190,7 +190,7 @@ template: -# spec: -# containers: -# - name: runner -#- image: ghcr.io/actions/actions-runner:latest -#+ image: ghcr.io/actions/actions-runner:2.306.0 -# command: ["/home/runner/run.sh"] -# -# ## Optional controller service account that needs to have required Role and RoleBinding -#EOF -# -#git apply pin-tags-and-dind.patch -#rm -v pin-tags-and-dind.patch +cd "$HOME" \ +&& git clone https://github.com/actions/actions-runner-controller.git \ +&& cd actions-runner-controller \ +&& git checkout e0a7e142e0fcd446c58e7875d4d44a7eea6e72f2 + + +mkdir -pv ~/arc-configuration/{controller,runner-scale-set-1,runner-scale-set-2} \ +&& cd ~/arc-configuration + + +cd ~/actions-runner-controller/charts \ +&& cp -v actions-runner-controller/values.yaml ~/arc-configuration/controller \ +&& cp -v gha-runner-scale-set/values.yaml ~/arc-configuration/runner-scale-set-1 \ +&& cp -v gha-runner-scale-set/values.yaml ~/arc-configuration/runner-scale-set-2 +``` +```bash cd "$HOME" \ && git clone https://github.com/actions/actions-runner-controller.git \ && cd actions-runner-controller \ @@ -555,15 +575,6 @@ wk8s ``` -```bash -INSTALLATION_NAME="arc-runner-set" -NAMESPACE="arc-runners" - -helm uninstall "${INSTALLATION_NAME}" \ - --namespace "${NAMESPACE}" -``` - - #### Kaniko @@ -573,7 +584,6 @@ https://snyk.io/blog/building-docker-images-kubernetes/ ```bash - NAMESPACE="arc-systems" helm install arc \ @@ -681,8 +691,8 @@ helm install kaniko-worker \ kubectl get pods -n "test-runners" ``` -### +### helm install --dry-run --debug @@ -696,9 +706,14 @@ arc \ --namespace "${NAMESPACE}" \ --create-namespace \ oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller \ +--set image.tag="0.8.1" \ +--version "0.8.1" \ -o yaml > gha-runner-scale-set-controller.yml +``` +Quabrado! +```bash INSTALLATION_NAME="arc-runner-set" NAMESPACE="arc-runners" GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" @@ -713,41 +728,27 @@ install \ --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ --set githubConfigSecret.github_token="${GITHUB_PAT}" \ oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \ +--set image.tag="0.8.1" \ +--version "0.8.1" \ -o yaml > gha-runner-scale-set.yml - -wk8s ``` ```bash -helm install --dry-run --debug - -#kubectl create namespace arc-runners -#kubectl create secret generic pre-defined-secret \ -#--namespace=arc-runners \ -#--from-literal=github_token="${GITHUB_PAT}" - - - --set image.tag="0.4.0" \ - --version "0.4.0" \ +kubectl create namespace arc-runners +kubectl create secret generic pre-defined-secret \ +--namespace=arc-runners \ +--from-literal=github_token="${GITHUB_PAT}" ``` ```bash -cd "$HOME" \ -&& git clone https://github.com/actions/actions-runner-controller.git \ -&& cd actions-runner-controller \ -&& git checkout 1f9b7541e6545a9d5ffa052481a84aad7ba4aa4d - - -mkdir -pv ~/arc-configuration/{controller,runner-scale-set-1,runner-scale-set-2} \ -&& cd ~/arc-configuration +INSTALLATION_NAME="arc-runner-set" +NAMESPACE="arc-runners" +helm uninstall "${INSTALLATION_NAME}" \ + --namespace "${NAMESPACE}" +``` -cd ~/actions-runner-controller/charts \ -&& cp -v actions-runner-controller/values.yaml ~/arc-configuration/controller \ -&& cp -v gha-runner-scale-set/values.yaml ~/arc-configuration/runner-scale-set-1 \ -&& cp -v gha-runner-scale-set/values.yaml ~/arc-configuration/runner-scale-set-2 -``` diff --git a/flake.nix b/flake.nix index 4f464ca..cb84f5e 100644 --- a/flake.nix +++ b/flake.nix @@ -393,6 +393,7 @@ nix-direnv nixos-option oh-my-zsh + xclip zsh zsh-autosuggestions zsh-completions @@ -459,8 +460,19 @@ environment.variables.KUBECONFIG = "/etc/kubernetes/cluster-admin.kubeconfig"; - # services.kubernetes.kubelet.seedDockerImages = []; - # ghcr.io/actions/actions-runner:2.311.0 + # services.kubernetes.kubelet.seedDockerImages = [ + # (pkgs.dockerTools.pullImage { + # name = "ghcr.io/actions"; + # imageName = "actions-runner:2.311.0"; + # # tag = "2.311.0"; + # sha256 = "sha256-BVnOXiYRUg3ukjYJBYbazOfrIrzQt7aRB2LWPf1b+ZE="; + # # podman inspect docker.io/sickcodes/docker-osx:latest | jq ".[].Digest" + # # imageDigest = "sha256:e505724e0dbb070454912b159645932f74dda9aed03c48c463f33e270a6cbfa1"; + # imageDigest = ""; + # }) + # ]; + # docker pull ghcr.io/actions/actions-runner:2.311.0 + # docker inspect docker.io/sickcodes/docker-osx:latest | jq ".[].Digest" # dockerTools.examples.redis services.kubernetes.roles = [ "master" "node" ]; services.kubernetes.masterAddress = "nixos"; From a0949aef73274ef6ca7a7b7b36b79ab7589f1969 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 27 Dec 2023 11:11:34 -0300 Subject: [PATCH 39/86] =?UTF-8?q?Adiciona=20anota=C3=A7=C3=B5es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 11 ++++++++++- kubernetes-examples/flask-gunicorn/notes.md | 3 +++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d1ead90..c16f5d4 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,7 @@ GITHUB_PAT=ghp_yyyyyyyyyyyyyyy ### O mais simples (que deveria funcionar!) +https://docs.github.com/en/enterprise-server@3.10/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/deploying-runner-scale-sets-with-actions-runner-controller#using-docker-in-docker-or-kubernetes-mode-for-containers ```bash cd "$HOME" \ @@ -191,7 +192,13 @@ wk8s docker:dind -> docker:24.0.7-dind-alpine3.18 actions-runner:latest -> actions-runner:2.311.0 - + +image.dindSidecarRepositoryAndTag +https://github.com/actions/actions-runner-controller/issues/3159#issuecomment-1857989007 + + +TODO: `dindSidecarRepositoryAndTag: "docker:24.0.7-dind-alpine3.18"` +https://github.com/actions/actions-runner-controller/issues/3159#issuecomment-1860616220 https://github.com/actions/actions-runner-controller/issues/3159#issuecomment-1864952928 @@ -280,6 +287,8 @@ kubectl -n arc-systems get pod arc-gha-rs-controller-58d944bbdb-rkdzh -o yaml Muda de `+ type: "dind"` para `+ type: ""` +Ver: https://github.com/actions/actions-runner-controller/issues/3159#issuecomment-1869454029 + ```bash cd "$HOME" \ && git clone https://github.com/actions/actions-runner-controller.git \ diff --git a/kubernetes-examples/flask-gunicorn/notes.md b/kubernetes-examples/flask-gunicorn/notes.md index 2eb88c4..d7764f5 100644 --- a/kubernetes-examples/flask-gunicorn/notes.md +++ b/kubernetes-examples/flask-gunicorn/notes.md @@ -136,3 +136,6 @@ curl 34.74.203.201:8080/ ``` Pelo que entendi não funciona externamente. + + + From 2590f3aca818f559c097502d4bb1c417c5b49f5a Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 27 Dec 2023 11:48:47 -0300 Subject: [PATCH 40/86] Adiciona como testar DinD --- README.md | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c16f5d4..92d3544 100644 --- a/README.md +++ b/README.md @@ -486,7 +486,9 @@ kubectl get pods -n arc-runners -### Tentativa +### Tentativa de reproduzir o que é feito no vídeo + + ```bash @@ -701,6 +703,42 @@ kubectl get pods -n "test-runners" ``` +### Testando DinD via Docker + + +```bash +docker run --device=/dev/kvm -d -t \ +--name=action-container --rm \ +-v /var/run/docker.sock:/var/run/docker.sock \ +summerwind/actions-runner-dind:v2.308.0-ubuntu-22.04 tail -f /dev/null + +docker exec -it action-container /bin/bash -c 'cat /etc/os-release +``` + + +```bash +docker run --device=/dev/kvm -d -t \ +--name=action-container --rm \ +-v /var/run/docker.sock:/var/run/docker.sock \ +docker:24.0.7-dind-alpine3.18 tail -f /dev/null + +docker exec -it action-container /bin/sh \ +-c 'docker run -it --rm alpine cat /etc/os-release' +``` + + +```bash +docker run --device=/dev/kvm -d -t \ +--name=action-container --rm \ +-v /var/run/docker.sock:/var/run/docker.sock \ +ghcr.io/actions/actions-runner:2.311.0 tail -f /dev/null + +docker exec -it action-container /bin/sh \ +-c 'docker run -it --rm alpine cat /etc/os-release' +``` + + + ### helm install --dry-run --debug @@ -721,7 +759,7 @@ oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-cont ``` -Quabrado! +Quebrado! ```bash INSTALLATION_NAME="arc-runner-set" NAMESPACE="arc-runners" From a818cc8ec6fb455f1d39639bfde171ba06755cae Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 27 Dec 2023 11:55:50 -0300 Subject: [PATCH 41/86] =?UTF-8?q?Adiciona=20link=20para=20v=C3=ADdeo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 92d3544..9926ae4 100644 --- a/README.md +++ b/README.md @@ -489,7 +489,9 @@ kubectl get pods -n arc-runners ### Tentativa de reproduzir o que é feito no vídeo +[GitHub Actions: Dive into actions-runner-controller (ARC) || Advanced installation & configuration](https://www.youtube.com/watch?v=_F5ocPrv6io) +> Bem complicado pinar cada coisa de modo exatamente igual! ```bash cd "$HOME" \ From f5f08b204b77d515ba78cee8ddf35365a0ff5a42 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 27 Dec 2023 12:02:21 -0300 Subject: [PATCH 42/86] Adiciona TODO sobre hardening --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 9926ae4..7ec4e8b 100644 --- a/README.md +++ b/README.md @@ -739,6 +739,8 @@ docker exec -it action-container /bin/sh \ -c 'docker run -it --rm alpine cat /etc/os-release' ``` +TODO: hardening e security +https://github.com/actions/actions-runner-controller/issues/1288 ### helm install --dry-run --debug From 2a92ed794bbe320da6c102b498dea5cf403eefd5 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 27 Dec 2023 14:17:54 -0300 Subject: [PATCH 43/86] Adds example --- README.md | 3 +++ kubernetes-examples/security-context-demo/notes.md | 13 +++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 kubernetes-examples/security-context-demo/notes.md diff --git a/README.md b/README.md index 7ec4e8b..b8cc9dc 100644 --- a/README.md +++ b/README.md @@ -708,6 +708,9 @@ kubectl get pods -n "test-runners" ### Testando DinD via Docker +TODO: ajudar nessa issue? +https://github.com/actions/actions-runner-controller/issues/2696#issuecomment-1701155470 + ```bash docker run --device=/dev/kvm -d -t \ --name=action-container --rm \ diff --git a/kubernetes-examples/security-context-demo/notes.md b/kubernetes-examples/security-context-demo/notes.md new file mode 100644 index 0000000..5c6deb9 --- /dev/null +++ b/kubernetes-examples/security-context-demo/notes.md @@ -0,0 +1,13 @@ + + + +```bash +kubectl apply -f https://k8s.io/examples/pods/security/security-context.yaml +``` +Refs.: +- https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod + + +```bash +wk8s +``` From 14387780510614d1bc1f0d9c92554d5938d71fb5 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 27 Dec 2023 16:50:25 -0300 Subject: [PATCH 44/86] Adiciona test-kaniko.yml --- .github/workflows/test-kaniko.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/test-kaniko.yml diff --git a/.github/workflows/test-kaniko.yml b/.github/workflows/test-kaniko.yml new file mode 100644 index 0000000..e261300 --- /dev/null +++ b/.github/workflows/test-kaniko.yml @@ -0,0 +1,31 @@ +name: 🧪 Test building with Kaniko + +on: + workflow_dispatch: + +jobs: + build: + runs-on: [kaniko-worker] + container: + image: gcr.io/kaniko-project/executor:debug + permissions: + contents: read + packages: write + + steps: + - name: Build and push container test + run: | + # Write config file + AUTH=$(echo -n ${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} | base64) + echo "{\"auths\": {\"ghcr.io\": {\"auth\": \"${AUTH}\"}}}" > /kaniko/.docker/config.json + + # Configure git + export GIT_USERNAME="kaniko-bot" + export GIT_PASSWORD="${{ secrets.GITHUB_TOKEN }}" + + # Build and push + /kaniko/executor --dockerfile="./images/ubi9.Dockerfile" \ + --context="${{ github.repositoryUrl }}#${{ github.ref }}#${{ github.sha }}" \ + --destination="ghcr.io/some-natalie/kubernoodles/kaniko-build:test" \ + --push-retry 5 \ + --image-name-with-digest-file /workspace/image-digest.txt From 74f20527d5890324c3f7d164af30f00e41292857 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 27 Dec 2023 16:56:01 -0300 Subject: [PATCH 45/86] Comenta touch /dev/kvm --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 528514e..65fdca5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,7 +34,7 @@ jobs: - name: Testes sobre docker run: | # env | sort - touch /dev/kvm + # touch /dev/kvm which docker docker --version sudo docker info From 348067022cf5810d4385304051e889dacb427487 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 27 Dec 2023 16:59:25 -0300 Subject: [PATCH 46/86] Workaround runs-on: kaniko-worker --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 65fdca5..0556eb8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,7 +14,7 @@ on: workflow_dispatch jobs: test: - runs-on: arc-runner-set + runs-on: kaniko-worker name: Testes steps: - name: checkout PR merge commit From 4db6a5a5216dcb3fe754f1e3ea364c2902219096 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 27 Dec 2023 17:27:54 -0300 Subject: [PATCH 47/86] Testa nodeSelector e volta com runs-on: arc-runner-set --- .github/workflows/tests.yml | 2 +- README.md | 26 ++++++++++++++------------ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0556eb8..65fdca5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,7 +14,7 @@ on: workflow_dispatch jobs: test: - runs-on: kaniko-worker + runs-on: arc-runner-set name: Testes steps: - name: checkout PR merge commit diff --git a/README.md b/README.md index b8cc9dc..6c50f90 100644 --- a/README.md +++ b/README.md @@ -605,16 +605,16 @@ helm install arc \ oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller -INSTALLATION_NAME="arc-runner-set" -NAMESPACE="arc-runners" -GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" - -helm install "${INSTALLATION_NAME}" \ - --namespace "${NAMESPACE}" \ - --create-namespace \ - --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ - --set githubConfigSecret.github_token="${GITHUB_PAT}" \ - oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set +#INSTALLATION_NAME="arc-runner-set" +#NAMESPACE="arc-runners" +#GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" +# +#helm install "${INSTALLATION_NAME}" \ +# --namespace "${NAMESPACE}" \ +# --create-namespace \ +# --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ +# --set githubConfigSecret.github_token="${GITHUB_PAT}" \ +# oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set cat > k8s-storage.yml << 'EOF' @@ -659,6 +659,8 @@ kubectl apply -f k8s-storage.yml cat > helm-kaniko.yml << 'EOF' template: spec: + nodeSelector: + size: linux initContainers: # needed to set permissions to use the PVC - name: kube-init image: ghcr.io/actions/actions-runner:latest @@ -694,14 +696,14 @@ containerMode: EOF helm install kaniko-worker \ - --namespace "test-runners" \ + --namespace "arc-runner-set" \ --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ --set githubConfigSecret.github_token="${GITHUB_PAT}" \ -f helm-kaniko.yml \ oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \ --version 0.8.1 -kubectl get pods -n "test-runners" +kubectl get pods -n "arc-runner-set" ``` From 0826f06ae79f2da4beab82f038189118c162a8df Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 27 Dec 2023 17:34:03 -0300 Subject: [PATCH 48/86] =?UTF-8?q?Volta=20com=20instala=C3=A7=C3=A3o=20part?= =?UTF-8?q?e=202=20com=20helm?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 6c50f90..fafc92f 100644 --- a/README.md +++ b/README.md @@ -605,16 +605,16 @@ helm install arc \ oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller -#INSTALLATION_NAME="arc-runner-set" -#NAMESPACE="arc-runners" -#GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" -# -#helm install "${INSTALLATION_NAME}" \ -# --namespace "${NAMESPACE}" \ -# --create-namespace \ -# --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ -# --set githubConfigSecret.github_token="${GITHUB_PAT}" \ -# oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set +INSTALLATION_NAME="arc-runner-set" +NAMESPACE="arc-runners" +GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" + +helm install "${INSTALLATION_NAME}" \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ + --set githubConfigSecret.github_token="${GITHUB_PAT}" \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set cat > k8s-storage.yml << 'EOF' @@ -652,10 +652,9 @@ EOF kubectl create namespace test-runners kubectl apply -f k8s-storage.yml -``` -```bash + cat > helm-kaniko.yml << 'EOF' template: spec: @@ -703,7 +702,7 @@ helm install kaniko-worker \ oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \ --version 0.8.1 -kubectl get pods -n "arc-runner-set" +wk8s ``` From 07ab3d1c2b146a49bf3a9a66fc2ebc4715e4ab8b Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 27 Dec 2023 18:24:35 -0300 Subject: [PATCH 49/86] Ajustes --- README.md | 46 +++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index fafc92f..75bbcf1 100644 --- a/README.md +++ b/README.md @@ -605,16 +605,16 @@ helm install arc \ oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller -INSTALLATION_NAME="arc-runner-set" -NAMESPACE="arc-runners" -GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" - -helm install "${INSTALLATION_NAME}" \ - --namespace "${NAMESPACE}" \ - --create-namespace \ - --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ - --set githubConfigSecret.github_token="${GITHUB_PAT}" \ - oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set +#INSTALLATION_NAME="arc-runner-set" +#NAMESPACE="arc-runners" +#GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" +# +#helm install "${INSTALLATION_NAME}" \ +# --namespace "${NAMESPACE}" \ +# --create-namespace \ +# --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ +# --set githubConfigSecret.github_token="${GITHUB_PAT}" \ +# oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set cat > k8s-storage.yml << 'EOF' @@ -623,7 +623,7 @@ apiVersion: storage.k8s.io/v1 metadata: name: k8s-mode namespace: test-runners # just showing the test namespace -provisioner: file.csi.azure.com # change this to your provisioner +provisioner: kubernetes.io/example allowVolumeExpansion: true # probably not strictly necessary reclaimPolicy: Delete mountOptions: @@ -654,12 +654,9 @@ kubectl create namespace test-runners kubectl apply -f k8s-storage.yml - cat > helm-kaniko.yml << 'EOF' template: spec: - nodeSelector: - size: linux initContainers: # needed to set permissions to use the PVC - name: kube-init image: ghcr.io/actions/actions-runner:latest @@ -694,18 +691,33 @@ containerMode: storage: 1Gi EOF -helm install kaniko-worker \ - --namespace "arc-runner-set" \ + +INSTALLATION_NAME="kaniko-worker" +NAMESPACE="test-runners" +GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" + +helm install "${INSTALLATION_NAME}" \ + --namespace "${NAMESPACE}" \ --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ --set githubConfigSecret.github_token="${GITHUB_PAT}" \ -f helm-kaniko.yml \ oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \ - --version 0.8.1 + --set image.tag="0.8.1" \ + --version "0.8.1" wk8s ``` +```bash +provisioner: kubernetes.io/example +``` + +```bash + nodeSelector: + kubernetes.io/hostname: nixos +``` + ### Testando DinD via Docker From 633575311cecf8b423005b6643faf0ab84909910 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Fri, 29 Dec 2023 17:43:57 -0300 Subject: [PATCH 50/86] runs-on: nixos --- .github/workflows/tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 65fdca5..e12351b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,7 +14,8 @@ on: workflow_dispatch jobs: test: - runs-on: arc-runner-set + # runs-on: arc-runner-set + runs-on: nixos name: Testes steps: - name: checkout PR merge commit From 5a1874366573332a7d436414d995b96c02c5a0c5 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Fri, 29 Dec 2023 17:45:38 -0300 Subject: [PATCH 51/86] runs-on: self-hosted --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e12351b..bdbbd35 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,7 +15,7 @@ on: workflow_dispatch jobs: test: # runs-on: arc-runner-set - runs-on: nixos + runs-on: self-hosted name: Testes steps: - name: checkout PR merge commit From 50d276e68f011f266c6873a7f2f63df335f4f81b Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Fri, 29 Dec 2023 17:47:12 -0300 Subject: [PATCH 52/86] Testando ambiente --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bdbbd35..72ee7f6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -30,7 +30,7 @@ jobs: # cat /etc/group uname -a id - sudo id + # sudo id - name: Testes sobre docker run: | From 900b6313a5ba1b5381719c5eb52f2915862a5c47 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Fri, 29 Dec 2023 17:58:51 -0300 Subject: [PATCH 53/86] Testando ambiente --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 72ee7f6..ed27fa2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,9 +36,9 @@ jobs: run: | # env | sort # touch /dev/kvm - which docker + # which docker docker --version - sudo docker info + # sudo docker info docker info docker images From 319c5565eb81b3c5ea58f4363b27f6eb3c5cfabb Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Fri, 29 Dec 2023 18:35:20 -0300 Subject: [PATCH 54/86] Testando ambiente --- .github/workflows/tests.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ed27fa2..a19ea92 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,6 +31,8 @@ jobs: uname -a id # sudo id + cat $HOME/logs.txt || true + echo FOO-BAR > $HOME/logs.txt - name: Testes sobre docker run: | @@ -42,7 +44,8 @@ jobs: docker info docker images - - name: Executa instalação via apt para teste mínimo + - name: Executa instalação via nix para teste mínimo run: | - sudo apt-get update && sudo apt-get install -y curl nix-bin hello xz-utils + nix profile install nixpkgs#hello nixpkgs#python3 hello + python3 -c 'import this' From eb07b6e8d92c27306060e251aad0ad96f33bee34 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Fri, 29 Dec 2023 18:41:31 -0300 Subject: [PATCH 55/86] Testando ambiente --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a19ea92..2baaded 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -43,6 +43,8 @@ jobs: # sudo docker info docker info docker images + docker run -it --rm alpine cat /etc/os-release + docker images - name: Executa instalação via nix para teste mínimo run: | From e590713719fc3eae4f9e9760e9421adf0d3dd26c Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Fri, 29 Dec 2023 18:45:35 -0300 Subject: [PATCH 56/86] Testando ambiente --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2baaded..6a5cd33 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -43,7 +43,7 @@ jobs: # sudo docker info docker info docker images - docker run -it --rm alpine cat /etc/os-release + docker run --rm alpine cat /etc/os-release docker images - name: Executa instalação via nix para teste mínimo From 28851884cd6e463698468b4268b8ad6bb76f7a4b Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Fri, 29 Dec 2023 18:50:24 -0300 Subject: [PATCH 57/86] Testando ambiente --- .github/workflows/tests.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6a5cd33..59005f8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,8 +37,9 @@ jobs: - name: Testes sobre docker run: | # env | sort - # touch /dev/kvm - # which docker + touch /dev/kvm + which docker + readlink -f $(which docker) docker --version # sudo docker info docker info @@ -46,8 +47,8 @@ jobs: docker run --rm alpine cat /etc/os-release docker images - - name: Executa instalação via nix para teste mínimo + - name: Mais metadados run: | - nix profile install nixpkgs#hello nixpkgs#python3 - hello + nix flake --version + nix flake metadata nixpkgs python3 -c 'import this' From 1a9354c1b3f22bfa088a22332f74edc8a09539fd Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Fri, 29 Dec 2023 18:51:22 -0300 Subject: [PATCH 58/86] Testando ambiente --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 59005f8..5efd415 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,7 +37,7 @@ jobs: - name: Testes sobre docker run: | # env | sort - touch /dev/kvm + # touch /dev/kvm which docker readlink -f $(which docker) docker --version From e49b77cc123602e03f37e5a40c970acd4431dc75 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Fri, 29 Dec 2023 18:56:23 -0300 Subject: [PATCH 59/86] Testando ambiente --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5efd415..43d3b90 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -38,8 +38,8 @@ jobs: run: | # env | sort # touch /dev/kvm - which docker - readlink -f $(which docker) + # which docker + # readlink -f $(which docker) docker --version # sudo docker info docker info From b82e1391d73c06724802177837e9785d7989da3b Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Fri, 29 Dec 2023 18:57:17 -0300 Subject: [PATCH 60/86] Testando ambiente --- .github/workflows/tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 43d3b90..7a22559 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -51,4 +51,3 @@ jobs: run: | nix flake --version nix flake metadata nixpkgs - python3 -c 'import this' From 64ee16086119e3dfa7a40d5895066e2b77bd3bea Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 3 Jan 2024 14:13:17 -0300 Subject: [PATCH 61/86] WIP: github-runner funciona --- README.md | 121 +++++++++++++++++++++++++++++++-- flake.lock | 6 +- flake.nix | 196 +++++++++++++++++++++++++++++++++++++++++------------ 3 files changed, 271 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index 75bbcf1..e03fb00 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ Links: - https://docs.github.com/en/enterprise-server@3.11/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/using-actions-runner-controller-runners-in-a-workflow#about-using-arc-runners-in-a-workflow-file - https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/about-actions-runner-controller#assets-and-releases - https://github.com/actions/runner/blob/9e3e57ff90c089641a3a5833c2211841da1a37f8/images/Dockerfile - +- https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/removing-self-hosted-runners ## DinD @@ -363,7 +363,7 @@ wk8s -#### Expação do template manualmente +#### Expanção do template manualmente TODO: WIP @@ -476,13 +476,107 @@ wk8s ``` Refs.: - https://docs.github.com/en/enterprise-server@3.11/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/deploying-runner-scale-sets-with-actions-runner-controller#using-docker-in-docker-mode -- + + ```bash kubectl get pods -n arc-systems kubectl get pods -n arc-runners ``` +### kubectl apply -f dind.yml + + +```bash +NAMESPACE="arc-systems" + +helm install arc \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller \ + --set image.tag="0.8.1" \ + --version "0.8.1" + +INSTALLATION_NAME="arc-runner-set" +NAMESPACE="arc-runners" +GITHUB_CONFIG_URL="https://github.com/Imobanco/github-ci-runner" + +helm install "${INSTALLATION_NAME}" \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + --set githubConfigUrl="${GITHUB_CONFIG_URL}" \ + --set githubConfigSecret.github_token="${GITHUB_PAT}" \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \ + --set image.tag="0.8.1" \ + --version "0.8.1" +wk8s +``` + +TODO: acho que o problema foi o tipo de entidade que fiz o `-o yaml` + +```bash +cat << 'EOF' > dind.yml +apiVersion: v1 +kind: Pod +metadata: + name: arc-gha-rs-controller-58d944bbdb-kn77b +spec: + initContainers: + - name: init-dind-externals + image: ghcr.io/actions/actions-runner:2.311.0 + command: ["cp", "-r", "-v", "/home/runner/externals/.", "/home/runner/tmpDir/"] + volumeMounts: + - name: dind-externals + mountPath: /home/runner/tmpDir + containers: + - name: runner + image: ghcr.io/actions/actions-runner:2.311.0 + command: ["/home/runner/run.sh"] + env: + - name: DOCKER_HOST + value: unix:///var/run/docker.sock + volumeMounts: + - name: work + mountPath: /home/runner/_work + - name: dind-sock + mountPath: /var/run + readOnly: true + - name: dind + image: docker:24.0.7-dind-alpine3.18 + args: + - dockerd + - --host=unix:///var/run/docker.sock + - --group=$(DOCKER_GROUP_GID) + env: + - name: DOCKER_GROUP_GID + value: "123" + securityContext: + privileged: true + volumeMounts: + - name: work + mountPath: /home/runner/_work + - name: dind-sock + mountPath: /var/run + - name: dind-externals + mountPath: /home/runner/externals + volumes: + - name: work + emptyDir: {} + - name: dind-sock + emptyDir: {} + - name: dind-externals + emptyDir: {} +EOF + +kubectl apply -f dind.yml +``` + + +```bash +github runner is invalid: +spec.containers[1].securityContext.privileged: +Forbidden: disallowed by cluster policy +``` @@ -595,6 +689,9 @@ wk8s https://some-natalie.dev/blog/kaniko-in-arc/ https://snyk.io/blog/building-docker-images-kubernetes/ +Modifiquei adicionando o PersistentVolume. Excelentes explicações: +https://stackoverflow.com/a/74372350 +https://stackoverflow.com/a/44891419 ```bash NAMESPACE="arc-systems" @@ -623,7 +720,7 @@ apiVersion: storage.k8s.io/v1 metadata: name: k8s-mode namespace: test-runners # just showing the test namespace -provisioner: kubernetes.io/example +provisioner: file.csi.azure.com allowVolumeExpansion: true # probably not strictly necessary reclaimPolicy: Delete mountOptions: @@ -636,6 +733,21 @@ mountOptions: - actimeo=30 --- apiVersion: v1 +kind: PersistentVolume +metadata: + name: task-pv-volume + labels: + type: local +spec: + storageClassName: "k8s-mode" + capacity: + storage: 1Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/home/runner/_work" +--- +apiVersion: v1 kind: PersistentVolumeClaim metadata: name: test-k8s-cache-pvc @@ -647,6 +759,7 @@ spec: requests: storage: 1Gi storageClassName: k8s-mode # we'll need this in the runner Helm chart + EOF diff --git a/flake.lock b/flake.lock index dfe4bdf..a3adfbe 100644 --- a/flake.lock +++ b/flake.lock @@ -32,11 +32,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1688392541, - "narHash": "sha256-lHrKvEkCPTUO+7tPfjIcb7Trk6k31rz18vkyqmkeJfY=", + "lastModified": 1703875886, + "narHash": "sha256-WJ0s5WWOBUnXAfPwxLm0cI4XjeLXPdI95Znwjrsfd2A=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ea4c80b39be4c09702b0cb3b42eab59e2ba4f24b", + "rev": "0fce51a5077927de454b94ffae41c2749fa15ba2", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index cb84f5e..bf5f381 100644 --- a/flake.nix +++ b/flake.nix @@ -17,6 +17,25 @@ , nixpkgs , ... }: + { + inherit (self) outputs; + + overlays.default = final: prev: { + inherit self final prev; + + foo-bar = prev.hello; + + # https://fnordig.de/2023/07/24/old-ruby-on-modern-nix/ + # nodejs_16 = prev.nodejs_16.meta // { insecure = false; knownVulnerabilities = []; }; + github-runner = + let + ignoringVulns = x: x // { meta = (x.meta // { knownVulnerabilities = [ ]; }); }; + in + prev.github-runner.override { + nodejs_16 = prev.nodejs_16.overrideAttrs ignoringVulns; + }; + }; + } // allAttrs.flake-utils.lib.eachDefaultSystem (system: let @@ -24,7 +43,10 @@ pkgsAllowUnfree = import nixpkgs { inherit system; - config = { allowUnfree = true; }; + overlays = [ self.overlays.default ]; + config = { + allowUnfree = true; + }; }; hack = pkgsAllowUnfree.writeShellScriptBin "hack" '' @@ -88,10 +110,10 @@ test -d .profiles || mkdir -v .profiles test -L .profiles/dev \ - || nix develop .# --profile .profiles/dev --command true + || nix develop --impure .# --profile .profiles/dev --command true test -L .profiles/dev-shell-default \ - || nix build $(nix eval --impure --raw .#devShells."$system".default.drvPath) --out-link .profiles/dev-shell-"$system"-default + || nix build --impure $(nix eval --impure --raw .#devShells."$system".default.drvPath) --out-link .profiles/dev-shell-"$system"-default test -L .profiles/nixosConfigurations."$system".vm.config.system.build.vm \ || nix build --impure --out-link .profiles/nixosConfigurations."$system".vm.config.system.build.vm .#nixosConfigurations.vm.config.system.build.vm @@ -141,52 +163,42 @@ boot.loader.systemd-boot.enable = true; fileSystems."/" = { device = "/dev/hda1"; }; - virtualisation.vmVariant = { - - virtualisation.useNixStoreImage = false; # TODO: hardening - virtualisation.writableStore = true; # TODO: hardening + virtualisation.vmVariant = + { - virtualisation.docker.enable = true; + virtualisation.useNixStoreImage = false; # TODO: hardening + virtualisation.writableStore = true; # TODO: hardening - programs.dconf.enable = true; - # security.polkit.enable = true; # TODO: hardening? + virtualisation.docker.enable = true; - virtualisation.memorySize = 1024 * 8; # Use MiB memory. - virtualisation.diskSize = 1024 * 50; # Use MiB memory. - virtualisation.cores = 8; # Number of cores. - virtualisation.graphics = true; + programs.dconf.enable = true; + # security.polkit.enable = true; # TODO: hardening? - virtualisation.resolution = lib.mkForce { x = 1024; y = 768; }; + virtualisation.memorySize = 1024 * 8; # Use MiB memory. + virtualisation.diskSize = 1024 * 50; # Use MiB memory. + virtualisation.cores = 8; # Number of cores. + virtualisation.graphics = true; - virtualisation.qemu.options = [ - # Better display option - # TODO: -display sdl,gl=on - # https://gitlab.com/qemu-project/qemu/-/issues/761 - "-vga virtio" - "-display gtk,zoom-to-fit=false" - # Enable copy/paste - # https://www.kraxel.org/blog/2021/05/qemu-cut-paste/ - "-chardev qemu-vdagent,id=ch1,name=vdagent,clipboard=on" - "-device virtio-serial-pci" - "-device virtserialport,chardev=ch1,id=ch1,name=com.redhat.spice.0" + virtualisation.resolution = lib.mkForce { x = 1024; y = 768; }; - # https://serverfault.com/a/1119403 - # "-device intel-iommu,intremap=on" + virtualisation.qemu.options = [ + # Better display option + # TODO: -display sdl,gl=on + # https://gitlab.com/qemu-project/qemu/-/issues/761 + "-vga virtio" + "-display gtk,zoom-to-fit=false" + # Enable copy/paste + # https://www.kraxel.org/blog/2021/05/qemu-cut-paste/ + "-chardev qemu-vdagent,id=ch1,name=vdagent,clipboard=on" + "-device virtio-serial-pci" + "-device virtserialport,chardev=ch1,id=ch1,name=com.redhat.spice.0" - # "-net user,hostfwd=tcp::8090-::8080" - ]; + # https://serverfault.com/a/1119403 + # "-device intel-iommu,intremap=on" - # https://discourse.nixos.org/t/nixpkgs-support-for-linux-builders-running-on-macos/24313/2 - virtualisation.forwardPorts = [ - { - from = "host"; - # host.address = "127.0.0.1"; - host.port = 8090; - # guest.address = "34.74.203.201"; - guest.port = 30163; - } - ]; - }; + # "-net user,hostfwd=tcp::8090-::8080" + ]; + }; users.users.root = { password = "root"; @@ -236,6 +248,7 @@ openssl starship which + foo-bar ]; shell = pkgs.zsh; @@ -251,6 +264,61 @@ ]; }; + /* + https://github.com/NixOS/nixpkgs/issues/169812 + https://github.com/actions/runner/issues/1882#issuecomment-1427930611 + nix shell nixpkgs#github-runner --command \ + sh \ + -c \ + 'config.sh --url https://github.com/imobanco/github-ci-runner --pat "$PAT" --ephemeral && run.sh' + config.sh --url https://github.com/imobanco/github-ci-runner --pat "$PAT" --ephemeral && run.sh + TODO: https://www.youtube.com/watch?v=G5f6GC7SnhU + */ + services.github-runner.enable = true; + services.github-runner.ephemeral = true; + services.github-runner.user = "nixuser"; + # services.github-runner.runnerGroup = "nixgroup"; + services.github-runner.url = "https://github.com/imobanco/github-ci-runner"; + # services.github-runner.tokenFile = config.sops.secrets."github-runner/token".path; + services.github-runner.tokenFile = "/run/secrets/github-runner/nixos.token"; + services.github-runner.extraPackages = with pkgs; [ config.virtualisation.docker.package ]; + virtualisation.docker.enable = true; + systemd.services.github-runner.serviceConfig.SupplementaryGroups = [ "docker" ]; + # systemd.services.github-runner."nixos".extraPackages = with pkgs; [ which sudo ]; + + systemd.user.services.populate-history-vagrant = { + script = '' + echo "Started" + + DESTINATION=/home/nixuser/.zsh_history + + # TODO: + echo "systemctl status github-runner-nixos.service" >> "$DESTINATION" + echo "github-runner && sudo systemctl restart github-runner-nixos.service" >> "$DESTINATION" + + echo "Ended" + ''; + wantedBy = [ "default.target" ]; + }; + + /* + https://github.com/vimjoyer/sops-nix-video/tree/25e5698044e60841a14dcd64955da0b1b66957a2 + https://github.com/Mic92/sops-nix/issues/65#issuecomment-929082304 + https://discourse.nixos.org/t/qmenu-secrets-sops-and-nixos/13621/8 + https://www.youtube.com/watch?v=1BquzE3Yb4I + https://github.com/FiloSottile/age#encrypting-to-a-github-user + https://devops.datenkollektiv.de/using-sops-with-age-and-git-like-a-pro.html + sudo cat /run/secrets/example-key + */ + /* + sops.defaultSopsFile = ./secrets/secrets.yaml.encrypted; + sops.defaultSopsFormat = "yaml"; + sops.gnupg.sshKeyPaths = []; + sops.age.sshKeyPaths = []; + sops.age.keyFile = ./secrets/keys.txt; + sops.secrets.example-key = { }; + */ + # https://github.com/NixOS/nixpkgs/blob/3a44e0112836b777b176870bb44155a2c1dbc226/nixos/modules/programs/zsh/oh-my-zsh.nix#L119 # https://discourse.nixos.org/t/nix-completions-for-zsh/5532 # https://github.com/NixOS/nixpkgs/blob/09aa1b23bb5f04dfc0ac306a379a464584fc8de7/nixos/modules/programs/zsh/zsh.nix#L230-L231 @@ -357,8 +425,10 @@ services.xserver.displayManager.autoLogin.user = "nixuser"; services.xserver.displayManager.sessionCommands = '' - exo-open --launch TerminalEmulator --zoom=-3 - export ABC_XYZ=42 + exo-open \ + --launch TerminalEmulator \ + --zoom=-3 \ + --geometry 154x24 ''; # https://nixos.org/manual/nixos/stable/#sec-xfce @@ -370,7 +440,7 @@ # For copy/paste to work services.spice-vdagentd.enable = true; - nixpkgs.config.allowUnfree = true; + # nixpkgs.config.allowUnfree = true; nix = { extraOptions = "experimental-features = nix-command flakes"; @@ -397,6 +467,8 @@ zsh zsh-autosuggestions zsh-completions + firefox + which # Looks like kubernetes needs at least all this kubectl @@ -413,6 +485,15 @@ iptables socat + ( + writeScriptBin "run-github-runner" '' + #! ${pkgs.runtimeShell} -e + sudo mkdir -pv -m 0700 /run/secrets/github-runner + sudo chown $(id -u):$(id -g) /run/secrets/github-runner + echo -n ghp_yyyy > /run/secrets/github-runner/nixos.token + '' + ) + ( writeScriptBin "fix-k8s-cluster-admin-key" '' #! ${pkgs.runtimeShell} -e @@ -530,10 +611,32 @@ wantedBy = [ "multi-user.target" ]; }; + # journalctl --user --unit create-custom-desktop-icons.service -b -f + systemd.user.services.create-custom-desktop-icons = { + script = '' + #! ${pkgs.runtimeShell} -e + + echo "Started" + + ln \ + -sfv \ + "${pkgs.xfce.xfce4-settings}"/share/applications/xfce4-terminal-emulator.desktop \ + /home/nixuser/Desktop/xfce4-terminal-emulator.desktop + + ln \ + -sfv \ + "${pkgs.firefox}"/share/applications/firefox.desktop \ + /home/nixuser/Desktop/firefox.desktop + + echo "Ended" + ''; + wantedBy = [ "xfce4-notifyd.service" ]; + }; + # https://discourse.nixos.org/t/nixos-firewall-with-kubernetes/23673/2 # networking.firewall.trustedInterfaces ?? # networking.firewall.allowedTCPPorts = [ 80 8000 8080 8443 9000 9443 ]; - networking.firewall.enable = false; + networking.firewall.enable = false; # TODO: hardening environment.etc."containers/registries.conf" = { mode = "0644"; @@ -555,7 +658,10 @@ system.stateVersion = "22.11"; }) + { nixpkgs.overlays = [ self.overlays.default ]; } + ]; + specialArgs = { inherit nixpkgs allAttrs; }; }; From decb76c70ff38c009a4575ae7882fc56202e2811 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 3 Jan 2024 14:15:37 -0300 Subject: [PATCH 62/86] Testa --privileged --- .github/workflows/tests.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7a22559..a8a9980 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -47,6 +47,12 @@ jobs: docker run --rm alpine cat /etc/os-release docker images + - name: Outros testes sobre docker + run: | + id + docker run --privileged=true --rm alpine touch /dev/kvm + docker images + - name: Mais metadados run: | nix flake --version From e0b97382c3a9e1d3066cdcd4f1ee0e052c58eafb Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 3 Jan 2024 14:27:15 -0300 Subject: [PATCH 63/86] Testa --privileged --- .github/workflows/tests.yml | 4 ++-- flake.nix | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a8a9980..1af06fb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -50,8 +50,8 @@ jobs: - name: Outros testes sobre docker run: | id - docker run --privileged=true --rm alpine touch /dev/kvm - docker images + docker run --privileged=true --rm \ + alpine sh -c 'id && touch /dev/kvm && stat /dev/kvm && file /dev/kvm' - name: Mais metadados run: | diff --git a/flake.nix b/flake.nix index bf5f381..22be549 100644 --- a/flake.nix +++ b/flake.nix @@ -294,7 +294,7 @@ # TODO: echo "systemctl status github-runner-nixos.service" >> "$DESTINATION" - echo "github-runner && sudo systemctl restart github-runner-nixos.service" >> "$DESTINATION" + echo "run-github-runner && sudo systemctl restart github-runner-nixos.service" >> "$DESTINATION" echo "Ended" ''; @@ -490,7 +490,7 @@ #! ${pkgs.runtimeShell} -e sudo mkdir -pv -m 0700 /run/secrets/github-runner sudo chown $(id -u):$(id -g) /run/secrets/github-runner - echo -n ghp_yyyy > /run/secrets/github-runner/nixos.token + echo -n ghp_yyyyy > /run/secrets/github-runner/nixos.token '' ) From 5c4dd199630f31a0368a9064726e6676a1cbe2b7 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 3 Jan 2024 14:29:17 -0300 Subject: [PATCH 64/86] Testa --privileged --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1af06fb..162aafa 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -51,7 +51,7 @@ jobs: run: | id docker run --privileged=true --rm \ - alpine sh -c 'id && touch /dev/kvm && stat /dev/kvm && file /dev/kvm' + alpine sh -c 'id && touch /dev/kvm && stat /dev/kvm' - name: Mais metadados run: | From 3c7454acf36bffc99b77ae85490bd04fb8e9a16b Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 3 Jan 2024 14:37:24 -0300 Subject: [PATCH 65/86] Testa outros metadados do ambiente --- .github/workflows/tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 162aafa..0d2f9da 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,7 +21,7 @@ jobs: - name: checkout PR merge commit uses: actions/checkout@v3 - - name: Rodar um echo + - name: Executa um echo run: echo 'Olá mundo!' - name: Metadatas @@ -30,6 +30,8 @@ jobs: # cat /etc/group uname -a id + pwd + ls -alh # sudo id cat $HOME/logs.txt || true echo FOO-BAR > $HOME/logs.txt From c0f7aa59d335d2751a11a56102e025398a7cb1f9 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 3 Jan 2024 14:47:18 -0300 Subject: [PATCH 66/86] Testa docker build Dockerfile --- .github/workflows/tests.yml | 8 +++++++- Dockerfile | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 Dockerfile diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0d2f9da..8d01778 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,6 +36,12 @@ jobs: cat $HOME/logs.txt || true echo FOO-BAR > $HOME/logs.txt + - name: Testes sobre docker build + run: | + id + docker build --tag custom-python . + docker images + - name: Testes sobre docker run: | # env | sort @@ -49,7 +55,7 @@ jobs: docker run --rm alpine cat /etc/os-release docker images - - name: Outros testes sobre docker + - name: Testes sobre /dev/kvm run: | id docker run --privileged=true --rm \ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d7657f1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +FROM python:3.9-slim-buster + + +# Set python environment variables +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 +ENV PIP_NO_CACHE_DIR 0 +ENV PIP_DISABLE_PIP_VERSION_CHECK 1 + +ENV USER app_user + +WORKDIR /home/app_user + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --no-install-suggests -y \ + ca-certificates \ + && apt-get -y autoremove \ + && apt-get -y clean \ + && rm -rf /var/lib/apt/lists/* + +RUN addgroup app_group \ + && adduser \ + --quiet \ + --disabled-password \ + --shell /bin/bash \ + --home /home/app_user \ + --gecos "User" app_user \ + --ingroup app_group \ + && chmod 0700 /home/app_user \ + && chown --recursive app_user:app_group /home/app_user + +CMD ["/bin/bash"] From b1b6a4e109e6a8ee6fa55d9488de7123386cb089 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 3 Jan 2024 15:33:49 -0300 Subject: [PATCH 67/86] Pequenos ajustes --- flake.nix | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 22be549..4358202 100644 --- a/flake.nix +++ b/flake.nix @@ -292,8 +292,9 @@ DESTINATION=/home/nixuser/.zsh_history - # TODO: - echo "systemctl status github-runner-nixos.service" >> "$DESTINATION" + # TODO: https://stackoverflow.com/a/67169387 + echo "journalctl -xeu github-runner-nixos.service" >> "$DESTINATION" + echo "systemctl status github-runner-nixos.service | cat" >> "$DESTINATION" echo "run-github-runner && sudo systemctl restart github-runner-nixos.service" >> "$DESTINATION" echo "Ended" @@ -428,7 +429,7 @@ exo-open \ --launch TerminalEmulator \ --zoom=-3 \ - --geometry 154x24 + --geometry 154x40 ''; # https://nixos.org/manual/nixos/stable/#sec-xfce From 50a152e73a70ffcbe99f8a4b2b501d7b2f71ea87 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 3 Jan 2024 15:39:34 -0300 Subject: [PATCH 68/86] Testa GNU hello --- .github/workflows/tests.yml | 3 +++ flake.nix | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8d01778..3144087 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,6 +24,9 @@ jobs: - name: Executa um echo run: echo 'Olá mundo!' + - name: Executa GNU hello + run: hello + - name: Metadatas run: | # cat /etc/os*release diff --git a/flake.nix b/flake.nix index 4358202..34fe470 100644 --- a/flake.nix +++ b/flake.nix @@ -281,7 +281,7 @@ services.github-runner.url = "https://github.com/imobanco/github-ci-runner"; # services.github-runner.tokenFile = config.sops.secrets."github-runner/token".path; services.github-runner.tokenFile = "/run/secrets/github-runner/nixos.token"; - services.github-runner.extraPackages = with pkgs; [ config.virtualisation.docker.package ]; + services.github-runner.extraPackages = with pkgs; [ config.virtualisation.docker.package hello ]; virtualisation.docker.enable = true; systemd.services.github-runner.serviceConfig.SupplementaryGroups = [ "docker" ]; # systemd.services.github-runner."nixos".extraPackages = with pkgs; [ which sudo ]; From f31f930ea99eda9ebeded532c8d839c85d7fdb5b Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 3 Jan 2024 17:27:00 -0300 Subject: [PATCH 69/86] Testa python39 --- .github/workflows/tests.yml | 8 ++++++-- flake.nix | 7 ++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3144087..44afd4a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,8 +24,10 @@ jobs: - name: Executa um echo run: echo 'Olá mundo!' - - name: Executa GNU hello - run: hello + - name: Executa GNU hello e python3 + run: | + hello + python --version - name: Metadatas run: | @@ -35,6 +37,8 @@ jobs: id pwd ls -alh + nproc + free -h # sudo id cat $HOME/logs.txt || true echo FOO-BAR > $HOME/logs.txt diff --git a/flake.nix b/flake.nix index 34fe470..592928e 100644 --- a/flake.nix +++ b/flake.nix @@ -281,7 +281,12 @@ services.github-runner.url = "https://github.com/imobanco/github-ci-runner"; # services.github-runner.tokenFile = config.sops.secrets."github-runner/token".path; services.github-runner.tokenFile = "/run/secrets/github-runner/nixos.token"; - services.github-runner.extraPackages = with pkgs; [ config.virtualisation.docker.package hello ]; + services.github-runner.extraPackages = with pkgs; [ + config.virtualisation.docker.package + hello + # sudo + python39 + ]; virtualisation.docker.enable = true; systemd.services.github-runner.serviceConfig.SupplementaryGroups = [ "docker" ]; # systemd.services.github-runner."nixos".extraPackages = with pkgs; [ which sudo ]; From fe1ae72bb9bd1f4abc86f9a1d6c3e7f5ecbff16a Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 3 Jan 2024 17:29:04 -0300 Subject: [PATCH 70/86] Testa free proveniente do pacote procps --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index 592928e..51c2113 100644 --- a/flake.nix +++ b/flake.nix @@ -285,6 +285,7 @@ config.virtualisation.docker.package hello # sudo + procps python39 ]; virtualisation.docker.enable = true; From c78fe2e0550bd0f7188bc79132f1505c02ab9542 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Thu, 4 Jan 2024 20:50:43 -0300 Subject: [PATCH 71/86] Volta para nixpkgs 22.11 --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index a3adfbe..dfe4bdf 100644 --- a/flake.lock +++ b/flake.lock @@ -32,11 +32,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1703875886, - "narHash": "sha256-WJ0s5WWOBUnXAfPwxLm0cI4XjeLXPdI95Znwjrsfd2A=", + "lastModified": 1688392541, + "narHash": "sha256-lHrKvEkCPTUO+7tPfjIcb7Trk6k31rz18vkyqmkeJfY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "0fce51a5077927de454b94ffae41c2749fa15ba2", + "rev": "ea4c80b39be4c09702b0cb3b42eab59e2ba4f24b", "type": "github" }, "original": { From 4235e37418d2581f8252f6eafca2e052a3fa7850 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Thu, 4 Jan 2024 20:54:35 -0300 Subject: [PATCH 72/86] Refactor --- flake.nix | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/flake.nix b/flake.nix index 51c2113..2cce68a 100644 --- a/flake.nix +++ b/flake.nix @@ -25,15 +25,6 @@ foo-bar = prev.hello; - # https://fnordig.de/2023/07/24/old-ruby-on-modern-nix/ - # nodejs_16 = prev.nodejs_16.meta // { insecure = false; knownVulnerabilities = []; }; - github-runner = - let - ignoringVulns = x: x // { meta = (x.meta // { knownVulnerabilities = [ ]; }); }; - in - prev.github-runner.override { - nodejs_16 = prev.nodejs_16.overrideAttrs ignoringVulns; - }; }; } // allAttrs.flake-utils.lib.eachDefaultSystem @@ -492,15 +483,6 @@ iptables socat - ( - writeScriptBin "run-github-runner" '' - #! ${pkgs.runtimeShell} -e - sudo mkdir -pv -m 0700 /run/secrets/github-runner - sudo chown $(id -u):$(id -g) /run/secrets/github-runner - echo -n ghp_yyyyy > /run/secrets/github-runner/nixos.token - '' - ) - ( writeScriptBin "fix-k8s-cluster-admin-key" '' #! ${pkgs.runtimeShell} -e From a4cc0d14e20bb6e04af35c81d4c150dd00918598 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 10 Jan 2024 16:02:39 -0300 Subject: [PATCH 73/86] =?UTF-8?q?Super=20atualiza=C3=A7=C3=A3o=2022.11=20-?= =?UTF-8?q?>=2023.11=20+=20VNC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/tests.yml | 4 +- flake.lock | 36 +++-- flake.nix | 300 +++++++++++++++++++----------------- 3 files changed, 187 insertions(+), 153 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 44afd4a..8c0555d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,8 +14,8 @@ on: workflow_dispatch jobs: test: - # runs-on: arc-runner-set - runs-on: self-hosted + runs-on: arc-runner-set + # runs-on: self-hosted name: Testes steps: - name: checkout PR merge commit diff --git a/flake.lock b/flake.lock index dfe4bdf..b6ff632 100644 --- a/flake.lock +++ b/flake.lock @@ -1,12 +1,15 @@ { "nodes": { "flake-utils": { + "inputs": { + "systems": "systems" + }, "locked": { - "lastModified": 1667395993, - "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", + "lastModified": 1701680307, + "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=", "owner": "numtide", "repo": "flake-utils", - "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", + "rev": "4022d587cbbfd70fe950c1e2083a02621806a725", "type": "github" }, "original": { @@ -32,11 +35,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1688392541, - "narHash": "sha256-lHrKvEkCPTUO+7tPfjIcb7Trk6k31rz18vkyqmkeJfY=", + "lastModified": 1704420045, + "narHash": "sha256-C36QmoJd5tdQ5R9MC1jM7fBkZW9zBUqbUCsgwS6j4QU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ea4c80b39be4c09702b0cb3b42eab59e2ba4f24b", + "rev": "c1be43e8e837b8dbee2b3665a007e761680f0c3d", "type": "github" }, "original": { @@ -53,11 +56,11 @@ ] }, "locked": { - "lastModified": 1671737304, - "narHash": "sha256-BdxYdQvZu+B8+UdsPdnPnkTOCmSqZNvFoNbbJOhty6Y=", + "lastModified": 1695950484, + "narHash": "sha256-8Vbgh8LZVvVObf/Bf1zjms9HSwIkG1KUs3wJF9ll5u8=", "owner": "ES-Nix", "repo": "podman-rootless", - "rev": "83ff27be5616e3029c2aec2de595f86da081e857", + "rev": "16d1060736366cbda79cc65c347175a4c3ac84a3", "type": "github" }, "original": { @@ -73,6 +76,21 @@ "nixpkgs": "nixpkgs", "podman-rootless": "podman-rootless" } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 2cce68a..5982e6d 100644 --- a/flake.nix +++ b/flake.nix @@ -25,105 +25,150 @@ foo-bar = prev.hello; + # https://fnordig.de/2023/07/24/old-ruby-on-modern-nix/ + # nodejs_16 = prev.nodejs_16.meta // { insecure = false; knownVulnerabilities = []; }; +# github-runner = +# let +# ignoringVulns = x: x // { meta = (x.meta // { knownVulnerabilities = [ ]; }); }; +# in +# prev.github-runner.override { +# nodejs_16 = prev.nodejs_20.overrideAttrs ignoringVulns; +# }; + }; } // - allAttrs.flake-utils.lib.eachDefaultSystem - (system: + ( let - name = "github-ci-runner"; + # nix flake show --allow-import-from-derivation --impure --refresh .# + suportedSystems = [ + "x86_64-linux" + "aarch64-linux" + # "aarch64-darwin" + ]; - pkgsAllowUnfree = import nixpkgs { - inherit system; - overlays = [ self.overlays.default ]; - config = { - allowUnfree = true; + in + allAttrs.flake-utils.lib.eachSystem suportedSystems + (system: + let + name = "github-ci-runner"; + + pkgsAllowUnfree = import nixpkgs { + inherit system; + overlays = [ self.overlays.default ]; + config = { + allowUnfree = true; + }; }; - }; - hack = pkgsAllowUnfree.writeShellScriptBin "hack" '' - # Dont overwrite customised configuration + hack = pkgsAllowUnfree.writeShellScriptBin "hack" '' + # Dont overwrite customised configuration - # https://dev.to/ifenna__/adding-colors-to-bash-scripts-48g4 - echo -e '\n\n\n\e[32m\tAmbiente pronto!\e[0m\n' - echo -e '\n\t\e[33mignore as proximas linhas...\e[0m\n\n\n' - ''; - - # https://gist.github.com/tpwrules/34db43e0e2e9d0b72d30534ad2cda66d#file-flake-nix-L28 - pleaseKeepMyInputs = pkgsAllowUnfree.writeTextDir "bin/.please-keep-my-inputs" - (builtins.concatStringsSep " " (builtins.attrValues allAttrs)); - in - rec { - - packages.vm = self.nixosConfigurations.vm.config.system.build.toplevel; - - /* - # Utilized by `nix run .#` - - rm -fv nixos.qcow2 - nix run --impure --refresh --verbose .#vm - - # Open the QMEU VM terminal and: - start-github-runner-with-pat "$PAT" - */ - apps.vm = { - type = "app"; - program = "${self.nixosConfigurations.vm.config.system.build.vm}/bin/run-nixos-vm"; - }; - - # nix fmt - formatter = pkgsAllowUnfree.nixpkgs-fmt; - - devShells.default = pkgsAllowUnfree.mkShell { - buildInputs = with pkgsAllowUnfree; [ - age - allAttrs.podman-rootless.packages.${system}.podman - bashInteractive - coreutils - curl - gettext - gh - gnumake - hack - httpie - jq - patchelf - sops - ssh-to-age - ]; - - shellHook = '' - # TODO: documentar esse comportamento, - # devo abrir issue no github do nixpkgs - export TMPDIR=/tmp - - echo "Entering the nix devShell no github-ci-runner" - - test -d .profiles || mkdir -v .profiles - - test -L .profiles/dev \ - || nix develop --impure .# --profile .profiles/dev --command true - - test -L .profiles/dev-shell-default \ - || nix build --impure $(nix eval --impure --raw .#devShells."$system".default.drvPath) --out-link .profiles/dev-shell-"$system"-default - - test -L .profiles/nixosConfigurations."$system".vm.config.system.build.vm \ - || nix build --impure --out-link .profiles/nixosConfigurations."$system".vm.config.system.build.vm .#nixosConfigurations.vm.config.system.build.vm - - # For SOPS - # test -d ~/.config/sops/age || mkdir -pv ~/.config/sops/age - # test -f ~/.config/sops/age/keys.txt || age-keygen -o ~/.config/sops/age/keys.txt - # https://github.com/getsops/sops/pull/860/files#diff-7b3ed02bc73dc06b7db906cf97aa91dec2b2eb21f2d92bc5caa761df5bbc168fR192 - # test -d secrets || mkdir -v secrets - # test -f secrets/secrets.yaml.encrypted \ - # || sops \ - # --encrypt \ - # --age $(age-keygen -y ~/.config/sops/age/keys.txt) \ - # secrets/secrets.yaml > secrets/secrets.yaml.encrypted - - hack + # https://dev.to/ifenna__/adding-colors-to-bash-scripts-48g4 + echo -e '\n\n\n\e[32m\tAmbiente pronto!\e[0m\n' + echo -e '\n\t\e[33mignore as proximas linhas...\e[0m\n\n\n' ''; - }; - }) + + # https://gist.github.com/tpwrules/34db43e0e2e9d0b72d30534ad2cda66d#file-flake-nix-L28 + pleaseKeepMyInputs = pkgsAllowUnfree.writeTextDir "bin/.please-keep-my-inputs" + (builtins.concatStringsSep " " (builtins.attrValues allAttrs)); + in + rec { + + packages.vm = self.nixosConfigurations.vm.config.system.build.toplevel; + + /* + # Utilized by `nix run .#` + + rm -fv nixos.qcow2 + nix run --impure --refresh --verbose .#vm + + # Open the QMEU VM terminal and: + start-github-runner-with-pat "$PAT" + */ + apps.vm = { + type = "app"; + program = "${self.nixosConfigurations.vm.config.system.build.vm}/bin/run-nixos-vm"; + }; + + packages.automatic-vm = pkgsAllowUnfree.writeShellApplication { + name = "run-nixos-vm"; + runtimeInputs = with pkgsAllowUnfree; [ libcanberra libcanberra-gtk2 virt-viewer ]; + text = '' + ${self.nixosConfigurations.vm.config.system.build.vm}/bin/run-nixos-vm & PID_QEMU="$!" + + for _ in web{0..10};do + if remote-viewer spice://localhost:3001 + then + break + fi + date +'%d/%m/%Y %H:%M:%S:%3N' + sleep 0.5 + done; + # remote-viewer spice://127.0.0.1:5930 + kill $PID_QEMU + ''; + }; + + apps.run-github-runner = { + type = "app"; + program = "${self.packages."${system}".automatic-vm}/bin/run-nixos-vm"; + }; + + # nix fmt + formatter = pkgsAllowUnfree.nixpkgs-fmt; + + devShells.default = pkgsAllowUnfree.mkShell { + buildInputs = with pkgsAllowUnfree; [ + age + allAttrs.podman-rootless.packages.${system}.podman + bashInteractive + coreutils + curl + gettext + gh + gnumake + hack + httpie + jq + patchelf + sops + ssh-to-age + ]; + + shellHook = '' + # TODO: documentar esse comportamento, + # devo abrir issue no github do nixpkgs + export TMPDIR=/tmp + + echo "Entering the nix devShell no github-ci-runner" + + test -d .profiles || mkdir -v .profiles + + test -L .profiles/dev \ + || nix develop --impure .# --profile .profiles/dev --command true + + test -L .profiles/dev-shell-default \ + || nix build --impure $(nix eval --impure --raw .#devShells."$system".default.drvPath) --out-link .profiles/dev-shell-"$system"-default + + test -L .profiles/nixosConfigurations."$system".vm.config.system.build.vm \ + || nix build --impure --out-link .profiles/nixosConfigurations."$system".vm.config.system.build.vm .#nixosConfigurations.vm.config.system.build.vm + + # For SOPS + # test -d ~/.config/sops/age || mkdir -pv ~/.config/sops/age + # test -f ~/.config/sops/age/keys.txt || age-keygen -o ~/.config/sops/age/keys.txt + # https://github.com/getsops/sops/pull/860/files#diff-7b3ed02bc73dc06b7db906cf97aa91dec2b2eb21f2d92bc5caa761df5bbc168fR192 + # test -d secrets || mkdir -v secrets + # test -f secrets/secrets.yaml.encrypted \ + # || sops \ + # --encrypt \ + # --age $(age-keygen -y ~/.config/sops/age/keys.txt) \ + # secrets/secrets.yaml > secrets/secrets.yaml.encrypted + + hack + ''; + }; + }) + ) // { nixosConfigurations.vm = nixpkgs.lib.nixosSystem { # About system and maybe --impure @@ -154,6 +199,9 @@ boot.loader.systemd-boot.enable = true; fileSystems."/" = { device = "/dev/hda1"; }; + # O Kernel de Fonseca é 5.* + boot.kernelPackages = pkgs.linuxKernel.packages.linux_rt_5_15; + virtualisation.vmVariant = { @@ -173,21 +221,16 @@ virtualisation.resolution = lib.mkForce { x = 1024; y = 768; }; virtualisation.qemu.options = [ - # Better display option - # TODO: -display sdl,gl=on - # https://gitlab.com/qemu-project/qemu/-/issues/761 - "-vga virtio" - "-display gtk,zoom-to-fit=false" - # Enable copy/paste - # https://www.kraxel.org/blog/2021/05/qemu-cut-paste/ - "-chardev qemu-vdagent,id=ch1,name=vdagent,clipboard=on" - "-device virtio-serial-pci" - "-device virtserialport,chardev=ch1,id=ch1,name=com.redhat.spice.0" - - # https://serverfault.com/a/1119403 - # "-device intel-iommu,intremap=on" - - # "-net user,hostfwd=tcp::8090-::8080" + # https://www.spice-space.org/spice-user-manual.html#Running_qemu_manually + # remote-viewer spice://localhost:3001 + + # "-daemonize" # How to save the QEMU PID? + "-machine vmport=off" + "-vga qxl" + "-spice port=3001,disable-ticketing=on" + "-device virtio-serial" + "-chardev spicevmc,id=vdagent,debug=0,name=vdagent" + "-device virtserialport,chardev=vdagent,name=com.redhat.spice.0" ]; }; @@ -255,35 +298,7 @@ ]; }; - /* - https://github.com/NixOS/nixpkgs/issues/169812 - https://github.com/actions/runner/issues/1882#issuecomment-1427930611 - nix shell nixpkgs#github-runner --command \ - sh \ - -c \ - 'config.sh --url https://github.com/imobanco/github-ci-runner --pat "$PAT" --ephemeral && run.sh' - config.sh --url https://github.com/imobanco/github-ci-runner --pat "$PAT" --ephemeral && run.sh - TODO: https://www.youtube.com/watch?v=G5f6GC7SnhU - */ - services.github-runner.enable = true; - services.github-runner.ephemeral = true; - services.github-runner.user = "nixuser"; - # services.github-runner.runnerGroup = "nixgroup"; - services.github-runner.url = "https://github.com/imobanco/github-ci-runner"; - # services.github-runner.tokenFile = config.sops.secrets."github-runner/token".path; - services.github-runner.tokenFile = "/run/secrets/github-runner/nixos.token"; - services.github-runner.extraPackages = with pkgs; [ - config.virtualisation.docker.package - hello - # sudo - procps - python39 - ]; - virtualisation.docker.enable = true; - systemd.services.github-runner.serviceConfig.SupplementaryGroups = [ "docker" ]; - # systemd.services.github-runner."nixos".extraPackages = with pkgs; [ which sudo ]; - - systemd.user.services.populate-history-vagrant = { + systemd.user.services.populate-history = { script = '' echo "Started" @@ -361,11 +376,11 @@ fonts = { fontDir.enable = true; - fonts = with pkgs; [ + packages = with pkgs; [ powerline powerline-fonts ]; - enableDefaultFonts = true; + enableDefaultPackages = true; enableGhostscriptFonts = true; }; @@ -400,11 +415,11 @@ # https://github.com/NixOS/nixpkgs/issues/21332#issuecomment-268730694 services.openssh = { allowSFTP = true; - kbdInteractiveAuthentication = false; + settings.KbdInteractiveAuthentication = false; enable = true; - forwardX11 = false; - passwordAuthentication = false; - permitRootLogin = "yes"; + # settings.ForwardX11 = false; + settings.PasswordAuthentication = false; + settings.PermitRootLogin = "yes"; ports = [ 10022 ]; authorizedKeysFiles = [ "${ pkgs.writeText "nixuser-keys.pub" "${toString nixuserKeys}" }" @@ -440,10 +455,11 @@ # nixpkgs.config.allowUnfree = true; + boot.readOnlyNixStore = true; + nix = { extraOptions = "experimental-features = nix-command flakes"; package = pkgs.nixVersions.nix_2_10; - readOnlyStore = true; registry.nixpkgs.flake = nixpkgs; # https://bou.ke/blog/nix-tips/ nixPath = [ "nixpkgs=${pkgs.path}" ]; }; From 32fbaaa18c5ec8a2b5c6e6d143c0003115ea850d Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 10 Jan 2024 16:04:30 -0300 Subject: [PATCH 74/86] =?UTF-8?q?Super=20atualiza=C3=A7=C3=A3o=2022.11=20-?= =?UTF-8?q?>=2023.11=20+=20VNC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8c0555d..e0d9df1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,7 +31,7 @@ jobs: - name: Metadatas run: | - # cat /etc/os*release + cat /etc/os*release # cat /etc/group uname -a id From ba13697b052109d5031c816207e6e9e6ad6a23c8 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Wed, 10 Jan 2024 16:12:35 -0300 Subject: [PATCH 75/86] =?UTF-8?q?Super=20atualiza=C3=A7=C3=A3o=2022.11=20-?= =?UTF-8?q?>=2023.11=20+=20VNC,=203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/tests.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e0d9df1..6d7e7b0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,11 +24,6 @@ jobs: - name: Executa um echo run: echo 'Olá mundo!' - - name: Executa GNU hello e python3 - run: | - hello - python --version - - name: Metadatas run: | cat /etc/os*release From fb088403fe8487e4bd6f840f39c6c21d2c7e6ec8 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Mon, 15 Jan 2024 15:14:43 -0300 Subject: [PATCH 76/86] =?UTF-8?q?Remove=20overlay=20n=C3=A3o=20usado?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- flake.nix | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/flake.nix b/flake.nix index 5982e6d..f4e44c2 100644 --- a/flake.nix +++ b/flake.nix @@ -25,16 +25,6 @@ foo-bar = prev.hello; - # https://fnordig.de/2023/07/24/old-ruby-on-modern-nix/ - # nodejs_16 = prev.nodejs_16.meta // { insecure = false; knownVulnerabilities = []; }; -# github-runner = -# let -# ignoringVulns = x: x // { meta = (x.meta // { knownVulnerabilities = [ ]; }); }; -# in -# prev.github-runner.override { -# nodejs_16 = prev.nodejs_20.overrideAttrs ignoringVulns; -# }; - }; } // ( From 9caab680b03e360c502a233e3877ffb1a774df7f Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Tue, 16 Jan 2024 19:26:33 -0300 Subject: [PATCH 77/86] =?UTF-8?q?Reduz=20RAM=20e=20n=C3=BAmero=20de=20core?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- flake.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index f4e44c2..c5498c0 100644 --- a/flake.nix +++ b/flake.nix @@ -203,9 +203,9 @@ programs.dconf.enable = true; # security.polkit.enable = true; # TODO: hardening? - virtualisation.memorySize = 1024 * 8; # Use MiB memory. + virtualisation.memorySize = 1024 * 3; # Use MiB memory. virtualisation.diskSize = 1024 * 50; # Use MiB memory. - virtualisation.cores = 8; # Number of cores. + virtualisation.cores = 2; # Number of cores. virtualisation.graphics = true; virtualisation.resolution = lib.mkForce { x = 1024; y = 768; }; From afcf7be676f16ca70b14d3925b994ae9c892d235 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Tue, 16 Jan 2024 19:49:25 -0300 Subject: [PATCH 78/86] Corrige comando de executar a VM --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e03fb00..2374253 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ Gerar token: ```bash -rm -fv nixos.qcow2; nix run --impure --refresh --verbose .#vm +rm -fv nixos.qcow2; nix run --impure --refresh --verbose .#run-github-runner ``` From bf0fafe491a220375ba5a6d662f8d28db43cb708 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Tue, 16 Jan 2024 21:23:14 -0300 Subject: [PATCH 79/86] Adiciona TODO --- flake.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flake.nix b/flake.nix index c5498c0..97e0e39 100644 --- a/flake.nix +++ b/flake.nix @@ -87,6 +87,8 @@ ${self.nixosConfigurations.vm.config.system.build.vm}/bin/run-nixos-vm & PID_QEMU="$!" for _ in web{0..10};do + # Aparentemente pode ocorrer uma condição de corrida + # TODO: https://unix.stackexchange.com/a/698488 if remote-viewer spice://localhost:3001 then break From 12c362da665ae124ef0878e1dfa8a9722ea68033 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Tue, 16 Jan 2024 21:37:08 -0300 Subject: [PATCH 80/86] =?UTF-8?q?Ajusta=20hist=C3=B3rico?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- flake.nix | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 97e0e39..ed3939c 100644 --- a/flake.nix +++ b/flake.nix @@ -297,9 +297,8 @@ DESTINATION=/home/nixuser/.zsh_history # TODO: https://stackoverflow.com/a/67169387 - echo "journalctl -xeu github-runner-nixos.service" >> "$DESTINATION" - echo "systemctl status github-runner-nixos.service | cat" >> "$DESTINATION" - echo "run-github-runner && sudo systemctl restart github-runner-nixos.service" >> "$DESTINATION" + echo "GITHUB_PAT=ghp_yyyyyyyyyyyyyyy" >> "$DESTINATION" + echo "bash -lc 'read -sp "Please enter your github PAT:" GITHUB_PAT'" >> "$DESTINATION" echo "Ended" ''; From 129819e6b58f4e1f571770c5c65cced9fdefc59c Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Tue, 16 Jan 2024 21:53:09 -0300 Subject: [PATCH 81/86] nc -vz localhost 3001 --- flake.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index ed3939c..ccca457 100644 --- a/flake.nix +++ b/flake.nix @@ -89,6 +89,7 @@ for _ in web{0..10};do # Aparentemente pode ocorrer uma condição de corrida # TODO: https://unix.stackexchange.com/a/698488 + # nc -vz localhost 3001 https://serverfault.com/questions/1059834/cant-access-vnc-port-from-local-network if remote-viewer spice://localhost:3001 then break @@ -298,7 +299,8 @@ # TODO: https://stackoverflow.com/a/67169387 echo "GITHUB_PAT=ghp_yyyyyyyyyyyyyyy" >> "$DESTINATION" - echo "bash -lc 'read -sp "Please enter your github PAT:" GITHUB_PAT'" >> "$DESTINATION" + + # echo "bash -lc 'read -sp "Please enter your github PAT:" GITHUB_PAT'" >> "$DESTINATION" echo "Ended" ''; From 26f7fb9751c20c462e17ca5f30bd34e1b3b8dd49 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Tue, 16 Jan 2024 22:02:38 -0300 Subject: [PATCH 82/86] Muda ordem do teste do docker --- .github/workflows/tests.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6d7e7b0..77b309c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -38,12 +38,6 @@ jobs: cat $HOME/logs.txt || true echo FOO-BAR > $HOME/logs.txt - - name: Testes sobre docker build - run: | - id - docker build --tag custom-python . - docker images - - name: Testes sobre docker run: | # env | sort @@ -57,6 +51,12 @@ jobs: docker run --rm alpine cat /etc/os-release docker images + - name: Testes sobre docker build + run: | + id + docker build --tag custom-python . + docker images + - name: Testes sobre /dev/kvm run: | id From 84d7a17ce126d5766e56269b7f790fdbc7e2461a Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Thu, 18 Jan 2024 11:21:16 -0300 Subject: [PATCH 83/86] Resolve race condition, hopefully --- flake.nix | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/flake.nix b/flake.nix index ccca457..2b1d497 100644 --- a/flake.nix +++ b/flake.nix @@ -82,22 +82,34 @@ packages.automatic-vm = pkgsAllowUnfree.writeShellApplication { name = "run-nixos-vm"; - runtimeInputs = with pkgsAllowUnfree; [ libcanberra libcanberra-gtk2 virt-viewer ]; + runtimeInputs = with pkgsAllowUnfree; [ curl virt-viewer ]; + /* + Pode ocorrer uma condição de corrida de seguinte forma: + a VM inicializa (o processo não é bloqueante, executa em background) + o spice/VNC interno a VM inicializa + o remote-viewer tenta conectar, mas o spice não está pronto ainda + + TODO: idealmente não deveria ser preciso ter mais uma dependência (o curl) + para poder sincronizar o cliente e o server. Será que no caso de + ambos estarem na mesma máquina seria melhor usar virt-viewer -fw? + https://unix.stackexchange.com/a/698488 + */ text = '' ${self.nixosConfigurations.vm.config.system.build.vm}/bin/run-nixos-vm & PID_QEMU="$!" - for _ in web{0..10};do - # Aparentemente pode ocorrer uma condição de corrida - # TODO: https://unix.stackexchange.com/a/698488 - # nc -vz localhost 3001 https://serverfault.com/questions/1059834/cant-access-vnc-port-from-local-network - if remote-viewer spice://localhost:3001 + export VNC_PORT=3001 + + for _ in web{0..50}; do + if [[ $(curl --fail --silent http://localhost:"$VNC_PORT") -eq 1 ]]; then break fi - date +'%d/%m/%Y %H:%M:%S:%3N' - sleep 0.5 + # date +'%d/%m/%Y %H:%M:%S:%3N' + sleep 0.2 done; - # remote-viewer spice://127.0.0.1:5930 + + remote-viewer spice://localhost:"$VNC_PORT" + kill $PID_QEMU ''; }; From bd5b34448eae5614d3e8452e24273f8651354630 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Thu, 18 Jan 2024 11:24:01 -0300 Subject: [PATCH 84/86] Remove atributo antigo --- flake.nix | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/flake.nix b/flake.nix index 2b1d497..f53d4db 100644 --- a/flake.nix +++ b/flake.nix @@ -66,20 +66,6 @@ packages.vm = self.nixosConfigurations.vm.config.system.build.toplevel; - /* - # Utilized by `nix run .#` - - rm -fv nixos.qcow2 - nix run --impure --refresh --verbose .#vm - - # Open the QMEU VM terminal and: - start-github-runner-with-pat "$PAT" - */ - apps.vm = { - type = "app"; - program = "${self.nixosConfigurations.vm.config.system.build.vm}/bin/run-nixos-vm"; - }; - packages.automatic-vm = pkgsAllowUnfree.writeShellApplication { name = "run-nixos-vm"; runtimeInputs = with pkgsAllowUnfree; [ curl virt-viewer ]; From 62d154951aaf3a09f65ddd31b0e5ed0b3907a0ee Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Thu, 18 Jan 2024 14:42:37 -0300 Subject: [PATCH 85/86] Deletes k8s helpers examples --- flake.nix | 50 ------- kubernetes-examples/appvia/deployment.yaml | 21 --- kubernetes-examples/appvia/exec-liveness.yaml | 21 --- kubernetes-examples/appvia/ingress.yaml | 18 --- kubernetes-examples/appvia/notes.md | 16 -- kubernetes-examples/appvia/service.yaml | 11 -- kubernetes-examples/flask-gunicorn/notes.md | 141 ------------------ .../minimal-pod-with-busybox-example.yaml | 9 -- .../minimal-pod-with-busybox-example/notes.md | 12 -- kubernetes-examples/nodejs-node-port/notes.md | 82 ---------- .../official-examples/notes.md | 90 ----------- .../security-context-demo/notes.md | 13 -- 12 files changed, 484 deletions(-) delete mode 100644 kubernetes-examples/appvia/deployment.yaml delete mode 100644 kubernetes-examples/appvia/exec-liveness.yaml delete mode 100644 kubernetes-examples/appvia/ingress.yaml delete mode 100644 kubernetes-examples/appvia/notes.md delete mode 100644 kubernetes-examples/appvia/service.yaml delete mode 100644 kubernetes-examples/flask-gunicorn/notes.md delete mode 100644 kubernetes-examples/minimal-pod-with-busybox-example/minimal-pod-with-busybox-example.yaml delete mode 100644 kubernetes-examples/minimal-pod-with-busybox-example/notes.md delete mode 100644 kubernetes-examples/nodejs-node-port/notes.md delete mode 100644 kubernetes-examples/official-examples/notes.md delete mode 100644 kubernetes-examples/security-context-demo/notes.md diff --git a/flake.nix b/flake.nix index f53d4db..a674995 100644 --- a/flake.nix +++ b/flake.nix @@ -557,56 +557,6 @@ flannel.enable = true; }; - # TODO: refatorar, talvez usar self? - environment.etc."kubernets/kubernetes-examples/minimal-pod-with-busybox-example/minimal-pod-with-busybox-example.yaml" = { - mode = "0644"; - text = "${builtins.readFile ./kubernetes-examples/minimal-pod-with-busybox-example/minimal-pod-with-busybox-example.yaml}"; - }; - - environment.etc."kubernets/kubernetes-examples/minimal-pod-with-busybox-example/notes.md" = { - mode = "0644"; - text = "${builtins.readFile ./kubernetes-examples/minimal-pod-with-busybox-example/notes.md}"; - }; - - environment.etc."kubernets/kubernetes-examples/appvia/deployment.yaml" = { - mode = "0644"; - text = "${builtins.readFile ./kubernetes-examples/appvia/deployment.yaml}"; - }; - - environment.etc."kubernets/kubernetes-examples/appvia/service.yaml" = { - mode = "0644"; - text = "${builtins.readFile ./kubernetes-examples/appvia/service.yaml}"; - }; - - environment.etc."kubernets/kubernetes-examples/appvia/ingress.yaml" = { - mode = "0644"; - text = "${builtins.readFile ./kubernetes-examples/appvia/ingress.yaml}"; - }; - - environment.etc."kubernets/kubernetes-examples/appvia/notes.md" = { - mode = "0644"; - text = "${builtins.readFile ./kubernetes-examples/appvia/notes.md}"; - }; - - # journalctl -u move-kubernetes-examples.service -b - systemd.services.move-kubernetes-examples = { - script = '' - echo "Started move-kubernets-examples" - - # cp -rv ''\${./kubernetes-examples} /home/nixuser/ - cp -Rv /etc/kubernets/kubernetes-examples/ /home/nixuser/ - - chown -Rv nixuser:nixgroup /home/nixuser/kubernetes-examples - - kubectl \ - apply \ - --file /home/nixuser/kubernetes-examples/deployment.yaml \ - --file /home/nixuser/kubernetes-examples/service.yaml \ - --file /home/nixuser/kubernetes-examples/ingress.yaml - ''; - wantedBy = [ "multi-user.target" ]; - }; - # journalctl --user --unit create-custom-desktop-icons.service -b -f systemd.user.services.create-custom-desktop-icons = { script = '' diff --git a/kubernetes-examples/appvia/deployment.yaml b/kubernetes-examples/appvia/deployment.yaml deleted file mode 100644 index c85e7bb..0000000 --- a/kubernetes-examples/appvia/deployment.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: podinfo - labels: - app: podinfo -spec: - selector: - matchLabels: - app: podinfo - template: - metadata: - labels: - app: podinfo - spec: - containers: - - name: podinfo-container - image: stefanprodan/podinfo - imagePullPolicy: IfNotPresent - ports: - - containerPort: 9898 diff --git a/kubernetes-examples/appvia/exec-liveness.yaml b/kubernetes-examples/appvia/exec-liveness.yaml deleted file mode 100644 index 7d6ca96..0000000 --- a/kubernetes-examples/appvia/exec-liveness.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - labels: - test: liveness - name: liveness-exec -spec: - containers: - - name: liveness - image: registry.k8s.io/busybox - args: - - /bin/sh - - -c - - touch /tmp/healthy; sleep 30; rm -f /tmp/healthy; sleep 600 - livenessProbe: - exec: - command: - - cat - - /tmp/healthy - initialDelaySeconds: 5 - periodSeconds: 5 diff --git a/kubernetes-examples/appvia/ingress.yaml b/kubernetes-examples/appvia/ingress.yaml deleted file mode 100644 index 7ca267c..0000000 --- a/kubernetes-examples/appvia/ingress.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: podinfo - annotations: - nginx.ingress.kubernetes.io/rewrite-target: / -spec: - ingressClassName: nginx-example - rules: - - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: podinfo - port: - number: 3000 diff --git a/kubernetes-examples/appvia/notes.md b/kubernetes-examples/appvia/notes.md deleted file mode 100644 index 2865eba..0000000 --- a/kubernetes-examples/appvia/notes.md +++ /dev/null @@ -1,16 +0,0 @@ - - -TODO: qual a fonte desse exemplo? Perdi e não encontrei novamente. - -```bash -cd ~/kubernetes-examples/appvia \ -&& kubectl apply \ - -f deployment.yaml \ - -f service.yaml \ - -f ingress.yaml -``` - - -```bash -wk8s -``` diff --git a/kubernetes-examples/appvia/service.yaml b/kubernetes-examples/appvia/service.yaml deleted file mode 100644 index e38a178..0000000 --- a/kubernetes-examples/appvia/service.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: my-service -spec: - selector: - app.kubernetes.io/name: my-podinfo - ports: - - protocol: TCP - port: 3000 - targetPort: 9898 diff --git a/kubernetes-examples/flask-gunicorn/notes.md b/kubernetes-examples/flask-gunicorn/notes.md deleted file mode 100644 index d7764f5..0000000 --- a/kubernetes-examples/flask-gunicorn/notes.md +++ /dev/null @@ -1,141 +0,0 @@ - - - - -Adaptado para usar `nodePort`: -```bash -cat > flask-node-port.yml <<-'EOF' -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: hello-service - name: hello-service -spec: - replicas: 3 - selector: - matchLabels: - app: hello-service - template: - metadata: - labels: - app: hello-service - spec: - containers: - - name: hello-service - image: quay.io/lordofthejars/hello-flask:1.0.0 - ports: - - containerPort: 8080 ---- -apiVersion: v1 -kind: Service -metadata: - name: hello-service - labels: - app: hello-service -spec: - # Expose the service on a static port on each node - # so that we can access the service from outside the cluster - type: NodePort - - # When the node receives a request on the static port (30163) - # "select pods with the label 'app' set to 'echo-hostname'" - # and forward the request to one of them - selector: - app: hello-service - - ports: - # Three types of ports for a service - # nodePort - a static port assigned on each the node - # port - port exposed internally in the cluster - # targetPort - the container port to send requests to - - nodePort: 30163 - port: 8080 - targetPort: 8080 -EOF - -kubectl apply -f flask-node-port.yml -``` -Refs.: -- [How to deploy a Flask application in Python with Gunicorn](https://developers.redhat.com/articles/2023/08/17/how-deploy-flask-application-python-gunicorn#) -- https://matthewpalmer.net/kubernetes-app-developer/articles/service-kubernetes-example-tutorial.html -- https://medium.com/google-cloud/kubernetes-nodeport-vs-loadbalancer-vs-ingress-when-should-i-use-what-922f010849e0 - - - -```bash -wk8s -``` - -Em um terminal na VM NixOS: -```bash -curl localhost:30163/ -``` - - -Em um terminal no host: -```bash -curl localhost:8090/ -``` - - -## Usando LoadBalancer e ip hardcoded - -```bash -cat > flask-load-balancer.yml <<-'EOF' -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: hello-service - name: hello-service -spec: - replicas: 1 - selector: - matchLabels: - app: hello-service - template: - metadata: - labels: - app: hello-service - spec: - containers: - - name: hello-service - image: quay.io/lordofthejars/hello-flask:1.0.0 - ports: - - containerPort: 8080 ---- -apiVersion: v1 -kind: Service -metadata: - name: hello-service - labels: - app: hello-service -spec: - ports: - - name: http - port: 8080 - selector: - app: hello-service - type: LoadBalancer - externalIPs: - - "34.74.203.201" -EOF - - -kubectl apply -f flask-load-balancer.yml - -``` -Refs.: -- https://paul-boone.medium.com/kubernetes-loadbalancer-ip-stuck-in-pending-6ddea72b8ff5 - - -No terminal da VM NixOS: -```bash -curl 34.74.203.201:8080/ -``` - -Pelo que entendi não funciona externamente. - - - diff --git a/kubernetes-examples/minimal-pod-with-busybox-example/minimal-pod-with-busybox-example.yaml b/kubernetes-examples/minimal-pod-with-busybox-example/minimal-pod-with-busybox-example.yaml deleted file mode 100644 index 4bc3ee4..0000000 --- a/kubernetes-examples/minimal-pod-with-busybox-example/minimal-pod-with-busybox-example.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: test-pod -spec: - containers: - - name: test-pod - image: busybox - command: ['sh', '-c', "while ! false; do echo $(date +'%d/%m/%Y %H:%M:%S:%3N'); sleep 1; done"] diff --git a/kubernetes-examples/minimal-pod-with-busybox-example/notes.md b/kubernetes-examples/minimal-pod-with-busybox-example/notes.md deleted file mode 100644 index bbb6a9e..0000000 --- a/kubernetes-examples/minimal-pod-with-busybox-example/notes.md +++ /dev/null @@ -1,12 +0,0 @@ - - -```bash -cd ~/kubernetes-examples/minimal-pod-with-busybox-example \ -&& kubectl apply \ --f minimal-pod-with-busybox-example.yaml -``` - -```bash -wk8s -``` - diff --git a/kubernetes-examples/nodejs-node-port/notes.md b/kubernetes-examples/nodejs-node-port/notes.md deleted file mode 100644 index e53bfd9..0000000 --- a/kubernetes-examples/nodejs-node-port/notes.md +++ /dev/null @@ -1,82 +0,0 @@ - - -Atualmente quebrado! - -https://kubernetes.io/docs/tutorials/stateless-application/expose-external-ip-address/#creating-a-service-for-an-application-running-in-five-pods - -Adaptado para usar `nodePort`: -```bash -cat > nodejs-node-port.yml <<-'EOF' -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app.kubernetes.io/name: load-balancer-example - name: hello-world -spec: - replicas: 5 - selector: - matchLabels: - app.kubernetes.io/name: load-balancer-example - template: - metadata: - labels: - app.kubernetes.io/name: load-balancer-example - spec: - containers: - - image: gcr.io/google-samples/node-hello:1.0 - name: hello-world - ports: - - containerPort: 8080 ---- -apiVersion: v1 -kind: Service -metadata: - name: hello-world - labels: - app: hello-world -spec: - # Expose the service on a static port on each node - # so that we can access the service from outside the cluster - type: NodePort - - # When the node receives a request on the static port (30163) - # "select pods with the label 'app' set to 'echo-hostname'" - # and forward the request to one of them - selector: - app: hello-world - - ports: - # Three types of ports for a service - # nodePort - a static port assigned on each the node - # port - port exposed internally in the cluster - # targetPort - the container port to send requests to - - nodePort: 30163 - port: 8080 - targetPort: 8080 -EOF - -kubectl apply -f nodejs-node-port.yml -``` -Refs.: -- [How to deploy a Flask application in Python with Gunicorn](https://developers.redhat.com/articles/2023/08/17/how-deploy-flask-application-python-gunicorn#) -- https://matthewpalmer.net/kubernetes-app-developer/articles/service-kubernetes-example-tutorial.html -- https://medium.com/google-cloud/kubernetes-nodeport-vs-loadbalancer-vs-ingress-when-should-i-use-what-922f010849e0 - - - -```bash -wk8s -``` - -Em um terminal na VM NixOS: -```bash -curl localhost:30163/ -``` - - -Em um terminal no host: -```bash -curl localhost:8090/ -``` - diff --git a/kubernetes-examples/official-examples/notes.md b/kubernetes-examples/official-examples/notes.md deleted file mode 100644 index f7a4030..0000000 --- a/kubernetes-examples/official-examples/notes.md +++ /dev/null @@ -1,90 +0,0 @@ - - -Originalmente encontrado aqui: -https://kubernetes.io/docs/tutorials/stateless-application/guestbook - -Está quebrado! Pelo menos uma das imagens usadas não existem mais! -```bash -docker pull gcr.io/google_samples/gb-frontend:v5 -``` -Refs.: -- https://kubernetes.io/docs/tutorials/stateless-application/guestbook/#creating-the-guestbook-frontend-deployment - -```bash -journalctl --unit docker -``` - - -Encontrei o repo original, mas também não consegui fazer funcionar 100%. - - -https://cloud.google.com/kubernetes-engine/docs/tutorials/guestbook - - -```bash -git clone https://github.com/GoogleCloudPlatform/kubernetes-engine-samples \ -&& cd kubernetes-engine-samples/quickstarts/guestbook \ -&& BASE_URL='https://k8s.io/examples/application/guestbook' -kubectl apply \ --f redis-leader-deployment.yaml \ --f redis-leader-service.yaml \ --f redis-follower-deployment.yaml \ --f redis-follower-service.yaml \ --f frontend-service.yaml -``` - -```bash -kubectl get service frontend -``` - -```bash -wk8s -``` - - - - -```bash -cat > static-docker-example.yml <<-'EOF' ---- -apiVersion: v1 -kind: Pod -metadata: - name: static-docker-example -spec: - volumes: - - name: dockersocket - emptyDir: {} - - containers: - - # This is going to be our docker service container. - - name: docker-service - image: docker:dind-rootless - - # IMPORTANT! This is security related. - # Read up about running privileged containers - securityContext: - privileged: true - - volumeMounts: - - name: dockersocket - mountPath: /run/user/1000/ - - # We will run commands in this one. - - name: docker-commander - image: docker:dind-rootless - # Just keep the container running - command: [ "/bin/sh", "-c", "sleep 86000s" ] - volumeMounts: - - name: dockersocket - mountPath: /var/run -EOF - - -kubectl apply -f static-docker-example.yml - -``` -Refs.: -- https://discuss.kubernetes.io/t/can-k8s-or-k8s-api-build-image-with-dockerfile/16059/2 - diff --git a/kubernetes-examples/security-context-demo/notes.md b/kubernetes-examples/security-context-demo/notes.md deleted file mode 100644 index 5c6deb9..0000000 --- a/kubernetes-examples/security-context-demo/notes.md +++ /dev/null @@ -1,13 +0,0 @@ - - - -```bash -kubectl apply -f https://k8s.io/examples/pods/security/security-context.yaml -``` -Refs.: -- https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod - - -```bash -wk8s -``` From 93ccd57759b92ff7a306e9c66a8ede131570bd60 Mon Sep 17 00:00:00 2001 From: "Pedro O. A. Regis" Date: Fri, 26 Jan 2024 00:37:16 -0300 Subject: [PATCH 86/86] Salva no remoto --- flake.nix | 112 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 98 insertions(+), 14 deletions(-) diff --git a/flake.nix b/flake.nix index a674995..bc8f294 100644 --- a/flake.nix +++ b/flake.nix @@ -25,6 +25,86 @@ foo-bar = prev.hello; + cachedOCIImage1 = prev.dockerTools.pullImage { + finalImageTag = "latest"; + imageDigest = "sha256:075975296016084fc66b59c35c9d4504765d95aadcd5469f28d2b75750348fc5"; + imageName = "quay.io/podman/hello"; + name = "quay.io/podman"; + sha256 = "sha256-GBZWj/pp5wzfTrAwaMnZ9xjZ1imqkRmoPAxosXrEiZM="; + }; + + cachedOCIImage2 = prev.dockerTools.pullImage { + finalImageTag = "1.28"; + imageDigest = "sha256:141c253bc4c3fd0a201d32dc1f493bcf3fff003b6df416dea4f41046e0f37d47"; + imageName = "busybox"; + name = "docker.io/busybox"; + sha256 = "sha256-qWfC8wZBUhwHEPkf6ns4ST7KUyE9RlKaatzr+DCz+3E="; + }; + + cachedOCIImage3 = prev.dockerTools.pullImage { + # skopeo inspect docker://ghcr.io/actions/actions-runner:2.311.0 | jq -r '.Digest' + finalImageTag = "2.311.0"; + imageDigest = "sha256:e505724e0dbb070454912b159645932f74dda9aed03c48c463f33e270a6cbfa1"; + imageName = "ghcr.io/actions/actions-runner"; + name = "ghcr.io/actions"; + sha256 = "sha256-pHkSrrUaqZhgm/+UviuiKEJrjDXue5ww85qOrG6ZFzY="; + }; + + cachedOCIImage4 = prev.dockerTools.pullImage { + # skopeo inspect docker://docker.io/docker:24.0.7-dind-alpine3.18 | jq -r '.Digest' + finalImageTag = "24.0.7-dind-alpine3.18"; + imageDigest = "sha256:c90e58d30700470fc59bdaaf802340fd25c1db628756d7bf74e100c566ba9589"; + imageName = "docker.io/docker"; + name = "docker.io/docker"; + sha256 = "sha256-RRXGM3xWuB67hCfZCa623utFWYMpkqYW6UB/G8GQsOw="; + }; + + cachedOCIImage5 = prev.dockerTools.streamLayeredImage { + name = "chromium"; + tag = "${prev.chromium.version}"; + config = { + Cmd = [ + "${prev.chromium}/bin/chromium" + "--headless" + "--no-default-browser-check" + "--no-first-run" + "--disable-extensions" + "--disable-background-networking" + "--disable-background-timer-throttling" + "--disable-backgrounding-occluded-windows" + "--disable-renderer-backgrounding" + "--disable-breakpad" + "--disable-client-side-phishing-detection" + "--disable-crash-reporter" + "--disable-default-apps" + "--disable-dev-shm-usage" + "--disable-device-discovery-notifications" + "--disable-namespace-sandbox" + "--user-data-dir=/tmp/chrome-data-dir" + "--disable-translate" + "--autoplay-policy=no-user-gesture-required" + "--window-size=1366x768" + "--remote-debugging-address=127.0.0.1" + "--remote-debugging-port=0" + "--no-sandbox" + "--disable-gpu" + ]; + Env = [ + "FONTCONFIG_FILE=${prev.fontconfig.out}/etc/fonts/fonts.conf" + "FONTCONFIG_PATH=${prev.fontconfig.out}/etc/fonts/" + ]; + }; + }; + #cachedOCIImage5 = prev.dockerTools.pullImage { + # # skopeo inspect docker://docker.io/docker:24.0.7-dind-alpine3.18 | jq -r '.Digest' + # finalImageTag = "2024-01-08"; + # imageDigest = "sha256:113439c25115736fbc26ad0c740349816079003587d1bc63d7eaf48a0b3a55c3"; + # imageName = "quay.io/jupyter/scipy-notebook"; + # name = "quay.io/jupyter"; + # sha256 = "sha256-wYc4v8jBrViU9qBZt1J2P812b8gqi8qQpA3Bo8YwwKM="; + # # diskSize = 2 * 1024; + # # buildVMMemorySize = 2 * 512; + #}; }; } // ( @@ -81,6 +161,12 @@ https://unix.stackexchange.com/a/698488 */ text = '' + + # https://unix.stackexchange.com/a/230442 + # export NO_AT_BRIDGE=1 + # https://gist.github.com/eoli3n/93111f23dbb1233f2f00f460663f99e2#file-rootless-podman-wayland-sh-L25 + export LD_LIBRARY_PATH="${pkgsAllowUnfree.libcanberra-gtk3}"/lib/gtk-3.0/modules + ${self.nixosConfigurations.vm.config.system.build.vm}/bin/run-nixos-vm & PID_QEMU="$!" export VNC_PORT=3001 @@ -123,6 +209,7 @@ jq patchelf sops + skopeo ssh-to-age ]; @@ -467,6 +554,7 @@ neovim nix-direnv nixos-option + skopeo oh-my-zsh xclip zsh @@ -537,20 +625,16 @@ environment.variables.KUBECONFIG = "/etc/kubernetes/cluster-admin.kubeconfig"; - # services.kubernetes.kubelet.seedDockerImages = [ - # (pkgs.dockerTools.pullImage { - # name = "ghcr.io/actions"; - # imageName = "actions-runner:2.311.0"; - # # tag = "2.311.0"; - # sha256 = "sha256-BVnOXiYRUg3ukjYJBYbazOfrIrzQt7aRB2LWPf1b+ZE="; - # # podman inspect docker.io/sickcodes/docker-osx:latest | jq ".[].Digest" - # # imageDigest = "sha256:e505724e0dbb070454912b159645932f74dda9aed03c48c463f33e270a6cbfa1"; - # imageDigest = ""; - # }) - # ]; - # docker pull ghcr.io/actions/actions-runner:2.311.0 - # docker inspect docker.io/sickcodes/docker-osx:latest | jq ".[].Digest" - # dockerTools.examples.redis + # nix eval --impure --json \ + # '.#nixosConfigurations.vm.config.services.kubernetes.kubelet.seedDockerImages' + # services.kubernetes.kubelet.seedDockerImages = (with pkgs; [ + # cachedOCIImage1 + # cachedOCIImage2 + # cachedOCIImage3 + # cachedOCIImage4 + # cachedOCIImage5 + # ]); + services.kubernetes.roles = [ "master" "node" ]; services.kubernetes.masterAddress = "nixos"; services.kubernetes = {