Upgrade ildkule (!36)
Eval nix flake / evals (push) Failing after 3h12m29s Details

This PR is made while moving Ildkule from PVE on joshua, to Openstack on stack.it.ntnu.no.

- The main monitoring dashboard is moved from https://ildkule.pvv.ntnu.no to https://grafana.pvv.ntnu.no.
- A new service is added: uptime-kuma on https://uptime.pvv.ntnu.no.
- The (hardware) configuration for ildkule is updated to fit the new virtualization environment, boot loader, network interfaces, etc.
- Metrics exporters on other hosts should be updated to allow connections from the new host

As this is the first proper server running on openstack, and therefore outside our main IP range, we might discover challenges in our network structure. For example, the database servers usually only allow connections from this range, so Ildkule can no longer access it. This should be explored, documented and/or fixed as we move more services.

Reviewed-on: #36
Co-authored-by: Felix Albrigtsen <felix@albrigtsen.it>
Co-committed-by: Felix Albrigtsen <felix@albrigtsen.it>
fix-gitea-ci-runner-network-issues
Felix Albrigtsen 2024-04-21 23:36:25 +02:00 committed by Felix Albrigtsen
parent 19d5ddc688
commit 55e8f01d1d
23 changed files with 81 additions and 107 deletions

View File

@ -7,7 +7,7 @@ keys:
# Hosts
- &host_jokum age1gp8ye4g2mmw3may5xg0zsy7mm04glfz3788mmdx9cvcsdxs9hg0s0cc9kt
- &host_ildkule age1hn45n46ypyrvypv0mwfnpt9ddrlmw34dwlpf33n8v67jexr3lucq6ahc9x
- &host_ildkule age1x28hmzvuv6f2n66c0jtqcca3h9rput8d7j5uek6jcpx8n9egd52sqpejq0
- &host_bekkalokk age12nj59tguy9wg882updc2vjdusx5srnxmjyfaqve4zx6jnnsaw3qsyjq6zd
- &host_bicep age1sl43gc9cw939z5tgha2lpwf0xxxgcnlw7w4xem4sqgmt2pt264vq0dmwx2

View File

@ -6,8 +6,8 @@
../../base.nix
../../misc/metrics-exporters.nix
./services/monitoring
./services/nginx
./services/metrics
];
sops.defaultSopsFile = ../../secrets/ildkule/ildkule.yaml;
@ -15,28 +15,21 @@
sops.age.keyFile = "/var/lib/sops-nix/key.txt";
sops.age.generateKey = true;
boot.loader.systemd-boot.enable = true;
boot.loader.efi.canTouchEfiVariables = true;
boot.loader.grub.device = "/dev/vda";
boot.tmp.cleanOnBoot = true;
zramSwap.enable = true;
networking.hostName = "ildkule"; # Define your hostname.
systemd.network.networks."30-ens18" = values.defaultNetworkConfig // {
matchConfig.Name = "ens18";
address = with values.hosts.ildkule; [ (ipv4 + "/25") (ipv6 + "/64") ];
systemd.network.networks."30-all" = values.defaultNetworkConfig // {
matchConfig.Name = "en*";
DHCP = "yes";
gateway = [ ];
};
# List packages installed in system profile
environment.systemPackages = with pkgs; [
];
# List services that you want to enable:
# This value determines the NixOS release from which the default
# settings for stateful data, like file locations and database versions
# on your system were taken. Its perfectly fine and recommended to leave
# this value at the release version of the first install of this system.
# Before changing this value read the documentation for this option
# (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
system.stateVersion = "21.11"; # Did you read the comment?
system.stateVersion = "23.11"; # Did you read the comment?
}

View File

@ -1,37 +1,9 @@
# Do not modify this file! It was generated by nixos-generate-config
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:
{ modulesPath, lib, ... }:
{
imports =
[ (modulesPath + "/profiles/qemu-guest.nix")
];
imports = [ (modulesPath + "/profiles/qemu-guest.nix") ];
boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "xen_blkfront" "vmw_pvscsi" ];
boot.initrd.kernelModules = [ "nvme" ];
fileSystems."/" = { device = "/dev/vda1"; fsType = "ext4"; };
boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "virtio_pci" "sr_mod" "virtio_blk" ];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ ];
boot.extraModulePackages = [ ];
fileSystems."/" =
{ device = "/dev/disk/by-uuid/afe70fe4-681a-4675-8cbd-e5d08cdcf5b5";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/B71A-E5CD";
fsType = "vfat";
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.ens18.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}

View File

@ -2,8 +2,9 @@
{
imports = [
./prometheus
./grafana.nix
./loki.nix
./prometheus
./uptime-kuma.nix
];
}

View File

@ -7,7 +7,6 @@ in {
in {
"keys/grafana/secret_key" = { inherit owner group; };
"keys/grafana/admin_password" = { inherit owner group; };
"keys/postgres/grafana" = { inherit owner group; };
};
services.grafana = {
@ -18,7 +17,7 @@ in {
secretFile = path: "$__file{${path}}";
in {
server = {
domain = "ildkule.pvv.ntnu.no";
domain = "grafana.pvv.ntnu.no";
http_port = 2342;
http_addr = "127.0.0.1";
};
@ -27,13 +26,6 @@ in {
secret_key = secretFile config.sops.secrets."keys/grafana/secret_key".path;
admin_password = secretFile config.sops.secrets."keys/grafana/admin_password".path;
};
database = {
type = "postgres";
user = "grafana";
host = "${values.hosts.bicep.ipv4}:5432";
password = secretFile config.sops.secrets."keys/postgres/grafana".path;
};
};
provision = {

View File

@ -0,0 +1,20 @@
{ config, pkgs, lib, ... }:
let
cfg = config.services.uptime-kuma;
domain = "status.pvv.ntnu.no";
in {
services.uptime-kuma = {
enable = true;
settings = {
PORT = "5059";
HOST = "127.0.1.2";
};
};
services.nginx.virtualHosts.${domain} = {
enableACME = true;
forceSSL = true;
kTLS = true;
locations."/".proxyPass = "http://${cfg.settings.HOST}:${cfg.settings.PORT}";
};
}

View File

@ -3,13 +3,5 @@
services.nginx = {
enable = true;
enableReload = true;
defaultListenAddresses = [
values.hosts.ildkule.ipv4
"[${values.hosts.ildkule.ipv6}]"
"127.0.0.1"
"127.0.0.2"
"[::1]"
];
};
}

View File

@ -14,6 +14,8 @@
"::1"
values.hosts.ildkule.ipv4
values.hosts.ildkule.ipv6
values.hosts.ildkule.ipv4_global
values.hosts.ildkule.ipv6_global
];
};

View File

@ -15,7 +15,6 @@ keys:
secret_key: ENC[AES256_GCM,data:+WoAJbDBEgKs0RoHT+7oEELAVQ+/2Xt+5RTMSXg23moCqVRx+Gzll9P5Drw=,iv:AkRn/Y20iEe5i1T+84wAgLCTFtAox2G3giyawAkltAw=,tag:BZbt5Wb5lYLIJBm/pfP4GQ==,type:str]
admin_password: ENC[AES256_GCM,data:ttKwfC4WuXeL/6x4,iv:x1X+e3z08CR992GzC62YnFIN7SGrE81/nDNrgcgVzx0=,tag:YajUoy61kYbpeGeC7yNrXQ==,type:str]
postgres:
grafana: ENC[AES256_GCM,data:D6qkg98WZYzKYegSNBb31v8o+KHisGmJ+ab5Ut7EMtsJz36kUup5RS4EbtM=,iv:rfE1uH1QycKMTpSq2p1ntQ2BIvptAh2J3l/QcQhiuLo=,tag:QxmGFcekjFRPf6orN86IxQ==,type:str]
postgres_exporter_env: ENC[AES256_GCM,data:8MEoikoA6tFNm9qZbk0DFWANd7nRs5QSqrsGLoLKPIc1xykJaXTlyP5v8ywVGR8j7bfPs4p6QfpUIWK8CCnfQ1QhsFPXUMksl8p+K+xuMakYZr9OoWigGqvOHpFb9blfBN1FBdRrk38REXWAMUn74KSRI9v+0i5lpC4=,iv:anpjWVUadKfSAm9XbkeAKu+jAk+LxcpVYQ+gUe5szYw=,tag:4tzb/8B/e1uVoqTsQGlcKA==,type:str]
postgres_exporter_knakelibrak_env: ENC[AES256_GCM,data:xjC7DGXrW2GIJq8XioIZb+jSe/Hzcz0tv9cUHmX/n1nhI+D64lYt+EKnq1+RX/vJzU4sTaKjveKBh88Qqnv6RQm+MZC//dIxcvnnAdl50qnHZyBCaFFEzSNI8I8vGyArMk8Ja72clBq3kMpUz/pLBP0qDrjblKDoWkU=,iv:ZW98hJy8A5t4Oxtu17R3tM7gou183VLbgBsHA8LFuJY=,tag:VMOvQz3X/XDylV1YFg2Jsg==,type:str]
sops:
@ -24,55 +23,55 @@ sops:
azure_kv: []
hc_vault: []
age:
- recipient: age1mrnldl334l2nszuta6ywvewng0fswv2dz9l5g4qcwe3nj4yxf92qjskdx6
- recipient: age1x28hmzvuv6f2n66c0jtqcca3h9rput8d7j5uek6jcpx8n9egd52sqpejq0
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrN3lJM2xWTUZ3UkRBaENI
VmJiWDlQbHd0VUNYdllPdURyQmUvL3lKMzJzCkZlRFVxbmNLOVNqUFg1akJQQlBP
VmdOMUdjZ1M4U2lLVEpGaGI5NjNTR2MKLS0tIDRlQUtucEZhZmRYbmpadVdKK01v
cWxCQlBRR1VaZTBDQnkzNGE0WGttWm8KK5s/coWNsdCP5lKQ8LMK7/3ku179+Lg1
4ujTVn4LhvXy6JvgGTWS/UbMmJjJebVxkulzf5St3YMMs2mcIYjOtA==
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBURkY4WTZhQzJoREpxV1Vr
aUExZ1dxNkIyMkJtUXpqOWtTT1J0MGpmMkY4ClR4Wm1FTmhKN2pIMENRdERrWVY2
SUlHblpEc3VackMrbFpHUUJwM2ltZHcKLS0tIEovMEtiOWc1L2tzZDh3ekZKbStr
NEFkcW03ZTRJODNxTlVuUnFlcFFUUncKEZzOeUtRsZiuugTLzG2xU4eJ3XtVuop7
hhlDBL/YoFn/CO3HjqFdCVv33QoPu7KKMeV52pbVEnv93mvdEeFxVA==
-----END AGE ENCRYPTED FILE-----
- recipient: age17tagmpwqjk3mdy45rfesrfey6h863x8wfq38wh33tkrlrywxducs0k6tpq
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBSa25taGsxdlhrUS96cXBi
cUo3WDVmdEhKN256THJhS2tHSitDRkVraDJNCmhGZzlFUDFkN0JKNkFWUlVLVzcz
MjFhcDdmcmpxdTA3V3JRREFNVmNUbEEKLS0tIFNSU2xNZzN2Y1ZzR2hFM0dOK0Zy
Tmk4bXd0ZHhPemxDSDREb3IvSjFza1EKsjtC6J3kYGRe8oLAoUZmg1BUmpkMyC98
uYq+IQmfJt48R/MKDei00j1w3zIK5+E5GU4o8+jILzwfpzYUUZWwiA==
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBSY3cxSGFvdDdWcFVLRTRy
Zng2VnhjZlFkc1RQN0NqUjJGeW02WlFaMlFZCjVZc2x2UXNXS1I2WDBxeHdjNUdr
WnZGc0l5NlArekUwUGU3Qkdub25EVm8KLS0tIDB2bGo3ZURtZ0pSZjFzcGpOdW5D
aTI3aTBUS0d1MzFmMTVMbUlFYTR4VlUKzOvNCAzan1GTXjoRxeySkUYIYtI4Mpvu
MC0Q8e350SyoOsrF7fUvw+Ru68fDMLW27H6Ly36xP7D3eo/h4eZVXw==
-----END AGE ENCRYPTED FILE-----
- recipient: age1hn45n46ypyrvypv0mwfnpt9ddrlmw34dwlpf33n8v67jexr3lucq6ahc9x
- recipient: age1mrnldl334l2nszuta6ywvewng0fswv2dz9l5g4qcwe3nj4yxf92qjskdx6
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPb09qTTc4cjRMcjIzRmxu
RzZWTDBNTGdvaEc2VFJPakYvakRMK1RnS1FnCktHRVkwZGlUUXl4UTBRcGxMQzdn
QVBCYVdlWEw5NW9tNytJTGIzRlpwa0UKLS0tIGdDdUtFMUgyT0phMXBxZE41Y1h4
a2hQVVprakt5NURpNXdQUjREczJKWTgKn60yrLqco9brlqigAolO8rEkww9z3y3u
KmefLVZCGfoko+fnKLVE9UKFS/tAowqgPS1qE76u1Mmkk6yqZoG9rg==
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFbDBYMDNQcWkySC8vN05t
U3hLMjlYVUE3Zms3U2R4R0VnMUtFcmVQclZvClY4aWZEYWZPdkltMElkUWxQeUtP
TEF0a0txbVQ4d3lrelp3cG9TbG5OSkEKLS0tIHR1V3JIVEwwUjM3RVdES2pQUmhP
T1MwME1tbGQ2NysrOEVNYVZRT1R0YmcKFpfe9GfH7s779CNQswRm/W7zwYO6wK11
z6IGPxtBlUGdshYiHA1BEz7fMVg3ZolL2D98cTNMM24U89Gssiw9qw==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2023-07-08T12:46:19Z"
mac: ENC[AES256_GCM,data:bQWG/GgSIv5LdhGTsyx3ENOAywtYVKjzK6nxOnUEZvD+RSi6jxj9Wze7qOhXvgjKWCz/cZj5oSuMQNRyoI8p8xJdxf0+UNdX8uPT05HiKuF7CBcXzprjKri/H6yFp87epOM9fMdn7ZUACn/iT1IZBo+5OuMtDnqVUm/GEmMcsog=,iv:ll/vEeiXsD3crbbxEFsJlxGbm9dZDUPC4GeO95RZZX4=,tag:TGCA721vK9EY3xlY6zIeIg==,type:str]
lastmodified: "2024-04-20T23:41:59Z"
mac: ENC[AES256_GCM,data:38Ask+adT2FshF8DYEfCWeVWt4KiaJsTXhF7Ib3xxdfQ6vAixM2OXTaK/qqUvN6gQok9TFF+HMJBJ+jezV00nVcKUYn04FaU2/D2zdam44eEEYEEovmfAZ6vbC+CiDv4d/DCc3hnYtDZCEgUTfP4gsZ9rLZFAOwaOFWRJxcDi6Y=,iv:BzuWdTjn6LhscNeouHjM7IYKxTahA8PzzlHSCYZ618s=,tag:BWtPbNwzdOJb788eOO5ZNA==,type:str]
pgp:
- created_at: "2023-01-21T19:52:08Z"
enc: |
- created_at: "2024-04-20T23:15:17Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hQIMA0av/duuklWYAQ//Sw5EHNbC9iPXcHSULYVmSMOQCAH7GSGvaaFvey/KffPD
5gbFr00vIi1JfjYXmYfn3KKpUfs/mMMo5NzYU2Ou5fWcPsqFLXOwubebuf61X6p7
7YfLYQMnjgBzkpb972AJl2tWUlcBcOz89tIw3oMi8R5vvXjRjEdDY8Yp+Z2Apj9V
YJCoSIe6RLBlubMs4I6VIOaTaKIM1DWthg95dozlShXYsEgFTYaJ6FbN9RuZOZPa
KzFs2DXtbylXXJtiCArQCHnOgA1Jnp80VvMYLO1ldteQhqGdmnxnqwjETx/uqy4l
QE31LcRf2JFKi0BBJdQfEqBGW9LD4Mjfwi6tWbHq4Mn29u8IT6z5HJIB99JRAV/9
RfBPzF7UVLq2baWxDwG/M6TvZlVJPdAyhJ5QqhkVdrWir7D1D108u+cgtJWw+vlS
cP3hT73LWCo2bXUrHXxFnrWdDQQSDpew/x2cTHUNvqdqLZgMJWdZgh+mXOQLjzHP
xGkjt0ae5/CEnUIse/Qt3SyoKN3rGVKJgoQ4D0AeBFU5z7NEOx7Ebl9t6IgVnJIB
sDJXg+7jJ8A0V1xGan6BP8dFi7m0aAJH0xi8RB9jC1ZRVNxUjFow3Szh0JQ7u2P5
4jZ3FT4tWzPzLQsgJUd/H41QyKSd3ke4VMf97mEKULJ7prtXdyxQfRDcE93UgVXS
XAF0u7pIl+O2RlJtki+UvuwVDszPBRSmGmfiQa4vsYfXahO4fmBjhdl2hdLtz82F
dh+dPu+RSD9OKwIhUwsDLtWWlI/4BvIB1yXbQxP2MyjZm3uVf1CtgUHyjWw8
=rri5
hQIMA0av/duuklWYAQ/+LSTWjii2dblTAkuqHan3uuuRRpt1ppmHEgHYkQZD+RzE
g+ljNaM/BPqci7Kr1NHFDw+cU2MYm/40Tz63l1cvfE3NEoVefsmoA5voNI3G/bx/
LTAe2aacPwO/TNoLtrCgRkzNyKXluUkM9OoIvkvB5DEGjYbe82+gI5Zi+NbW9N/p
5ilr9Cc1jvIivjZMGGPLRgkAc/twOOuyrZlsFd9kddAL9YFO7wpd/dko886y1jE5
jz9n9F4SKYOcgLPqZuG1iZ8qaA2zGT2bP2caai/QJAmL90stQCiRWtQgB8KeWugm
nRFBm5BLamtoqjXXwzdtXGKbFAhvL5/h+kPxnJDjylfFVbgCpoWJ/fxdE5xxxZtq
zCcGCQQsaa85eWkBByhu7TdwyAW7bJCm8z6kfFPGqhNDkS8ifxnEWm6ulgYVokiL
WVBvuQCd1s8KSExs6zNWGcGlqgvcbovHXyVlmLeqZfBA7i/vYqksZtBT47rG7nCS
YGfHy69yVrMdj4KrLuMXNfjtS92hkQqWmCyl5X5zOSJXqEL2dorMzSZn89gK4nL4
V4zOKkKtsj2MqynYn/XAoUf3AfYs2wtRhJiU+r/q+rx9Hx31H8mnUuUerT58yQCY
mAkjIhTzvZcWIalQo7xnZhos4p1IYaA7MAuGC6HxuWVaOsyiFkRaKwB9svWyZ/DS
XAFID3fQ1xfNyYsW8nvXQmvZubnhE+dAQPaiAFP9ujY4RVXWBFOrV6NAs7y/LID/
89lpfWN87JWSJWUk6DCD3AQ+1GiBCFy7uswUJkG4zou1RQBSl7X88ziVDILU
=tXkN
-----END PGP MESSAGE-----
fp: F7D37890228A907440E1FD4846B9228E814A2AAC
unencrypted_suffix: _unencrypted
version: 3.7.3
version: 3.8.1

View File

@ -30,8 +30,11 @@ in rec {
ipv6 = pvv-ipv6 168;
};
ildkule = {
ipv4 = pvv-ipv4 187;
ipv6 = pvv-ipv6 "1:187";
ipv4 = "10.212.25.209";
ipv6 = "2001:700:300:6025:f816:3eff:feee:812d";
ipv4_global = "129.241.153.213";
ipv6_global = "2001:700:300:6026:f816:3eff:fe58:f1e8";
};
bicep = {
ipv4 = pvv-ipv4 209;