From 157aee00a5b599027f5daa90b699735441ace1c8 Mon Sep 17 00:00:00 2001 From: tomberek Date: Sat, 5 Jun 2021 14:42:51 -0400 Subject: [PATCH] nixos/sourcehut: init (#113244) * nixos/sourcehut: init * sourcehut: default nginx setup * sourcehut: documentation * sourcehut: re-structure settings * sourcehut: tests * nixos/sourcehut: adopt StateDirectory * Apply suggestions from code review Co-authored-by: Aaron Andersen Co-authored-by: Thibaut Marty Co-authored-by: malte-v <34393802+malte-v@users.noreply.github.com> * nixos/sourcehut: PR suggestions * nixos/sourcehut: malte-v patch * nixos/sourcehut: add base virtualhost * nixos/sourcehut: remove superfluous key * nixos/sourcehut: use default from cfg * nixos/sourcehut: use originBase for logs * nixos/sourcehut: use toPythonApplication in systemPackages * nixos/sourcehut: directly use ExecStart * nixos/sourcehut: update docs Co-authored-by: Aaron Andersen Co-authored-by: Thibaut Marty Co-authored-by: malte-v <34393802+malte-v@users.noreply.github.com> --- nixos/modules/module-list.nix | 1 + .../services/misc/sourcehut/builds.nix | 220 ++++++++++++++++++ .../services/misc/sourcehut/default.nix | 198 ++++++++++++++++ .../services/misc/sourcehut/dispatch.nix | 125 ++++++++++ nixos/modules/services/misc/sourcehut/git.nix | 214 +++++++++++++++++ nixos/modules/services/misc/sourcehut/hg.nix | 173 ++++++++++++++ nixos/modules/services/misc/sourcehut/hub.nix | 118 ++++++++++ .../modules/services/misc/sourcehut/lists.nix | 185 +++++++++++++++ nixos/modules/services/misc/sourcehut/man.nix | 122 ++++++++++ .../modules/services/misc/sourcehut/meta.nix | 211 +++++++++++++++++ .../modules/services/misc/sourcehut/paste.nix | 133 +++++++++++ .../services/misc/sourcehut/service.nix | 66 ++++++ .../services/misc/sourcehut/sourcehut.xml | 115 +++++++++ .../modules/services/misc/sourcehut/todo.nix | 161 +++++++++++++ nixos/tests/sourcehut.nix | 29 +++ .../version-management/sourcehut/default.nix | 1 + 16 files changed, 2072 insertions(+) create mode 100644 nixos/modules/services/misc/sourcehut/builds.nix create mode 100644 nixos/modules/services/misc/sourcehut/default.nix create mode 100644 nixos/modules/services/misc/sourcehut/dispatch.nix create mode 100644 nixos/modules/services/misc/sourcehut/git.nix create mode 100644 nixos/modules/services/misc/sourcehut/hg.nix create mode 100644 nixos/modules/services/misc/sourcehut/hub.nix create mode 100644 nixos/modules/services/misc/sourcehut/lists.nix create mode 100644 nixos/modules/services/misc/sourcehut/man.nix create mode 100644 nixos/modules/services/misc/sourcehut/meta.nix create mode 100644 nixos/modules/services/misc/sourcehut/paste.nix create mode 100644 nixos/modules/services/misc/sourcehut/service.nix create mode 100644 nixos/modules/services/misc/sourcehut/sourcehut.xml create mode 100644 nixos/modules/services/misc/sourcehut/todo.nix create mode 100644 nixos/tests/sourcehut.nix diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 42f0471c4cf..7cc83e5734c 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -553,6 +553,7 @@ ./services/misc/siproxd.nix ./services/misc/snapper.nix ./services/misc/sonarr.nix + ./services/misc/sourcehut ./services/misc/spice-vdagentd.nix ./services/misc/ssm-agent.nix ./services/misc/sssd.nix diff --git a/nixos/modules/services/misc/sourcehut/builds.nix b/nixos/modules/services/misc/sourcehut/builds.nix new file mode 100644 index 00000000000..e228665784e --- /dev/null +++ b/nixos/modules/services/misc/sourcehut/builds.nix @@ -0,0 +1,220 @@ +{ config, lib, pkgs, ... }: + +with lib; +let + cfg = config.services.sourcehut; + scfg = cfg.builds; + rcfg = config.services.redis; + iniKey = "builds.sr.ht"; + + drv = pkgs.sourcehut.buildsrht; +in +{ + options.services.sourcehut.builds = { + user = mkOption { + type = types.str; + default = "buildsrht"; + description = '' + User for builds.sr.ht. + ''; + }; + + port = mkOption { + type = types.port; + default = 5002; + description = '' + Port on which the "builds" module should listen. + ''; + }; + + database = mkOption { + type = types.str; + default = "builds.sr.ht"; + description = '' + PostgreSQL database name for builds.sr.ht. + ''; + }; + + statePath = mkOption { + type = types.path; + default = "${cfg.statePath}/buildsrht"; + description = '' + State path for builds.sr.ht. + ''; + }; + + enableWorker = mkOption { + type = types.bool; + default = false; + description = '' + Run workers for builds.sr.ht. + Perform manually on machine: `cd ${scfg.statePath}/images; docker build -t qemu -f qemu/Dockerfile .` + ''; + }; + + images = mkOption { + type = types.attrsOf (types.attrsOf (types.attrsOf types.package)); + default = { }; + example = lib.literalExample ''(let + # Pinning unstable to allow usage with flakes and limit rebuilds. + pkgs_unstable = builtins.fetchGit { + url = "https://github.com/NixOS/nixpkgs"; + rev = "ff96a0fa5635770390b184ae74debea75c3fd534"; + ref = "nixos-unstable"; + }; + image_from_nixpkgs = pkgs_unstable: (import ("${pkgs.sourcehut.buildsrht}/lib/images/nixos/image.nix") { + pkgs = (import pkgs_unstable {}); + }); + in + { + nixos.unstable.x86_64 = image_from_nixpkgs pkgs_unstable; + } + )''; + description = '' + Images for builds.sr.ht. Each package should be distro.release.arch and point to a /nix/store/package/root.img.qcow2. + ''; + }; + + }; + + config = with scfg; let + image_dirs = lib.lists.flatten ( + lib.attrsets.mapAttrsToList + (distro: revs: + lib.attrsets.mapAttrsToList + (rev: archs: + lib.attrsets.mapAttrsToList + (arch: image: + pkgs.runCommandNoCC "buildsrht-images" { } '' + mkdir -p $out/${distro}/${rev}/${arch} + ln -s ${image}/*.qcow2 $out/${distro}/${rev}/${arch}/root.img.qcow2 + '') + archs) + revs) + scfg.images); + image_dir_pre = pkgs.symlinkJoin { + name = "builds.sr.ht-worker-images-pre"; + paths = image_dirs ++ [ + "${pkgs.sourcehut.buildsrht}/lib/images" + ]; + }; + image_dir = pkgs.runCommandNoCC "builds.sr.ht-worker-images" { } '' + mkdir -p $out/images + cp -Lr ${image_dir_pre}/* $out/images + ''; + in + lib.mkIf (cfg.enable && elem "builds" cfg.services) { + users = { + users = { + "${user}" = { + isSystemUser = true; + group = user; + extraGroups = lib.optionals cfg.builds.enableWorker [ "docker" ]; + description = "builds.sr.ht user"; + }; + }; + + groups = { + "${user}" = { }; + }; + }; + + services.postgresql = { + authentication = '' + local ${database} ${user} trust + ''; + ensureDatabases = [ database ]; + ensureUsers = [ + { + name = user; + ensurePermissions = { "DATABASE \"${database}\"" = "ALL PRIVILEGES"; }; + } + ]; + }; + + systemd = { + tmpfiles.rules = [ + "d ${statePath} 0755 ${user} ${user} -" + ] ++ (lib.optionals cfg.builds.enableWorker + [ "d ${statePath}/logs 0775 ${user} ${user} - -" ] + ); + + services = { + buildsrht = import ./service.nix { inherit config pkgs lib; } scfg drv iniKey + { + after = [ "postgresql.service" "network.target" ]; + requires = [ "postgresql.service" ]; + wantedBy = [ "multi-user.target" ]; + + description = "builds.sr.ht website service"; + + serviceConfig.ExecStart = "${cfg.python}/bin/gunicorn ${drv.pname}.app:app -b ${cfg.address}:${toString port}"; + + # Hack to bypass this hack: https://git.sr.ht/~sircmpwn/core.sr.ht/tree/master/item/srht-update-profiles#L6 + } // { preStart = " "; }; + + buildsrht-worker = { + enable = scfg.enableWorker; + after = [ "postgresql.service" "network.target" ]; + requires = [ "postgresql.service" ]; + wantedBy = [ "multi-user.target" ]; + partOf = [ "buildsrht.service" ]; + description = "builds.sr.ht worker service"; + path = [ pkgs.openssh pkgs.docker ]; + serviceConfig = { + Type = "simple"; + User = user; + Group = "nginx"; + Restart = "always"; + }; + serviceConfig.ExecStart = "${pkgs.sourcehut.buildsrht}/bin/builds.sr.ht-worker"; + }; + }; + }; + + services.sourcehut.settings = { + # URL builds.sr.ht is being served at (protocol://domain) + "builds.sr.ht".origin = mkDefault "http://builds.${cfg.originBase}"; + # Address and port to bind the debug server to + "builds.sr.ht".debug-host = mkDefault "0.0.0.0"; + "builds.sr.ht".debug-port = mkDefault port; + # Configures the SQLAlchemy connection string for the database. + "builds.sr.ht".connection-string = mkDefault "postgresql:///${database}?user=${user}&host=/var/run/postgresql"; + # Set to "yes" to automatically run migrations on package upgrade. + "builds.sr.ht".migrate-on-upgrade = mkDefault "yes"; + # builds.sr.ht's OAuth client ID and secret for meta.sr.ht + # Register your client at meta.example.org/oauth + "builds.sr.ht".oauth-client-id = mkDefault null; + "builds.sr.ht".oauth-client-secret = mkDefault null; + # The redis connection used for the celery worker + "builds.sr.ht".redis = mkDefault "redis://${rcfg.bind}:${toString rcfg.port}/3"; + # The shell used for ssh + "builds.sr.ht".shell = mkDefault "runner-shell"; + # Register the builds.sr.ht dispatcher + "git.sr.ht::dispatch".${builtins.unsafeDiscardStringContext "${pkgs.sourcehut.buildsrht}/bin/buildsrht-keys"} = mkDefault "${user}:${user}"; + + # Location for build logs, images, and control command + } // lib.attrsets.optionalAttrs scfg.enableWorker { + # Default worker stores logs that are accessible via this address:port + "builds.sr.ht::worker".name = mkDefault "127.0.0.1:5020"; + "builds.sr.ht::worker".buildlogs = mkDefault "${scfg.statePath}/logs"; + "builds.sr.ht::worker".images = mkDefault "${image_dir}/images"; + "builds.sr.ht::worker".controlcmd = mkDefault "${image_dir}/images/control"; + "builds.sr.ht::worker".timeout = mkDefault "3m"; + }; + + services.nginx.virtualHosts."logs.${cfg.originBase}" = + if scfg.enableWorker then { + listen = with builtins; let address = split ":" cfg.settings."builds.sr.ht::worker".name; + in [{ addr = elemAt address 0; port = lib.toInt (elemAt address 2); }]; + locations."/logs".root = "${scfg.statePath}"; + } else { }; + + services.nginx.virtualHosts."builds.${cfg.originBase}" = { + forceSSL = true; + locations."/".proxyPass = "http://${cfg.address}:${toString port}"; + locations."/query".proxyPass = "http://${cfg.address}:${toString (port + 100)}"; + locations."/static".root = "${pkgs.sourcehut.buildsrht}/${pkgs.sourcehut.python.sitePackages}/buildsrht"; + }; + }; +} diff --git a/nixos/modules/services/misc/sourcehut/default.nix b/nixos/modules/services/misc/sourcehut/default.nix new file mode 100644 index 00000000000..9c812d6b043 --- /dev/null +++ b/nixos/modules/services/misc/sourcehut/default.nix @@ -0,0 +1,198 @@ +{ config, pkgs, lib, ... }: + +with lib; +let + cfg = config.services.sourcehut; + cfgIni = cfg.settings; + settingsFormat = pkgs.formats.ini { }; + + # Specialized python containing all the modules + python = pkgs.sourcehut.python.withPackages (ps: with ps; [ + gunicorn + # Sourcehut services + srht + buildsrht + dispatchsrht + gitsrht + hgsrht + hubsrht + listssrht + mansrht + metasrht + pastesrht + todosrht + ]); +in +{ + imports = + [ + ./git.nix + ./hg.nix + ./hub.nix + ./todo.nix + ./man.nix + ./meta.nix + ./paste.nix + ./builds.nix + ./lists.nix + ./dispatch.nix + (mkRemovedOptionModule [ "services" "sourcehut" "nginx" "enable" ] '' + The sourcehut module supports `nginx` as a local reverse-proxy by default and doesn't + support other reverse-proxies officially. + + However it's possible to use an alternative reverse-proxy by + + * disabling nginx + * adjusting the relevant settings for server addresses and ports directly + + Further details about this can be found in the `Sourcehut`-section of the NixOS-manual. + '') + ]; + + options.services.sourcehut = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Enable sourcehut - git hosting, continuous integration, mailing list, ticket tracking, + task dispatching, wiki and account management services + ''; + }; + + services = mkOption { + type = types.nonEmptyListOf (types.enum [ "builds" "dispatch" "git" "hub" "hg" "lists" "man" "meta" "paste" "todo" ]); + default = [ "man" "meta" "paste" ]; + example = [ "builds" "dispatch" "git" "hub" "hg" "lists" "man" "meta" "paste" "todo" ]; + description = '' + Services to enable on the sourcehut network. + ''; + }; + + originBase = mkOption { + type = types.str; + default = with config.networking; hostName + lib.optionalString (domain != null) ".${domain}"; + description = '' + Host name used by reverse-proxy and for default settings. Will host services at git."''${originBase}". For example: git.sr.ht + ''; + }; + + address = mkOption { + type = types.str; + default = "127.0.0.1"; + description = '' + Address to bind to. + ''; + }; + + python = mkOption { + internal = true; + type = types.package; + default = python; + description = '' + The python package to use. It should contain references to the *srht modules and also + gunicorn. + ''; + }; + + statePath = mkOption { + type = types.path; + default = "/var/lib/sourcehut"; + description = '' + Root state path for the sourcehut network. If left as the default value + this directory will automatically be created before the sourcehut server + starts, otherwise the sysadmin is responsible for ensuring the + directory exists with appropriate ownership and permissions. + ''; + }; + + settings = mkOption { + type = lib.types.submodule { + freeformType = settingsFormat.type; + }; + default = { }; + description = '' + The configuration for the sourcehut network. + ''; + }; + }; + + config = mkIf cfg.enable { + assertions = + [ + { + assertion = with cfgIni.webhooks; private-key != null && stringLength private-key == 44; + message = "The webhook's private key must be defined and of a 44 byte length."; + } + + { + assertion = hasAttrByPath [ "meta.sr.ht" "origin" ] cfgIni && cfgIni."meta.sr.ht".origin != null; + message = "meta.sr.ht's origin must be defined."; + } + ]; + + virtualisation.docker.enable = true; + environment.etc."sr.ht/config.ini".source = + settingsFormat.generate "sourcehut-config.ini" (mapAttrsRecursive + ( + path: v: if v == null then "" else v + ) + cfg.settings); + + environment.systemPackages = [ pkgs.sourcehut.coresrht ]; + + # PostgreSQL server + services.postgresql.enable = mkOverride 999 true; + # Mail server + services.postfix.enable = mkOverride 999 true; + # Cron daemon + services.cron.enable = mkOverride 999 true; + # Redis server + services.redis.enable = mkOverride 999 true; + services.redis.bind = mkOverride 999 "127.0.0.1"; + + services.sourcehut.settings = { + # The name of your network of sr.ht-based sites + "sr.ht".site-name = mkDefault "sourcehut"; + # The top-level info page for your site + "sr.ht".site-info = mkDefault "https://sourcehut.org"; + # {{ site-name }}, {{ site-blurb }} + "sr.ht".site-blurb = mkDefault "the hacker's forge"; + # If this != production, we add a banner to each page + "sr.ht".environment = mkDefault "development"; + # Contact information for the site owners + "sr.ht".owner-name = mkDefault "Drew DeVault"; + "sr.ht".owner-email = mkDefault "sir@cmpwn.com"; + # The source code for your fork of sr.ht + "sr.ht".source-url = mkDefault "https://git.sr.ht/~sircmpwn/srht"; + # A secret key to encrypt session cookies with + "sr.ht".secret-key = mkDefault null; + "sr.ht".global-domain = mkDefault null; + + # Outgoing SMTP settings + mail.smtp-host = mkDefault null; + mail.smtp-port = mkDefault null; + mail.smtp-user = mkDefault null; + mail.smtp-password = mkDefault null; + mail.smtp-from = mkDefault null; + # Application exceptions are emailed to this address + mail.error-to = mkDefault null; + mail.error-from = mkDefault null; + # Your PGP key information (DO NOT mix up pub and priv here) + # You must remove the password from your secret key, if present. + # You can do this with gpg --edit-key [key-id], then use the passwd + # command and do not enter a new password. + mail.pgp-privkey = mkDefault null; + mail.pgp-pubkey = mkDefault null; + mail.pgp-key-id = mkDefault null; + + # base64-encoded Ed25519 key for signing webhook payloads. This should be + # consistent for all *.sr.ht sites, as we'll use this key to verify signatures + # from other sites in your network. + # + # Use the srht-webhook-keygen command to generate a key. + webhooks.private-key = mkDefault null; + }; + }; + meta.doc = ./sourcehut.xml; + meta.maintainers = with maintainers; [ tomberek ]; +} diff --git a/nixos/modules/services/misc/sourcehut/dispatch.nix b/nixos/modules/services/misc/sourcehut/dispatch.nix new file mode 100644 index 00000000000..a9db17bebe8 --- /dev/null +++ b/nixos/modules/services/misc/sourcehut/dispatch.nix @@ -0,0 +1,125 @@ +{ config, lib, pkgs, ... }: + +with lib; +let + cfg = config.services.sourcehut; + cfgIni = cfg.settings; + scfg = cfg.dispatch; + iniKey = "dispatch.sr.ht"; + + drv = pkgs.sourcehut.dispatchsrht; +in +{ + options.services.sourcehut.dispatch = { + user = mkOption { + type = types.str; + default = "dispatchsrht"; + description = '' + User for dispatch.sr.ht. + ''; + }; + + port = mkOption { + type = types.port; + default = 5005; + description = '' + Port on which the "dispatch" module should listen. + ''; + }; + + database = mkOption { + type = types.str; + default = "dispatch.sr.ht"; + description = '' + PostgreSQL database name for dispatch.sr.ht. + ''; + }; + + statePath = mkOption { + type = types.path; + default = "${cfg.statePath}/dispatchsrht"; + description = '' + State path for dispatch.sr.ht. + ''; + }; + }; + + config = with scfg; lib.mkIf (cfg.enable && elem "dispatch" cfg.services) { + + users = { + users = { + "${user}" = { + isSystemUser = true; + group = user; + description = "dispatch.sr.ht user"; + }; + }; + + groups = { + "${user}" = { }; + }; + }; + + services.postgresql = { + authentication = '' + local ${database} ${user} trust + ''; + ensureDatabases = [ database ]; + ensureUsers = [ + { + name = user; + ensurePermissions = { "DATABASE \"${database}\"" = "ALL PRIVILEGES"; }; + } + ]; + }; + + systemd = { + tmpfiles.rules = [ + "d ${statePath} 0750 ${user} ${user} -" + ]; + + services.dispatchsrht = import ./service.nix { inherit config pkgs lib; } scfg drv iniKey { + after = [ "postgresql.service" "network.target" ]; + requires = [ "postgresql.service" ]; + wantedBy = [ "multi-user.target" ]; + + description = "dispatch.sr.ht website service"; + + serviceConfig.ExecStart = "${cfg.python}/bin/gunicorn ${drv.pname}.app:app -b ${cfg.address}:${toString port}"; + }; + }; + + services.sourcehut.settings = { + # URL dispatch.sr.ht is being served at (protocol://domain) + "dispatch.sr.ht".origin = mkDefault "http://dispatch.${cfg.originBase}"; + # Address and port to bind the debug server to + "dispatch.sr.ht".debug-host = mkDefault "0.0.0.0"; + "dispatch.sr.ht".debug-port = mkDefault port; + # Configures the SQLAlchemy connection string for the database. + "dispatch.sr.ht".connection-string = mkDefault "postgresql:///${database}?user=${user}&host=/var/run/postgresql"; + # Set to "yes" to automatically run migrations on package upgrade. + "dispatch.sr.ht".migrate-on-upgrade = mkDefault "yes"; + # dispatch.sr.ht's OAuth client ID and secret for meta.sr.ht + # Register your client at meta.example.org/oauth + "dispatch.sr.ht".oauth-client-id = mkDefault null; + "dispatch.sr.ht".oauth-client-secret = mkDefault null; + + # Github Integration + "dispatch.sr.ht::github".oauth-client-id = mkDefault null; + "dispatch.sr.ht::github".oauth-client-secret = mkDefault null; + + # Gitlab Integration + "dispatch.sr.ht::gitlab".enabled = mkDefault null; + "dispatch.sr.ht::gitlab".canonical-upstream = mkDefault "gitlab.com"; + "dispatch.sr.ht::gitlab".repo-cache = mkDefault "./repo-cache"; + # "dispatch.sr.ht::gitlab"."gitlab.com" = mkDefault "GitLab:application id:secret"; + }; + + services.nginx.virtualHosts."dispatch.${cfg.originBase}" = { + forceSSL = true; + locations."/".proxyPass = "http://${cfg.address}:${toString port}"; + locations."/query".proxyPass = "http://${cfg.address}:${toString (port + 100)}"; + locations."/static".root = "${pkgs.sourcehut.dispatchsrht}/${pkgs.sourcehut.python.sitePackages}/dispatchsrht"; + }; + }; +} diff --git a/nixos/modules/services/misc/sourcehut/git.nix b/nixos/modules/services/misc/sourcehut/git.nix new file mode 100644 index 00000000000..99b9aec0612 --- /dev/null +++ b/nixos/modules/services/misc/sourcehut/git.nix @@ -0,0 +1,214 @@ +{ config, lib, pkgs, ... }: + +with lib; +let + cfg = config.services.sourcehut; + scfg = cfg.git; + iniKey = "git.sr.ht"; + + rcfg = config.services.redis; + drv = pkgs.sourcehut.gitsrht; +in +{ + options.services.sourcehut.git = { + user = mkOption { + type = types.str; + visible = false; + internal = true; + readOnly = true; + default = "git"; + description = '' + User for git.sr.ht. + ''; + }; + + port = mkOption { + type = types.port; + default = 5001; + description = '' + Port on which the "git" module should listen. + ''; + }; + + database = mkOption { + type = types.str; + default = "git.sr.ht"; + description = '' + PostgreSQL database name for git.sr.ht. + ''; + }; + + statePath = mkOption { + type = types.path; + default = "${cfg.statePath}/gitsrht"; + description = '' + State path for git.sr.ht. + ''; + }; + + package = mkOption { + type = types.package; + default = pkgs.git; + example = literalExample "pkgs.gitFull"; + description = '' + Git package for git.sr.ht. This can help silence collisions. + ''; + }; + }; + + config = with scfg; lib.mkIf (cfg.enable && elem "git" cfg.services) { + # sshd refuses to run with `Unsafe AuthorizedKeysCommand ... bad ownership or modes for directory /nix/store` + environment.etc."ssh/gitsrht-dispatch" = { + mode = "0755"; + text = '' + #! ${pkgs.stdenv.shell} + ${cfg.python}/bin/gitsrht-dispatch "$@" + ''; + }; + + # Needs this in the $PATH when sshing into the server + environment.systemPackages = [ cfg.git.package ]; + + users = { + users = { + "${user}" = { + isSystemUser = true; + group = user; + # https://stackoverflow.com/questions/22314298/git-push-results-in-fatal-protocol-error-bad-line-length-character-this + # Probably could use gitsrht-shell if output is restricted to just parameters... + shell = pkgs.bash; + description = "git.sr.ht user"; + }; + }; + + groups = { + "${user}" = { }; + }; + }; + + services = { + cron.systemCronJobs = [ "*/20 * * * * ${cfg.python}/bin/gitsrht-periodic" ]; + fcgiwrap.enable = true; + + openssh.authorizedKeysCommand = ''/etc/ssh/gitsrht-dispatch "%u" "%h" "%t" "%k"''; + openssh.authorizedKeysCommandUser = "root"; + openssh.extraConfig = '' + PermitUserEnvironment SRHT_* + ''; + + postgresql = { + authentication = '' + local ${database} ${user} trust + ''; + ensureDatabases = [ database ]; + ensureUsers = [ + { + name = user; + ensurePermissions = { "DATABASE \"${database}\"" = "ALL PRIVILEGES"; }; + } + ]; + }; + }; + + systemd = { + tmpfiles.rules = [ + # /var/log is owned by root + "f /var/log/git-srht-shell 0644 ${user} ${user} -" + + "d ${statePath} 0750 ${user} ${user} -" + "d ${cfg.settings."${iniKey}".repos} 2755 ${user} ${user} -" + ]; + + services = { + gitsrht = import ./service.nix { inherit config pkgs lib; } scfg drv iniKey { + after = [ "redis.service" "postgresql.service" "network.target" ]; + requires = [ "redis.service" "postgresql.service" ]; + wantedBy = [ "multi-user.target" ]; + + # Needs internally to create repos at the very least + path = [ pkgs.git ]; + description = "git.sr.ht website service"; + + serviceConfig.ExecStart = "${cfg.python}/bin/gunicorn ${drv.pname}.app:app -b ${cfg.address}:${toString port}"; + }; + + gitsrht-webhooks = { + after = [ "postgresql.service" "network.target" ]; + requires = [ "postgresql.service" ]; + wantedBy = [ "multi-user.target" ]; + + description = "git.sr.ht webhooks service"; + serviceConfig = { + Type = "simple"; + User = user; + Restart = "always"; + }; + + serviceConfig.ExecStart = "${cfg.python}/bin/celery -A ${drv.pname}.webhooks worker --loglevel=info"; + }; + }; + }; + + services.sourcehut.settings = { + # URL git.sr.ht is being served at (protocol://domain) + "git.sr.ht".origin = mkDefault "http://git.${cfg.originBase}"; + # Address and port to bind the debug server to + "git.sr.ht".debug-host = mkDefault "0.0.0.0"; + "git.sr.ht".debug-port = mkDefault port; + # Configures the SQLAlchemy connection string for the database. + "git.sr.ht".connection-string = mkDefault "postgresql:///${database}?user=${user}&host=/var/run/postgresql"; + # Set to "yes" to automatically run migrations on package upgrade. + "git.sr.ht".migrate-on-upgrade = mkDefault "yes"; + # The redis connection used for the webhooks worker + "git.sr.ht".webhooks = mkDefault "redis://${rcfg.bind}:${toString rcfg.port}/1"; + + # A post-update script which is installed in every git repo. + "git.sr.ht".post-update-script = mkDefault "${pkgs.sourcehut.gitsrht}/bin/gitsrht-update-hook"; + + # git.sr.ht's OAuth client ID and secret for meta.sr.ht + # Register your client at meta.example.org/oauth + "git.sr.ht".oauth-client-id = mkDefault null; + "git.sr.ht".oauth-client-secret = mkDefault null; + # Path to git repositories on disk + "git.sr.ht".repos = mkDefault "/var/lib/git"; + + "git.sr.ht".outgoing-domain = mkDefault "http://git.${cfg.originBase}"; + + # The authorized keys hook uses this to dispatch to various handlers + # The format is a program to exec into as the key, and the user to match as the + # value. When someone tries to log in as this user, this program is executed + # and is expected to omit an AuthorizedKeys file. + # + # Discard of the string context is in order to allow derivation-derived strings. + # This is safe if the relevant package is installed which will be the case if the setting is utilized. + "git.sr.ht::dispatch".${builtins.unsafeDiscardStringContext "${pkgs.sourcehut.gitsrht}/bin/gitsrht-keys"} = mkDefault "${user}:${user}"; + }; + + services.nginx.virtualHosts."git.${cfg.originBase}" = { + forceSSL = true; + locations."/".proxyPass = "http://${cfg.address}:${toString port}"; + locations."/query".proxyPass = "http://${cfg.address}:${toString (port + 100)}"; + locations."/static".root = "${pkgs.sourcehut.gitsrht}/${pkgs.sourcehut.python.sitePackages}/gitsrht"; + extraConfig = '' + location = /authorize { + proxy_pass http://${cfg.address}:${toString port}; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-URI $request_uri; + } + location ~ ^/([^/]+)/([^/]+)/(HEAD|info/refs|objects/info/.*|git-upload-pack).*$ { + auth_request /authorize; + root /var/lib/git; + fastcgi_pass unix:/run/fcgiwrap.sock; + fastcgi_param SCRIPT_FILENAME ${pkgs.git}/bin/git-http-backend; + fastcgi_param PATH_INFO $uri; + fastcgi_param GIT_PROJECT_ROOT $document_root; + fastcgi_read_timeout 500s; + include ${pkgs.nginx}/conf/fastcgi_params; + gzip off; + } + ''; + + }; + }; +} diff --git a/nixos/modules/services/misc/sourcehut/hg.nix b/nixos/modules/services/misc/sourcehut/hg.nix new file mode 100644 index 00000000000..5cd36bb0455 --- /dev/null +++ b/nixos/modules/services/misc/sourcehut/hg.nix @@ -0,0 +1,173 @@ +{ config, lib, pkgs, ... }: + +with lib; +let + cfg = config.services.sourcehut; + scfg = cfg.hg; + iniKey = "hg.sr.ht"; + + rcfg = config.services.redis; + drv = pkgs.sourcehut.hgsrht; +in +{ + options.services.sourcehut.hg = { + user = mkOption { + type = types.str; + internal = true; + readOnly = true; + default = "hg"; + description = '' + User for hg.sr.ht. + ''; + }; + + port = mkOption { + type = types.port; + default = 5010; + description = '' + Port on which the "hg" module should listen. + ''; + }; + + database = mkOption { + type = types.str; + default = "hg.sr.ht"; + description = '' + PostgreSQL database name for hg.sr.ht. + ''; + }; + + statePath = mkOption { + type = types.path; + default = "${cfg.statePath}/hgsrht"; + description = '' + State path for hg.sr.ht. + ''; + }; + + cloneBundles = mkOption { + type = types.bool; + default = false; + description = '' + Generate clonebundles (which require more disk space but dramatically speed up cloning large repositories). + ''; + }; + }; + + config = with scfg; lib.mkIf (cfg.enable && elem "hg" cfg.services) { + # In case it ever comes into being + environment.etc."ssh/hgsrht-dispatch" = { + mode = "0755"; + text = '' + #! ${pkgs.stdenv.shell} + ${cfg.python}/bin/gitsrht-dispatch $@ + ''; + }; + + environment.systemPackages = [ pkgs.mercurial ]; + + users = { + users = { + "${user}" = { + isSystemUser = true; + group = user; + # Assuming hg.sr.ht needs this too + shell = pkgs.bash; + description = "hg.sr.ht user"; + }; + }; + + groups = { + "${user}" = { }; + }; + }; + + services = { + cron.systemCronJobs = [ "*/20 * * * * ${cfg.python}/bin/hgsrht-periodic" ] + ++ optional cloneBundles "0 * * * * ${cfg.python}/bin/hgsrht-clonebundles"; + + openssh.authorizedKeysCommand = ''/etc/ssh/hgsrht-dispatch "%u" "%h" "%t" "%k"''; + openssh.authorizedKeysCommandUser = "root"; + openssh.extraConfig = '' + PermitUserEnvironment SRHT_* + ''; + + postgresql = { + authentication = '' + local ${database} ${user} trust + ''; + ensureDatabases = [ database ]; + ensureUsers = [ + { + name = user; + ensurePermissions = { "DATABASE \"${database}\"" = "ALL PRIVILEGES"; }; + } + ]; + }; + }; + + systemd = { + tmpfiles.rules = [ + # /var/log is owned by root + "f /var/log/hg-srht-shell 0644 ${user} ${user} -" + + "d ${statePath} 0750 ${user} ${user} -" + "d ${cfg.settings."${iniKey}".repos} 2755 ${user} ${user} -" + ]; + + services.hgsrht = import ./service.nix { inherit config pkgs lib; } scfg drv iniKey { + after = [ "redis.service" "postgresql.service" "network.target" ]; + requires = [ "redis.service" "postgresql.service" ]; + wantedBy = [ "multi-user.target" ]; + + path = [ pkgs.mercurial ]; + description = "hg.sr.ht website service"; + + serviceConfig.ExecStart = "${cfg.python}/bin/gunicorn ${drv.pname}.app:app -b ${cfg.address}:${toString port}"; + }; + }; + + services.sourcehut.settings = { + # URL hg.sr.ht is being served at (protocol://domain) + "hg.sr.ht".origin = mkDefault "http://hg.${cfg.originBase}"; + # Address and port to bind the debug server to + "hg.sr.ht".debug-host = mkDefault "0.0.0.0"; + "hg.sr.ht".debug-port = mkDefault port; + # Configures the SQLAlchemy connection string for the database. + "hg.sr.ht".connection-string = mkDefault "postgresql:///${database}?user=${user}&host=/var/run/postgresql"; + # The redis connection used for the webhooks worker + "hg.sr.ht".webhooks = mkDefault "redis://${rcfg.bind}:${toString rcfg.port}/1"; + # A post-update script which is installed in every mercurial repo. + "hg.sr.ht".changegroup-script = mkDefault "${cfg.python}/bin/hgsrht-hook-changegroup"; + # hg.sr.ht's OAuth client ID and secret for meta.sr.ht + # Register your client at meta.example.org/oauth + "hg.sr.ht".oauth-client-id = mkDefault null; + "hg.sr.ht".oauth-client-secret = mkDefault null; + # Path to mercurial repositories on disk + "hg.sr.ht".repos = mkDefault "/var/lib/hg"; + # Path to the srht mercurial extension + # (defaults to where the hgsrht code is) + # "hg.sr.ht".srhtext = mkDefault null; + # .hg/store size (in MB) past which the nightly job generates clone bundles. + # "hg.sr.ht".clone_bundle_threshold = mkDefault 50; + # Path to hg-ssh (if not in $PATH) + # "hg.sr.ht".hg_ssh = mkDefault /path/to/hg-ssh; + + # The authorized keys hook uses this to dispatch to various handlers + # The format is a program to exec into as the key, and the user to match as the + # value. When someone tries to log in as this user, this program is executed + # and is expected to omit an AuthorizedKeys file. + # + # Uncomment the relevant lines to enable the various sr.ht dispatchers. + "hg.sr.ht::dispatch"."/run/current-system/sw/bin/hgsrht-keys" = mkDefault "${user}:${user}"; + }; + + # TODO: requires testing and addition of hg-specific requirements + services.nginx.virtualHosts."hg.${cfg.originBase}" = { + forceSSL = true; + locations."/".proxyPass = "http://${cfg.address}:${toString port}"; + locations."/query".proxyPass = "http://${cfg.address}:${toString (port + 100)}"; + locations."/static".root = "${pkgs.sourcehut.hgsrht}/${pkgs.sourcehut.python.sitePackages}/hgsrht"; + }; + }; +} diff --git a/nixos/modules/services/misc/sourcehut/hub.nix b/nixos/modules/services/misc/sourcehut/hub.nix new file mode 100644 index 00000000000..be3ea21011c --- /dev/null +++ b/nixos/modules/services/misc/sourcehut/hub.nix @@ -0,0 +1,118 @@ +{ config, lib, pkgs, ... }: + +with lib; +let + cfg = config.services.sourcehut; + cfgIni = cfg.settings; + scfg = cfg.hub; + iniKey = "hub.sr.ht"; + + drv = pkgs.sourcehut.hubsrht; +in +{ + options.services.sourcehut.hub = { + user = mkOption { + type = types.str; + default = "hubsrht"; + description = '' + User for hub.sr.ht. + ''; + }; + + port = mkOption { + type = types.port; + default = 5014; + description = '' + Port on which the "hub" module should listen. + ''; + }; + + database = mkOption { + type = types.str; + default = "hub.sr.ht"; + description = '' + PostgreSQL database name for hub.sr.ht. + ''; + }; + + statePath = mkOption { + type = types.path; + default = "${cfg.statePath}/hubsrht"; + description = '' + State path for hub.sr.ht. + ''; + }; + }; + + config = with scfg; lib.mkIf (cfg.enable && elem "hub" cfg.services) { + users = { + users = { + "${user}" = { + isSystemUser = true; + group = user; + description = "hub.sr.ht user"; + }; + }; + + groups = { + "${user}" = { }; + }; + }; + + services.postgresql = { + authentication = '' + local ${database} ${user} trust + ''; + ensureDatabases = [ database ]; + ensureUsers = [ + { + name = user; + ensurePermissions = { "DATABASE \"${database}\"" = "ALL PRIVILEGES"; }; + } + ]; + }; + + systemd = { + tmpfiles.rules = [ + "d ${statePath} 0750 ${user} ${user} -" + ]; + + services.hubsrht = import ./service.nix { inherit config pkgs lib; } scfg drv iniKey { + after = [ "postgresql.service" "network.target" ]; + requires = [ "postgresql.service" ]; + wantedBy = [ "multi-user.target" ]; + + description = "hub.sr.ht website service"; + + serviceConfig.ExecStart = "${cfg.python}/bin/gunicorn ${drv.pname}.app:app -b ${cfg.address}:${toString port}"; + }; + }; + + services.sourcehut.settings = { + # URL hub.sr.ht is being served at (protocol://domain) + "hub.sr.ht".origin = mkDefault "http://hub.${cfg.originBase}"; + # Address and port to bind the debug server to + "hub.sr.ht".debug-host = mkDefault "0.0.0.0"; + "hub.sr.ht".debug-port = mkDefault port; + # Configures the SQLAlchemy connection string for the database. + "hub.sr.ht".connection-string = mkDefault "postgresql:///${database}?user=${user}&host=/var/run/postgresql"; + # Set to "yes" to automatically run migrations on package upgrade. + "hub.sr.ht".migrate-on-upgrade = mkDefault "yes"; + # hub.sr.ht's OAuth client ID and secret for meta.sr.ht + # Register your client at meta.example.org/oauth + "hub.sr.ht".oauth-client-id = mkDefault null; + "hub.sr.ht".oauth-client-secret = mkDefault null; + }; + + services.nginx.virtualHosts."${cfg.originBase}" = { + forceSSL = true; + locations."/".proxyPass = "http://${cfg.address}:${toString port}"; + locations."/query".proxyPass = "http://${cfg.address}:${toString (port + 100)}"; + locations."/static".root = "${pkgs.sourcehut.hubsrht}/${pkgs.sourcehut.python.sitePackages}/hubsrht"; + }; + services.nginx.virtualHosts."hub.${cfg.originBase}" = { + globalRedirect = "${cfg.originBase}"; + forceSSL = true; + }; + }; +} diff --git a/nixos/modules/services/misc/sourcehut/lists.nix b/nixos/modules/services/misc/sourcehut/lists.nix new file mode 100644 index 00000000000..7b1fe9fd463 --- /dev/null +++ b/nixos/modules/services/misc/sourcehut/lists.nix @@ -0,0 +1,185 @@ +# Email setup is fairly involved, useful references: +# https://drewdevault.com/2018/08/05/Local-mail-server.html + +{ config, lib, pkgs, ... }: + +with lib; +let + cfg = config.services.sourcehut; + cfgIni = cfg.settings; + scfg = cfg.lists; + iniKey = "lists.sr.ht"; + + rcfg = config.services.redis; + drv = pkgs.sourcehut.listssrht; +in +{ + options.services.sourcehut.lists = { + user = mkOption { + type = types.str; + default = "listssrht"; + description = '' + User for lists.sr.ht. + ''; + }; + + port = mkOption { + type = types.port; + default = 5006; + description = '' + Port on which the "lists" module should listen. + ''; + }; + + database = mkOption { + type = types.str; + default = "lists.sr.ht"; + description = '' + PostgreSQL database name for lists.sr.ht. + ''; + }; + + statePath = mkOption { + type = types.path; + default = "${cfg.statePath}/listssrht"; + description = '' + State path for lists.sr.ht. + ''; + }; + }; + + config = with scfg; lib.mkIf (cfg.enable && elem "lists" cfg.services) { + users = { + users = { + "${user}" = { + isSystemUser = true; + group = user; + extraGroups = [ "postfix" ]; + description = "lists.sr.ht user"; + }; + }; + groups = { + "${user}" = { }; + }; + }; + + services.postgresql = { + authentication = '' + local ${database} ${user} trust + ''; + ensureDatabases = [ database ]; + ensureUsers = [ + { + name = user; + ensurePermissions = { "DATABASE \"${database}\"" = "ALL PRIVILEGES"; }; + } + ]; + }; + + systemd = { + tmpfiles.rules = [ + "d ${statePath} 0750 ${user} ${user} -" + ]; + + services = { + listssrht = import ./service.nix { inherit config pkgs lib; } scfg drv iniKey { + after = [ "postgresql.service" "network.target" ]; + requires = [ "postgresql.service" ]; + wantedBy = [ "multi-user.target" ]; + + description = "lists.sr.ht website service"; + + serviceConfig.ExecStart = "${cfg.python}/bin/gunicorn ${drv.pname}.app:app -b ${cfg.address}:${toString port}"; + }; + + listssrht-process = { + after = [ "postgresql.service" "network.target" ]; + requires = [ "postgresql.service" ]; + wantedBy = [ "multi-user.target" ]; + + description = "lists.sr.ht process service"; + serviceConfig = { + Type = "simple"; + User = user; + Restart = "always"; + ExecStart = "${cfg.python}/bin/celery -A ${drv.pname}.process worker --loglevel=info"; + }; + }; + + listssrht-lmtp = { + after = [ "postgresql.service" "network.target" ]; + requires = [ "postgresql.service" ]; + wantedBy = [ "multi-user.target" ]; + + description = "lists.sr.ht process service"; + serviceConfig = { + Type = "simple"; + User = user; + Restart = "always"; + ExecStart = "${cfg.python}/bin/listssrht-lmtp"; + }; + }; + + + listssrht-webhooks = { + after = [ "postgresql.service" "network.target" ]; + requires = [ "postgresql.service" ]; + wantedBy = [ "multi-user.target" ]; + + description = "lists.sr.ht webhooks service"; + serviceConfig = { + Type = "simple"; + User = user; + Restart = "always"; + ExecStart = "${cfg.python}/bin/celery -A ${drv.pname}.webhooks worker --loglevel=info"; + }; + }; + }; + }; + + services.sourcehut.settings = { + # URL lists.sr.ht is being served at (protocol://domain) + "lists.sr.ht".origin = mkDefault "http://lists.${cfg.originBase}"; + # Address and port to bind the debug server to + "lists.sr.ht".debug-host = mkDefault "0.0.0.0"; + "lists.sr.ht".debug-port = mkDefault port; + # Configures the SQLAlchemy connection string for the database. + "lists.sr.ht".connection-string = mkDefault "postgresql:///${database}?user=${user}&host=/var/run/postgresql"; + # Set to "yes" to automatically run migrations on package upgrade. + "lists.sr.ht".migrate-on-upgrade = mkDefault "yes"; + # lists.sr.ht's OAuth client ID and secret for meta.sr.ht + # Register your client at meta.example.org/oauth + "lists.sr.ht".oauth-client-id = mkDefault null; + "lists.sr.ht".oauth-client-secret = mkDefault null; + # Outgoing email for notifications generated by users + "lists.sr.ht".notify-from = mkDefault "CHANGEME@example.org"; + # The redis connection used for the webhooks worker + "lists.sr.ht".webhooks = mkDefault "redis://${rcfg.bind}:${toString rcfg.port}/2"; + # The redis connection used for the celery worker + "lists.sr.ht".redis = mkDefault "redis://${rcfg.bind}:${toString rcfg.port}/4"; + # Network-key + "lists.sr.ht".network-key = mkDefault null; + # Allow creation + "lists.sr.ht".allow-new-lists = mkDefault "no"; + # Posting Domain + "lists.sr.ht".posting-domain = mkDefault "lists.${cfg.originBase}"; + + # Path for the lmtp daemon's unix socket. Direct incoming mail to this socket. + # Alternatively, specify IP:PORT and an SMTP server will be run instead. + "lists.sr.ht::worker".sock = mkDefault "/tmp/lists.sr.ht-lmtp.sock"; + # The lmtp daemon will make the unix socket group-read/write for users in this + # group. + "lists.sr.ht::worker".sock-group = mkDefault "postfix"; + "lists.sr.ht::worker".reject-url = mkDefault "https://man.sr.ht/lists.sr.ht/etiquette.md"; + "lists.sr.ht::worker".reject-mimetypes = mkDefault "text/html"; + + }; + + services.nginx.virtualHosts."lists.${cfg.originBase}" = { + forceSSL = true; + locations."/".proxyPass = "http://${cfg.address}:${toString port}"; + locations."/query".proxyPass = "http://${cfg.address}:${toString (port + 100)}"; + locations."/static".root = "${pkgs.sourcehut.listssrht}/${pkgs.sourcehut.python.sitePackages}/listssrht"; + }; + }; +} diff --git a/nixos/modules/services/misc/sourcehut/man.nix b/nixos/modules/services/misc/sourcehut/man.nix new file mode 100644 index 00000000000..7693396d187 --- /dev/null +++ b/nixos/modules/services/misc/sourcehut/man.nix @@ -0,0 +1,122 @@ +{ config, lib, pkgs, ... }: + +with lib; +let + cfg = config.services.sourcehut; + cfgIni = cfg.settings; + scfg = cfg.man; + iniKey = "man.sr.ht"; + + drv = pkgs.sourcehut.mansrht; +in +{ + options.services.sourcehut.man = { + user = mkOption { + type = types.str; + default = "mansrht"; + description = '' + User for man.sr.ht. + ''; + }; + + port = mkOption { + type = types.port; + default = 5004; + description = '' + Port on which the "man" module should listen. + ''; + }; + + database = mkOption { + type = types.str; + default = "man.sr.ht"; + description = '' + PostgreSQL database name for man.sr.ht. + ''; + }; + + statePath = mkOption { + type = types.path; + default = "${cfg.statePath}/mansrht"; + description = '' + State path for man.sr.ht. + ''; + }; + }; + + config = with scfg; lib.mkIf (cfg.enable && elem "man" cfg.services) { + assertions = + [ + { + assertion = hasAttrByPath [ "git.sr.ht" "oauth-client-id" ] cfgIni; + message = "man.sr.ht needs access to git.sr.ht."; + } + ]; + + users = { + users = { + "${user}" = { + isSystemUser = true; + group = user; + description = "man.sr.ht user"; + }; + }; + + groups = { + "${user}" = { }; + }; + }; + + services.postgresql = { + authentication = '' + local ${database} ${user} trust + ''; + ensureDatabases = [ database ]; + ensureUsers = [ + { + name = user; + ensurePermissions = { "DATABASE \"${database}\"" = "ALL PRIVILEGES"; }; + } + ]; + }; + + systemd = { + tmpfiles.rules = [ + "d ${statePath} 0750 ${user} ${user} -" + ]; + + services.mansrht = import ./service.nix { inherit config pkgs lib; } scfg drv iniKey { + after = [ "postgresql.service" "network.target" ]; + requires = [ "postgresql.service" ]; + wantedBy = [ "multi-user.target" ]; + + description = "man.sr.ht website service"; + + serviceConfig.ExecStart = "${cfg.python}/bin/gunicorn ${drv.pname}.app:app -b ${cfg.address}:${toString port}"; + }; + }; + + services.sourcehut.settings = { + # URL man.sr.ht is being served at (protocol://domain) + "man.sr.ht".origin = mkDefault "http://man.${cfg.originBase}"; + # Address and port to bind the debug server to + "man.sr.ht".debug-host = mkDefault "0.0.0.0"; + "man.sr.ht".debug-port = mkDefault port; + # Configures the SQLAlchemy connection string for the database. + "man.sr.ht".connection-string = mkDefault "postgresql:///${database}?user=${user}&host=/var/run/postgresql"; + # Set to "yes" to automatically run migrations on package upgrade. + "man.sr.ht".migrate-on-upgrade = mkDefault "yes"; + # man.sr.ht's OAuth client ID and secret for meta.sr.ht + # Register your client at meta.example.org/oauth + "man.sr.ht".oauth-client-id = mkDefault null; + "man.sr.ht".oauth-client-secret = mkDefault null; + }; + + services.nginx.virtualHosts."man.${cfg.originBase}" = { + forceSSL = true; + locations."/".proxyPass = "http://${cfg.address}:${toString port}"; + locations."/query".proxyPass = "http://${cfg.address}:${toString (port + 100)}"; + locations."/static".root = "${pkgs.sourcehut.mansrht}/${pkgs.sourcehut.python.sitePackages}/mansrht"; + }; + }; +} diff --git a/nixos/modules/services/misc/sourcehut/meta.nix b/nixos/modules/services/misc/sourcehut/meta.nix new file mode 100644 index 00000000000..56127a824eb --- /dev/null +++ b/nixos/modules/services/misc/sourcehut/meta.nix @@ -0,0 +1,211 @@ +{ config, lib, pkgs, ... }: + +with lib; +let + cfg = config.services.sourcehut; + cfgIni = cfg.settings; + scfg = cfg.meta; + iniKey = "meta.sr.ht"; + + rcfg = config.services.redis; + drv = pkgs.sourcehut.metasrht; +in +{ + options.services.sourcehut.meta = { + user = mkOption { + type = types.str; + default = "metasrht"; + description = '' + User for meta.sr.ht. + ''; + }; + + port = mkOption { + type = types.port; + default = 5000; + description = '' + Port on which the "meta" module should listen. + ''; + }; + + database = mkOption { + type = types.str; + default = "meta.sr.ht"; + description = '' + PostgreSQL database name for meta.sr.ht. + ''; + }; + + statePath = mkOption { + type = types.path; + default = "${cfg.statePath}/metasrht"; + description = '' + State path for meta.sr.ht. + ''; + }; + }; + + config = with scfg; lib.mkIf (cfg.enable && elem "meta" cfg.services) { + assertions = + [ + { + assertion = with cfgIni."meta.sr.ht::billing"; enabled == "yes" -> (stripe-public-key != null && stripe-secret-key != null); + message = "If meta.sr.ht::billing is enabled, the keys should be defined."; + } + ]; + + users = { + users = { + ${user} = { + isSystemUser = true; + group = user; + description = "meta.sr.ht user"; + }; + }; + + groups = { + "${user}" = { }; + }; + }; + + services.cron.systemCronJobs = [ "0 0 * * * ${cfg.python}/bin/metasrht-daily" ]; + services.postgresql = { + authentication = '' + local ${database} ${user} trust + ''; + ensureDatabases = [ database ]; + ensureUsers = [ + { + name = user; + ensurePermissions = { "DATABASE \"${database}\"" = "ALL PRIVILEGES"; }; + } + ]; + }; + + systemd = { + tmpfiles.rules = [ + "d ${statePath} 0750 ${user} ${user} -" + ]; + + services = { + metasrht = import ./service.nix { inherit config pkgs lib; } scfg drv iniKey { + after = [ "postgresql.service" "network.target" ]; + requires = [ "postgresql.service" ]; + wantedBy = [ "multi-user.target" ]; + + description = "meta.sr.ht website service"; + + preStart = '' + # Configure client(s) as "preauthorized" + ${concatMapStringsSep "\n\n" + (attr: '' + if ! test -e "${statePath}/${attr}.oauth" || [ "$(cat ${statePath}/${attr}.oauth)" != "${cfgIni."${attr}".oauth-client-id}" ]; then + # Configure ${attr}'s OAuth client as "preauthorized" + psql ${database} \ + -c "UPDATE oauthclient SET preauthorized = true WHERE client_id = '${cfgIni."${attr}".oauth-client-id}'" + + printf "%s" "${cfgIni."${attr}".oauth-client-id}" > "${statePath}/${attr}.oauth" + fi + '') + (builtins.attrNames (filterAttrs + (k: v: !(hasInfix "::" k) && builtins.hasAttr "oauth-client-id" v && v.oauth-client-id != null) + cfg.settings))} + ''; + + serviceConfig.ExecStart = "${cfg.python}/bin/gunicorn ${drv.pname}.app:app -b ${cfg.address}:${toString port}"; + }; + + metasrht-api = import ./service.nix { inherit config pkgs lib; } scfg drv iniKey { + after = [ "postgresql.service" "network.target" ]; + requires = [ "postgresql.service" ]; + wantedBy = [ "multi-user.target" ]; + + description = "meta.sr.ht api service"; + + preStart = '' + # Configure client(s) as "preauthorized" + ${concatMapStringsSep "\n\n" + (attr: '' + if ! test -e "${statePath}/${attr}.oauth" || [ "$(cat ${statePath}/${attr}.oauth)" != "${cfgIni."${attr}".oauth-client-id}" ]; then + # Configure ${attr}'s OAuth client as "preauthorized" + psql ${database} \ + -c "UPDATE oauthclient SET preauthorized = true WHERE client_id = '${cfgIni."${attr}".oauth-client-id}'" + + printf "%s" "${cfgIni."${attr}".oauth-client-id}" > "${statePath}/${attr}.oauth" + fi + '') + (builtins.attrNames (filterAttrs + (k: v: !(hasInfix "::" k) && builtins.hasAttr "oauth-client-id" v && v.oauth-client-id != null) + cfg.settings))} + ''; + + serviceConfig.ExecStart = "${pkgs.sourcehut.metasrht}/bin/metasrht-api -b :${toString (port + 100)}"; + }; + + metasrht-webhooks = { + after = [ "postgresql.service" "network.target" ]; + requires = [ "postgresql.service" ]; + wantedBy = [ "multi-user.target" ]; + + description = "meta.sr.ht webhooks service"; + serviceConfig = { + Type = "simple"; + User = user; + Restart = "always"; + ExecStart = "${cfg.python}/bin/celery -A ${drv.pname}.webhooks worker --loglevel=info"; + }; + + }; + }; + }; + + services.sourcehut.settings = { + # URL meta.sr.ht is being served at (protocol://domain) + "meta.sr.ht".origin = mkDefault "https://meta.${cfg.originBase}"; + # Address and port to bind the debug server to + "meta.sr.ht".debug-host = mkDefault "0.0.0.0"; + "meta.sr.ht".debug-port = mkDefault port; + # Configures the SQLAlchemy connection string for the database. + "meta.sr.ht".connection-string = mkDefault "postgresql:///${database}?user=${user}&host=/var/run/postgresql"; + # Set to "yes" to automatically run migrations on package upgrade. + "meta.sr.ht".migrate-on-upgrade = mkDefault "yes"; + # If "yes", the user will be sent the stock sourcehut welcome emails after + # signup (requires cron to be configured properly). These are specific to the + # sr.ht instance so you probably want to patch these before enabling this. + "meta.sr.ht".welcome-emails = mkDefault "no"; + + # The redis connection used for the webhooks worker + "meta.sr.ht".webhooks = mkDefault "redis://${rcfg.bind}:${toString rcfg.port}/6"; + + # If "no", public registration will not be permitted. + "meta.sr.ht::settings".registration = mkDefault "no"; + # Where to redirect new users upon registration + "meta.sr.ht::settings".onboarding-redirect = mkDefault "https://meta.${cfg.originBase}"; + # How many invites each user is issued upon registration (only applicable if + # open registration is disabled) + "meta.sr.ht::settings".user-invites = mkDefault 5; + + # Origin URL for API, 100 more than web + "meta.sr.ht".api-origin = mkDefault "http://localhost:5100"; + + # You can add aliases for the client IDs of commonly used OAuth clients here. + # + # Example: + "meta.sr.ht::aliases" = mkDefault { }; + # "meta.sr.ht::aliases"."git.sr.ht" = 12345; + + # "yes" to enable the billing system + "meta.sr.ht::billing".enabled = mkDefault "no"; + # Get your keys at https://dashboard.stripe.com/account/apikeys + "meta.sr.ht::billing".stripe-public-key = mkDefault null; + "meta.sr.ht::billing".stripe-secret-key = mkDefault null; + }; + + services.nginx.virtualHosts."meta.${cfg.originBase}" = { + forceSSL = true; + locations."/".proxyPass = "http://${cfg.address}:${toString port}"; + locations."/query".proxyPass = "http://${cfg.address}:${toString (port + 100)}"; + locations."/static".root = "${pkgs.sourcehut.metasrht}/${pkgs.sourcehut.python.sitePackages}/metasrht"; + }; + }; +} diff --git a/nixos/modules/services/misc/sourcehut/paste.nix b/nixos/modules/services/misc/sourcehut/paste.nix new file mode 100644 index 00000000000..b2d5151969e --- /dev/null +++ b/nixos/modules/services/misc/sourcehut/paste.nix @@ -0,0 +1,133 @@ +{ config, lib, pkgs, ... }: + +with lib; +let + cfg = config.services.sourcehut; + cfgIni = cfg.settings; + scfg = cfg.paste; + iniKey = "paste.sr.ht"; + + rcfg = config.services.redis; + drv = pkgs.sourcehut.pastesrht; +in +{ + options.services.sourcehut.paste = { + user = mkOption { + type = types.str; + default = "pastesrht"; + description = '' + User for paste.sr.ht. + ''; + }; + + port = mkOption { + type = types.port; + default = 5011; + description = '' + Port on which the "paste" module should listen. + ''; + }; + + database = mkOption { + type = types.str; + default = "paste.sr.ht"; + description = '' + PostgreSQL database name for paste.sr.ht. + ''; + }; + + statePath = mkOption { + type = types.path; + default = "${cfg.statePath}/pastesrht"; + description = '' + State path for pastesrht.sr.ht. + ''; + }; + }; + + config = with scfg; lib.mkIf (cfg.enable && elem "paste" cfg.services) { + users = { + users = { + "${user}" = { + isSystemUser = true; + group = user; + description = "paste.sr.ht user"; + }; + }; + + groups = { + "${user}" = { }; + }; + }; + + services.postgresql = { + authentication = '' + local ${database} ${user} trust + ''; + ensureDatabases = [ database ]; + ensureUsers = [ + { + name = user; + ensurePermissions = { "DATABASE \"${database}\"" = "ALL PRIVILEGES"; }; + } + ]; + }; + + systemd = { + tmpfiles.rules = [ + "d ${statePath} 0750 ${user} ${user} -" + ]; + + services = { + pastesrht = import ./service.nix { inherit config pkgs lib; } scfg drv iniKey { + after = [ "postgresql.service" "network.target" ]; + requires = [ "postgresql.service" ]; + wantedBy = [ "multi-user.target" ]; + + description = "paste.sr.ht website service"; + + serviceConfig.ExecStart = "${cfg.python}/bin/gunicorn ${drv.pname}.app:app -b ${cfg.address}:${toString port}"; + }; + + pastesrht-webhooks = { + after = [ "postgresql.service" "network.target" ]; + requires = [ "postgresql.service" ]; + wantedBy = [ "multi-user.target" ]; + + description = "paste.sr.ht webhooks service"; + serviceConfig = { + Type = "simple"; + User = user; + Restart = "always"; + ExecStart = "${cfg.python}/bin/celery -A ${drv.pname}.webhooks worker --loglevel=info"; + }; + + }; + }; + }; + + services.sourcehut.settings = { + # URL paste.sr.ht is being served at (protocol://domain) + "paste.sr.ht".origin = mkDefault "http://paste.${cfg.originBase}"; + # Address and port to bind the debug server to + "paste.sr.ht".debug-host = mkDefault "0.0.0.0"; + "paste.sr.ht".debug-port = mkDefault port; + # Configures the SQLAlchemy connection string for the database. + "paste.sr.ht".connection-string = mkDefault "postgresql:///${database}?user=${user}&host=/var/run/postgresql"; + # Set to "yes" to automatically run migrations on package upgrade. + "paste.sr.ht".migrate-on-upgrade = mkDefault "yes"; + # paste.sr.ht's OAuth client ID and secret for meta.sr.ht + # Register your client at meta.example.org/oauth + "paste.sr.ht".oauth-client-id = mkDefault null; + "paste.sr.ht".oauth-client-secret = mkDefault null; + "paste.sr.ht".webhooks = mkDefault "redis://${rcfg.bind}:${toString rcfg.port}/5"; + }; + + services.nginx.virtualHosts."paste.${cfg.originBase}" = { + forceSSL = true; + locations."/".proxyPass = "http://${cfg.address}:${toString port}"; + locations."/query".proxyPass = "http://${cfg.address}:${toString (port + 100)}"; + locations."/static".root = "${pkgs.sourcehut.pastesrht}/${pkgs.sourcehut.python.sitePackages}/pastesrht"; + }; + }; +} diff --git a/nixos/modules/services/misc/sourcehut/service.nix b/nixos/modules/services/misc/sourcehut/service.nix new file mode 100644 index 00000000000..65b4ad020f9 --- /dev/null +++ b/nixos/modules/services/misc/sourcehut/service.nix @@ -0,0 +1,66 @@ +{ config, pkgs, lib }: +serviceCfg: serviceDrv: iniKey: attrs: +let + cfg = config.services.sourcehut; + cfgIni = cfg.settings."${iniKey}"; + pgSuperUser = config.services.postgresql.superUser; + + setupDB = pkgs.writeScript "${serviceDrv.pname}-gen-db" '' + #! ${cfg.python}/bin/python + from ${serviceDrv.pname}.app import db + db.create() + ''; +in +with serviceCfg; with lib; recursiveUpdate +{ + environment.HOME = statePath; + path = [ config.services.postgresql.package ] ++ (attrs.path or [ ]); + restartTriggers = [ config.environment.etc."sr.ht/config.ini".source ]; + serviceConfig = { + Type = "simple"; + User = user; + Group = user; + Restart = "always"; + WorkingDirectory = statePath; + } // (if (cfg.statePath == "/var/lib/sourcehut/${serviceDrv.pname}") then { + StateDirectory = [ "sourcehut/${serviceDrv.pname}" ]; + } else {}) + ; + + preStart = '' + if ! test -e ${statePath}/db; then + # Setup the initial database + ${setupDB} + + # Set the initial state of the database for future database upgrades + if test -e ${cfg.python}/bin/${serviceDrv.pname}-migrate; then + # Run alembic stamp head once to tell alembic the schema is up-to-date + ${cfg.python}/bin/${serviceDrv.pname}-migrate stamp head + fi + + printf "%s" "${serviceDrv.version}" > ${statePath}/db + fi + + # Update copy of each users' profile to the latest + # See https://lists.sr.ht/~sircmpwn/sr.ht-admins/<20190302181207.GA13778%40cirno.my.domain> + if ! test -e ${statePath}/webhook; then + # Update ${iniKey}'s users' profile copy to the latest + ${cfg.python}/bin/srht-update-profiles ${iniKey} + + touch ${statePath}/webhook + fi + + ${optionalString (builtins.hasAttr "migrate-on-upgrade" cfgIni && cfgIni.migrate-on-upgrade == "yes") '' + if [ "$(cat ${statePath}/db)" != "${serviceDrv.version}" ]; then + # Manage schema migrations using alembic + ${cfg.python}/bin/${serviceDrv.pname}-migrate -a upgrade head + + # Mark down current package version + printf "%s" "${serviceDrv.version}" > ${statePath}/db + fi + ''} + + ${attrs.preStart or ""} + ''; +} + (builtins.removeAttrs attrs [ "path" "preStart" ]) diff --git a/nixos/modules/services/misc/sourcehut/sourcehut.xml b/nixos/modules/services/misc/sourcehut/sourcehut.xml new file mode 100644 index 00000000000..ab9a8c6cb4b --- /dev/null +++ b/nixos/modules/services/misc/sourcehut/sourcehut.xml @@ -0,0 +1,115 @@ + + Sourcehut + + Sourcehut is an open-source, + self-hostable software development platform. The server setup can be automated using + services.sourcehut. + + +
+ Basic usage + + Sourcehut is a Python and Go based set of applications. + services.sourcehut + by default will use + services.nginx, + services.redis, + services.cron, + and + services.postgresql. + + + + A very basic configuration may look like this: + +{ pkgs, ... }: +let + fqdn = + let + join = hostName: domain: hostName + optionalString (domain != null) ".${domain}"; + in join config.networking.hostName config.networking.domain; +in { + + networking = { + hostName = "srht"; + domain = "tld"; + firewall.allowedTCPPorts = [ 22 80 443 ]; + }; + + services.sourcehut = { + enable = true; + originBase = fqdn; + services = [ "meta" "man" "git" ]; + settings = { + "sr.ht" = { + environment = "production"; + global-domain = fqdn; + origin = "https://${fqdn}"; + # Produce keys with srht-keygen from sourcehut.coresrht. + network-key = "SECRET"; + service-key = "SECRET"; + }; + webhooks.private-key= "SECRET"; + }; + }; + + security.acme.certs."${fqdn}".extraDomainNames = [ + "meta.${fqdn}" + "man.${fqdn}" + "git.${fqdn}" + ]; + + services.nginx = { + enable = true; + # only recommendedProxySettings are strictly required, but the rest make sense as well. + recommendedTlsSettings = true; + recommendedOptimisation = true; + recommendedGzipSettings = true; + recommendedProxySettings = true; + + # Settings to setup what certificates are used for which endpoint. + virtualHosts = { + "${fqdn}".enableACME = true; + "meta.${fqdn}".useACMEHost = fqdn: + "man.${fqdn}".useACMEHost = fqdn: + "git.${fqdn}".useACMEHost = fqdn: + }; + }; +} + + + + + The hostName option is used internally to configure the nginx + reverse-proxy. The settings attribute set is + used by the configuration generator and the result is placed in /etc/sr.ht/config.ini. + +
+ +
+ Configuration + + + All configuration parameters are also stored in + /etc/sr.ht/config.ini which is generated by + the module and linked from the store to ensure that all values from config.ini + can be modified by the module. + + +
+ +
+ Using an alternative webserver as reverse-proxy (e.g. <literal>httpd</literal>) + + By default, nginx is used as reverse-proxy for sourcehut. + However, it's possible to use e.g. httpd by explicitly disabling + nginx using and fixing the + settings. + +
+ + diff --git a/nixos/modules/services/misc/sourcehut/todo.nix b/nixos/modules/services/misc/sourcehut/todo.nix new file mode 100644 index 00000000000..aec773b0669 --- /dev/null +++ b/nixos/modules/services/misc/sourcehut/todo.nix @@ -0,0 +1,161 @@ +{ config, lib, pkgs, ... }: + +with lib; +let + cfg = config.services.sourcehut; + cfgIni = cfg.settings; + scfg = cfg.todo; + iniKey = "todo.sr.ht"; + + rcfg = config.services.redis; + drv = pkgs.sourcehut.todosrht; +in +{ + options.services.sourcehut.todo = { + user = mkOption { + type = types.str; + default = "todosrht"; + description = '' + User for todo.sr.ht. + ''; + }; + + port = mkOption { + type = types.port; + default = 5003; + description = '' + Port on which the "todo" module should listen. + ''; + }; + + database = mkOption { + type = types.str; + default = "todo.sr.ht"; + description = '' + PostgreSQL database name for todo.sr.ht. + ''; + }; + + statePath = mkOption { + type = types.path; + default = "${cfg.statePath}/todosrht"; + description = '' + State path for todo.sr.ht. + ''; + }; + }; + + config = with scfg; lib.mkIf (cfg.enable && elem "todo" cfg.services) { + users = { + users = { + "${user}" = { + isSystemUser = true; + group = user; + extraGroups = [ "postfix" ]; + description = "todo.sr.ht user"; + }; + }; + groups = { + "${user}" = { }; + }; + }; + + services.postgresql = { + authentication = '' + local ${database} ${user} trust + ''; + ensureDatabases = [ database ]; + ensureUsers = [ + { + name = user; + ensurePermissions = { "DATABASE \"${database}\"" = "ALL PRIVILEGES"; }; + } + ]; + }; + + systemd = { + tmpfiles.rules = [ + "d ${statePath} 0750 ${user} ${user} -" + ]; + + services = { + todosrht = import ./service.nix { inherit config pkgs lib; } scfg drv iniKey { + after = [ "postgresql.service" "network.target" ]; + requires = [ "postgresql.service" ]; + wantedBy = [ "multi-user.target" ]; + + description = "todo.sr.ht website service"; + + serviceConfig.ExecStart = "${cfg.python}/bin/gunicorn ${drv.pname}.app:app -b ${cfg.address}:${toString port}"; + }; + + todosrht-lmtp = { + after = [ "postgresql.service" "network.target" ]; + bindsTo = [ "postgresql.service" ]; + wantedBy = [ "multi-user.target" ]; + + description = "todo.sr.ht process service"; + serviceConfig = { + Type = "simple"; + User = user; + Restart = "always"; + ExecStart = "${cfg.python}/bin/todosrht-lmtp"; + }; + }; + + todosrht-webhooks = { + after = [ "postgresql.service" "network.target" ]; + requires = [ "postgresql.service" ]; + wantedBy = [ "multi-user.target" ]; + + description = "todo.sr.ht webhooks service"; + serviceConfig = { + Type = "simple"; + User = user; + Restart = "always"; + ExecStart = "${cfg.python}/bin/celery -A ${drv.pname}.webhooks worker --loglevel=info"; + }; + + }; + }; + }; + + services.sourcehut.settings = { + # URL todo.sr.ht is being served at (protocol://domain) + "todo.sr.ht".origin = mkDefault "http://todo.${cfg.originBase}"; + # Address and port to bind the debug server to + "todo.sr.ht".debug-host = mkDefault "0.0.0.0"; + "todo.sr.ht".debug-port = mkDefault port; + # Configures the SQLAlchemy connection string for the database. + "todo.sr.ht".connection-string = mkDefault "postgresql:///${database}?user=${user}&host=/var/run/postgresql"; + # Set to "yes" to automatically run migrations on package upgrade. + "todo.sr.ht".migrate-on-upgrade = mkDefault "yes"; + # todo.sr.ht's OAuth client ID and secret for meta.sr.ht + # Register your client at meta.example.org/oauth + "todo.sr.ht".oauth-client-id = mkDefault null; + "todo.sr.ht".oauth-client-secret = mkDefault null; + # Outgoing email for notifications generated by users + "todo.sr.ht".notify-from = mkDefault "CHANGEME@example.org"; + # The redis connection used for the webhooks worker + "todo.sr.ht".webhooks = mkDefault "redis://${rcfg.bind}:${toString rcfg.port}/1"; + # Network-key + "todo.sr.ht".network-key = mkDefault null; + + # Path for the lmtp daemon's unix socket. Direct incoming mail to this socket. + # Alternatively, specify IP:PORT and an SMTP server will be run instead. + "todo.sr.ht::mail".sock = mkDefault "/tmp/todo.sr.ht-lmtp.sock"; + # The lmtp daemon will make the unix socket group-read/write for users in this + # group. + "todo.sr.ht::mail".sock-group = mkDefault "postfix"; + + "todo.sr.ht::mail".posting-domain = mkDefault "todo.${cfg.originBase}"; + }; + + services.nginx.virtualHosts."todo.${cfg.originBase}" = { + forceSSL = true; + locations."/".proxyPass = "http://${cfg.address}:${toString port}"; + locations."/query".proxyPass = "http://${cfg.address}:${toString (port + 100)}"; + locations."/static".root = "${pkgs.sourcehut.todosrht}/${pkgs.sourcehut.python.sitePackages}/todosrht"; + }; + }; +} diff --git a/nixos/tests/sourcehut.nix b/nixos/tests/sourcehut.nix new file mode 100644 index 00000000000..b56a14ebf85 --- /dev/null +++ b/nixos/tests/sourcehut.nix @@ -0,0 +1,29 @@ +import ./make-test-python.nix ({ pkgs, ... }: + +{ + name = "sourcehut"; + + meta.maintainers = [ pkgs.lib.maintainers.tomberek ]; + + machine = { config, pkgs, ... }: { + virtualisation.memorySize = 2048; + networking.firewall.allowedTCPPorts = [ 80 ]; + + services.sourcehut = { + enable = true; + services = [ "meta" ]; + originBase = "sourcehut"; + settings."sr.ht".service-key = "8888888888888888888888888888888888888888888888888888888888888888"; + settings."sr.ht".network-key = "0000000000000000000000000000000000000000000="; + settings.webhooks.private-key = "0000000000000000000000000000000000000000000="; + }; + }; + + testScript = '' + start_all() + machine.wait_for_unit("multi-user.target") + machine.wait_for_unit("metasrht.service") + machine.wait_for_open_port(5000) + machine.succeed("curl -sL http://localhost:5000 | grep meta.sourcehut") + ''; +}) diff --git a/pkgs/applications/version-management/sourcehut/default.nix b/pkgs/applications/version-management/sourcehut/default.nix index 43d783e1934..401a1437b7d 100644 --- a/pkgs/applications/version-management/sourcehut/default.nix +++ b/pkgs/applications/version-management/sourcehut/default.nix @@ -31,6 +31,7 @@ let in with python.pkgs; recurseIntoAttrs { inherit python; + coresrht = toPythonApplication srht; buildsrht = toPythonApplication buildsrht; dispatchsrht = toPythonApplication dispatchsrht; gitsrht = toPythonApplication gitsrht;