From f5f8341c76ffad22ae52c622be97c94ccbd2a847 Mon Sep 17 00:00:00 2001 From: talyz Date: Wed, 28 Apr 2021 16:56:06 +0200 Subject: [PATCH] nixos/geoipupdate: Replace the old `geoip-updater` module Our old bespoke GeoIP updater doesn't seem to be working anymore. Instead of trying to fix it, replace it with the official updater from MaxMind. --- .../from_md/release-notes/rl-2111.section.xml | 20 +- .../manual/release-notes/rl-2111.section.md | 7 + nixos/modules/misc/ids.nix | 2 +- nixos/modules/module-list.nix | 2 +- nixos/modules/services/misc/geoip-updater.nix | 306 ------------------ nixos/modules/services/misc/geoipupdate.nix | 145 +++++++++ 6 files changed, 171 insertions(+), 311 deletions(-) delete mode 100644 nixos/modules/services/misc/geoip-updater.nix create mode 100644 nixos/modules/services/misc/geoipupdate.nix diff --git a/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml index e861b4fe7e2..e0b52ffa342 100644 --- a/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml +++ b/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml @@ -19,18 +19,32 @@
New Services - - + + + + geoipupdate, + a GeoIP database updater from MaxMind. Available as + services.geoipupdate. + + +
Backward Incompatibilities - + The staticjinja package has been upgraded from 1.0.4 to 2.0.0 + + + services.geoip-updater was broken and has + been replaced by + services.geoipupdate. + +
diff --git a/nixos/doc/manual/release-notes/rl-2111.section.md b/nixos/doc/manual/release-notes/rl-2111.section.md index 9a6da7f22bd..d4604447f83 100644 --- a/nixos/doc/manual/release-notes/rl-2111.section.md +++ b/nixos/doc/manual/release-notes/rl-2111.section.md @@ -8,8 +8,15 @@ In addition to numerous new and upgraded packages, this release has the followin ## New Services +* [geoipupdate](https://github.com/maxmind/geoipupdate), a GeoIP + database updater from MaxMind. Available as + [services.geoipupdate](options.html#opt-services.geoipupdate.enable). + ## Backward Incompatibilities * The `staticjinja` package has been upgraded from 1.0.4 to 2.0.0 +* `services.geoip-updater` was broken and has been replaced by + [services.geoipupdate](options.html#opt-services.geoipupdate.enable). + ## Other Notable Changes diff --git a/nixos/modules/misc/ids.nix b/nixos/modules/misc/ids.nix index 05cc5002aaf..7ea2940292b 100644 --- a/nixos/modules/misc/ids.nix +++ b/nixos/modules/misc/ids.nix @@ -300,7 +300,7 @@ in #pdns-recursor = 269; # dynamically allocated as of 2020-20-18 #kresd = 270; # switched to "knot-resolver" with dynamic ID rpc = 271; - geoip = 272; + #geoip = 272; # new module uses DynamicUser fcron = 273; sonarr = 274; radarr = 275; diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index ebafb5ef5aa..42f0471c4cf 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -492,7 +492,7 @@ ./services/misc/freeswitch.nix ./services/misc/fstrim.nix ./services/misc/gammu-smsd.nix - ./services/misc/geoip-updater.nix + ./services/misc/geoipupdate.nix ./services/misc/gitea.nix #./services/misc/gitit.nix ./services/misc/gitlab.nix diff --git a/nixos/modules/services/misc/geoip-updater.nix b/nixos/modules/services/misc/geoip-updater.nix deleted file mode 100644 index baf0a8d73d1..00000000000 --- a/nixos/modules/services/misc/geoip-updater.nix +++ /dev/null @@ -1,306 +0,0 @@ -{ config, lib, pkgs, ... }: - -with lib; - -let - cfg = config.services.geoip-updater; - - dbBaseUrl = "https://geolite.maxmind.com/download/geoip/database"; - - randomizedTimerDelaySec = "3600"; - - # Use writeScriptBin instead of writeScript, so that argv[0] (logged to the - # journal) doesn't include the long nix store path hash. (Prefixing the - # ExecStart= command with '@' doesn't work because we start a shell (new - # process) that creates a new argv[0].) - geoip-updater = pkgs.writeScriptBin "geoip-updater" '' - #!${pkgs.runtimeShell} - skipExisting=0 - debug() - { - echo "<7>$@" - } - info() - { - echo "<6>$@" - } - error() - { - echo "<3>$@" - } - die() - { - error "$@" - exit 1 - } - waitNetworkOnline() - { - ret=1 - for i in $(seq 6); do - curl_out=$("${pkgs.curl.bin}/bin/curl" \ - --silent --fail --show-error --max-time 60 "${dbBaseUrl}" 2>&1) - if [ $? -eq 0 ]; then - debug "Server is reachable (try $i)" - ret=0 - break - else - debug "Server is unreachable (try $i): $curl_out" - sleep 10 - fi - done - return $ret - } - dbFnameTmp() - { - dburl=$1 - echo "${cfg.databaseDir}/.$(basename "$dburl")" - } - dbFnameTmpDecompressed() - { - dburl=$1 - echo "${cfg.databaseDir}/.$(basename "$dburl")" | sed 's/\.\(gz\|xz\)$//' - } - dbFname() - { - dburl=$1 - echo "${cfg.databaseDir}/$(basename "$dburl")" | sed 's/\.\(gz\|xz\)$//' - } - downloadDb() - { - dburl=$1 - curl_out=$("${pkgs.curl.bin}/bin/curl" \ - --silent --fail --show-error --max-time 900 -L -o "$(dbFnameTmp "$dburl")" "$dburl" 2>&1) - if [ $? -ne 0 ]; then - error "Failed to download $dburl: $curl_out" - return 1 - fi - } - decompressDb() - { - fn=$(dbFnameTmp "$1") - ret=0 - case "$fn" in - *.gz) - cmd_out=$("${pkgs.gzip}/bin/gzip" --decompress --force "$fn" 2>&1) - ;; - *.xz) - cmd_out=$("${pkgs.xz.bin}/bin/xz" --decompress --force "$fn" 2>&1) - ;; - *) - cmd_out=$(echo "File \"$fn\" is neither a .gz nor .xz file") - false - ;; - esac - if [ $? -ne 0 ]; then - error "$cmd_out" - ret=1 - fi - } - atomicRename() - { - dburl=$1 - mv "$(dbFnameTmpDecompressed "$dburl")" "$(dbFname "$dburl")" - } - removeIfNotInConfig() - { - # Arg 1 is the full path of an installed DB. - # If the corresponding database is not specified in the NixOS config we - # remove it. - db=$1 - for cdb in ${lib.concatStringsSep " " cfg.databases}; do - confDb=$(echo "$cdb" | sed 's/\.\(gz\|xz\)$//') - if [ "$(basename "$db")" = "$(basename "$confDb")" ]; then - return 0 - fi - done - rm "$db" - if [ $? -eq 0 ]; then - debug "Removed $(basename "$db") (not listed in services.geoip-updater.databases)" - else - error "Failed to remove $db" - fi - } - removeUnspecifiedDbs() - { - for f in "${cfg.databaseDir}/"*; do - test -f "$f" || continue - case "$f" in - *.dat|*.mmdb|*.csv) - removeIfNotInConfig "$f" - ;; - *) - debug "Not removing \"$f\" (unknown file extension)" - ;; - esac - done - } - downloadAndInstall() - { - dburl=$1 - if [ "$skipExisting" -eq 1 -a -f "$(dbFname "$dburl")" ]; then - debug "Skipping existing file: $(dbFname "$dburl")" - return 0 - fi - downloadDb "$dburl" || return 1 - decompressDb "$dburl" || return 1 - atomicRename "$dburl" || return 1 - info "Updated $(basename "$(dbFname "$dburl")")" - } - for arg in "$@"; do - case "$arg" in - --skip-existing) - skipExisting=1 - info "Option --skip-existing is set: not updating existing databases" - ;; - *) - error "Unknown argument: $arg";; - esac - done - waitNetworkOnline || die "Network is down (${dbBaseUrl} is unreachable)" - test -d "${cfg.databaseDir}" || die "Database directory (${cfg.databaseDir}) doesn't exist" - debug "Starting update of GeoIP databases in ${cfg.databaseDir}" - all_ret=0 - for db in ${lib.concatStringsSep " \\\n " cfg.databases}; do - downloadAndInstall "${dbBaseUrl}/$db" || all_ret=1 - done - removeUnspecifiedDbs || all_ret=1 - if [ $all_ret -eq 0 ]; then - info "Completed GeoIP database update in ${cfg.databaseDir}" - else - error "Completed GeoIP database update in ${cfg.databaseDir}, with error(s)" - fi - # Hack to work around systemd journal race: - # https://github.com/systemd/systemd/issues/2913 - sleep 2 - exit $all_ret - ''; - -in - -{ - options = { - services.geoip-updater = { - enable = mkOption { - default = false; - type = types.bool; - description = '' - Whether to enable periodic downloading of GeoIP databases from - maxmind.com. You might want to enable this if you, for instance, use - ntopng or Wireshark. - ''; - }; - - interval = mkOption { - type = types.str; - default = "weekly"; - description = '' - Update the GeoIP databases at this time / interval. - The format is described in - systemd.time - 7. - To prevent load spikes on maxmind.com, the timer interval is - randomized by an additional delay of ${randomizedTimerDelaySec} - seconds. Setting a shorter interval than this is not recommended. - ''; - }; - - databaseDir = mkOption { - type = types.path; - default = "/var/lib/geoip-databases"; - description = '' - Directory that will contain GeoIP databases. - ''; - }; - - databases = mkOption { - type = types.listOf types.str; - default = [ - "GeoLiteCountry/GeoIP.dat.gz" - "GeoIPv6.dat.gz" - "GeoLiteCity.dat.xz" - "GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz" - "asnum/GeoIPASNum.dat.gz" - "asnum/GeoIPASNumv6.dat.gz" - "GeoLite2-Country.mmdb.gz" - "GeoLite2-City.mmdb.gz" - ]; - description = '' - Which GeoIP databases to update. The full URL is ${dbBaseUrl}/ + - the_database. - ''; - }; - - }; - - }; - - config = mkIf cfg.enable { - - assertions = [ - { assertion = (builtins.filter - (x: builtins.match ".*\\.(gz|xz)$" x == null) cfg.databases) == []; - message = '' - services.geoip-updater.databases supports only .gz and .xz databases. - - Current value: - ${toString cfg.databases} - - Offending element(s): - ${toString (builtins.filter (x: builtins.match ".*\\.(gz|xz)$" x == null) cfg.databases)}; - ''; - } - ]; - - users.users.geoip = { - group = "root"; - description = "GeoIP database updater"; - uid = config.ids.uids.geoip; - }; - - systemd.timers.geoip-updater = - { description = "GeoIP Updater Timer"; - partOf = [ "geoip-updater.service" ]; - wantedBy = [ "timers.target" ]; - timerConfig.OnCalendar = cfg.interval; - timerConfig.Persistent = "true"; - timerConfig.RandomizedDelaySec = randomizedTimerDelaySec; - }; - - systemd.services.geoip-updater = { - description = "GeoIP Updater"; - after = [ "network-online.target" "nss-lookup.target" ]; - wants = [ "network-online.target" ]; - preStart = '' - mkdir -p "${cfg.databaseDir}" - chmod 755 "${cfg.databaseDir}" - chown geoip:root "${cfg.databaseDir}" - ''; - serviceConfig = { - ExecStart = "${geoip-updater}/bin/geoip-updater"; - User = "geoip"; - PermissionsStartOnly = true; - }; - }; - - systemd.services.geoip-updater-setup = { - description = "GeoIP Updater Setup"; - after = [ "network-online.target" "nss-lookup.target" ]; - wants = [ "network-online.target" ]; - wantedBy = [ "multi-user.target" ]; - conflicts = [ "geoip-updater.service" ]; - preStart = '' - mkdir -p "${cfg.databaseDir}" - chmod 755 "${cfg.databaseDir}" - chown geoip:root "${cfg.databaseDir}" - ''; - serviceConfig = { - ExecStart = "${geoip-updater}/bin/geoip-updater --skip-existing"; - User = "geoip"; - PermissionsStartOnly = true; - # So it won't be (needlessly) restarted: - RemainAfterExit = true; - }; - }; - - }; -} diff --git a/nixos/modules/services/misc/geoipupdate.nix b/nixos/modules/services/misc/geoipupdate.nix new file mode 100644 index 00000000000..5d87be928d9 --- /dev/null +++ b/nixos/modules/services/misc/geoipupdate.nix @@ -0,0 +1,145 @@ +{ config, lib, pkgs, ... }: + +let + cfg = config.services.geoipupdate; +in +{ + imports = [ + (lib.mkRemovedOptionModule [ "services" "geoip-updater" ] "services.geoip-updater has been removed, use services.geoipupdate instead.") + ]; + + options = { + services.geoipupdate = { + enable = lib.mkEnableOption '' + periodic downloading of GeoIP databases using + geoipupdate. + ''; + + interval = lib.mkOption { + type = lib.types.str; + default = "weekly"; + description = '' + Update the GeoIP databases at this time / interval. + The format is described in + systemd.time + 7. + ''; + }; + + settings = lib.mkOption { + description = '' + geoipupdate configuration + options. See + + for a full list of available options. + ''; + type = lib.types.submodule { + freeformType = + with lib.types; + let + type = oneOf [str int bool]; + in + attrsOf (either type (listOf type)); + + options = { + + AccountID = lib.mkOption { + type = lib.types.int; + description = '' + Your MaxMind account ID. + ''; + }; + + EditionIDs = lib.mkOption { + type = with lib.types; listOf (either str int); + example = [ + "GeoLite2-ASN" + "GeoLite2-City" + "GeoLite2-Country" + ]; + description = '' + List of database edition IDs. This includes new string + IDs like GeoIP2-City and old + numeric IDs like 106. + ''; + }; + + LicenseKey = lib.mkOption { + type = lib.types.path; + description = '' + A file containing the MaxMind + license key. + ''; + }; + + DatabaseDirectory = lib.mkOption { + type = lib.types.path; + default = "/var/lib/GeoIP"; + example = "/run/GeoIP"; + description = '' + The directory to store the database files in. The + directory will be automatically created, the owner + changed to geoip and permissions + set to world readable. This applies if the directory + already exists as well, so don't use a directory with + sensitive contents. + ''; + }; + + }; + }; + }; + }; + + }; + + config = lib.mkIf cfg.enable { + + services.geoipupdate.settings = { + LockFile = "/run/geoipupdate/.lock"; + }; + + systemd.services.geoipupdate = { + description = "GeoIP Updater"; + after = [ "network-online.target" "nss-lookup.target" ]; + wants = [ "network-online.target" ]; + startAt = cfg.interval; + serviceConfig = { + ExecStartPre = + let + geoipupdateKeyValue = lib.generators.toKeyValue { + mkKeyValue = lib.flip lib.generators.mkKeyValueDefault " " rec { + mkValueString = v: with builtins; + if isInt v then toString v + else if isString v then v + else if true == v then "1" + else if false == v then "0" + else if isList v then lib.concatMapStringsSep " " mkValueString v + else throw "unsupported type ${typeOf v}: ${(lib.generators.toPretty {}) v}"; + }; + }; + + geoipupdateConf = pkgs.writeText "discourse.conf" (geoipupdateKeyValue cfg.settings); + + script = '' + mkdir -p "${cfg.settings.DatabaseDirectory}" + chmod 755 "${cfg.settings.DatabaseDirectory}" + chown geoip "${cfg.settings.DatabaseDirectory}" + + cp ${geoipupdateConf} /run/geoipupdate/GeoIP.conf + ${pkgs.replace-secret}/bin/replace-secret '${cfg.settings.LicenseKey}' \ + '${cfg.settings.LicenseKey}' \ + /run/geoipupdate/GeoIP.conf + ''; + in + "+${pkgs.writeShellScript "start-pre-full-privileges" script}"; + ExecStart = "${pkgs.geoipupdate}/bin/geoipupdate -f /run/geoipupdate/GeoIP.conf"; + User = "geoip"; + DynamicUser = true; + ReadWritePaths = cfg.settings.DatabaseDirectory; + RuntimeDirectory = "geoipupdate"; + RuntimeDirectoryMode = 0700; + }; + }; + }; +}