Moritz Mühlenhoff | 1475946 | 2020-04-07 12:02:30 +0200 | [diff] [blame] | 1 | #!/bin/bash |
| 2 | |
| 3 | ############################################################################## |
| 4 | # WMF Update production known hosts |
| 5 | # |
| 6 | # DESCRIPTION: |
| 7 | # - Populate a known_hosts file with all the production hosts and services |
| 8 | # in the Wikimedia Foundation production infrastructure for easy |
| 9 | # autocompletion while keeping StrictHostKeyChecking active: |
| 10 | # - sync all the known hosts from a bastion |
| 11 | # - clean the hostname without FQDN in it |
| 12 | # - optionally generate known hosts for services defined as CNAMEs in the |
| 13 | # DNS repository, see PARAMS below. This allows for the autocompletion of |
| 14 | # active/passive services like icinga.wikimedia.org. |
| 15 | # - Silently ignore all CNAMEs to the main DYNA record (dyna.wikimedia.org.) |
| 16 | # - Keeps a backup file with the previous known hosts |
| 17 | # - Show a diff between the new known hosts and the current ones |
| 18 | # |
| 19 | # It saves the known hosts into KNOWN_HOST_FILE, adjust this and/or the |
| 20 | # UserKnownHostsFile parameter in your ~/.ssh/config in order for them to |
| 21 | # match. A warning will be shown if they don't match. |
| 22 | # |
| 23 | # By default only the hosts from the choosen BASTION_HOST known_hosts file |
| 24 | # will be imported, cleaning the hostname (not the FQDN) to ease the auto- |
| 25 | # completion when ssh-ing. |
| 26 | # |
| 27 | # PARAMS: |
| 28 | # It accept one positional argument that, if specified, must be the path to |
| 29 | # a local clone of the Operations DNS repository, (either from Gerrit or from |
| 30 | # GitHub): |
| 31 | # https://gerrit.wikimedia.org/r/operations/dns |
| 32 | # In this case also the services defined as CNAMEs in the wikimedia.org and |
| 33 | # wmnet zone files will be added with the identity of the target host, if |
| 34 | # that is found in the known_hosts file, skipping the missing ones. |
| 35 | # |
| 36 | # USAGE: |
| 37 | # wmf-update-prod-known-hosts [PATH_TO_DNS_REPOSITORY] |
| 38 | # |
| 39 | # Author: Riccardo Coccioli <rcoccioli@wikimedia.org> |
| 40 | # Date: 2017-06-21 |
| 41 | # Last update: 2019-11-05 |
| 42 | # Dependencies: colordiff |
| 43 | # Version: 1.2 |
| 44 | # License: GPLv3+ |
| 45 | ############################################################################## |
| 46 | |
| 47 | set -e |
| 48 | |
| 49 | DNS_REPO_PATH="${1}" |
| 50 | KNOWN_HOSTS_PATH="${HOME}/.ssh/known_hosts.d" |
| 51 | KNOWN_HOST_FILE="${KNOWN_HOSTS_PATH}/wmf-prod" |
Moritz Mühlenhoff | 221df3d | 2021-02-12 09:44:40 +0100 | [diff] [blame^] | 52 | BASTION_HOST="bast3005.wikimedia.org" |
Moritz Mühlenhoff | 1475946 | 2020-04-07 12:02:30 +0200 | [diff] [blame] | 53 | MAIN_DYNA_RECORD="dyna.wikimedia.org." |
| 54 | |
| 55 | if [[ ! -d "${KNOWN_HOSTS_PATH}" ]]; then |
| 56 | echo "ERROR: KNOWN_HOSTS_PATH '${KNOWN_HOSTS_PATH}' is not a directory, you might want to adjust the constant in the script or create it" |
| 57 | exit 1 |
| 58 | fi |
| 59 | |
| 60 | if [[ -n "${DNS_REPO_PATH}" ]]; then |
| 61 | if [[ ! -d "${DNS_REPO_PATH}" ]]; then |
| 62 | echo "ERROR: DNS_REPO_PATH '${DNS_REPO_PATH}' is not a directory" |
| 63 | exit 2 |
| 64 | fi |
| 65 | if ! git -C "${DNS_REPO_PATH}" remote -v | egrep '(gerrit.wikimedia.org|github.com\/wikimedia)' | grep -cq 'operations[/-]dns'; then |
| 66 | echo "ERROR: DNS_REPO_PATH '${DNS_REPO_PATH}' doesn't seems to be a checkout of the operations/dns repository" |
| 67 | exit 3 |
| 68 | fi |
| 69 | fi |
| 70 | |
| 71 | function parse_line() { |
| 72 | local line="${1}" |
| 73 | local domain="${2}" |
| 74 | local name |
| 75 | local target |
| 76 | local found |
| 77 | |
| 78 | name="$(echo "${line}" | cut -d' ' -f1)" |
| 79 | target="$(echo "${line}" | cut -d' ' -f2)" |
| 80 | |
| 81 | if [[ "${target}" == "${MAIN_DYNA_RECORD}" ]]; then |
| 82 | # Silently ignore CNAMEs to the MAIN_DYNA_RECORD |
| 83 | return |
| 84 | fi |
| 85 | |
| 86 | sep="\." |
| 87 | if [[ "${target: -1}" == '.' ]]; then |
| 88 | target="${target%?}" |
| 89 | sep="," |
| 90 | fi |
| 91 | |
| 92 | set +e |
| 93 | found=$(grep -c "^${target}${sep}" "${KNOWN_HOST_FILE}.new") |
| 94 | set -e |
| 95 | if [[ "${found}" -eq "0" || "${found}" -gt "1" ]]; then |
| 96 | >&2 echo "Skipping '${target}' CNAME target, found ${found}/1 matches" |
| 97 | return |
| 98 | fi |
| 99 | |
| 100 | grep "^${target}${sep}" "${KNOWN_HOST_FILE}.new" | awk -v name="${name}" -v domain="${domain}" '{ printf name"."domain; for (i = 2; i <= NF; i++) printf FS$i; print NL }' |
| 101 | } |
| 102 | |
| 103 | function extract_cnames_from_zone() { |
| 104 | local zone_file |
| 105 | local origin |
| 106 | local boundaries |
| 107 | local start |
| 108 | local end |
| 109 | local domain |
| 110 | |
| 111 | zone_file="${1}" |
| 112 | if [[ ! -f "${zone_file}" ]]; then |
| 113 | >&2 echo "Unable to find zone file ${zone_file}, skipping..." |
| 114 | return |
| 115 | fi |
| 116 | |
| 117 | origin="${2}" |
| 118 | if [[ -n "${origin}" ]]; then |
| 119 | boundaries="$(grep -n "\$ORIGIN" "${zone_file}" | grep -A 1 "\$ORIGIN ${origin}\.$")" |
| 120 | start=$(echo "${boundaries}" | head -n1 | cut -d':' -f1) |
| 121 | end=$(echo "${boundaries}" | tail -n1 | cut -d':' -f1) |
| 122 | domain="${origin}" |
| 123 | |
| 124 | head -n "${end}" "${zone_file}" | tail -n "$((end - start))" | awk '/ CNAME / { print $1, $5 }' | while read -r line; do |
| 125 | parse_line "${line}" "${domain}" >> "${KNOWN_HOST_FILE}.new" |
| 126 | done |
| 127 | else |
| 128 | domain="$(basename "${zone_file}")" |
| 129 | awk '/ CNAME / { print $1, $5 }' "${zone_file}" | while read -r line; do |
| 130 | parse_line "${line}" "${domain}" >> "${KNOWN_HOST_FILE}.new" |
| 131 | done |
| 132 | fi |
| 133 | } |
| 134 | |
| 135 | # Get new known hosts |
| 136 | echo "===> SSHing to ${BASTION_HOST} (if a smartcard input is needed, check it now)" |
| 137 | ssh "${BASTION_HOST}" 'cat /etc/ssh/ssh_known_hosts' > "${KNOWN_HOST_FILE}.new" |
| 138 | |
| 139 | # Remove the non-FQDN hostnames to avoid multiple autocompletions |
| 140 | awk -F ',' '{ printf $1; for (i = 3; i <= NF; i++) printf FS$i; print NL }' "${KNOWN_HOST_FILE}.new" > "${KNOWN_HOST_FILE}.new.clean" |
| 141 | mv -f "${KNOWN_HOST_FILE}.new.clean" "${KNOWN_HOST_FILE}.new" |
| 142 | |
| 143 | if [[ -n "${DNS_REPO_PATH}" ]]; then |
| 144 | extract_cnames_from_zone "${DNS_REPO_PATH}/templates/wikimedia.org" |
| 145 | extract_cnames_from_zone "${DNS_REPO_PATH}/templates/wmnet" "eqiad.wmnet" |
| 146 | fi |
| 147 | |
| 148 | PREV_COUNT=0 |
| 149 | PREV_FILE=/dev/null |
| 150 | if [[ -f "${KNOWN_HOST_FILE}" ]]; then |
| 151 | PREV_COUNT="$(wc -l "${KNOWN_HOST_FILE}")" |
| 152 | PREV_FILE="${KNOWN_HOST_FILE}" |
| 153 | fi |
| 154 | |
| 155 | echo "==== DIFFERENCES ====" |
| 156 | colordiff --fakeexitcode "${PREV_FILE}" "${KNOWN_HOST_FILE}.new" |
| 157 | echo "=====================" |
| 158 | echo "Going from ${PREV_COUNT} to $(wc -l "${KNOWN_HOST_FILE}.new") known hosts and services" |
| 159 | |
| 160 | if [[ -f "${KNOWN_HOST_FILE}" ]]; then |
| 161 | mv -vf "${KNOWN_HOST_FILE}" "${KNOWN_HOST_FILE}.old" |
| 162 | echo "Backup file is ${KNOWN_HOST_FILE}.old" |
| 163 | fi |
| 164 | mv -v "${KNOWN_HOST_FILE}.new" "${KNOWN_HOST_FILE}" |
| 165 | echo "New file generated at ${KNOWN_HOST_FILE}" |
| 166 | |
| 167 | if ! egrep -cq "UserKnownHostsFile .*/wmf-prod( |$)" "${HOME}/.ssh/config"; then |
| 168 | echo "WARNING: You may need to add/update 'UserKnownHostsFile ${KNOWN_HOST_FILE}' to your ~/.ssh/config" |
| 169 | fi |
| 170 | |
| 171 | if [[ "${SHELL}" == '/usr/bin/zsh' ]]; then |
| 172 | echo 'Add this line to your .zshrc to tab-complete remote hosts:' |
| 173 | echo "zstyle ':completion:*:hosts' known-hosts-files ${HOME}/.ssh/known_hosts ${KNOWN_HOST_FILE}" |
| 174 | fi |
| 175 | |
Moritz Mühlenhoff | 221df3d | 2021-02-12 09:44:40 +0100 | [diff] [blame^] | 176 | exit 0 |