gerrit.wikimedia.org
Gitiles
Code ReviewSign In
gerrit.wikimedia.org / operations / debs / wmf-sre-laptop / ab18ff7a42963c6d9e83ff6e6bdca3a286651815 / . / scripts / wmf-update-known-hosts-production
blob: 864b15c4ff33a219bc28abebd044ae4b6384c79d [file] [log] [blame]
Moritz Mühlenhoff14759462020-04-07 12:02:30 +0200[diff] [blame]
1
#!/bin/bash
2
3
##############################################################################
4
# WMF Update production known hosts
5
#
6
# DESCRIPTION:
7
# - Populate a known_hosts file with all the production hosts and services
8
# in the Wikimedia Foundation production infrastructure for easy
9
# autocompletion while keeping StrictHostKeyChecking active:
John Bondac002dc2021-02-23 17:05:33 +0100[diff] [blame]
10
# - sync all the known hosts from https://config-master.wikimedia.org/known_hosts.ecdsa
Moritz Mühlenhoff14759462020-04-07 12:02:30 +0200[diff] [blame]
11
# - clean the hostname without FQDN in it
12
# - optionally generate known hosts for services defined as CNAMEs in the
13
# DNS repository, see PARAMS below. This allows for the autocompletion of
14
# active/passive services like icinga.wikimedia.org.
15
# - Silently ignore all CNAMEs to the main DYNA record (dyna.wikimedia.org.)
16
# - Keeps a backup file with the previous known hosts
17
# - Show a diff between the new known hosts and the current ones
18
#
19
# It saves the known hosts into KNOWN_HOST_FILE, adjust this and/or the
20
# UserKnownHostsFile parameter in your ~/.ssh/config in order for them to
21
# match. A warning will be shown if they don't match.
22
#
Moritz Mühlenhoff14759462020-04-07 12:02:30 +0200[diff] [blame]
23
# PARAMS:
24
# It accept one positional argument that, if specified, must be the path to
25
# a local clone of the Operations DNS repository, (either from Gerrit or from
26
# GitHub):
27
# https://gerrit.wikimedia.org/r/operations/dns
28
# In this case also the services defined as CNAMEs in the wikimedia.org and
29
# wmnet zone files will be added with the identity of the target host, if
30
# that is found in the known_hosts file, skipping the missing ones.
31
#
32
# USAGE:
33
# wmf-update-prod-known-hosts [PATH_TO_DNS_REPOSITORY]
34
#
35
# Author: Riccardo Coccioli <rcoccioli@wikimedia.org>
36
# Date: 2017-06-21
37
# Last update: 2019-11-05
38
# Dependencies: colordiff
John Bondac002dc2021-02-23 17:05:33 +0100[diff] [blame]
39
# Version: 1.3
Moritz Mühlenhoff14759462020-04-07 12:02:30 +0200[diff] [blame]
40
# License: GPLv3+
41
##############################################################################
42
43
set -e
44
45
DNS_REPO_PATH="${1}"
46
KNOWN_HOSTS_PATH​=​"${HOME}/.ssh/known_hosts.d"
47
KNOWN_HOST_FILE​=​"${KNOWN_HOSTS_PATH}/wmf-prod"
John Bondac002dc2021-02-23 17:05:33 +0100[diff] [blame]
48
KNOWN_HOST_URL​=​"https://config-master.wikimedia.org/known_hosts.ecdsa"
Moritz Mühlenhoff14759462020-04-07 12:02:30 +0200[diff] [blame]
49
MAIN_DYNA_RECORD​=​"dyna.wikimedia.org."
50
51
if [[ ! -d "${KNOWN_HOSTS_PATH}" ]]; then
52
 echo "ERROR: KNOWN_HOSTS_PATH '${KNOWN_HOSTS_PATH}' is not a directory, you might want to adjust the constant in the script or create it"
53
 exit 1
54
fi
55
56
if [[ -n "${DNS_REPO_PATH}" ]]; then
57
 if [[ ! -d "${DNS_REPO_PATH}" ]]; then
58
 echo "ERROR: DNS_REPO_PATH '${DNS_REPO_PATH}' is not a directory"
59
 exit 2
60
 fi
John Bondac002dc2021-02-23 17:05:33 +0100[diff] [blame]
61
 if ! git -C "${DNS_REPO_PATH}" remote -v | grep -E '(gerrit.wikimedia.org|github.com\/wikimedia)' | grep -cq 'operations[/-]dns'; then
Moritz Mühlenhoff14759462020-04-07 12:02:30 +0200[diff] [blame]
62
 echo "ERROR: DNS_REPO_PATH '${DNS_REPO_PATH}' doesn't seems to be a checkout of the operations/dns repository"
63
 exit 3
64
 fi
65
fi
66
67
function parse_line() {
68
 local line="${1}"
69
 local domain="${2}"
70
 local name
71
 local target
72
 local found
73
74
 name="$(echo "${line}" | cut -d' ' -f1)"
75
 target="$(echo "${line}" | cut -d' ' -f2)"
76
77
 if [[ "${target}" == "${MAIN_DYNA_RECORD}" ]]; then
78
 # Silently ignore CNAMEs to the MAIN_DYNA_RECORD
79
 return
80
 fi
81
John Bondac002dc2021-02-23 17:05:33 +0100[diff] [blame]
82
 sep="\\."
Moritz Mühlenhoff14759462020-04-07 12:02:30 +0200[diff] [blame]
83
 if [[ "${target: -1}" == '.' ]]; then
84
 target="${target%?}"
85
 sep=","
86
 fi
87
88
 set +e
89
 found=$(grep -c "^${target}${sep}" "${KNOWN_HOST_FILE}.new"​)
90
 set -e
91
 if [[ "${found}" -eq "0" || "${found}" -gt "1" ]]; then
92
 >&2 echo "Skipping '${target}' CNAME target, found ${found}/1 matches"
93
 return
94
 fi
95
96
 grep "^${target}${sep}" "${KNOWN_HOST_FILE}.new" | awk -v name="${name}" -v domain="${domain}" '{ printf name"."domain; for (i = 2; i <= NF; i++) printf FS$i; print NL }'
97
}
98
99
function extract_cnames_from_zone() {
100
 local zone_file
101
 local origin
102
 local boundaries
103
 local start
104
 local end
105
 local domain
106
107
 zone_file="${1}"
108
 if [[ ! -f "${zone_file}" ]]; then
109
 >&2 echo "Unable to find zone file ${zone_file}, skipping..."
110
 return
111
 fi
112
113
 origin="${2}"
114
 if [[ -n "${origin}" ]]; then
John Bondac002dc2021-02-23 17:05:33 +0100[diff] [blame]
115
 boundaries="$(grep -n "\$ORIGIN" "${zone_file}" | grep -A 1 "\$ORIGIN ${origin}\\.$")"
Moritz Mühlenhoff14759462020-04-07 12:02:30 +0200[diff] [blame]
116
 start=$(echo "${boundaries}" | head -n1 | cut -d':' -f1)
117
 end=$(echo "${boundaries}" | tail -n1 | cut -d':' -f1)
118
 domain="${origin}"
119
120
 head -n "${end}" "${zone_file}" | tail -n "$((end - start))" | awk '/ CNAME / { print $1, $5 }' | while read -r line; do
121
 parse_line "${line}" "${domain}" >> "${KNOWN_HOST_FILE}.new"
122
 done
123
 else
124
 domain="$(basename "${zone_file}")"
125
 awk '/ CNAME / { print $1, $5 }' "${zone_file}" | while read -r line; do
126
 parse_line "${line}" "${domain}" >> "${KNOWN_HOST_FILE}.new"
127
 done
128
 fi
129
}
130
131
# Get new known hosts
John Bondac002dc2021-02-23 17:05:33 +0100[diff] [blame]
132
echo "===> Fetching from ${KNOWN_HOST_URL}"
133
curl "${KNOWN_HOST_URL}" -o "${KNOWN_HOST_FILE}.new"
Moritz Mühlenhoff14759462020-04-07 12:02:30 +0200[diff] [blame]
134
135
if [[ -n "${DNS_REPO_PATH}" ]]; then
136
 extract_cnames_from_zone "${DNS_REPO_PATH}/templates/wikimedia.org"
137
 extract_cnames_from_zone "${DNS_REPO_PATH}/templates/wmnet" "eqiad.wmnet"
138
fi
139
140
PREV_COUNT=0
141
PREV_FILE=/dev/null
142
if [[ -f "${KNOWN_HOST_FILE}" ]]; then
143
 PREV_COUNT="$(wc -l "${KNOWN_HOST_FILE}")"
144
 PREV_FILE​=​"${KNOWN_HOST_FILE}"
145
fi
146
147
echo "==== DIFFERENCES ===="
148
colordiff --fakeexitcode "${PREV_FILE}" "${KNOWN_HOST_FILE}.new"
149
echo "====================="
150
echo "Going from ${PREV_COUNT} to $(wc -l "​$​{​KNOWN_HOST_FILE​}.​new​") known hosts and services"
151
152
if [[ -f "${KNOWN_HOST_FILE}" ]]; then
153
 mv -vf "${KNOWN_HOST_FILE}" "${KNOWN_HOST_FILE}.old"
154
 echo "Backup file is ${KNOWN_HOST_FILE}.old"
155
fi
156
mv -v "${KNOWN_HOST_FILE}.new" "${KNOWN_HOST_FILE}"
157
echo "New file generated at ${KNOWN_HOST_FILE}"
158
John Bondac002dc2021-02-23 17:05:33 +0100[diff] [blame]
159
if ! grep -Ecq "UserKnownHostsFile .*/wmf-prod( |$)" "${HOME}/.ssh/config"; then
Moritz Mühlenhoff14759462020-04-07 12:02:30 +0200[diff] [blame]
160
 echo "WARNING: You may need to add/update 'UserKnownHostsFile ${KNOWN_HOST_FILE}' to your ~/.ssh/config"
161
fi
162
163
if [[ "${SHELL}" == '/usr/bin/zsh' ]]; then
164
 echo 'Add this line to your .zshrc to tab-complete remote hosts:'
165
 echo "zstyle ':completion:*:hosts' known-hosts-files ${HOME}/.ssh/known_hosts ${KNOWN_HOST_FILE}"
166
fi
167
Moritz Mühlenhoff221df3d2021-02-12 09:44:40 +0100[diff] [blame]
168
exit 0
Powered by Gitiles
txt
json