gerrit.wikimedia.org
Gitiles
Code ReviewSign In
gerrit.wikimedia.org / operations / debs / wmf-sre-laptop / refs/heads/master / . / scripts / wmf-update-known-hosts-production
blob: 864b15c4ff33a219bc28abebd044ae4b6384c79d [file] [log] [blame]
Moritz Mühlenhoff14759462020-04-07 12:02:30 +0200[diff] [blame]
1
#!/bin/bash
2
3
##############################################################################
4
# WMF Update production known hosts
5
#
6
# DESCRIPTION:
7
# - Populate a known_hosts file with all the production hosts and services
8
# in the Wikimedia Foundation production infrastructure for easy
9
# autocompletion while keeping StrictHostKeyChecking active:
John Bondac002dc2021-02-23 17:05:33 +0100[diff] [blame]
10
# - sync all the known hosts from https://config-master.wikimedia.org/known_hosts.ecdsa
Moritz Mühlenhoff14759462020-04-07 12:02:30 +0200[diff] [blame]
11
# - clean the hostname without FQDN in it
12
# - optionally generate known hosts for services defined as CNAMEs in the
13
# DNS repository, see PARAMS below. This allows for the autocompletion of
14
# active/passive services like icinga.wikimedia.org.
15
# - Silently ignore all CNAMEs to the main DYNA record (dyna.wikimedia.org.)
16
# - Keeps a backup file with the previous known hosts
17
# - Show a diff between the new known hosts and the current ones
18
#
19
# It saves the known hosts into KNOWN_HOST_FILE, adjust this and/or the
20
# UserKnownHostsFile parameter in your ~/.ssh/config in order for them to
21
# match. A warning will be shown if they don't match.
22
#
Moritz Mühlenhoff14759462020-04-07 12:02:30 +0200[diff] [blame]
23
# PARAMS:
24
# It accept one positional argument that, if specified, must be the path to
25
# a local clone of the Operations DNS repository, (either from Gerrit or from
26
# GitHub):
27
# https://gerrit.wikimedia.org/r/operations/dns
28
# In this case also the services defined as CNAMEs in the wikimedia.org and
29
# wmnet zone files will be added with the identity of the target host, if
30
# that is found in the known_hosts file, skipping the missing ones.
31
#
32
# USAGE:
33
# wmf-update-prod-known-hosts [PATH_TO_DNS_REPOSITORY]
34
#
35
# Author: Riccardo Coccioli <rcoccioli@wikimedia.org>
36
# Date: 2017-06-21
37
# Last update: 2019-11-05
38
# Dependencies: colordiff
John Bondac002dc2021-02-23 17:05:33 +0100[diff] [blame]
39
# Version: 1.3
Moritz Mühlenhoff14759462020-04-07 12:02:30 +0200[diff] [blame]
40
# License: GPLv3+
41
##############################################################################
42
43
set -e
44
45
DNS_REPO_PATH="${1}"
46
KNOWN_HOSTS_PATH​=​"${HOME}/.ssh/known_hosts.d"
47
KNOWN_HOST_FILE​=​"${KNOWN_HOSTS_PATH}/wmf-prod"
John Bondac002dc2021-02-23 17:05:33 +0100[diff] [blame]
48
KNOWN_HOST_URL​=​"https://config-master.wikimedia.org/known_hosts.ecdsa"
Moritz Mühlenhoff14759462020-04-07 12:02:30 +0200[diff] [blame]
49
MAIN_DYNA_RECORD​=​"dyna.wikimedia.org."
50
51
if [[ ! -d "${KNOWN_HOSTS_PATH}" ]]; then
52
echo "ERROR: KNOWN_HOSTS_PATH '${KNOWN_HOSTS_PATH}' is not a directory, you might want to adjust the constant in the script or create it"
53
exit 1
54
fi
55
56
if [[ -n "${DNS_REPO_PATH}" ]]; then
57
if [[ ! -d "${DNS_REPO_PATH}" ]]; then
58
echo "ERROR: DNS_REPO_PATH '${DNS_REPO_PATH}' is not a directory"
59
exit 2
60
fi
John Bondac002dc2021-02-23 17:05:33 +0100[diff] [blame]
61
if ! git -C "${DNS_REPO_PATH}" remote -v | grep -E '(gerrit.wikimedia.org|github.com\/wikimedia)' | grep -cq 'operations[/-]dns'; then
Moritz Mühlenhoff14759462020-04-07 12:02:30 +0200[diff] [blame]
62
echo "ERROR: DNS_REPO_PATH '${DNS_REPO_PATH}' doesn't seems to be a checkout of the operations/dns repository"
63
exit 3
64
fi
65
fi
66
67
function parse_line() {
68
local line="${1}"
69
local domain="${2}"
70
local name
71
local target
72
local found
73
74
name="$(echo "${line}" | cut -d' ' -f1)"
75
target="$(echo "${line}" | cut -d' ' -f2)"
76
77
if [[ "${target}" == "${MAIN_DYNA_RECORD}" ]]; then
78
# Silently ignore CNAMEs to the MAIN_DYNA_RECORD
79
return
80
fi
81
John Bondac002dc2021-02-23 17:05:33 +0100[diff] [blame]
82
sep="\\."
Moritz Mühlenhoff14759462020-04-07 12:02:30 +0200[diff] [blame]
83
if [[ "${target: -1}" == '.' ]]; then
84
target="${target%?}"
85
sep=","
86
fi
87
88
set +e
89
found=$(grep -c "^${target}${sep}" "${KNOWN_HOST_FILE}.new"​)
90
set -e
91
if [[ "${found}" -eq "0" || "${found}" -gt "1" ]]; then
92
>&2 echo "Skipping '${target}' CNAME target, found ${found}/1 matches"
93
return
94
fi
95
96
grep "^${target}${sep}" "${KNOWN_HOST_FILE}.new" | awk -v name="${name}" -v domain="${domain}" '{ printf name"."domain; for (i = 2; i <= NF; i++) printf FS$i; print NL }'
97
}
98
99
function extract_cnames_from_zone() {
100
local zone_file
101
local origin
102
local boundaries
103
local start
104
local end
105
local domain
106
107
zone_file="${1}"
108
if [[ ! -f "${zone_file}" ]]; then
109
>&2 echo "Unable to find zone file ${zone_file}, skipping..."
110
return
111
fi
112
113
origin="${2}"
114
if [[ -n "${origin}" ]]; then
John Bondac002dc2021-02-23 17:05:33 +0100[diff] [blame]
115
boundaries="$(grep -n "\$ORIGIN" "${zone_file}" | grep -A 1 "\$ORIGIN ${origin}\\.$")"
Moritz Mühlenhoff14759462020-04-07 12:02:30 +0200[diff] [blame]
116
start=$(echo "${boundaries}" | head -n1 | cut -d':' -f1)
117
end=$(echo "${boundaries}" | tail -n1 | cut -d':' -f1)
118
domain="${origin}"
119
120
head -n "${end}" "${zone_file}" | tail -n "$((end - start))" | awk '/ CNAME / { print $1, $5 }' | while read -r line; do
121
parse_line "${line}" "${domain}" >> "${KNOWN_HOST_FILE}.new"
122
done
123
else
124
domain="$(basename "${zone_file}")"
125
awk '/ CNAME / { print $1, $5 }' "${zone_file}" | while read -r line; do
126
parse_line "${line}" "${domain}" >> "${KNOWN_HOST_FILE}.new"
127
done
128
fi
129
}
130
131
# Get new known hosts
John Bondac002dc2021-02-23 17:05:33 +0100[diff] [blame]
132
echo "===> Fetching from ${KNOWN_HOST_URL}"
133
curl "${KNOWN_HOST_URL}" -o "${KNOWN_HOST_FILE}.new"
Moritz Mühlenhoff14759462020-04-07 12:02:30 +0200[diff] [blame]
134
135
if [[ -n "${DNS_REPO_PATH}" ]]; then
136
extract_cnames_from_zone "${DNS_REPO_PATH}/templates/wikimedia.org"
137
extract_cnames_from_zone "${DNS_REPO_PATH}/templates/wmnet" "eqiad.wmnet"
138
fi
139
140
PREV_COUNT=0
141
PREV_FILE=/dev/null
142
if [[ -f "${KNOWN_HOST_FILE}" ]]; then
143
PREV_COUNT="$(wc -l "${KNOWN_HOST_FILE}")"
144
PREV_FILE​=​"${KNOWN_HOST_FILE}"
145
fi
146
147
echo "==== DIFFERENCES ===="
148
colordiff --fakeexitcode "${PREV_FILE}" "${KNOWN_HOST_FILE}.new"
149
echo "====================="
150
echo "Going from ${PREV_COUNT} to $(wc -l "​$​{​KNOWN_HOST_FILE​}.​new​") known hosts and services"
151
152
if [[ -f "${KNOWN_HOST_FILE}" ]]; then
153
mv -vf "${KNOWN_HOST_FILE}" "${KNOWN_HOST_FILE}.old"
154
echo "Backup file is ${KNOWN_HOST_FILE}.old"
155
fi
156
mv -v "${KNOWN_HOST_FILE}.new" "${KNOWN_HOST_FILE}"
157
echo "New file generated at ${KNOWN_HOST_FILE}"
158
John Bondac002dc2021-02-23 17:05:33 +0100[diff] [blame]
159
if ! grep -Ecq "UserKnownHostsFile .*/wmf-prod( |$)" "${HOME}/.ssh/config"; then
Moritz Mühlenhoff14759462020-04-07 12:02:30 +0200[diff] [blame]
160
echo "WARNING: You may need to add/update 'UserKnownHostsFile ${KNOWN_HOST_FILE}' to your ~/.ssh/config"
161
fi
162
163
if [[ "${SHELL}" == '/usr/bin/zsh' ]]; then
164
echo 'Add this line to your .zshrc to tab-complete remote hosts:'
165
echo "zstyle ':completion:*:hosts' known-hosts-files ${HOME}/.ssh/known_hosts ${KNOWN_HOST_FILE}"
166
fi
167
Moritz Mühlenhoff221df3d2021-02-12 09:44:40 +0100[diff] [blame]
168
exit 0
Powered by Gitiles
txt
json