#!/usr/bin/env bash # This basically just takes all of the instructions from DigitalOcean's "How To Set Up an OpenVPN Server on Ubuntu 16.04" and puts them into a single script. # Define functions. function exportReplace { ek="export KEY_${1}=" sed -i "s/${ek}\".*\"/${ek}\"${2}\"/g" vars } function getPublicInterface { pubInterface="$(ip route | grep default)" pubInterface="$(echo $pubInterface | grep -oP -m 1 "dev .*?( |$)" | head -1)" pubInterfaceLen=${#pubInterface}; if [ "${pubInterface:$pubInterfaceLen:1}" = " " ]; then ((pubInterfaceLen--)) fi ((pubInterfaceLen-=4)) pubInterface=${pubInterface:4:$pubInterfaceLen} echo $pubInterface } function getIPAddress { ip route get 1.1.1.1 | awk '{print $NF; exit}' } function getAdditionToBeforeRules { pubInterface="$(getPublicInterface)" echoStr="" echoStr=$echoStr"\n# START OPENVPN RULES" echoStr=$echoStr"\n# NAT table rules" echoStr=$echoStr"\n*nat" echoStr=$echoStr"\n:POSTROUTING ACCEPT [0:0]" echoStr=$echoStr"\n# Allow traffic from OpenVPN client to ${pubInterface}" echoStr=$echoStr"\n-A POSTROUTING -s 10.8.0.0/8 -o ${pubInterface} -j MASQUERADE" echoStr=$echoStr"\nCOMMIT" echoStr=$echoStr"\n# END OPENVPN Rules" echoStr=$echoStr"\n" echoStr=$echoStr"\n" echo -e $echoStr } # Check root privileges. if [[ $(id -u) -ne 0 ]] ; then echo "This script must be run as root." exit 1 fi # Step 1: Install OpenVPN apt-get update apt-get install -y openvpn easy-rsa # Step 2: Set up the CA directory and switch to it. make-cadir ~/openvpn-ca cd ~/openvpn-ca # Step 3: Configure the CA Variables # Get information from user to customize vars. printf "\n\nSetup will need to customize the VPN server. Some information will need" printf "\nto be collected." printf "\nDon't use quotation marks in any of these. It'll mess stuff up." printf "\nDon't leave anything blank." printf "\n\nPlease enter email address:" read emailAddress printf "\n\nPlease enter country code (i.e., \"US\",\"CA\", etc):" read countryCode printf "\n\nPlease enter state or province code (i.e., \"NY\",\"MI\"):" read provinceCode printf "\n\nPlease enter city name (but remove spaces, i.e., SanFrancisco):" read cityName printf "\n\nPlease enter organization name (could be The Illuminati for all I care):" read orgName printf "\n\nPlease enter organizational unit (Seriously, what does that even mean?):" read OUName printf "\n\nIn a short while, another program is going to ask you to confirm these." printf "\nIt will do this multiple times. You can just press Enter on all of them." printf "\nto confirm." printf "\nIMPORTANT! One of the setup programs will ask you to enter a" printf "\n\"challenge\" password. Leave it blank and just hit [Enter]." #printf "\nHowever, when asked to create a PEM password, be sure to use a strong one." printf "\nWhen a setup program asks you a y/n question, respond with \"y\"." printf "\nPress [Enter] now to continue with install." read -p "" # Customize vars file and run it. exportReplace "COUNTRY" $countryCode exportReplace "PROVINCE" $provinceCode exportReplace "CITY" $cityName exportReplace "ORG" $orgName exportReplace "EMAIL" $emailAddress exportReplace "OU" $OUName exportReplace "NAME" "server" # Step 4: Build the Certificate Authority source vars ./clean-all ./build-ca # Step 5: Create the Server Certificate, Key, and Encryption Files ./build-key-server server ./build-dh openvpn --genkey --secret keys/ta.key # Step 6: Generate a Client Certificate and Key Pair source vars ./build-key client1 # Step 7: Configure the OpenVPN Service # Copy the Files to the OpenVPN Directory cd ~/openvpn-ca/keys sudo cp ca.crt ca.key server.crt server.key ta.key dh2048.pem /etc/openvpn gunzip -c /usr/share/doc/openvpn/examples/sample-config-files/server.conf.gz | sudo tee /etc/openvpn/server.conf # Adjust the OpenVPN Configuration confFile='/etc/openvpn/server.conf' #Note! This var is used later in the script, not just in this section. subString="tls-auth ta.key 0 # This file is secret" oldString=";${subString}" newString="${subString}\nkey-direction 0" sed -i "s/${oldString}/${newString}/g" $confFile subString="cipher AES-128-CBC" oldString=";${subString}" newString="${subString}\nauth SHA256" sed -i "s/${oldString}/${newString}/g" $confFile subString="user nobody" oldString=";${subString}" newString="${subString}" sed -i "s/${oldString}/${newString}/g" $confFile subString="group nogroup" oldString=";${subString}" newString="${subString}" sed -i "s/${oldString}/${newString}/g" $confFile subString="push \"redirect-gateway def1 bypass-dhcp\"" oldString=";${subString}" newString="${subString}" sed -i "s/${oldString}/${newString}/g" $confFile subString='push "dhcp-option DNS 208.67.222.222"' oldString=";${subString//./\\.}" newString="${subString}" sed -i "s/${oldString}/${newString}/g" $confFile subString="push \"dhcp-option DNS 208.67.220.220\"" oldString=";${subString//./\\.}" newString="${subString}" sed -i "s/${oldString}/${newString}/g" $confFile # Step 8: Adjust the Server Networking Configuration. # Allow IP Forwarding subString="net.ipv4.ip_forward=1" oldString="#${subString//./\\.}" newString="${subString}" sed -i "s/${oldString}/${newString}/g" /etc/sysctl.conf sudo sysctl -p # Adjust the UFW Rules to Masquerade Client Connections fileName="/etc/ufw/before.rules" newRules="$(getAdditionToBeforeRules)" fileContents="$(awk -v "n=10" -v "s=${newRules}" '(NR==n) { print s } 1' ${fileName})" echo "${fileContents}" > $fileName oldString="DEFAULT_FORWARD_POLICY=\"DROP\"" newString="DEFAULT_FORWARD_POLICY=\"ACCEPT\"" sed -i "s/${oldString}/${newString}/g" /etc/default/ufw # Open the OpenVPN Port and Enable the Changes ufw allow 1194/udp ufw allow OpenSSH ufw disable ufw enable # Step 9: Start and Enable the OpenVPN Service systemctl start openvpn@server systemctl enable openvpn@server # Step 10: Create Client Configuration Infrastructure # Creating the Client Config Directory Structure mkdir -p ~/client-configs/files chmod 700 ~/client-configs/files # Creating a Base Configuration fileName="${HOME}/client-configs/base.conf" cp /usr/share/doc/openvpn/examples/sample-config-files/client.conf $fileName ipAddress="$(getIPAddress)" sed -i "s/my-server-1/${ipAddress}/g" $fileName subString="user" oldString=";${subString}" newString="${subString}" sed -i "s/${oldString}/${newString}/g" $fileName subString="group" oldString=";${subString}" newString="${subString}" sed -i "s/${oldString}/${newString}/g" $fileName subString="ca ca.crt" oldString="${subString/\./\\.}" newString="#${subString}" sed -i "s/${oldString}/${newString}/g" $fileName subString="cert client.crt" oldString="${subString/\./\\.}" newString="#${subString}" sed -i "s/${oldString}/${newString}/g" $fileName subString="key client.key" oldString="${subString/\./\\.}" newString="#${subString}" sed -i "s/${oldString}/${newString}/g" $fileName echo "">> "${fileName}" echo "cipher AES-128-CBC">> "${fileName}" echo "auth SHA256" >> "${fileName}" echo "key-direction 1" >> "${fileName}" echo "#script-security 2" >> "${fileName}" echo "#up /etc/openvpn/update-resolv-conf" >> "${fileName}" echo "#down /etc/openvpn/update-resolv-conf" >> "${fileName}" # Creating a Configuration Generation Script makeConfigStr=" # First argument: Client identifier KEY_DIR=~/openvpn-ca/keys OUTPUT_DIR=~/client-configs/files BASE_CONFIG=~/client-configs/base.conf cat \${BASE_CONFIG} \\ <(echo -e '<ca>') \\ \${KEY_DIR}/ca.crt \\ <(echo -e '</ca>\n<cert>') \\ \${KEY_DIR}/\${1}.crt \\ <(echo -e '</cert>\n<key>') \\ \${KEY_DIR}/\${1}.key \\ <(echo -e '</key>\n<tls-auth>') \\ \${KEY_DIR}/ta.key \\ <(echo -e '</tls-auth>') \\ > \${OUTPUT_DIR}/\${1}.ovpn " echo "#!/bin/bash" > "${HOME}/client-configs/make_config.sh" echo "${makeConfigStr}" >> "${HOME}/client-configs/make_config.sh" chmod 700 ~/client-configs/make_config.sh # Step 11: Generate Client Configurations cd ~/client-configs ./make_config.sh client1 # Copy VPN configuration to /root, so it's the same path for everyone. cp ~/client-configs/files/client1.ovpn /root

#!/usr/bin/env bash PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:~/bin export PATH #=================================================================# # System Required: CentOS 6+, Debian 7+, Ubuntu 12+ # # Description: One click Install Shadowsocks-Python server # # Author: Teddysun <i@teddysun.com> # # Thanks: @clowwindy <https://twitter.com/clowwindy> # # Intro: https://teddysun.com/342.html # #=================================================================# clear echo echo "#############################################################" echo "# One click Install Shadowsocks-Python server #" echo "# Intro: https://teddysun.com/342.html #" echo "# Author: Teddysun <i@teddysun.com> #" echo "# Github: https://github.com/shadowsocks/shadowsocks #" echo "#############################################################" echo libsodium_file="libsodium-1.0.16" libsodium_url="https://github.com/jedisct1/libsodium/releases/download/1.0.16/libsodium-1.0.16.tar.gz" # Current folder cur_dir=`pwd` # Stream Ciphers ciphers=( aes-256-gcm aes-192-gcm aes-128-gcm aes-256-ctr aes-192-ctr aes-128-ctr aes-256-cfb aes-192-cfb aes-128-cfb camellia-128-cfb camellia-192-cfb camellia-256-cfb chacha20-ietf-poly1305 chacha20-ietf chacha20 rc4-md5 ) # Color red='\033[0;31m' green='\033[0;32m' yellow='\033[0;33m' plain='\033[0m' # Make sure only root can run our script [[ $EUID -ne 0 ]] && echo -e "[${red}Error${plain}] This script must be run as root!" && exit 1 # Disable selinux disable_selinux(){ if [ -s /etc/selinux/config ] && grep 'SELINUX=enforcing' /etc/selinux/config; then sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config setenforce 0 fi } #Check system check_sys(){ local checkType=$1 local value=$2 local release='' local systemPackage='' if [[ -f /etc/redhat-release ]]; then release="centos" systemPackage="yum" elif cat /etc/issue | grep -Eqi "debian"; then release="debian" systemPackage="apt" elif cat /etc/issue | grep -Eqi "ubuntu"; then release="ubuntu" systemPackage="apt" elif cat /etc/issue | grep -Eqi "centos|red hat|redhat"; then release="centos" systemPackage="yum" elif cat /proc/version | grep -Eqi "debian"; then release="debian" systemPackage="apt" elif cat /proc/version | grep -Eqi "ubuntu"; then release="ubuntu" systemPackage="apt" elif cat /proc/version | grep -Eqi "centos|red hat|redhat"; then release="centos" systemPackage="yum" fi if [[ ${checkType} == "sysRelease" ]]; then if [ "$value" == "$release" ]; then return 0 else return 1 fi elif [[ ${checkType} == "packageManager" ]]; then if [ "$value" == "$systemPackage" ]; then return 0 else return 1 fi fi } # Get version getversion(){ if [[ -s /etc/redhat-release ]]; then grep -oE "[0-9.]+" /etc/redhat-release else grep -oE "[0-9.]+" /etc/issue fi } # CentOS version centosversion(){ if check_sys sysRelease centos; then local code=$1 local version="$(getversion)" local main_ver=${version%%.*} if [ "$main_ver" == "$code" ]; then return 0 else return 1 fi else return 1 fi } # Get public IP address get_ip(){ local IP=$( ip addr | egrep -o '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' | egrep -v "^192\.168|^172\.1[6-9]\.|^172\.2[0-9]\.|^172\.3[0-2]\.|^10\.|^127\.|^255\.|^0\." | head -n 1 ) [ -z ${IP} ] && IP=$( wget -qO- -t1 -T2 ipv4.icanhazip.com ) [ -z ${IP} ] && IP=$( wget -qO- -t1 -T2 ipinfo.io/ip ) [ ! -z ${IP} ] && echo ${IP} || echo } get_char(){ SAVEDSTTY=`stty -g` stty -echo stty cbreak dd if=/dev/tty bs=1 count=1 2> /dev/null stty -raw stty echo stty $SAVEDSTTY } # Pre-installation settings pre_install(){ if check_sys packageManager yum || check_sys packageManager apt; then # Not support CentOS 5 if centosversion 5; then echo -e "$[{red}Error${plain}] Not supported CentOS 5, please change to CentOS 6+/Debian 7+/Ubuntu 12+ and try again." exit 1 fi else echo -e "[${red}Error${plain}] Your OS is not supported. please change OS to CentOS/Debian/Ubuntu and try again." exit 1 fi # Set shadowsocks config password echo "Please enter password for shadowsocks-python" read -p "(Default password: teddysun.com):" shadowsockspwd [ -z "${shadowsockspwd}" ] && shadowsockspwd="teddysun.com" echo echo "---------------------------" echo "password = ${shadowsockspwd}" echo "---------------------------" echo # Set shadowsocks config port while true do dport=$(shuf -i 9000-19999 -n 1) echo "Please enter a port for shadowsocks-python [1-65535]" read -p "(Default port: ${dport}):" shadowsocksport [ -z "$shadowsocksport" ] && shadowsocksport=${dport} expr ${shadowsocksport} + 1 &>/dev/null if [ $? -eq 0 ]; then if [ ${shadowsocksport} -ge 1 ] && [ ${shadowsocksport} -le 65535 ] && [ ${shadowsocksport:0:1} != 0 ]; then echo echo "---------------------------" echo "port = ${shadowsocksport}" echo "---------------------------" echo break fi fi echo -e "[${red}Error${plain}] Please enter a correct number [1-65535]" done # Set shadowsocks config stream ciphers while true do echo -e "Please select stream cipher for shadowsocks-python:" for ((i=1;i<=${#ciphers[@]};i++ )); do hint="${ciphers[$i-1]}" echo -e "${green}${i}${plain}) ${hint}" done read -p "Which cipher you'd select(Default: ${ciphers[0]}):" pick [ -z "$pick" ] && pick=1 expr ${pick} + 1 &>/dev/null if [ $? -ne 0 ]; then echo -e "[${red}Error${plain}] Please enter a number" continue fi if [[ "$pick" -lt 1 || "$pick" -gt ${#ciphers[@]} ]]; then echo -e "[${red}Error${plain}] Please enter a number between 1 and ${#ciphers[@]}" continue fi shadowsockscipher=${ciphers[$pick-1]} echo echo "---------------------------" echo "cipher = ${shadowsockscipher}" echo "---------------------------" echo break done echo echo "Press any key to start...or Press Ctrl+C to cancel" char=`get_char` # Install necessary dependencies if check_sys packageManager yum; then yum install -y python python-devel python-setuptools openssl openssl-devel curl wget unzip gcc automake autoconf make libtool elif check_sys packageManager apt; then apt-get -y update apt-get -y install python python-dev python-setuptools openssl libssl-dev curl wget unzip gcc automake autoconf make libtool fi cd ${cur_dir} } # Download files download_files(){ # Download libsodium file if ! wget --no-check-certificate -O ${libsodium_file}.tar.gz ${libsodium_url}; then echo -e "[${red}Error${plain}] Failed to download ${libsodium_file}.tar.gz!" exit 1 fi # Download Shadowsocks file if ! wget --no-check-certificate -O shadowsocks-master.zip https://github.com/shadowsocks/shadowsocks/archive/master.zip; then echo -e "[${red}Error${plain}] Failed to download shadowsocks python file!" exit 1 fi # Download Shadowsocks init script if check_sys packageManager yum; then if ! wget --no-check-certificate https://raw.githubusercontent.com/teddysun/shadowsocks_install/master/shadowsocks -O /etc/init.d/shadowsocks; then echo -e "[${red}Error${plain}] Failed to download shadowsocks chkconfig file!" exit 1 fi elif check_sys packageManager apt; then if ! wget --no-check-certificate https://raw.githubusercontent.com/teddysun/shadowsocks_install/master/shadowsocks-debian -O /etc/init.d/shadowsocks; then echo -e "[${red}Error${plain}] Failed to download shadowsocks chkconfig file!" exit 1 fi fi } # Config shadowsocks config_shadowsocks(){ cat > /etc/shadowsocks.json<<-EOF { "server":"0.0.0.0", "server_port":${shadowsocksport}, "local_address":"127.0.0.1", "local_port":1080, "password":"${shadowsockspwd}", "timeout":300, "method":"${shadowsockscipher}", "fast_open":false } EOF } # Firewall set firewall_set(){ echo -e "[${green}Info${plain}] firewall set start..." if centosversion 6; then /etc/init.d/iptables status > /dev/null 2>&1 if [ $? -eq 0 ]; then iptables -L -n | grep -i ${shadowsocksport} > /dev/null 2>&1 if [ $? -ne 0 ]; then iptables -I INPUT -m state --state NEW -m tcp -p tcp --dport ${shadowsocksport} -j ACCEPT iptables -I INPUT -m state --state NEW -m udp -p udp --dport ${shadowsocksport} -j ACCEPT /etc/init.d/iptables save /etc/init.d/iptables restart else echo -e "[${green}Info${plain}] port ${shadowsocksport} has already been set up." fi else echo -e "[${yellow}Warning${plain}] iptables looks like shutdown or not installed, please manually set it if necessary." fi elif centosversion 7; then systemctl status firewalld > /dev/null 2>&1 if [ $? -eq 0 ]; then firewall-cmd --permanent --zone=public --add-port=${shadowsocksport}/tcp firewall-cmd --permanent --zone=public --add-port=${shadowsocksport}/udp firewall-cmd --reload else echo -e "[${yellow}Warning${plain}] firewalld looks like not running or not installed, please enable port ${shadowsocksport} manually if necessary." fi fi echo -e "[${green}Info${plain}] firewall set completed..." } # Install Shadowsocks install(){ # Install libsodium if [ ! -f /usr/lib/libsodium.a ]; then cd ${cur_dir} tar zxf ${libsodium_file}.tar.gz cd ${libsodium_file} ./configure --prefix=/usr && make && make install if [ $? -ne 0 ]; then echo -e "[${red}Error${plain}] libsodium install failed!" install_cleanup exit 1 fi fi ldconfig # Install Shadowsocks cd ${cur_dir} unzip -q shadowsocks-master.zip if [ $? -ne 0 ];then echo -e "[${red}Error${plain}] unzip shadowsocks-master.zip failed! please check unzip command." install_cleanup exit 1 fi cd ${cur_dir}/shadowsocks-master python setup.py install --record /usr/local/shadowsocks_install.log if [ -f /usr/bin/ssserver ] || [ -f /usr/local/bin/ssserver ]; then chmod +x /etc/init.d/shadowsocks if check_sys packageManager yum; then chkconfig --add shadowsocks chkconfig shadowsocks on elif check_sys packageManager apt; then update-rc.d -f shadowsocks defaults fi /etc/init.d/shadowsocks start else echo echo -e "[${red}Error${plain}] Shadowsocks install failed! please visit https://teddysun.com/342.html and contact." install_cleanup exit 1 fi clear echo echo -e "Congratulations, Shadowsocks-python server install completed!" echo -e "Your Server IP : \033[41;37m $(get_ip) \033[0m" echo -e "Your Server Port : \033[41;37m ${shadowsocksport} \033[0m" echo -e "Your Password : \033[41;37m ${shadowsockspwd} \033[0m" echo -e "Your Encryption Method: \033[41;37m ${shadowsockscipher} \033[0m" echo echo "Welcome to visit:https://teddysun.com/342.html" echo "Enjoy it!" echo } # Install cleanup install_cleanup(){ cd ${cur_dir} rm -rf shadowsocks-master.zip shadowsocks-master ${libsodium_file}.tar.gz ${libsodium_file} } # Uninstall Shadowsocks uninstall_shadowsocks(){ printf "Are you sure uninstall Shadowsocks? (y/n) " printf "\n" read -p "(Default: n):" answer [ -z ${answer} ] && answer="n" if [ "${answer}" == "y" ] || [ "${answer}" == "Y" ]; then ps -ef | grep -v grep | grep -i "ssserver" > /dev/null 2>&1 if [ $? -eq 0 ]; then /etc/init.d/shadowsocks stop fi if check_sys packageManager yum; then chkconfig --del shadowsocks elif check_sys packageManager apt; then update-rc.d -f shadowsocks remove fi # delete config file rm -f /etc/shadowsocks.json rm -f /var/run/shadowsocks.pid rm -f /etc/init.d/shadowsocks rm -f /var/log/shadowsocks.log if [ -f /usr/local/shadowsocks_install.log ]; then cat /usr/local/shadowsocks_install.log | xargs rm -rf fi echo "Shadowsocks uninstall success!" else echo echo "uninstall cancelled, nothing to do..." echo fi } # Install Shadowsocks-python install_shadowsocks(){ disable_selinux pre_install download_files config_shadowsocks if check_sys packageManager yum; then firewall_set fi install install_cleanup } # Initialization step action=$1 [ -z $1 ] && action=install case "$action" in install|uninstall) ${action}_shadowsocks ;; *) echo "Arguments error! [${action}]" echo "Usage: `basename $0` [install|uninstall]" ;; esac

#!/usr/bin/env bash #/ Usage: ghe-backup-repositories #/ Take an online, incremental snapshot of all Git repository data. #/ #/ Note: This command typically isn't called directly. It's invoked by #/ ghe-backup. set -e # This command is designed to allow for transferring active Git repository data # from a GitHub instance to a backup site in a way that ensures data is # captured in a consistent state even when being written to. # # - All Git GC operations are disabled on the GitHub instance for the duration of # the backup. This removes the possibly of objects or packs being removed # while the backup is in progress. # # - In progress Git GC operations are given a cooldown window to complete. The # script will sleep for up to 60 seconds waiting for GC operations to finish. # # - Git repository data is transferred in a specific order: auxiliary files, # packed refs, loose refs, reflogs, and finally objects and pack files in that # order. This ensures that all referenced objects are captured. # # - Git GC operations are re-enabled on the GitHub instance. # # The script uses multiple runs of rsync to transfer repository files. Each run # includes a list of filter rules that ensure only specific types of files are # transferred. # # See the "FILTER RULES" and "INCLUDE/EXCLUDE PATTERN RULES" sections of the # rsync(1) manual for more information: # <http://rsync.samba.org/ftp/rsync/rsync.html> # Bring in the backup configuration # shellcheck source=share/github-backup-utils/ghe-backup-config . "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" bm_start "$(basename $0)" # Set up remote host and root backup snapshot directory based on config host="$GHE_HOSTNAME" backup_dir="$GHE_SNAPSHOT_DIR/repositories" # Location of last good backup for rsync --link-dest backup_current="$GHE_DATA_DIR/current/repositories" # Verify rsync is available. if ! rsync --version 1>/dev/null 2>&1; then echo "Error: rsync not found." 1>&2 exit 1 fi # Perform a host-check and establish GHE_REMOTE_XXX variables. ghe_remote_version_required "$host" # Split host:port into parts port=$(ssh_port_part "$GHE_HOSTNAME") host=$(ssh_host_part "$GHE_HOSTNAME") # Add user / -l option user="${host%@*}" [ "$user" = "$host" ] && user="admin" hostnames=$host ssh_config_file_opt= tempdir=$(mktemp -d -t backup-utils-backup-XXXXXX) remote_tempdir=$(ghe-ssh "$GHE_HOSTNAME" -- mktemp -d -t backup-utils-backup-XXXXXX) routes_list=$tempdir/routes_list remote_routes_list=$remote_tempdir/remote_routes_list opts="$GHE_EXTRA_SSH_OPTS" # git server hostnames under cluster if [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then ssh_config_file="$tempdir/ssh_config" ssh_config_file_opt="-F $ssh_config_file" opts="$opts -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no" hostnames=$(ghe-cluster-nodes "$GHE_HOSTNAME" "git-server") ghe-ssh-config "$GHE_HOSTNAME" "$hostnames" > "$ssh_config_file" fi # Make sure root backup dir exists if this is the first run mkdir -p "$backup_dir" # Removes the remote sync-in-progress file on exit, re-enabling GC operations # on the remote instance. cleanup() { for pid in $(jobs -p); do kill -KILL $pid > /dev/null 2>&1 || true done # Enable remote GC operations for hostname in $hostnames; do ghe-gc-enable $ssh_config_file_opt $hostname:$port || true done ghe-ssh "$GHE_HOSTNAME" -- rm -rf $remote_tempdir rm -rf $tempdir } trap 'cleanup' EXIT trap 'exit $?' INT # ^C always terminate # Disable remote GC operations for hostname in $hostnames; do ghe-gc-disable $ssh_config_file_opt $hostname:$port done # If we have a previous increment, avoid transferring existing files via rsync's # --link-dest support. This also decreases physical space usage considerably. if [ -d "$backup_current" ]; then link_dest="--link-dest=../../current/repositories" fi # Calculate sync routes. This will store the healthy repo paths for each node # # This gets a repo path and stores the path in the $node.sync file # a/nw/a8/3f/02/100000855 dgit-node1 >> dgit-node1.sync # a/nw/a8/bc/8d/100000880 dgit-node3 >> dgit-node3.sync # a/nw/a5/06/81/100000659 dgit-node2 >> dgit-node2.sync # ... # One route per line. # # NOTE: The route generation is performed on the appliance as it is considerably # more performant than performing over an SSH pipe. # bm_start "$(basename $0) - Generating routes" echo "github-env ./bin/dgit-cluster-backup-routes > $remote_routes_list" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash ghe-ssh "$GHE_HOSTNAME" -- cat $remote_routes_list | ghe_debug bm_end "$(basename $0) - Generating routes" bm_start "$(basename $0) - Fetching routes" ghe-ssh "$GHE_HOSTNAME" -- gzip -c $remote_routes_list | gzip -d > $routes_list cat $routes_list | ghe_debug bm_end "$(basename $0) - Fetching routes" bm_start "$(basename $0) - Processing routes" if [ "$GHE_BACKUP_STRATEGY" != "cluster" ]; then server=$host fi cat $routes_list | awk -v tempdir="$tempdir" -v server="$server" '{ for(i=2;i<=NF;i++){ server != "" ? host=server : host=$i; print $1 > (tempdir"/"host".rsync") }}' ghe_debug "\n$(find "$tempdir" -maxdepth 1 -name '*.rsync')" bm_end "$(basename $0) - Processing routes" if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then echo "Warning: no routes found, skipping repositories backup ..." exit 0 fi # Transfer repository data from a GitHub instance to the current snapshot # directory, using a previous snapshot to avoid transferring files that have # already been transferred. A set of rsync filter rules are provided on stdin # for each invocation. rsync_repository_data () { port=$(ssh_port_part "$1") host=$(ssh_host_part "$1") #check if we are syncing from a given file list if [[ "$2" == *".rsync" ]]; then files_list="$2" shift shift ghe-rsync -avr \ -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" \ $link_dest "$@" \ --rsync-path='sudo -u git rsync' \ --include-from=- --exclude=\* \ --files-from="$files_list" \ --ignore-missing-args \ "$host:$GHE_REMOTE_DATA_USER_DIR/repositories/" \ "$backup_dir" 1>&3 2>&3 else shift ghe-rsync -avr \ -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" \ $link_dest "$@" \ --rsync-path='sudo -u git rsync' \ --include-from=- --exclude=\* \ --ignore-missing-args \ "$host:$GHE_REMOTE_DATA_USER_DIR/repositories/" \ "$backup_dir" 1>&3 2>&3 fi } sync_data (){ # Sync all auxiliary repository data. This includes files and directories like # HEAD, audit_log, config, description, info/, etc. No refs or object data # should be transferred here. echo 1>&3 echo "* Transferring auxiliary files ..." 1>&3 rsync_repository_data $1:122 $2 -z <<RULES - /__*__/ - /info/ + /*/ + /*/*.git - /*/*.git/objects - /*/*.git/refs - /*/*.git/packed-refs - /*/*.git/logs + /*/*.git/** + /*/??/ + /*/??/??/ + /*/??/??/??/ + /*/??/??/??/gist/ + /*/??/??/??/gist/*.git - /*/??/??/??/gist/*.git/objects - /*/??/??/??/gist/*.git/refs - /*/??/??/??/gist/*.git/packed-refs - /*/??/??/??/gist/*.git/logs + /*/??/??/??/gist/*.git/** + /*/nw/??/??/??/ + /*/nw/??/??/??/*/ + /*/nw/??/??/??/*/*.git - /*/nw/??/??/??/*/*.git/objects - /*/nw/??/??/??/*/*.git/refs - /*/nw/??/??/??/*/*.git/packed-refs - /*/nw/??/??/??/*/*.git/logs + /*/nw/??/??/??/*/*.git/** RULES # Sync packed refs files. This is performed before sync'ing loose refs since # loose refs trump packed-refs information. echo 1>&3 echo "* Transferring packed-refs files ..." 1>&3 rsync_repository_data $1:122 $2 -z <<RULES - /__*__/ - /info/ + /*/ + /*/*.git + /*/*.git/packed-refs + /*/??/ + /*/??/??/ + /*/??/??/??/ + /*/??/??/??/gist/ + /*/??/??/??/gist/*.git + /*/??/??/??/gist/*.git/packed-refs + /*/nw/??/ + /*/nw/??/??/ + /*/nw/??/??/??/ + /*/nw/??/??/??/*/ + /*/nw/??/??/??/*/*.git + /*/nw/??/??/??/*/*.git/packed-refs RULES # Sync loose refs and reflogs. This must be performed before object data is # transferred to ensure that all referenced objects are included. echo 1>&3 echo "* Transferring refs and reflogs ..." 1>&3 rsync_repository_data $1:122 $2 -z <<RULES - /__*__/ - /info/ + /*/ + /*/*.git + /*/*.git/refs + /*/*.git/refs/** + /*/*.git/logs + /*/*.git/logs/** + /*/??/ + /*/??/??/ + /*/??/??/??/ + /*/??/??/??/gist/ + /*/??/??/??/gist/*.git + /*/??/??/??/gist/*.git/refs + /*/??/??/??/gist/*.git/refs/** + /*/??/??/??/gist/*.git/logs + /*/??/??/??/gist/*.git/logs/** + /*/nw/??/ + /*/nw/??/??/ + /*/nw/??/??/??/ + /*/nw/??/??/??/*/ + /*/nw/??/??/??/*/*.git + /*/nw/??/??/??/*/*.git/refs + /*/nw/??/??/??/*/*.git/refs/** + /*/nw/??/??/??/*/*.git/logs + /*/nw/??/??/??/*/*.git/logs/** RULES # Sync git objects and pack files. Compression is disabled during this phase # since these files are already well compressed. echo 1>&3 echo "* Transferring objects and packs ..." 1>&3 rsync_repository_data $1:122 $2 -H <<RULES - /__*__/ - /info/ + /*/ + /*/*.git + /*/*.git/objects - /*/*.git/objects/**/tmp_* + /*/*.git/objects/** + /*/??/ + /*/??/??/ + /*/??/??/??/ + /*/??/??/??/gist/ + /*/??/??/??/gist/*.git + /*/??/??/??/gist/*.git/objects - /*/??/??/??/gist/*.git/objects/**/tmp_* + /*/??/??/??/gist/*.git/objects/** + /*/nw/??/ + /*/nw/??/??/ + /*/nw/??/??/??/ + /*/nw/??/??/??/*/ + /*/nw/??/??/??/*/*.git + /*/nw/??/??/??/*/*.git/objects - /*/nw/??/??/??/*/*.git/objects/**/tmp_* + /*/nw/??/??/??/*/*.git/objects/** RULES echo 1>&3 } # rsync all the repositories bm_start "$(basename $0) - Repo sync" for file_list in $tempdir/*.rsync; do hostname=$(basename $file_list .rsync) repo_num=$(cat $file_list | wc -l) ghe_verbose "* Transferring $repo_num repositories from $hostname" sync_data $hostname $file_list & done for pid in $(jobs -p); do wait $pid done bm_end "$(basename $0) - Repo sync" # Since there are no routes for special data directories, we need to do this # serially for all hostnames. Good candidate for future optimizations. bm_start "$(basename $0) - Special Data Directories Sync" for h in $hostnames; do # Sync __special__ data directories, including the __alambic_assets__, # __hookshot__, and __purgatory__ directories. The __nodeload_archives__, # __gitmon__, and __render__ directories are excludes since they act only as # caches. # # Under v2.x and greater, only the special __purgatory__ directory remains under # /data/repositories. All other special user data directories have been moved under # the /data/user directory. echo 1>&3 echo "* Transferring special data directories from $h..." 1>&3 rsync_repository_data $h:122 -z <<RULES - /__nodeload_archives__/ - /__gitmon__/ - /__render__/ + /__*__/ + /__*__/** + /info/ - /info/lost+found/ + /info/* RULES echo 1>&3 done bm_end "$(basename $0) - Special Data Directories Sync" if [ -z "$GHE_SKIP_ROUTE_VERIFICATION" ]; then bm_start "$(basename $0) - Verifying Routes" cat $tempdir/*.rsync | uniq | sort | uniq > $tempdir/source_routes (cd $backup_dir/ && find * -mindepth 5 -maxdepth 6 -type d -name \*.git | fix_paths_for_ghe_version | uniq | sort | uniq) > $tempdir/destination_routes git --no-pager diff --unified=0 --no-prefix -- $tempdir/source_routes $tempdir/destination_routes || echo "Warning: One or more repository networks and/or gists were not found on the source appliance. Please contact GitHub Enterprise Support for assistance." bm_end "$(basename $0) - Verifying Routes" fi bm_end "$(basename $0)"

header – File Header Management Script header is a Bash utility that automatically adds or updates a standardized metadata header at the top of files (YAML, Bash, Python, configs, etc.).It’s designed to help teams maintain consistent file documentation and versioning, while preserving shebangs (#!/usr/bin/env bash) and file permissions.

#!/usr/bin/env bash #/ Usage: ghe-backup-repositories #/ Take an online, incremental snapshot of all Git repository data. #/ #/ Note: This command typically isn't called directly. It's invoked by #/ ghe-backup. set -e # This command is designed to allow for transferring active Git repository data # from a GitHub instance to a backup site in a way that ensures data is # captured in a consistent state even when being written to. # # - All Git GC operations are disabled on the GitHub instance for the duration of # the backup. This removes the possibly of objects or packs being removed # while the backup is in progress. # # - In progress Git GC operations are given a cooldown window to complete. The # script will sleep for up to 60 seconds waiting for GC operations to finish. # # - Git repository data is transferred in a specific order: auxiliary files, # packed refs, loose refs, reflogs, and finally objects and pack files in that # order. This ensures that all referenced objects are captured. # # - Git GC operations are re-enabled on the GitHub instance. # # The script uses multiple runs of rsync to transfer repository files. Each run # includes a list of filter rules that ensure only specific types of files are # transferred. # # See the "FILTER RULES" and "INCLUDE/EXCLUDE PATTERN RULES" sections of the # rsync(1) manual for more information: # <http://rsync.samba.org/ftp/rsync/rsync.html> # Bring in the backup configuration # shellcheck source=share/github-backup-utils/ghe-backup-config . "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" bm_start "$(basename $0)" # Set up remote host and root backup snapshot directory based on config host="$GHE_HOSTNAME" backup_dir="$GHE_SNAPSHOT_DIR/repositories" # Location of last good backup for rsync --link-dest backup_current="$GHE_DATA_DIR/current/repositories" # Verify rsync is available. if ! rsync --version 1>/dev/null 2>&1; then echo "Error: rsync not found." 1>&2 exit 1 fi # Perform a host-check and establish GHE_REMOTE_XXX variables. ghe_remote_version_required "$host" # Split host:port into parts port=$(ssh_port_part "$GHE_HOSTNAME") host=$(ssh_host_part "$GHE_HOSTNAME") # Add user / -l option user="${host%@*}" [ "$user" = "$host" ] && user="admin" hostnames=$host ssh_config_file_opt= tempdir=$(mktemp -d -t backup-utils-backup-XXXXXX) remote_tempdir=$(ghe-ssh "$GHE_HOSTNAME" -- mktemp -d -t backup-utils-backup-XXXXXX) routes_list=$tempdir/routes_list remote_routes_list=$remote_tempdir/remote_routes_list opts="$GHE_EXTRA_SSH_OPTS" # git server hostnames under cluster if [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then ssh_config_file="$tempdir/ssh_config" ssh_config_file_opt="-F $ssh_config_file" opts="$opts -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no" hostnames=$(ghe-cluster-nodes "$GHE_HOSTNAME" "git-server") ghe-ssh-config "$GHE_HOSTNAME" "$hostnames" > "$ssh_config_file" fi # Make sure root backup dir exists if this is the first run mkdir -p "$backup_dir" # Removes the remote sync-in-progress file on exit, re-enabling GC operations # on the remote instance. cleanup() { for pid in $(jobs -p); do kill -KILL $pid > /dev/null 2>&1 || true done # Enable remote GC operations for hostname in $hostnames; do ghe-gc-enable $ssh_config_file_opt $hostname:$port || true done ghe-ssh "$GHE_HOSTNAME" -- rm -rf $remote_tempdir rm -rf $tempdir } trap 'cleanup' EXIT trap 'exit $?' INT # ^C always terminate # Disable remote GC operations for hostname in $hostnames; do ghe-gc-disable $ssh_config_file_opt $hostname:$port done # If we have a previous increment, avoid transferring existing files via rsync's # --link-dest support. This also decreases physical space usage considerably. if [ -d "$backup_current" ]; then link_dest="--link-dest=../../current/repositories" fi # Calculate sync routes. This will store the healthy repo paths for each node # # This gets a repo path and stores the path in the $node.sync file # a/nw/a8/3f/02/100000855 dgit-node1 >> dgit-node1.sync # a/nw/a8/bc/8d/100000880 dgit-node3 >> dgit-node3.sync # a/nw/a5/06/81/100000659 dgit-node2 >> dgit-node2.sync # ... # One route per line. # # NOTE: The route generation is performed on the appliance as it is considerably # more performant than performing over an SSH pipe. # bm_start "$(basename $0) - Generating routes" echo "github-env ./bin/dgit-cluster-backup-routes > $remote_routes_list" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash ghe-ssh "$GHE_HOSTNAME" -- cat $remote_routes_list | ghe_debug bm_end "$(basename $0) - Generating routes" bm_start "$(basename $0) - Fetching routes" ghe-ssh "$GHE_HOSTNAME" -- gzip -c $remote_routes_list | gzip -d > $routes_list cat $routes_list | ghe_debug bm_end "$(basename $0) - Fetching routes" bm_start "$(basename $0) - Processing routes" if [ "$GHE_BACKUP_STRATEGY" != "cluster" ]; then server=$host fi cat $routes_list | awk -v tempdir="$tempdir" -v server="$server" '{ for(i=2;i<=NF;i++){ server != "" ? host=server : host=$i; print $1 > (tempdir"/"host".rsync") }}' ghe_debug "\n$(find "$tempdir" -maxdepth 1 -name '*.rsync')" bm_end "$(basename $0) - Processing routes" if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then echo "Warning: no routes found, skipping repositories backup ..." exit 0 fi # Transfer repository data from a GitHub instance to the current snapshot # directory, using a previous snapshot to avoid transferring files that have # already been transferred. A set of rsync filter rules are provided on stdin # for each invocation. rsync_repository_data () { port=$(ssh_port_part "$1") host=$(ssh_host_part "$1") #check if we are syncing from a given file list if [[ "$2" == *".rsync" ]]; then files_list="$2" shift shift ghe-rsync -avr \ -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" \ $link_dest "$@" \ --rsync-path='sudo -u git rsync' \ --include-from=- --exclude=\* \ --files-from="$files_list" \ --ignore-missing-args \ "$host:$GHE_REMOTE_DATA_USER_DIR/repositories/" \ "$backup_dir" 1>&3 2>&3 else shift ghe-rsync -avr \ -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" \ $link_dest "$@" \ --rsync-path='sudo -u git rsync' \ --include-from=- --exclude=\* \ --ignore-missing-args \ "$host:$GHE_REMOTE_DATA_USER_DIR/repositories/" \ "$backup_dir" 1>&3 2>&3 fi } sync_data (){ # Sync all auxiliary repository data. This includes files and directories like # HEAD, audit_log, config, description, info/, etc. No refs or object data # should be transferred here. echo 1>&3 echo "* Transferring auxiliary files ..." 1>&3 rsync_repository_data $1:122 $2 -z <<RULES - /__*__/ - /info/ + /*/ + /*/*.git - /*/*.git/objects - /*/*.git/refs - /*/*.git/packed-refs - /*/*.git/logs + /*/*.git/** + /*/??/ + /*/??/??/ + /*/??/??/??/ + /*/??/??/??/gist/ + /*/??/??/??/gist/*.git - /*/??/??/??/gist/*.git/objects - /*/??/??/??/gist/*.git/refs - /*/??/??/??/gist/*.git/packed-refs - /*/??/??/??/gist/*.git/logs + /*/??/??/??/gist/*.git/** + /*/nw/??/??/??/ + /*/nw/??/??/??/*/ + /*/nw/??/??/??/*/*.git - /*/nw/??/??/??/*/*.git/objects - /*/nw/??/??/??/*/*.git/refs - /*/nw/??/??/??/*/*.git/packed-refs - /*/nw/??/??/??/*/*.git/logs + /*/nw/??/??/??/*/*.git/** RULES # Sync packed refs files. This is performed before sync'ing loose refs since # loose refs trump packed-refs information. echo 1>&3 echo "* Transferring packed-refs files ..." 1>&3 rsync_repository_data $1:122 $2 -z <<RULES - /__*__/ - /info/ + /*/ + /*/*.git + /*/*.git/packed-refs + /*/??/ + /*/??/??/ + /*/??/??/??/ + /*/??/??/??/gist/ + /*/??/??/??/gist/*.git + /*/??/??/??/gist/*.git/packed-refs + /*/nw/??/ + /*/nw/??/??/ + /*/nw/??/??/??/ + /*/nw/??/??/??/*/ + /*/nw/??/??/??/*/*.git + /*/nw/??/??/??/*/*.git/packed-refs RULES # Sync loose refs and reflogs. This must be performed before object data is # transferred to ensure that all referenced objects are included. echo 1>&3 echo "* Transferring refs and reflogs ..." 1>&3 rsync_repository_data $1:122 $2 -z <<RULES - /__*__/ - /info/ + /*/ + /*/*.git + /*/*.git/refs + /*/*.git/refs/** + /*/*.git/logs + /*/*.git/logs/** + /*/??/ + /*/??/??/ + /*/??/??/??/ + /*/??/??/??/gist/ + /*/??/??/??/gist/*.git + /*/??/??/??/gist/*.git/refs + /*/??/??/??/gist/*.git/refs/** + /*/??/??/??/gist/*.git/logs + /*/??/??/??/gist/*.git/logs/** + /*/nw/??/ + /*/nw/??/??/ + /*/nw/??/??/??/ + /*/nw/??/??/??/*/ + /*/nw/??/??/??/*/*.git + /*/nw/??/??/??/*/*.git/refs + /*/nw/??/??/??/*/*.git/refs/** + /*/nw/??/??/??/*/*.git/logs + /*/nw/??/??/??/*/*.git/logs/** RULES # Sync git objects and pack files. Compression is disabled during this phase # since these files are already well compressed. echo 1>&3 echo "* Transferring objects and packs ..." 1>&3 rsync_repository_data $1:122 $2 -H <<RULES - /__*__/ - /info/ + /*/ + /*/*.git + /*/*.git/objects - /*/*.git/objects/**/tmp_* + /*/*.git/objects/** + /*/??/ + /*/??/??/ + /*/??/??/??/ + /*/??/??/??/gist/ + /*/??/??/??/gist/*.git + /*/??/??/??/gist/*.git/objects - /*/??/??/??/gist/*.git/objects/**/tmp_* + /*/??/??/??/gist/*.git/objects/** + /*/nw/??/ + /*/nw/??/??/ + /*/nw/??/??/??/ + /*/nw/??/??/??/*/ + /*/nw/??/??/??/*/*.git + /*/nw/??/??/??/*/*.git/objects - /*/nw/??/??/??/*/*.git/objects/**/tmp_* + /*/nw/??/??/??/*/*.git/objects/** RULES echo 1>&3 } # rsync all the repositories bm_start "$(basename $0) - Repo sync" for file_list in $tempdir/*.rsync; do hostname=$(basename $file_list .rsync) repo_num=$(cat $file_list | wc -l) ghe_verbose "* Transferring $repo_num repositories from $hostname" sync_data $hostname $file_list & done for pid in $(jobs -p); do wait $pid done bm_end "$(basename $0) - Repo sync" # Since there are no routes for special data directories, we need to do this # serially for all hostnames. Good candidate for future optimizations. bm_start "$(basename $0) - Special Data Directories Sync" for h in $hostnames; do # Sync __special__ data directories, including the __alambic_assets__, # __hookshot__, and __purgatory__ directories. The __nodeload_archives__, # __gitmon__, and __render__ directories are excludes since they act only as # caches. # # Under v2.x and greater, only the special __purgatory__ directory remains under # /data/repositories. All other special user data directories have been moved under # the /data/user directory. echo 1>&3 echo "* Transferring special data directories from $h..." 1>&3 rsync_repository_data $h:122 -z <<RULES - /__nodeload_archives__/ - /__gitmon__/ - /__render__/ + /__*__/ + /__*__/** + /info/ - /info/lost+found/ + /info/* RULES echo 1>&3 done bm_end "$(basename $0) - Special Data Directories Sync" if [ -z "$GHE_SKIP_ROUTE_VERIFICATION" ]; then bm_start "$(basename $0) - Verifying Routes" cat $tempdir/*.rsync | uniq | sort | uniq > $tempdir/source_routes (cd $backup_dir/ && find * -mindepth 5 -maxdepth 6 -type d -name \*.git | fix_paths_for_ghe_version | uniq | sort | uniq) > $tempdir/destination_routes git --no-pager diff --unified=0 --no-prefix -- $tempdir/source_routes $tempdir/destination_routes || echo "Warning: One or more repository networks and/or gists were not found on the source appliance. Please contact GitHub Enterprise Support for assistance." bm_end "$(basename $0) - Verifying Routes" fi bm_end "$(basename $0)"

my #!/usr/bin/env bash scripts