From b7bba6a689d135a9de972393fee47a520e6c5406 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 12 Oct 2021 18:15:56 +0200 Subject: [PATCH 01/99] Validate when adding not when removing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/webpage.sh | 44 ++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 52c388f85c..b651bed5d9 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -709,7 +709,13 @@ AddCustomDNSAddress() { ip="${args[2]}" host="${args[3]}" - echo "${ip} ${host}" >> "${dnscustomfile}" + + if valid_ip "${ip}" || valid_ip6 "${ip}" ; then + echo "${ip} ${host}" >> "${dnscustomfile}" + else + echo -e " ${CROSS} Invalid IP has been passed" + exit 1 + fi # Restart dnsmasq to load new custom DNS entries RestartDNS @@ -721,12 +727,7 @@ RemoveCustomDNSAddress() { ip="${args[2]}" host="${args[3]}" - if valid_ip "${ip}" || valid_ip6 "${ip}" ; then - sed -i "/^${ip} ${host}$/d" "${dnscustomfile}" - else - echo -e " ${CROSS} Invalid IP has been passed" - exit 1 - fi + sed -i "/^${ip} ${host}$/d" "${dnscustomfile}" # Restart dnsmasq to update removed custom DNS entries RestartDNS @@ -738,8 +739,19 @@ AddCustomCNAMERecord() { domain="${args[2]}" target="${args[3]}" - echo "cname=${domain},${target}" >> "${dnscustomcnamefile}" - + validDomain="$(checkDomain "${domain}")" + if [[ -n "${validDomain}" ]]; then + validTarget="$(checkDomain "${target}")" + if [[ -n "${validTarget}" ]]; then + echo "cname=${validDomain},${validTarget}" >> "${dnscustomcnamefile}" + else + echo " ${CROSS} Invalid Target Passed!" + exit 1 + fi + else + echo " ${CROSS} Invalid Domain passed!" + exit 1 + fi # Restart dnsmasq to load new custom CNAME records RestartDNS } @@ -750,19 +762,7 @@ RemoveCustomCNAMERecord() { domain="${args[2]}" target="${args[3]}" - validDomain="$(checkDomain "${domain}")" - if [[ -n "${validDomain}" ]]; then - validTarget="$(checkDomain "${target}")" - if [[ -n "${validDomain}" ]]; then - sed -i "/cname=${validDomain},${validTarget}$/d" "${dnscustomcnamefile}" - else - echo " ${CROSS} Invalid Target Passed!" - exit 1 - fi - else - echo " ${CROSS} Invalid Domain passed!" - exit 1 - fi + sed -i "/cname=${domain},${target}$/d" "${dnscustomcnamefile}" # Restart dnsmasq to update removed custom CNAME records RestartDNS From bc21a7155de841b5f39ce8c3c3aa51a691d25fdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 12 Oct 2021 19:49:36 +0200 Subject: [PATCH 02/99] Add option to not reload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/webpage.sh | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index b651bed5d9..f382b4d15c 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -709,6 +709,7 @@ AddCustomDNSAddress() { ip="${args[2]}" host="${args[3]}" + reload="${args[4]}" if valid_ip "${ip}" || valid_ip6 "${ip}" ; then echo "${ip} ${host}" >> "${dnscustomfile}" @@ -717,8 +718,10 @@ AddCustomDNSAddress() { exit 1 fi - # Restart dnsmasq to load new custom DNS entries - RestartDNS + # Restart dnsmasq to load new custom DNS entries only if $reload not false + if [[ ! $reload == "false" ]]; then + RestartDNS + fi } RemoveCustomDNSAddress() { @@ -726,11 +729,14 @@ RemoveCustomDNSAddress() { ip="${args[2]}" host="${args[3]}" + reload="${args[4]}" sed -i "/^${ip} ${host}$/d" "${dnscustomfile}" - # Restart dnsmasq to update removed custom DNS entries - RestartDNS + # Restart dnsmasq to load new custom DNS entries only if reload is not false + if [[ ! $reload == "false" ]]; then + RestartDNS + fi } AddCustomCNAMERecord() { @@ -738,6 +744,7 @@ AddCustomCNAMERecord() { domain="${args[2]}" target="${args[3]}" + reload="${args[4]}" validDomain="$(checkDomain "${domain}")" if [[ -n "${validDomain}" ]]; then @@ -752,8 +759,10 @@ AddCustomCNAMERecord() { echo " ${CROSS} Invalid Domain passed!" exit 1 fi - # Restart dnsmasq to load new custom CNAME records - RestartDNS + # Restart dnsmasq to load new custom CNAME records only if reload is not false + if [[ ! $reload == "false" ]]; then + RestartDNS + fi } RemoveCustomCNAMERecord() { @@ -761,11 +770,14 @@ RemoveCustomCNAMERecord() { domain="${args[2]}" target="${args[3]}" + reload="${args[4]}" sed -i "/cname=${domain},${target}$/d" "${dnscustomcnamefile}" - # Restart dnsmasq to update removed custom CNAME records - RestartDNS + # Restart dnsmasq to update removed custom CNAME records only if $reload not false + if [[ ! $reload == "false" ]]; then + RestartDNS + fi } main() { From a872fabe7d8518a9d025887da70ad922251e2cf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 17 Oct 2021 20:51:59 +0200 Subject: [PATCH 03/99] Validate on removal as well MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/webpage.sh | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index f382b4d15c..463b12fe4e 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -731,7 +731,12 @@ RemoveCustomDNSAddress() { host="${args[3]}" reload="${args[4]}" - sed -i "/^${ip} ${host}$/d" "${dnscustomfile}" + if valid_ip "${ip}" || valid_ip6 "${ip}" ; then + sed -i "/^${ip} ${host}$/d" "${dnscustomfile}" + else + echo -e " ${CROSS} Invalid IP has been passed" + exit 1 + fi # Restart dnsmasq to load new custom DNS entries only if reload is not false if [[ ! $reload == "false" ]]; then @@ -772,7 +777,19 @@ RemoveCustomCNAMERecord() { target="${args[3]}" reload="${args[4]}" - sed -i "/cname=${domain},${target}$/d" "${dnscustomcnamefile}" + validDomain="$(checkDomain "${domain}")" + if [[ -n "${validDomain}" ]]; then + validTarget="$(checkDomain "${target}")" + if [[ -n "${validTarget}" ]]; then + sed -i "/cname=${validDomain},${validTarget}$/d" "${dnscustomcnamefile}" + else + echo " ${CROSS} Invalid Target Passed!" + exit 1 + fi + else + echo " ${CROSS} Invalid Domain passed!" + exit 1 + fi # Restart dnsmasq to update removed custom CNAME records only if $reload not false if [[ ! $reload == "false" ]]; then From 596689b4c99f794eba8ddd14b51e914d0eac5917 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 19 Oct 2021 21:34:16 +0200 Subject: [PATCH 04/99] Validate host/domain of Local DNS records as well MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/webpage.sh | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 463b12fe4e..a739d89814 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -711,12 +711,18 @@ AddCustomDNSAddress() { host="${args[3]}" reload="${args[4]}" - if valid_ip "${ip}" || valid_ip6 "${ip}" ; then - echo "${ip} ${host}" >> "${dnscustomfile}" - else - echo -e " ${CROSS} Invalid IP has been passed" - exit 1 - fi + validHost="$(checkDomain "${host}")" + if [[ -n "${validHost}" ]]; then + if valid_ip "${ip}" || valid_ip6 "${ip}" ; then + echo "${ip} ${validHost}" >> "${dnscustomfile}" + else + echo -e " ${CROSS} Invalid IP has been passed" + exit 1 + fi + else + echo " ${CROSS} Invalid Domain passed!" + exit 1 + fi # Restart dnsmasq to load new custom DNS entries only if $reload not false if [[ ! $reload == "false" ]]; then @@ -731,11 +737,17 @@ RemoveCustomDNSAddress() { host="${args[3]}" reload="${args[4]}" - if valid_ip "${ip}" || valid_ip6 "${ip}" ; then - sed -i "/^${ip} ${host}$/d" "${dnscustomfile}" - else - echo -e " ${CROSS} Invalid IP has been passed" - exit 1 + validHost="$(checkDomain "${host}")" + if [[ -n "${validHost}" ]]; then + if valid_ip "${ip}" || valid_ip6 "${ip}" ; then + sed -i "/^${ip} ${validHost}$/d" "${dnscustomfile}" + else + echo -e " ${CROSS} Invalid IP has been passed" + exit 1 + fi + else + echo " ${CROSS} Invalid Domain passed!" + exit 1 fi # Restart dnsmasq to load new custom DNS entries only if reload is not false From 8713135b018fd0464c55dee53393eaae23c195a0 Mon Sep 17 00:00:00 2001 From: Blayne Campbell Date: Sat, 23 Oct 2021 12:43:20 -0600 Subject: [PATCH 05/99] update tests: remove fedora 32, add fedora 34 (#4403) Signed-off-by: bcambl --- .github/workflows/test.yml | 2 +- test/{_fedora_32.Dockerfile => _fedora_34.Dockerfile} | 2 +- test/{tox.fedora_32.ini => tox.fedora_34.ini} | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename test/{_fedora_32.Dockerfile => _fedora_34.Dockerfile} (97%) rename test/{tox.fedora_32.ini => tox.fedora_34.ini} (78%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c2b4dbbca6..49f139e133 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,7 +28,7 @@ jobs: needs: smoke-test strategy: matrix: - distro: [debian_9, debian_10, debian_11, ubuntu_16, ubuntu_18, ubuntu_20, ubuntu_21, centos_7, centos_8, fedora_32, fedora_33] + distro: [debian_9, debian_10, debian_11, ubuntu_16, ubuntu_18, ubuntu_20, ubuntu_21, centos_7, centos_8, fedora_33, fedora_34] env: DISTRO: ${{matrix.distro}} steps: diff --git a/test/_fedora_32.Dockerfile b/test/_fedora_34.Dockerfile similarity index 97% rename from test/_fedora_32.Dockerfile rename to test/_fedora_34.Dockerfile index e9c2ff2a67..96de18da0e 100644 --- a/test/_fedora_32.Dockerfile +++ b/test/_fedora_34.Dockerfile @@ -1,4 +1,4 @@ -FROM fedora:32 +FROM fedora:34 ENV GITDIR /etc/.pihole ENV SCRIPTDIR /opt/pihole diff --git a/test/tox.fedora_32.ini b/test/tox.fedora_34.ini similarity index 78% rename from test/tox.fedora_32.ini rename to test/tox.fedora_34.ini index c68e0757aa..154662cff4 100644 --- a/test/tox.fedora_32.ini +++ b/test/tox.fedora_34.ini @@ -4,5 +4,5 @@ envlist = py37 [testenv] whitelist_externals = docker deps = -rrequirements.txt -commands = docker build -f _fedora_32.Dockerfile -t pytest_pihole:test_container ../ +commands = docker build -f _fedora_34.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_automated_install.py ./test_centos_fedora_common_support.py ./test_fedora_support.py From c1eb35a35e7afd212205c317ee229b968d020734 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 26 Oct 2021 22:46:52 +0200 Subject: [PATCH 06/99] Add custom.list (Local DNS Records) to debug log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index cd61582551..f033739a85 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -88,6 +88,7 @@ PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list" PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate" PIHOLE_SETUP_VARS_FILE="${PIHOLE_DIRECTORY}/setupVars.conf" PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole-FTL.conf" +PIHOLE_CUSTOM_HOSTS_FILE="${PIHOLE_DIRECTORY}/custom.list" # Read the value of an FTL config key. The value is printed to stdout. # @@ -179,7 +180,8 @@ REQUIRED_FILES=("${PIHOLE_CRON_FILE}" "${PIHOLE_WEB_SERVER_ACCESS_LOG_FILE}" "${PIHOLE_WEB_SERVER_ERROR_LOG_FILE}" "${RESOLVCONF}" -"${DNSMASQ_CONF}") +"${DNSMASQ_CONF}" +"${PIHOLE_CUSTOM_HOSTS_FILE}") DISCLAIMER="This process collects information from your Pi-hole, and optionally uploads it to a unique and random directory on tricorder.pi-hole.net. From c6da1a39183573261642fdc38555cebcf9f753c0 Mon Sep 17 00:00:00 2001 From: David Irvine Date: Thu, 28 Oct 2021 12:09:34 +0200 Subject: [PATCH 07/99] Fix documentation; add some missing zones Signed-off-by: David Irvine --- advanced/06-rfc6761.conf | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/advanced/06-rfc6761.conf b/advanced/06-rfc6761.conf index e03569e868..fcdd00108a 100644 --- a/advanced/06-rfc6761.conf +++ b/advanced/06-rfc6761.conf @@ -25,11 +25,12 @@ server=/localhost/ server=/invalid/ # The same RFC requests something similar for -# 16.172.in-addr.arpa. 22.172.in-addr.arpa. 27.172.in-addr.arpa. -# 17.172.in-addr.arpa. 30.172.in-addr.arpa. 28.172.in-addr.arpa. -# 18.172.in-addr.arpa. 23.172.in-addr.arpa. 29.172.in-addr.arpa. -# 19.172.in-addr.arpa. 24.172.in-addr.arpa. 31.172.in-addr.arpa. -# 20.172.in-addr.arpa. 25.172.in-addr.arpa. 168.192.in-addr.arpa. +# 10.in-addr.arpa. 21.172.in-addr.arpa. 27.172.in-addr.arpa. +# 16.172.in-addr.arpa. 22.172.in-addr.arpa. 28.172.in-addr.arpa. +# 17.172.in-addr.arpa. 23.172.in-addr.arpa. 29.172.in-addr.arpa. +# 18.172.in-addr.arpa. 24.172.in-addr.arpa. 30.172.in-addr.arpa. +# 19.172.in-addr.arpa. 25.172.in-addr.arpa. 31.172.in-addr.arpa. +# 20.172.in-addr.arpa. 26.172.in-addr.arpa. 168.192.in-addr.arpa. # Pi-hole implements this via the dnsmasq option "bogus-priv" (see # 01-pihole.conf) because this also covers IPv6. From e73355329542de074caace35fb38908343346955 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 2 Nov 2021 21:22:14 +0100 Subject: [PATCH 08/99] Remove unused code from webpage.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/webpage.sh | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index a739d89814..2a4065651b 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -532,25 +532,6 @@ CustomizeAdLists() { fi } -SetPrivacyMode() { - if [[ "${args[2]}" == "true" ]]; then - change_setting "API_PRIVACY_MODE" "true" - else - change_setting "API_PRIVACY_MODE" "false" - fi -} - -ResolutionSettings() { - typ="${args[2]}" - state="${args[3]}" - - if [[ "${typ}" == "forward" ]]; then - change_setting "API_GET_UPSTREAM_DNS_HOSTNAME" "${state}" - elif [[ "${typ}" == "clients" ]]; then - change_setting "API_GET_CLIENT_HOSTNAME" "${state}" - fi -} - AddDHCPStaticAddress() { mac="${args[2]}" ip="${args[3]}" @@ -829,8 +810,6 @@ main() { "layout" ) SetWebUILayout;; "theme" ) SetWebUITheme;; "-h" | "--help" ) helpFunc;; - "privacymode" ) SetPrivacyMode;; - "resolve" ) ResolutionSettings;; "addstaticdhcp" ) AddDHCPStaticAddress;; "removestaticdhcp" ) RemoveDHCPStaticAddress;; "-e" | "email" ) SetAdminEmail "$3";; From 996f8fff28237d916ab44f7f039202ac32ccd75d Mon Sep 17 00:00:00 2001 From: yubiuser Date: Thu, 4 Nov 2021 23:55:16 +0100 Subject: [PATCH 09/99] Recommend apt instead of apt-get if updating the package cache failed (#4421) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Only change the recommendation to use apt Signed-off-by: Christian König --- automated install/basic-install.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index babb821388..0f9f8cc07e 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1481,8 +1481,14 @@ update_package_cache() { printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" else # Otherwise, show an error and exit + + # In case we used apt-get and apt is also available, we use this as recommendation as we have seen it + # gives more user-friendly (interactive) advice + if [[ ${PKG_MANAGER} == "apt-get" ]] && is_command apt ; then + UPDATE_PKG_CACHE="apt update" + fi printf "%b %b %s\\n" "${OVER}" "${CROSS}" "${str}" - printf " %bError: Unable to update package cache. Please try \"%s\"%b" "${COL_LIGHT_RED}" "sudo ${UPDATE_PKG_CACHE}" "${COL_NC}" + printf " %bError: Unable to update package cache. Please try \"%s\"%b\\n" "${COL_LIGHT_RED}" "sudo ${UPDATE_PKG_CACHE}" "${COL_NC}" return 1 fi } From ac4a975be55a3533d6425eac72026d5b94dda5f1 Mon Sep 17 00:00:00 2001 From: yubiuser Date: Sat, 6 Nov 2021 21:32:03 +0100 Subject: [PATCH 10/99] Allow users to skip setting static IP adress (#4419) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Allow users to skip setting static IP adresss Signed-off-by: Christian König --- automated install/basic-install.sh | 91 +++++++++++++++++------------- 1 file changed, 51 insertions(+), 40 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 0f9f8cc07e..c806a6aea8 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -761,7 +761,6 @@ collect_v4andv6_information() { if [[ -f "/etc/dhcpcd.conf" ]]; then # configure networking via dhcpcd getStaticIPv4Settings - setDHCPCD fi find_IPv6_information printf " %b IPv6 address: %s\\n" "${INFO}" "${IPV6_ADDRESS}" @@ -770,47 +769,59 @@ collect_v4andv6_information() { getStaticIPv4Settings() { # Local, named variables local ipSettingsCorrect + local DHCPChoice # Ask if the user wants to use DHCP settings as their static IP # This is useful for users that are using DHCP reservations; then we can just use the information gathered via our functions - if whiptail --backtitle "Calibrating network interface" --title "Static IP Address" --yesno "Do you want to use your current network settings as a static address? - IP address: ${IPV4_ADDRESS} - Gateway: ${IPv4gw}" "${r}" "${c}"; then + DHCPChoice=$(whiptail --backtitle "Calibrating network interface" --title "Static IP Address" --menu --separate-output "Do you want to use your current network settings as a static address? \\n + IP address: ${IPV4_ADDRESS} \\n + Gateway: ${IPv4gw} \\n" "${r}" "${c}" 3\ + "Yes" "Set static IP using current values" \ + "No" "Set static IP using custom values" \ + "Skip" "I will set a static IP later, or have already done so" 3>&2 2>&1 1>&3) || \ + { printf " %bCancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}"; exit 1; } + + case ${DHCPChoice} in + "Yes") # If they choose yes, let the user know that the IP address will not be available via DHCP and may cause a conflict. whiptail --msgbox --backtitle "IP information" --title "FYI: IP Conflict" "It is possible your router could still try to assign this IP to a device, which would cause a conflict. But in most cases the router is smart enough to not do that. -If you are worried, either manually set the address, or modify the DHCP reservation pool so it does not include the IP you want. -It is also possible to use a DHCP reservation, but if you are going to do that, you might as well set a static address." "${r}" "${c}" - # Nothing else to do since the variables are already set above - else - # Otherwise, we need to ask the user to input their desired settings. - # Start by getting the IPv4 address (pre-filling it with info gathered from DHCP) - # Start a loop to let the user enter their information with the chance to go back and edit it if necessary - until [[ "${ipSettingsCorrect}" = True ]]; do - - # Ask for the IPv4 address - IPV4_ADDRESS=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 address" --inputbox "Enter your desired IPv4 address" "${r}" "${c}" "${IPV4_ADDRESS}" 3>&1 1>&2 2>&3) || \ - # Canceling IPv4 settings window - { ipSettingsCorrect=False; echo -e " ${COL_LIGHT_RED}Cancel was selected, exiting installer${COL_NC}"; exit 1; } - printf " %b Your static IPv4 address: %s\\n" "${INFO}" "${IPV4_ADDRESS}" - - # Ask for the gateway - IPv4gw=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 gateway (router)" --inputbox "Enter your desired IPv4 default gateway" "${r}" "${c}" "${IPv4gw}" 3>&1 1>&2 2>&3) || \ - # Canceling gateway settings window - { ipSettingsCorrect=False; echo -e " ${COL_LIGHT_RED}Cancel was selected, exiting installer${COL_NC}"; exit 1; } - printf " %b Your static IPv4 gateway: %s\\n" "${INFO}" "${IPv4gw}" - - # Give the user a chance to review their settings before moving on - if whiptail --backtitle "Calibrating network interface" --title "Static IP Address" --yesno "Are these settings correct? - IP address: ${IPV4_ADDRESS} - Gateway: ${IPv4gw}" "${r}" "${c}"; then - # After that's done, the loop ends and we move on - ipSettingsCorrect=True - else - # If the settings are wrong, the loop continues - ipSettingsCorrect=False - fi - done - # End the if statement for DHCP vs. static - fi + If you are worried, either manually set the address, or modify the DHCP reservation pool so it does not include the IP you want. + It is also possible to use a DHCP reservation, but if you are going to do that, you might as well set a static address." "${r}" "${c}" + # Nothing else to do since the variables are already set above + setDHCPCD + ;; + + "No") + # Otherwise, we need to ask the user to input their desired settings. + # Start by getting the IPv4 address (pre-filling it with info gathered from DHCP) + # Start a loop to let the user enter their information with the chance to go back and edit it if necessary + until [[ "${ipSettingsCorrect}" = True ]]; do + + # Ask for the IPv4 address + IPV4_ADDRESS=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 address" --inputbox "Enter your desired IPv4 address" "${r}" "${c}" "${IPV4_ADDRESS}" 3>&1 1>&2 2>&3) || \ + # Canceling IPv4 settings window + { ipSettingsCorrect=False; echo -e " ${COL_LIGHT_RED}Cancel was selected, exiting installer${COL_NC}"; exit 1; } + printf " %b Your static IPv4 address: %s\\n" "${INFO}" "${IPV4_ADDRESS}" + + # Ask for the gateway + IPv4gw=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 gateway (router)" --inputbox "Enter your desired IPv4 default gateway" "${r}" "${c}" "${IPv4gw}" 3>&1 1>&2 2>&3) || \ + # Canceling gateway settings window + { ipSettingsCorrect=False; echo -e " ${COL_LIGHT_RED}Cancel was selected, exiting installer${COL_NC}"; exit 1; } + printf " %b Your static IPv4 gateway: %s\\n" "${INFO}" "${IPv4gw}" + + # Give the user a chance to review their settings before moving on + if whiptail --backtitle "Calibrating network interface" --title "Static IP Address" --yesno "Are these settings correct? + IP address: ${IPV4_ADDRESS} + Gateway: ${IPv4gw}" "${r}" "${c}"; then + # After that's done, the loop ends and we move on + ipSettingsCorrect=True + else + # If the settings are wrong, the loop continues + ipSettingsCorrect=False + fi + done + setDHCPCD + ;; + esac } # Configure networking via dhcpcd @@ -2475,12 +2486,12 @@ main() { get_available_interfaces # Find interfaces and let the user choose one chooseInterface + # find IPv4 and IPv6 information of the device + collect_v4andv6_information # Decide what upstream DNS Servers to use setDNS # Give the user a choice of blocklists to include in their install. Or not. chooseBlocklists - # find IPv4 and IPv6 information of the device - collect_v4andv6_information # Let the user decide if they want the web interface to be installed automatically setAdminFlag # Let the user decide if they want query logging enabled... From cedd1a259105dc231744d58562f0bc80d4294c7a Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Thu, 11 Nov 2021 17:44:57 +0100 Subject: [PATCH 11/99] unit test for umask problems in #3177 and #2730 (#3191) * add test for file permissions of $webroot Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * changes sudo to su for running command as user www-data Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * installs PIHOLE_WEB_DEPS to create LIGHTTPD_USER Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * changes stdout to rc Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * use installPihole instead of installPiholeWeb in test Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * try installation process with main Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * mock systemctl Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * removes stickler errors Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * start lighttpd and make webpage test optional Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * test all files and directories in $webroot Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * fix stickler and codefactor warnings Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * set permission for /var/cache if it did not exist before Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * add test case for pihole files Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * fix stickler errors Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * revert "set permission for /var/cache if it did not exist before" and make lighttpd start work Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * add --add-cap=NET_ADMIN to enable FTL start Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * specify DNS server for cURL Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * check files created by FTL Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * reorder code and change nameserver in /etc/resolv.conf Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * resolve with dig instead of relying on /etc/resolv.conf Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * set IP to 127.0.0.1 in setupVars.conf for blockpage tests Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * resolve domain with dig and remove debug output Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * fix stickler errors Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * no git pull in Github Action runs for pull requests Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * --cap-add=ALL test Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * fix stickler errors Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * remove debug code Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * update_repo patch for CentOS 7 in Github Actions Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * removes TODOs and stickler warnings Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * adds trailing slash to domain Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * use only first result from dig Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * domain name resolution does not work reliably in docker container Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * repair executable permission Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> * Create mock_command_passthrough that allows intercepting of specific arguments - everything else is passed through to the proper command. Use this new command instead of making changes in basic-install.sh to make the tests pass. Signed-off-by: Adam Warner Co-authored-by: Adam Warner --- test/README.md | 4 +- test/conftest.py | 69 ++++- test/test_automated_install.py | 514 +++++++++++++++++++++++++++++++++ 3 files changed, 581 insertions(+), 6 deletions(-) diff --git a/test/README.md b/test/README.md index b4dd112235..692155b7d8 100644 --- a/test/README.md +++ b/test/README.md @@ -18,8 +18,8 @@ py.test -vv -n auto -m "build_stage" py.test -vv -n auto -m "not build_stage" ``` -The build_stage tests have to run first to create the docker images, followed by the actual tests which utilize said images. Unless you're changing your dockerfiles you shouldn't have to run the build_stage every time - but it's a good idea to rebuild at least once a day in case the base Docker images or packages change. +The build_stage tests have to run first to create the docker images, followed by the actual tests which utilize said images. Unless you're changing your dockerfiles you shouldn't have to run the build_stage every time - but it's a good idea to rebuild at least once a day in case the base Docker images or packages change. # How do I debug python? -Highly recommended: Setup PyCharm on a **Docker enabled** machine. Having a python debugger like PyCharm changes your life if you've never used it :) +Highly recommended: Setup PyCharm on a **Docker enabled** machine. Having a python debugger like PyCharm changes your life if you've never used it :) diff --git a/test/conftest.py b/test/conftest.py index 13731eb8e6..2d388fddf1 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -64,7 +64,7 @@ def args(request): ''' -t became required when tput began being used ''' - return '-t -d' + return '-t -d --cap-add=ALL' @pytest.fixture(params=[ @@ -100,7 +100,7 @@ def mock_command(script, args, container): in unit tests ''' full_script_path = '/usr/local/bin/{}'.format(script) - mock_script = dedent('''\ + mock_script = dedent(r'''\ #!/bin/bash -e echo "\$0 \$@" >> /var/log/{script} case "\$1" in'''.format(script=script)) @@ -121,13 +121,75 @@ def mock_command(script, args, container): scriptlog=script)) +def mock_command_passthrough(script, args, container): + ''' + Per other mock_command* functions, allows intercepting of commands we don't want to run for real + in unit tests, however also allows only specific arguments to be mocked. Anything not defined will + be passed through to the actual command. + + Example use-case: mocking `git pull` but still allowing `git clone` to work as intended + ''' + orig_script_path = check_output('which {}'.format(script)) + full_script_path = '/usr/local/bin/{}'.format(script) + mock_script = dedent(r'''\ + #!/bin/bash -e + echo "\$0 \$@" >> /var/log/{script} + case "\$1" in'''.format(script=script)) + for k, v in args.items(): + case = dedent(''' + {arg}) + echo {res} + exit {retcode} + ;;'''.format(arg=k, res=v[0], retcode=v[1])) + mock_script += case + mock_script += dedent(r''' + *) + {orig_script_path} "\$@" + ;;'''.format(orig_script_path=orig_script_path)) + mock_script += dedent(''' + esac''') + container.run(''' + cat < {script}\n{content}\nEOF + chmod +x {script} + rm -f /var/log/{scriptlog}'''.format(script=full_script_path, + content=mock_script, + scriptlog=script)) + + +def mock_command_run(script, args, container): + ''' + Allows for setup of commands we don't really want to have to run for real + in unit tests + ''' + full_script_path = '/usr/local/bin/{}'.format(script) + mock_script = dedent(r'''\ + #!/bin/bash -e + echo "\$0 \$@" >> /var/log/{script} + case "\$1 \$2" in'''.format(script=script)) + for k, v in args.items(): + case = dedent(''' + \"{arg}\") + echo {res} + exit {retcode} + ;;'''.format(arg=k, res=v[0], retcode=v[1])) + mock_script += case + mock_script += dedent(''' + esac''') + container.run(''' + cat < {script}\n{content}\nEOF + chmod +x {script} + rm -f /var/log/{scriptlog}'''.format(script=full_script_path, + content=mock_script, + scriptlog=script)) + + def mock_command_2(script, args, container): ''' Allows for setup of commands we don't really want to have to run for real in unit tests ''' full_script_path = '/usr/local/bin/{}'.format(script) - mock_script = dedent('''\ + mock_script = dedent(r'''\ #!/bin/bash -e echo "\$0 \$@" >> /var/log/{script} case "\$1 \$2" in'''.format(script=script)) @@ -147,7 +209,6 @@ def mock_command_2(script, args, container): content=mock_script, scriptlog=script)) - def run_script(Pihole, script): result = Pihole.run(script) assert result.rc == 0 diff --git a/test/test_automated_install.py b/test/test_automated_install.py index 37ebdad258..304a8783d9 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -1,3 +1,4 @@ +import pytest from textwrap import dedent import re from .conftest import ( @@ -6,7 +7,9 @@ info_box, cross_box, mock_command, + mock_command_run, mock_command_2, + mock_command_passthrough, run_script ) @@ -109,6 +112,7 @@ def test_installPiholeWeb_fresh_install_no_errors(Pihole): confirms all web page assets from Core repo are installed on a fresh build ''' installWeb = Pihole.run(''' + umask 0027 source /opt/pihole/basic-install.sh installPiholeWeb ''') @@ -129,6 +133,516 @@ def test_installPiholeWeb_fresh_install_no_errors(Pihole): assert 'blockingpage.css' in web_directory +def get_directories_recursive(Pihole, directory): + if directory is None: + return directory + ls = Pihole.run('ls -d {}'.format(directory + '/*/')) + directories = list(filter(bool, ls.stdout.splitlines())) + dirs = directories + for dirval in directories: + dir_rec = get_directories_recursive(Pihole, dirval) + if isinstance(dir_rec, str): + dirs.extend([dir_rec]) + else: + dirs.extend(dir_rec) + return dirs + + +def test_installPihole_fresh_install_readableFiles(Pihole): + ''' + confirms all neccessary files are readable by pihole user + ''' + # Whiptail dialog returns Cancel for user prompt + mock_command('whiptail', {'*': ('', '0')}, Pihole) + # mock git pull + mock_command_passthrough('git', {'pull': ('', '0')}, Pihole) + # mock systemctl to not start lighttpd and FTL + mock_command_2( + 'systemctl', + { + 'enable lighttpd': ( + '', + '0' + ), + 'restart lighttpd': ( + '', + '0' + ), + 'start lighttpd': ( + '', + '0' + ), + 'enable pihole-FTL': ( + '', + '0' + ), + 'restart pihole-FTL': ( + '', + '0' + ), + 'start pihole-FTL': ( + '', + '0' + ), + '*': ( + 'echo "systemctl call with $@"', + '0' + ), + }, + Pihole + ) + # try to install man + Pihole.run('command -v apt-get > /dev/null && apt-get install -qq man') + Pihole.run('command -v dnf > /dev/null && dnf install -y man') + Pihole.run('command -v yum > /dev/null && yum install -y man') + # create configuration file + setup_var_file = 'cat < /etc/pihole/setupVars.conf\n' + for k, v in SETUPVARS.items(): + setup_var_file += "{}={}\n".format(k, v) + setup_var_file += "INSTALL_WEB_SERVER=true\n" + setup_var_file += "INSTALL_WEB_INTERFACE=true\n" + setup_var_file += "EOF\n" + Pihole.run(setup_var_file) + install = Pihole.run(''' + export TERM=xterm + export DEBIAN_FRONTEND=noninteractive + umask 0027 + runUnattended=true + useUpdateVars=true + source /opt/pihole/basic-install.sh > /dev/null + runUnattended=true + useUpdateVars=true + main + ''') + assert 0 == install.rc + maninstalled = True + if (info_box + ' man not installed') in install.stdout: + maninstalled = False + piholeuser = 'pihole' + exit_status_success = 0 + test_cmd = 'su --shell /bin/bash --command "test -{0} {1}" -p {2}' + # check files in /etc/pihole for read, write and execute permission + check_etc = test_cmd.format('r', '/etc/pihole', piholeuser) + actual_rc = Pihole.run(check_etc).rc + assert exit_status_success == actual_rc + check_etc = test_cmd.format('x', '/etc/pihole', piholeuser) + actual_rc = Pihole.run(check_etc).rc + assert exit_status_success == actual_rc + # readable and writable dhcp.leases + check_leases = test_cmd.format('r', '/etc/pihole/dhcp.leases', piholeuser) + actual_rc = Pihole.run(check_leases).rc + assert exit_status_success == actual_rc + check_leases = test_cmd.format('w', '/etc/pihole/dhcp.leases', piholeuser) + actual_rc = Pihole.run(check_leases).rc + # readable dns-servers.conf + assert exit_status_success == actual_rc + check_servers = test_cmd.format( + 'r', '/etc/pihole/dns-servers.conf', piholeuser) + actual_rc = Pihole.run(check_servers).rc + assert exit_status_success == actual_rc + # readable GitHubVersions + check_version = test_cmd.format( + 'r', '/etc/pihole/GitHubVersions', piholeuser) + actual_rc = Pihole.run(check_version).rc + assert exit_status_success == actual_rc + # readable install.log + check_install = test_cmd.format( + 'r', '/etc/pihole/install.log', piholeuser) + actual_rc = Pihole.run(check_install).rc + assert exit_status_success == actual_rc + # readable localbranches + check_localbranch = test_cmd.format( + 'r', '/etc/pihole/localbranches', piholeuser) + actual_rc = Pihole.run(check_localbranch).rc + assert exit_status_success == actual_rc + # readable localversions + check_localversion = test_cmd.format( + 'r', '/etc/pihole/localversions', piholeuser) + actual_rc = Pihole.run(check_localversion).rc + assert exit_status_success == actual_rc + # readable logrotate + check_logrotate = test_cmd.format( + 'r', '/etc/pihole/logrotate', piholeuser) + actual_rc = Pihole.run(check_logrotate).rc + assert exit_status_success == actual_rc + # readable macvendor.db + check_macvendor = test_cmd.format( + 'r', '/etc/pihole/macvendor.db', piholeuser) + actual_rc = Pihole.run(check_macvendor).rc + assert exit_status_success == actual_rc + # readable and writeable pihole-FTL.conf + check_FTLconf = test_cmd.format( + 'r', '/etc/pihole/pihole-FTL.conf', piholeuser) + actual_rc = Pihole.run(check_FTLconf).rc + assert exit_status_success == actual_rc + check_FTLconf = test_cmd.format( + 'w', '/etc/pihole/pihole-FTL.conf', piholeuser) + actual_rc = Pihole.run(check_FTLconf).rc + assert exit_status_success == actual_rc + # readable setupVars.conf + check_setup = test_cmd.format( + 'r', '/etc/pihole/setupVars.conf', piholeuser) + actual_rc = Pihole.run(check_setup).rc + assert exit_status_success == actual_rc + # check dnsmasq files + # readable /etc/dnsmasq.conf + check_dnsmasqconf = test_cmd.format( + 'r', '/etc/dnsmasq.conf', piholeuser) + actual_rc = Pihole.run(check_dnsmasqconf).rc + assert exit_status_success == actual_rc + # readable /etc/dnsmasq.d/01-pihole.conf + check_dnsmasqconf = test_cmd.format( + 'r', '/etc/dnsmasq.d', piholeuser) + actual_rc = Pihole.run(check_dnsmasqconf).rc + assert exit_status_success == actual_rc + check_dnsmasqconf = test_cmd.format( + 'x', '/etc/dnsmasq.d', piholeuser) + actual_rc = Pihole.run(check_dnsmasqconf).rc + assert exit_status_success == actual_rc + check_dnsmasqconf = test_cmd.format( + 'r', '/etc/dnsmasq.d/01-pihole.conf', piholeuser) + actual_rc = Pihole.run(check_dnsmasqconf).rc + assert exit_status_success == actual_rc + # check readable and executable /etc/init.d/pihole-FTL + check_init = test_cmd.format( + 'x', '/etc/init.d/pihole-FTL', piholeuser) + actual_rc = Pihole.run(check_init).rc + assert exit_status_success == actual_rc + check_init = test_cmd.format( + 'r', '/etc/init.d/pihole-FTL', piholeuser) + actual_rc = Pihole.run(check_init).rc + assert exit_status_success == actual_rc + # check readable /etc/lighttpd/lighttpd.conf + check_lighttpd = test_cmd.format( + 'r', '/etc/lighttpd/lighttpd.conf', piholeuser) + actual_rc = Pihole.run(check_lighttpd).rc + assert exit_status_success == actual_rc + # check readable and executable manpages + if maninstalled is True: + check_man = test_cmd.format( + 'x', '/usr/local/share/man', piholeuser) + actual_rc = Pihole.run(check_man).rc + assert exit_status_success == actual_rc + check_man = test_cmd.format( + 'r', '/usr/local/share/man', piholeuser) + actual_rc = Pihole.run(check_man).rc + assert exit_status_success == actual_rc + check_man = test_cmd.format( + 'x', '/usr/local/share/man/man8', piholeuser) + actual_rc = Pihole.run(check_man).rc + assert exit_status_success == actual_rc + check_man = test_cmd.format( + 'r', '/usr/local/share/man/man8', piholeuser) + actual_rc = Pihole.run(check_man).rc + assert exit_status_success == actual_rc + check_man = test_cmd.format( + 'x', '/usr/local/share/man/man5', piholeuser) + actual_rc = Pihole.run(check_man).rc + assert exit_status_success == actual_rc + check_man = test_cmd.format( + 'r', '/usr/local/share/man/man5', piholeuser) + actual_rc = Pihole.run(check_man).rc + assert exit_status_success == actual_rc + check_man = test_cmd.format( + 'r', '/usr/local/share/man/man8/pihole.8', piholeuser) + actual_rc = Pihole.run(check_man).rc + assert exit_status_success == actual_rc + check_man = test_cmd.format( + 'r', '/usr/local/share/man/man8/pihole-FTL.8', piholeuser) + actual_rc = Pihole.run(check_man).rc + assert exit_status_success == actual_rc + check_man = test_cmd.format( + 'r', '/usr/local/share/man/man5/pihole-FTL.conf.5', piholeuser) + actual_rc = Pihole.run(check_man).rc + assert exit_status_success == actual_rc + # check not readable sudoers file + check_sudo = test_cmd.format( + 'r', '/etc/sudoers.d/pihole', piholeuser) + actual_rc = Pihole.run(check_sudo).rc + assert exit_status_success != actual_rc + # check not readable cron file + check_sudo = test_cmd.format( + 'x', '/etc/cron.d/', piholeuser) + actual_rc = Pihole.run(check_sudo).rc + assert exit_status_success == actual_rc + check_sudo = test_cmd.format( + 'r', '/etc/cron.d/', piholeuser) + actual_rc = Pihole.run(check_sudo).rc + assert exit_status_success == actual_rc + check_sudo = test_cmd.format( + 'r', '/etc/cron.d/pihole', piholeuser) + actual_rc = Pihole.run(check_sudo).rc + assert exit_status_success == actual_rc + directories = get_directories_recursive(Pihole, '/etc/.pihole/') + for directory in directories: + check_pihole = test_cmd.format('r', directory, piholeuser) + actual_rc = Pihole.run(check_pihole).rc + check_pihole = test_cmd.format('x', directory, piholeuser) + actual_rc = Pihole.run(check_pihole).rc + findfiles = 'find "{}" -maxdepth 1 -type f -exec echo {{}} \\;;' + filelist = Pihole.run(findfiles.format(directory)) + files = list(filter(bool, filelist.stdout.splitlines())) + for file in files: + check_pihole = test_cmd.format('r', file, piholeuser) + actual_rc = Pihole.run(check_pihole).rc + + +@pytest.mark.parametrize("test_webpage", [True]) +def test_installPihole_fresh_install_readableBlockpage(Pihole, test_webpage): + ''' + confirms all web page assets from Core repo are readable + by $LIGHTTPD_USER on a fresh build + ''' + piholeWebpage = [ + "127.0.0.1", + # "pi.hole" + ] + # Whiptail dialog returns Cancel for user prompt + mock_command('whiptail', {'*': ('', '0')}, Pihole) + # mock git pull + mock_command_passthrough('git', {'pull': ('', '0')}, Pihole) + # mock systemctl to start lighttpd and FTL + ligthttpdcommand = dedent(r'''\"\" + echo 'starting lighttpd with {}' + if [ command -v "apt-get" >/dev/null 2>&1 ]; then + LIGHTTPD_USER="www-data" + LIGHTTPD_GROUP="www-data" + else + LIGHTTPD_USER="lighttpd" + LIGHTTPD_GROUP="lighttpd" + fi + mkdir -p "{run}" + chown {usergroup} "{run}" + mkdir -p "{cache}" + chown {usergroup} "/var/cache" + chown {usergroup} "{cache}" + mkdir -p "{compress}" + chown {usergroup} "{compress}" + mkdir -p "{uploads}" + chown {usergroup} "{uploads}" + chmod 0777 /var + chmod 0777 /var/cache + chmod 0777 "{cache}" + find "{run}" -type d -exec chmod 0777 {chmodarg} \;; + find "{run}" -type f -exec chmod 0666 {chmodarg} \;; + find "{compress}" -type d -exec chmod 0777 {chmodarg} \;; + find "{compress}" -type f -exec chmod 0666 {chmodarg} \;; + find "{uploads}" -type d -exec chmod 0777 {chmodarg} \;; + find "{uploads}" -type f -exec chmod 0666 {chmodarg} \;; + /usr/sbin/lighttpd -tt -f '{config}' + /usr/sbin/lighttpd -f '{config}' + echo \"\"'''.format( + '{}', + usergroup='${{LIGHTTPD_USER}}:${{LIGHTTPD_GROUP}}', + chmodarg='{{}}', + config='/etc/lighttpd/lighttpd.conf', + run='/var/run/lighttpd', + cache='/var/cache/lighttpd', + uploads='/var/cache/lighttpd/uploads', + compress='/var/cache/lighttpd/compress' + ) + ) + FTLcommand = dedent('''\"\" + set -x + /etc/init.d/pihole-FTL restart + echo \"\"''') + mock_command_run( + 'systemctl', + { + 'enable lighttpd': ( + '', + '0' + ), + 'restart lighttpd': ( + ligthttpdcommand.format('restart'), + '0' + ), + 'start lighttpd': ( + ligthttpdcommand.format('start'), + '0' + ), + 'enable pihole-FTL': ( + '', + '0' + ), + 'restart pihole-FTL': ( + FTLcommand, + '0' + ), + 'start pihole-FTL': ( + FTLcommand, + '0' + ), + '*': ( + 'echo "systemctl call with $@"', + '0' + ), + }, + Pihole + ) + # create configuration file + setup_var_file = 'cat < /etc/pihole/setupVars.conf\n' + for k, v in SETUPVARS.items(): + setup_var_file += "{}={}\n".format(k, v) + setup_var_file += "INSTALL_WEB_SERVER=true\n" + setup_var_file += "INSTALL_WEB_INTERFACE=true\n" + setup_var_file += "IPV4_ADDRESS=127.0.0.1\n" + setup_var_file += "EOF\n" + Pihole.run(setup_var_file) + installWeb = Pihole.run(''' + export TERM=xterm + export DEBIAN_FRONTEND=noninteractive + umask 0027 + runUnattended=true + useUpdateVars=true + source /opt/pihole/basic-install.sh > /dev/null + runUnattended=true + useUpdateVars=true + main + echo "LIGHTTPD_USER=${LIGHTTPD_USER}" + echo "webroot=${webroot}" + echo "INSTALL_WEB_INTERFACE=${INSTALL_WEB_INTERFACE}" + echo "INSTALL_WEB_SERVER=${INSTALL_WEB_SERVER}" + ''') + assert 0 == installWeb.rc + piholeuser = 'pihole' + webuser = '' + user = re.findall( + r"^\s*LIGHTTPD_USER=.*$", installWeb.stdout, re.MULTILINE) + for match in user: + webuser = match.replace('LIGHTTPD_USER=', '').strip() + webroot = '' + user = re.findall( + r"^\s*webroot=.*$", installWeb.stdout, re.MULTILINE) + for match in user: + webroot = match.replace('webroot=', '').strip() + if not webroot.strip(): + webroot = '/var/www/html' + installWebInterface = True + interface = re.findall( + r"^\s*INSTALL_WEB_INTERFACE=.*$", installWeb.stdout, re.MULTILINE) + for match in interface: + testvalue = match.replace('INSTALL_WEB_INTERFACE=', '').strip().lower() + if not testvalue.strip(): + installWebInterface = testvalue == "true" + installWebServer = True + server = re.findall( + r"^\s*INSTALL_WEB_SERVER=.*$", installWeb.stdout, re.MULTILINE) + for match in server: + testvalue = match.replace('INSTALL_WEB_SERVER=', '').strip().lower() + if not testvalue.strip(): + installWebServer = testvalue == "true" + # if webserver install was not requested + # at least pihole must be able to read files + if installWebServer is False: + webuser = piholeuser + exit_status_success = 0 + test_cmd = 'su --shell /bin/bash --command "test -{0} {1}" -p {2}' + # check files that need a running FTL to be created + # readable and writeable pihole-FTL.db + check_FTLconf = test_cmd.format( + 'r', '/etc/pihole/pihole-FTL.db', piholeuser) + actual_rc = Pihole.run(check_FTLconf).rc + assert exit_status_success == actual_rc + check_FTLconf = test_cmd.format( + 'w', '/etc/pihole/pihole-FTL.db', piholeuser) + actual_rc = Pihole.run(check_FTLconf).rc + assert exit_status_success == actual_rc + # check directories above $webroot for read and execute permission + check_var = test_cmd.format('r', '/var', webuser) + actual_rc = Pihole.run(check_var).rc + assert exit_status_success == actual_rc + check_var = test_cmd.format('x', '/var', webuser) + actual_rc = Pihole.run(check_var).rc + assert exit_status_success == actual_rc + check_www = test_cmd.format('r', '/var/www', webuser) + actual_rc = Pihole.run(check_www).rc + assert exit_status_success == actual_rc + check_www = test_cmd.format('x', '/var/www', webuser) + actual_rc = Pihole.run(check_www).rc + assert exit_status_success == actual_rc + check_html = test_cmd.format('r', '/var/www/html', webuser) + actual_rc = Pihole.run(check_html).rc + assert exit_status_success == actual_rc + check_html = test_cmd.format('x', '/var/www/html', webuser) + actual_rc = Pihole.run(check_html).rc + assert exit_status_success == actual_rc + # check directories below $webroot for read and execute permission + check_admin = test_cmd.format('r', webroot + '/admin', webuser) + actual_rc = Pihole.run(check_admin).rc + assert exit_status_success == actual_rc + check_admin = test_cmd.format('x', webroot + '/admin', webuser) + actual_rc = Pihole.run(check_admin).rc + assert exit_status_success == actual_rc + directories = get_directories_recursive(Pihole, webroot + '/admin/*/') + for directory in directories: + check_pihole = test_cmd.format('r', directory, webuser) + actual_rc = Pihole.run(check_pihole).rc + check_pihole = test_cmd.format('x', directory, webuser) + actual_rc = Pihole.run(check_pihole).rc + findfiles = 'find "{}" -maxdepth 1 -type f -exec echo {{}} \\;;' + filelist = Pihole.run(findfiles.format(directory)) + files = list(filter(bool, filelist.stdout.splitlines())) + for file in files: + check_pihole = test_cmd.format('r', file, webuser) + actual_rc = Pihole.run(check_pihole).rc + # check web interface files + # change nameserver to pi-hole + # setting nameserver in /etc/resolv.conf to pi-hole does + # not work here because of the way docker uses this file + ns = Pihole.run( + r"sed -i 's/nameserver.*/nameserver 127.0.0.1/' /etc/resolv.conf") + pihole_is_ns = ns.rc == 0 + + def is_ip(address): + m = re.match(r"(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})", address) + return bool(m) + if installWebInterface is True: + check_pihole = test_cmd.format('r', webroot + '/pihole', webuser) + actual_rc = Pihole.run(check_pihole).rc + assert exit_status_success == actual_rc + check_pihole = test_cmd.format('x', webroot + '/pihole', webuser) + actual_rc = Pihole.run(check_pihole).rc + assert exit_status_success == actual_rc + # check most important files in $webroot for read permission + check_index = test_cmd.format( + 'r', webroot + '/pihole/index.php', webuser) + actual_rc = Pihole.run(check_index).rc + assert exit_status_success == actual_rc + check_blockpage = test_cmd.format( + 'r', webroot + '/pihole/blockingpage.css', webuser) + actual_rc = Pihole.run(check_blockpage).rc + assert exit_status_success == actual_rc + if test_webpage is True: + # check webpage for unreadable files + noPHPfopen = re.compile( + (r"PHP Error(%d+):\s+fopen([^)]+):\s+" + + r"failed to open stream: " + + r"Permission denied in"), + re.I) + # using cURL option --dns-servers is not possible + status = ( + 'curl -s --head "{}" | ' + + 'head -n 1 | ' + + 'grep "HTTP/1.[01] [23].." > /dev/null') + digcommand = r"dig A +short {} @127.0.0.1 | head -n 1" + pagecontent = 'curl --verbose -L "{}"' + for page in piholeWebpage: + testpage = "http://" + page + "/admin/" + resolvesuccess = True + if is_ip(page) is False: + dig = Pihole.run(digcommand.format(page)) + testpage = "http://" + dig.stdout.strip() + "/admin/" + resolvesuccess = dig.rc == 0 + if resolvesuccess or pihole_is_ns: + # check HTTP status of blockpage + actual_rc = Pihole.run(status.format(testpage)) + assert exit_status_success == actual_rc.rc + # check for PHP error + actual_output = Pihole.run(pagecontent.format(testpage)) + assert noPHPfopen.match(actual_output.stdout) is None + + def test_update_package_cache_success_no_errors(Pihole): ''' confirms package cache was updated without any errors From cdd4d9ea9e51c8110df832d5fadeb44bc90b5085 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Thu, 18 Nov 2021 01:03:37 +0000 Subject: [PATCH 12/99] Update the tests (#4427) * unpin the requirements and update all to latest available - needs more work still. see notes in `def host()` Signed-off-by: Adam Warner * fix py3 monkey patch of testinfra docker using bash Signed-off-by: Adam Hill * update the other test files to use `host` instead of `Pihole` Address some sticklr and codefactor update python version from 3.7 to 3.8 preload `git` onto the centos/fedora test images, and switch which with command -v in the passthrough mock testinfra is deprecated, use pytest-testinfra Signed-off-by: Adam Warner Co-authored-by: Adam Hill --- .github/workflows/test.yml | 4 +- .gitignore | 1 + test/_centos_7.Dockerfile | 1 + test/_centos_8.Dockerfile | 1 + test/_fedora_33.Dockerfile | 1 + test/_fedora_34.Dockerfile | 1 + test/conftest.py | 99 ++----- test/requirements.txt | 12 +- test/test_automated_install.py | 339 +++++++++++----------- test/test_centos_7_support.py | 22 +- test/test_centos_8_support.py | 22 +- test/test_centos_common_support.py | 48 +-- test/test_centos_fedora_common_support.py | 24 +- test/test_fedora_support.py | 8 +- test/tox.centos_7.ini | 2 +- test/tox.centos_8.ini | 2 +- test/tox.debian_10.ini | 2 +- test/tox.debian_11.ini | 2 +- test/tox.debian_9.ini | 2 +- test/tox.fedora_33.ini | 2 +- test/tox.fedora_34.ini | 2 +- test/tox.ubuntu_16.ini | 2 +- test/tox.ubuntu_18.ini | 2 +- test/tox.ubuntu_20.ini | 2 +- test/tox.ubuntu_21.ini | 2 +- 25 files changed, 284 insertions(+), 321 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 49f139e133..17557a87b4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,10 +36,10 @@ jobs: name: Checkout repository uses: actions/checkout@v2 - - name: Set up Python 3.7 + name: Set up Python 3.8 uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: 3.8 - name: Install dependencies run: pip install -r test/requirements.txt diff --git a/.gitignore b/.gitignore index c19555ed3b..8016472b10 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ __pycache__ *.egg-info .idea/ *.iml +.vscode/ diff --git a/test/_centos_7.Dockerfile b/test/_centos_7.Dockerfile index 434242bf8a..355f4fdb41 100644 --- a/test/_centos_7.Dockerfile +++ b/test/_centos_7.Dockerfile @@ -1,4 +1,5 @@ FROM centos:7 +RUN yum install -y git ENV GITDIR /etc/.pihole ENV SCRIPTDIR /opt/pihole diff --git a/test/_centos_8.Dockerfile b/test/_centos_8.Dockerfile index afd2dc8acc..fddb3ed14c 100644 --- a/test/_centos_8.Dockerfile +++ b/test/_centos_8.Dockerfile @@ -1,4 +1,5 @@ FROM centos:8 +RUN yum install -y git ENV GITDIR /etc/.pihole ENV SCRIPTDIR /opt/pihole diff --git a/test/_fedora_33.Dockerfile b/test/_fedora_33.Dockerfile index 9ae94c7082..5cdd66eeca 100644 --- a/test/_fedora_33.Dockerfile +++ b/test/_fedora_33.Dockerfile @@ -1,4 +1,5 @@ FROM fedora:33 +RUN dnf install -y git ENV GITDIR /etc/.pihole ENV SCRIPTDIR /opt/pihole diff --git a/test/_fedora_34.Dockerfile b/test/_fedora_34.Dockerfile index 96de18da0e..fbbaacd6da 100644 --- a/test/_fedora_34.Dockerfile +++ b/test/_fedora_34.Dockerfile @@ -1,4 +1,5 @@ FROM fedora:34 +RUN dnf install -y git ENV GITDIR /etc/.pihole ENV SCRIPTDIR /opt/pihole diff --git a/test/conftest.py b/test/conftest.py index 2d388fddf1..fb7e1eea33 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,10 +1,9 @@ import pytest import testinfra +import testinfra.backend.docker +import subprocess from textwrap import dedent -check_output = testinfra.get_backend( - "local://" -).get_module("Command").check_output SETUPVARS = { 'PIHOLE_INTERFACE': 'eth99', @@ -12,85 +11,42 @@ 'PIHOLE_DNS_2': '4.2.2.2' } +IMAGE = 'pytest_pihole:test_container' + tick_box = "[\x1b[1;32m\u2713\x1b[0m]" cross_box = "[\x1b[1;31m\u2717\x1b[0m]" info_box = "[i]" -@pytest.fixture -def Pihole(Docker): - ''' - used to contain some script stubbing, now pretty much an alias. - Also provides bash as the default run function shell - ''' - def run_bash(self, command, *args, **kwargs): - cmd = self.get_command(command, *args) - if self.user is not None: - out = self.run_local( - "docker exec -u %s %s /bin/bash -c %s", - self.user, self.name, cmd) - else: - out = self.run_local( - "docker exec %s /bin/bash -c %s", self.name, cmd) - out.command = self.encode(cmd) - return out - - funcType = type(Docker.run) - Docker.run = funcType(run_bash, Docker) - return Docker +# Monkeypatch sh to bash, if they ever support non hard code /bin/sh this can go away +# https://github.com/pytest-dev/pytest-testinfra/blob/master/testinfra/backend/docker.py +def run_bash(self, command, *args, **kwargs): + cmd = self.get_command(command, *args) + if self.user is not None: + out = self.run_local( + "docker exec -u %s %s /bin/bash -c %s", self.user, self.name, cmd + ) + else: + out = self.run_local("docker exec %s /bin/bash -c %s", self.name, cmd) + out.command = self.encode(cmd) + return out -@pytest.fixture -def Docker(request, args, image, cmd): - ''' - combine our fixtures into a docker run command and setup finalizer to - cleanup - ''' - assert 'docker' in check_output('id'), "Are you in the docker group?" - docker_run = "docker run {} {} {}".format(args, image, cmd) - docker_id = check_output(docker_run) - - def teardown(): - check_output("docker rm -f %s", docker_id) - request.addfinalizer(teardown) - - docker_container = testinfra.get_backend("docker://" + docker_id) - docker_container.id = docker_id - return docker_container +testinfra.backend.docker.DockerBackend.run = run_bash @pytest.fixture -def args(request): - ''' - -t became required when tput began being used - ''' - return '-t -d --cap-add=ALL' +def host(): + # run a container + docker_id = subprocess.check_output( + ['docker', 'run', '-t', '-d', '--cap-add=ALL', IMAGE]).decode().strip() + # return a testinfra connection to the container + docker_host = testinfra.get_host("docker://" + docker_id) -@pytest.fixture(params=[ - 'test_container' -]) -def tag(request): - ''' - consumed by image to make the test matrix - ''' - return request.param - - -@pytest.fixture() -def image(request, tag): - ''' - built by test_000_build_containers.py - ''' - return 'pytest_pihole:{}'.format(tag) - - -@pytest.fixture() -def cmd(request): - ''' - default to doing nothing by tailing null, but don't exit - ''' - return 'tail -f /dev/null' + yield docker_host + # at the end of the test suite, destroy the container + subprocess.check_call(['docker', 'rm', '-f', docker_id]) # Helper functions @@ -129,7 +85,7 @@ def mock_command_passthrough(script, args, container): Example use-case: mocking `git pull` but still allowing `git clone` to work as intended ''' - orig_script_path = check_output('which {}'.format(script)) + orig_script_path = container.check_output('command -v {}'.format(script)) full_script_path = '/usr/local/bin/{}'.format(script) mock_script = dedent(r'''\ #!/bin/bash -e @@ -209,6 +165,7 @@ def mock_command_2(script, args, container): content=mock_script, scriptlog=script)) + def run_script(Pihole, script): result = Pihole.run(script) assert result.rc == 0 diff --git a/test/requirements.txt b/test/requirements.txt index 17d62ad965..d65ee6a5cf 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ -docker-compose==1.23.2 -pytest==4.3.0 -pytest-xdist==1.26.1 -pytest-cov==2.6.1 -testinfra==1.19.0 -tox==3.7.0 +docker-compose +pytest +pytest-xdist +pytest-cov +pytest-testinfra +tox diff --git a/test/test_automated_install.py b/test/test_automated_install.py index 304a8783d9..911407fc7d 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -14,14 +14,14 @@ ) -def test_supported_package_manager(Pihole): +def test_supported_package_manager(host): ''' confirm installer exits when no supported package manager found ''' # break supported package managers - Pihole.run('rm -rf /usr/bin/apt-get') - Pihole.run('rm -rf /usr/bin/rpm') - package_manager_detect = Pihole.run(''' + host.run('rm -rf /usr/bin/apt-get') + host.run('rm -rf /usr/bin/rpm') + package_manager_detect = host.run(''' source /opt/pihole/basic-install.sh package_manager_detect ''') @@ -30,7 +30,7 @@ def test_supported_package_manager(Pihole): # assert package_manager_detect.rc == 1 -def test_setupVars_are_sourced_to_global_scope(Pihole): +def test_setupVars_are_sourced_to_global_scope(host): ''' currently update_dialogs sources setupVars with a dot, then various other functions use the variables. @@ -40,7 +40,7 @@ def test_setupVars_are_sourced_to_global_scope(Pihole): for k, v in SETUPVARS.items(): setup_var_file += "{}={}\n".format(k, v) setup_var_file += "EOF\n" - Pihole.run(setup_var_file) + host.run(setup_var_file) script = dedent('''\ set -e @@ -58,13 +58,13 @@ def test_setupVars_are_sourced_to_global_scope(Pihole): printSetupVars ''') - output = run_script(Pihole, script).stdout + output = run_script(host, script).stdout for k, v in SETUPVARS.items(): assert "{}={}".format(k, v) in output -def test_setupVars_saved_to_file(Pihole): +def test_setupVars_saved_to_file(host): ''' confirm saved settings are written to a file for future updates to re-use ''' @@ -72,7 +72,7 @@ def test_setupVars_saved_to_file(Pihole): set_setup_vars = '\n' for k, v in SETUPVARS.items(): set_setup_vars += " {}={}\n".format(k, v) - Pihole.run(set_setup_vars).stdout + host.run(set_setup_vars) script = dedent('''\ set -e @@ -87,17 +87,17 @@ def test_setupVars_saved_to_file(Pihole): cat /etc/pihole/setupVars.conf '''.format(set_setup_vars)) - output = run_script(Pihole, script).stdout + output = run_script(host, script).stdout for k, v in SETUPVARS.items(): assert "{}={}".format(k, v) in output -def test_selinux_not_detected(Pihole): +def test_selinux_not_detected(host): ''' confirms installer continues when SELinux configuration file does not exist ''' - check_selinux = Pihole.run(''' + check_selinux = host.run(''' rm -f /etc/selinux/config source /opt/pihole/basic-install.sh checkSelinux @@ -107,11 +107,11 @@ def test_selinux_not_detected(Pihole): assert check_selinux.rc == 0 -def test_installPiholeWeb_fresh_install_no_errors(Pihole): +def test_installPiholeWeb_fresh_install_no_errors(host): ''' confirms all web page assets from Core repo are installed on a fresh build ''' - installWeb = Pihole.run(''' + installWeb = host.run(''' umask 0027 source /opt/pihole/basic-install.sh installPiholeWeb @@ -128,19 +128,19 @@ def test_installPiholeWeb_fresh_install_no_errors(Pihole): assert expected_stdout in installWeb.stdout expected_stdout = tick_box + ' Installing sudoer file' assert expected_stdout in installWeb.stdout - web_directory = Pihole.run('ls -r /var/www/html/pihole').stdout + web_directory = host.run('ls -r /var/www/html/pihole').stdout assert 'index.php' in web_directory assert 'blockingpage.css' in web_directory -def get_directories_recursive(Pihole, directory): +def get_directories_recursive(host, directory): if directory is None: return directory - ls = Pihole.run('ls -d {}'.format(directory + '/*/')) + ls = host.run('ls -d {}'.format(directory + '/*/')) directories = list(filter(bool, ls.stdout.splitlines())) dirs = directories for dirval in directories: - dir_rec = get_directories_recursive(Pihole, dirval) + dir_rec = get_directories_recursive(host, dirval) if isinstance(dir_rec, str): dirs.extend([dir_rec]) else: @@ -148,14 +148,14 @@ def get_directories_recursive(Pihole, directory): return dirs -def test_installPihole_fresh_install_readableFiles(Pihole): +def test_installPihole_fresh_install_readableFiles(host): ''' confirms all neccessary files are readable by pihole user ''' # Whiptail dialog returns Cancel for user prompt - mock_command('whiptail', {'*': ('', '0')}, Pihole) + mock_command('whiptail', {'*': ('', '0')}, host) # mock git pull - mock_command_passthrough('git', {'pull': ('', '0')}, Pihole) + mock_command_passthrough('git', {'pull': ('', '0')}, host) # mock systemctl to not start lighttpd and FTL mock_command_2( 'systemctl', @@ -189,12 +189,12 @@ def test_installPihole_fresh_install_readableFiles(Pihole): '0' ), }, - Pihole + host ) # try to install man - Pihole.run('command -v apt-get > /dev/null && apt-get install -qq man') - Pihole.run('command -v dnf > /dev/null && dnf install -y man') - Pihole.run('command -v yum > /dev/null && yum install -y man') + host.run('command -v apt-get > /dev/null && apt-get install -qq man') + host.run('command -v dnf > /dev/null && dnf install -y man') + host.run('command -v yum > /dev/null && yum install -y man') # create configuration file setup_var_file = 'cat < /etc/pihole/setupVars.conf\n' for k, v in SETUPVARS.items(): @@ -202,8 +202,8 @@ def test_installPihole_fresh_install_readableFiles(Pihole): setup_var_file += "INSTALL_WEB_SERVER=true\n" setup_var_file += "INSTALL_WEB_INTERFACE=true\n" setup_var_file += "EOF\n" - Pihole.run(setup_var_file) - install = Pihole.run(''' + host.run(setup_var_file) + install = host.run(''' export TERM=xterm export DEBIAN_FRONTEND=noninteractive umask 0027 @@ -223,172 +223,172 @@ def test_installPihole_fresh_install_readableFiles(Pihole): test_cmd = 'su --shell /bin/bash --command "test -{0} {1}" -p {2}' # check files in /etc/pihole for read, write and execute permission check_etc = test_cmd.format('r', '/etc/pihole', piholeuser) - actual_rc = Pihole.run(check_etc).rc + actual_rc = host.run(check_etc).rc assert exit_status_success == actual_rc check_etc = test_cmd.format('x', '/etc/pihole', piholeuser) - actual_rc = Pihole.run(check_etc).rc + actual_rc = host.run(check_etc).rc assert exit_status_success == actual_rc # readable and writable dhcp.leases check_leases = test_cmd.format('r', '/etc/pihole/dhcp.leases', piholeuser) - actual_rc = Pihole.run(check_leases).rc + actual_rc = host.run(check_leases).rc assert exit_status_success == actual_rc check_leases = test_cmd.format('w', '/etc/pihole/dhcp.leases', piholeuser) - actual_rc = Pihole.run(check_leases).rc + actual_rc = host.run(check_leases).rc # readable dns-servers.conf assert exit_status_success == actual_rc check_servers = test_cmd.format( 'r', '/etc/pihole/dns-servers.conf', piholeuser) - actual_rc = Pihole.run(check_servers).rc + actual_rc = host.run(check_servers).rc assert exit_status_success == actual_rc # readable GitHubVersions check_version = test_cmd.format( 'r', '/etc/pihole/GitHubVersions', piholeuser) - actual_rc = Pihole.run(check_version).rc + actual_rc = host.run(check_version).rc assert exit_status_success == actual_rc # readable install.log check_install = test_cmd.format( 'r', '/etc/pihole/install.log', piholeuser) - actual_rc = Pihole.run(check_install).rc + actual_rc = host.run(check_install).rc assert exit_status_success == actual_rc # readable localbranches check_localbranch = test_cmd.format( 'r', '/etc/pihole/localbranches', piholeuser) - actual_rc = Pihole.run(check_localbranch).rc + actual_rc = host.run(check_localbranch).rc assert exit_status_success == actual_rc # readable localversions check_localversion = test_cmd.format( 'r', '/etc/pihole/localversions', piholeuser) - actual_rc = Pihole.run(check_localversion).rc + actual_rc = host.run(check_localversion).rc assert exit_status_success == actual_rc # readable logrotate check_logrotate = test_cmd.format( 'r', '/etc/pihole/logrotate', piholeuser) - actual_rc = Pihole.run(check_logrotate).rc + actual_rc = host.run(check_logrotate).rc assert exit_status_success == actual_rc # readable macvendor.db check_macvendor = test_cmd.format( 'r', '/etc/pihole/macvendor.db', piholeuser) - actual_rc = Pihole.run(check_macvendor).rc + actual_rc = host.run(check_macvendor).rc assert exit_status_success == actual_rc # readable and writeable pihole-FTL.conf check_FTLconf = test_cmd.format( 'r', '/etc/pihole/pihole-FTL.conf', piholeuser) - actual_rc = Pihole.run(check_FTLconf).rc + actual_rc = host.run(check_FTLconf).rc assert exit_status_success == actual_rc check_FTLconf = test_cmd.format( 'w', '/etc/pihole/pihole-FTL.conf', piholeuser) - actual_rc = Pihole.run(check_FTLconf).rc + actual_rc = host.run(check_FTLconf).rc assert exit_status_success == actual_rc # readable setupVars.conf check_setup = test_cmd.format( 'r', '/etc/pihole/setupVars.conf', piholeuser) - actual_rc = Pihole.run(check_setup).rc + actual_rc = host.run(check_setup).rc assert exit_status_success == actual_rc # check dnsmasq files # readable /etc/dnsmasq.conf check_dnsmasqconf = test_cmd.format( 'r', '/etc/dnsmasq.conf', piholeuser) - actual_rc = Pihole.run(check_dnsmasqconf).rc + actual_rc = host.run(check_dnsmasqconf).rc assert exit_status_success == actual_rc # readable /etc/dnsmasq.d/01-pihole.conf check_dnsmasqconf = test_cmd.format( 'r', '/etc/dnsmasq.d', piholeuser) - actual_rc = Pihole.run(check_dnsmasqconf).rc + actual_rc = host.run(check_dnsmasqconf).rc assert exit_status_success == actual_rc check_dnsmasqconf = test_cmd.format( 'x', '/etc/dnsmasq.d', piholeuser) - actual_rc = Pihole.run(check_dnsmasqconf).rc + actual_rc = host.run(check_dnsmasqconf).rc assert exit_status_success == actual_rc check_dnsmasqconf = test_cmd.format( 'r', '/etc/dnsmasq.d/01-pihole.conf', piholeuser) - actual_rc = Pihole.run(check_dnsmasqconf).rc + actual_rc = host.run(check_dnsmasqconf).rc assert exit_status_success == actual_rc # check readable and executable /etc/init.d/pihole-FTL check_init = test_cmd.format( 'x', '/etc/init.d/pihole-FTL', piholeuser) - actual_rc = Pihole.run(check_init).rc + actual_rc = host.run(check_init).rc assert exit_status_success == actual_rc check_init = test_cmd.format( 'r', '/etc/init.d/pihole-FTL', piholeuser) - actual_rc = Pihole.run(check_init).rc + actual_rc = host.run(check_init).rc assert exit_status_success == actual_rc # check readable /etc/lighttpd/lighttpd.conf check_lighttpd = test_cmd.format( 'r', '/etc/lighttpd/lighttpd.conf', piholeuser) - actual_rc = Pihole.run(check_lighttpd).rc + actual_rc = host.run(check_lighttpd).rc assert exit_status_success == actual_rc # check readable and executable manpages if maninstalled is True: check_man = test_cmd.format( 'x', '/usr/local/share/man', piholeuser) - actual_rc = Pihole.run(check_man).rc + actual_rc = host.run(check_man).rc assert exit_status_success == actual_rc check_man = test_cmd.format( 'r', '/usr/local/share/man', piholeuser) - actual_rc = Pihole.run(check_man).rc + actual_rc = host.run(check_man).rc assert exit_status_success == actual_rc check_man = test_cmd.format( 'x', '/usr/local/share/man/man8', piholeuser) - actual_rc = Pihole.run(check_man).rc + actual_rc = host.run(check_man).rc assert exit_status_success == actual_rc check_man = test_cmd.format( 'r', '/usr/local/share/man/man8', piholeuser) - actual_rc = Pihole.run(check_man).rc + actual_rc = host.run(check_man).rc assert exit_status_success == actual_rc check_man = test_cmd.format( 'x', '/usr/local/share/man/man5', piholeuser) - actual_rc = Pihole.run(check_man).rc + actual_rc = host.run(check_man).rc assert exit_status_success == actual_rc check_man = test_cmd.format( 'r', '/usr/local/share/man/man5', piholeuser) - actual_rc = Pihole.run(check_man).rc + actual_rc = host.run(check_man).rc assert exit_status_success == actual_rc check_man = test_cmd.format( 'r', '/usr/local/share/man/man8/pihole.8', piholeuser) - actual_rc = Pihole.run(check_man).rc + actual_rc = host.run(check_man).rc assert exit_status_success == actual_rc check_man = test_cmd.format( 'r', '/usr/local/share/man/man8/pihole-FTL.8', piholeuser) - actual_rc = Pihole.run(check_man).rc + actual_rc = host.run(check_man).rc assert exit_status_success == actual_rc check_man = test_cmd.format( 'r', '/usr/local/share/man/man5/pihole-FTL.conf.5', piholeuser) - actual_rc = Pihole.run(check_man).rc + actual_rc = host.run(check_man).rc assert exit_status_success == actual_rc # check not readable sudoers file check_sudo = test_cmd.format( 'r', '/etc/sudoers.d/pihole', piholeuser) - actual_rc = Pihole.run(check_sudo).rc + actual_rc = host.run(check_sudo).rc assert exit_status_success != actual_rc # check not readable cron file check_sudo = test_cmd.format( 'x', '/etc/cron.d/', piholeuser) - actual_rc = Pihole.run(check_sudo).rc + actual_rc = host.run(check_sudo).rc assert exit_status_success == actual_rc check_sudo = test_cmd.format( 'r', '/etc/cron.d/', piholeuser) - actual_rc = Pihole.run(check_sudo).rc + actual_rc = host.run(check_sudo).rc assert exit_status_success == actual_rc check_sudo = test_cmd.format( 'r', '/etc/cron.d/pihole', piholeuser) - actual_rc = Pihole.run(check_sudo).rc + actual_rc = host.run(check_sudo).rc assert exit_status_success == actual_rc - directories = get_directories_recursive(Pihole, '/etc/.pihole/') + directories = get_directories_recursive(host, '/etc/.pihole/') for directory in directories: check_pihole = test_cmd.format('r', directory, piholeuser) - actual_rc = Pihole.run(check_pihole).rc + actual_rc = host.run(check_pihole).rc check_pihole = test_cmd.format('x', directory, piholeuser) - actual_rc = Pihole.run(check_pihole).rc + actual_rc = host.run(check_pihole).rc findfiles = 'find "{}" -maxdepth 1 -type f -exec echo {{}} \\;;' - filelist = Pihole.run(findfiles.format(directory)) + filelist = host.run(findfiles.format(directory)) files = list(filter(bool, filelist.stdout.splitlines())) for file in files: check_pihole = test_cmd.format('r', file, piholeuser) - actual_rc = Pihole.run(check_pihole).rc + actual_rc = host.run(check_pihole).rc @pytest.mark.parametrize("test_webpage", [True]) -def test_installPihole_fresh_install_readableBlockpage(Pihole, test_webpage): +def test_installPihole_fresh_install_readableBlockpage(host, test_webpage): ''' confirms all web page assets from Core repo are readable by $LIGHTTPD_USER on a fresh build @@ -398,9 +398,10 @@ def test_installPihole_fresh_install_readableBlockpage(Pihole, test_webpage): # "pi.hole" ] # Whiptail dialog returns Cancel for user prompt - mock_command('whiptail', {'*': ('', '0')}, Pihole) + mock_command('whiptail', {'*': ('', '0')}, host) + # mock git pull - mock_command_passthrough('git', {'pull': ('', '0')}, Pihole) + mock_command_passthrough('git', {'pull': ('', '0')}, host) # mock systemctl to start lighttpd and FTL ligthttpdcommand = dedent(r'''\"\" echo 'starting lighttpd with {}' @@ -478,7 +479,7 @@ def test_installPihole_fresh_install_readableBlockpage(Pihole, test_webpage): '0' ), }, - Pihole + host ) # create configuration file setup_var_file = 'cat < /etc/pihole/setupVars.conf\n' @@ -488,8 +489,8 @@ def test_installPihole_fresh_install_readableBlockpage(Pihole, test_webpage): setup_var_file += "INSTALL_WEB_INTERFACE=true\n" setup_var_file += "IPV4_ADDRESS=127.0.0.1\n" setup_var_file += "EOF\n" - Pihole.run(setup_var_file) - installWeb = Pihole.run(''' + host.run(setup_var_file) + installWeb = host.run(''' export TERM=xterm export DEBIAN_FRONTEND=noninteractive umask 0027 @@ -542,55 +543,55 @@ def test_installPihole_fresh_install_readableBlockpage(Pihole, test_webpage): # readable and writeable pihole-FTL.db check_FTLconf = test_cmd.format( 'r', '/etc/pihole/pihole-FTL.db', piholeuser) - actual_rc = Pihole.run(check_FTLconf).rc + actual_rc = host.run(check_FTLconf).rc assert exit_status_success == actual_rc check_FTLconf = test_cmd.format( 'w', '/etc/pihole/pihole-FTL.db', piholeuser) - actual_rc = Pihole.run(check_FTLconf).rc + actual_rc = host.run(check_FTLconf).rc assert exit_status_success == actual_rc # check directories above $webroot for read and execute permission check_var = test_cmd.format('r', '/var', webuser) - actual_rc = Pihole.run(check_var).rc + actual_rc = host.run(check_var).rc assert exit_status_success == actual_rc check_var = test_cmd.format('x', '/var', webuser) - actual_rc = Pihole.run(check_var).rc + actual_rc = host.run(check_var).rc assert exit_status_success == actual_rc check_www = test_cmd.format('r', '/var/www', webuser) - actual_rc = Pihole.run(check_www).rc + actual_rc = host.run(check_www).rc assert exit_status_success == actual_rc check_www = test_cmd.format('x', '/var/www', webuser) - actual_rc = Pihole.run(check_www).rc + actual_rc = host.run(check_www).rc assert exit_status_success == actual_rc check_html = test_cmd.format('r', '/var/www/html', webuser) - actual_rc = Pihole.run(check_html).rc + actual_rc = host.run(check_html).rc assert exit_status_success == actual_rc check_html = test_cmd.format('x', '/var/www/html', webuser) - actual_rc = Pihole.run(check_html).rc + actual_rc = host.run(check_html).rc assert exit_status_success == actual_rc # check directories below $webroot for read and execute permission check_admin = test_cmd.format('r', webroot + '/admin', webuser) - actual_rc = Pihole.run(check_admin).rc + actual_rc = host.run(check_admin).rc assert exit_status_success == actual_rc check_admin = test_cmd.format('x', webroot + '/admin', webuser) - actual_rc = Pihole.run(check_admin).rc + actual_rc = host.run(check_admin).rc assert exit_status_success == actual_rc - directories = get_directories_recursive(Pihole, webroot + '/admin/*/') + directories = get_directories_recursive(host, webroot + '/admin/*/') for directory in directories: check_pihole = test_cmd.format('r', directory, webuser) - actual_rc = Pihole.run(check_pihole).rc + actual_rc = host.run(check_pihole).rc check_pihole = test_cmd.format('x', directory, webuser) - actual_rc = Pihole.run(check_pihole).rc + actual_rc = host.run(check_pihole).rc findfiles = 'find "{}" -maxdepth 1 -type f -exec echo {{}} \\;;' - filelist = Pihole.run(findfiles.format(directory)) + filelist = host.run(findfiles.format(directory)) files = list(filter(bool, filelist.stdout.splitlines())) for file in files: check_pihole = test_cmd.format('r', file, webuser) - actual_rc = Pihole.run(check_pihole).rc + actual_rc = host.run(check_pihole).rc # check web interface files # change nameserver to pi-hole # setting nameserver in /etc/resolv.conf to pi-hole does # not work here because of the way docker uses this file - ns = Pihole.run( + ns = host.run( r"sed -i 's/nameserver.*/nameserver 127.0.0.1/' /etc/resolv.conf") pihole_is_ns = ns.rc == 0 @@ -599,19 +600,19 @@ def is_ip(address): return bool(m) if installWebInterface is True: check_pihole = test_cmd.format('r', webroot + '/pihole', webuser) - actual_rc = Pihole.run(check_pihole).rc + actual_rc = host.run(check_pihole).rc assert exit_status_success == actual_rc check_pihole = test_cmd.format('x', webroot + '/pihole', webuser) - actual_rc = Pihole.run(check_pihole).rc + actual_rc = host.run(check_pihole).rc assert exit_status_success == actual_rc # check most important files in $webroot for read permission check_index = test_cmd.format( 'r', webroot + '/pihole/index.php', webuser) - actual_rc = Pihole.run(check_index).rc + actual_rc = host.run(check_index).rc assert exit_status_success == actual_rc check_blockpage = test_cmd.format( 'r', webroot + '/pihole/blockingpage.css', webuser) - actual_rc = Pihole.run(check_blockpage).rc + actual_rc = host.run(check_blockpage).rc assert exit_status_success == actual_rc if test_webpage is True: # check webpage for unreadable files @@ -631,23 +632,23 @@ def is_ip(address): testpage = "http://" + page + "/admin/" resolvesuccess = True if is_ip(page) is False: - dig = Pihole.run(digcommand.format(page)) + dig = host.run(digcommand.format(page)) testpage = "http://" + dig.stdout.strip() + "/admin/" resolvesuccess = dig.rc == 0 if resolvesuccess or pihole_is_ns: # check HTTP status of blockpage - actual_rc = Pihole.run(status.format(testpage)) + actual_rc = host.run(status.format(testpage)) assert exit_status_success == actual_rc.rc # check for PHP error - actual_output = Pihole.run(pagecontent.format(testpage)) + actual_output = host.run(pagecontent.format(testpage)) assert noPHPfopen.match(actual_output.stdout) is None -def test_update_package_cache_success_no_errors(Pihole): +def test_update_package_cache_success_no_errors(host): ''' confirms package cache was updated without any errors ''' - updateCache = Pihole.run(''' + updateCache = host.run(''' source /opt/pihole/basic-install.sh package_manager_detect update_package_cache @@ -657,12 +658,12 @@ def test_update_package_cache_success_no_errors(Pihole): assert 'error' not in updateCache.stdout.lower() -def test_update_package_cache_failure_no_errors(Pihole): +def test_update_package_cache_failure_no_errors(host): ''' confirms package cache was not updated ''' - mock_command('apt-get', {'update': ('', '1')}, Pihole) - updateCache = Pihole.run(''' + mock_command('apt-get', {'update': ('', '1')}, host) + updateCache = host.run(''' source /opt/pihole/basic-install.sh package_manager_detect update_package_cache @@ -672,12 +673,12 @@ def test_update_package_cache_failure_no_errors(Pihole): assert 'Error: Unable to update package cache.' in updateCache.stdout -def test_FTL_detect_aarch64_no_errors(Pihole): +def test_FTL_detect_aarch64_no_errors(host): ''' confirms only aarch64 package is downloaded for FTL engine ''' # mock uname to return aarch64 platform - mock_command('uname', {'-m': ('aarch64', '0')}, Pihole) + mock_command('uname', {'-m': ('aarch64', '0')}, host) # mock ldd to respond with aarch64 shared library mock_command( 'ldd', @@ -687,9 +688,9 @@ def test_FTL_detect_aarch64_no_errors(Pihole): '0' ) }, - Pihole + host ) - detectPlatform = Pihole.run(''' + detectPlatform = host.run(''' source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -705,15 +706,15 @@ def test_FTL_detect_aarch64_no_errors(Pihole): assert expected_stdout in detectPlatform.stdout -def test_FTL_detect_armv4t_no_errors(Pihole): +def test_FTL_detect_armv4t_no_errors(host): ''' confirms only armv4t package is downloaded for FTL engine ''' # mock uname to return armv4t platform - mock_command('uname', {'-m': ('armv4t', '0')}, Pihole) + mock_command('uname', {'-m': ('armv4t', '0')}, host) # mock ldd to respond with ld-linux shared library - mock_command('ldd', {'/bin/ls': ('/lib/ld-linux.so.3', '0')}, Pihole) - detectPlatform = Pihole.run(''' + mock_command('ldd', {'/bin/ls': ('/lib/ld-linux.so.3', '0')}, host) + detectPlatform = host.run(''' source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -729,15 +730,15 @@ def test_FTL_detect_armv4t_no_errors(Pihole): assert expected_stdout in detectPlatform.stdout -def test_FTL_detect_armv5te_no_errors(Pihole): +def test_FTL_detect_armv5te_no_errors(host): ''' confirms only armv5te package is downloaded for FTL engine ''' # mock uname to return armv5te platform - mock_command('uname', {'-m': ('armv5te', '0')}, Pihole) + mock_command('uname', {'-m': ('armv5te', '0')}, host) # mock ldd to respond with ld-linux shared library - mock_command('ldd', {'/bin/ls': ('/lib/ld-linux.so.3', '0')}, Pihole) - detectPlatform = Pihole.run(''' + mock_command('ldd', {'/bin/ls': ('/lib/ld-linux.so.3', '0')}, host) + detectPlatform = host.run(''' source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -753,15 +754,15 @@ def test_FTL_detect_armv5te_no_errors(Pihole): assert expected_stdout in detectPlatform.stdout -def test_FTL_detect_armv6l_no_errors(Pihole): +def test_FTL_detect_armv6l_no_errors(host): ''' confirms only armv6l package is downloaded for FTL engine ''' # mock uname to return armv6l platform - mock_command('uname', {'-m': ('armv6l', '0')}, Pihole) + mock_command('uname', {'-m': ('armv6l', '0')}, host) # mock ldd to respond with ld-linux-armhf shared library - mock_command('ldd', {'/bin/ls': ('/lib/ld-linux-armhf.so.3', '0')}, Pihole) - detectPlatform = Pihole.run(''' + mock_command('ldd', {'/bin/ls': ('/lib/ld-linux-armhf.so.3', '0')}, host) + detectPlatform = host.run(''' source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -778,15 +779,15 @@ def test_FTL_detect_armv6l_no_errors(Pihole): assert expected_stdout in detectPlatform.stdout -def test_FTL_detect_armv7l_no_errors(Pihole): +def test_FTL_detect_armv7l_no_errors(host): ''' confirms only armv7l package is downloaded for FTL engine ''' # mock uname to return armv7l platform - mock_command('uname', {'-m': ('armv7l', '0')}, Pihole) + mock_command('uname', {'-m': ('armv7l', '0')}, host) # mock ldd to respond with ld-linux-armhf shared library - mock_command('ldd', {'/bin/ls': ('/lib/ld-linux-armhf.so.3', '0')}, Pihole) - detectPlatform = Pihole.run(''' + mock_command('ldd', {'/bin/ls': ('/lib/ld-linux-armhf.so.3', '0')}, host) + detectPlatform = host.run(''' source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -803,15 +804,15 @@ def test_FTL_detect_armv7l_no_errors(Pihole): assert expected_stdout in detectPlatform.stdout -def test_FTL_detect_armv8a_no_errors(Pihole): +def test_FTL_detect_armv8a_no_errors(host): ''' confirms only armv8a package is downloaded for FTL engine ''' # mock uname to return armv8a platform - mock_command('uname', {'-m': ('armv8a', '0')}, Pihole) + mock_command('uname', {'-m': ('armv8a', '0')}, host) # mock ldd to respond with ld-linux-armhf shared library - mock_command('ldd', {'/bin/ls': ('/lib/ld-linux-armhf.so.3', '0')}, Pihole) - detectPlatform = Pihole.run(''' + mock_command('ldd', {'/bin/ls': ('/lib/ld-linux-armhf.so.3', '0')}, host) + detectPlatform = host.run(''' source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -827,11 +828,11 @@ def test_FTL_detect_armv8a_no_errors(Pihole): assert expected_stdout in detectPlatform.stdout -def test_FTL_detect_x86_64_no_errors(Pihole): +def test_FTL_detect_x86_64_no_errors(host): ''' confirms only x86_64 package is downloaded for FTL engine ''' - detectPlatform = Pihole.run(''' + detectPlatform = host.run(''' source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -847,11 +848,11 @@ def test_FTL_detect_x86_64_no_errors(Pihole): assert expected_stdout in detectPlatform.stdout -def test_FTL_detect_unknown_no_errors(Pihole): +def test_FTL_detect_unknown_no_errors(host): ''' confirms only generic package is downloaded for FTL engine ''' # mock uname to return generic platform - mock_command('uname', {'-m': ('mips', '0')}, Pihole) - detectPlatform = Pihole.run(''' + mock_command('uname', {'-m': ('mips', '0')}, host) + detectPlatform = host.run(''' source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -863,18 +864,18 @@ def test_FTL_detect_unknown_no_errors(Pihole): assert expected_stdout in detectPlatform.stdout -def test_FTL_download_aarch64_no_errors(Pihole): +def test_FTL_download_aarch64_no_errors(host): ''' confirms only aarch64 package is downloaded for FTL engine ''' # mock whiptail answers and ensure installer dependencies - mock_command('whiptail', {'*': ('', '0')}, Pihole) - Pihole.run(''' + mock_command('whiptail', {'*': ('', '0')}, host) + host.run(''' source /opt/pihole/basic-install.sh package_manager_detect install_dependent_packages ${INSTALLER_DEPS[@]} ''') - download_binary = Pihole.run(''' + download_binary = host.run(''' source /opt/pihole/basic-install.sh create_pihole_user FTLinstall "pihole-FTL-aarch64-linux-gnu" @@ -884,11 +885,11 @@ def test_FTL_download_aarch64_no_errors(Pihole): assert 'error' not in download_binary.stdout.lower() -def test_FTL_binary_installed_and_responsive_no_errors(Pihole): +def test_FTL_binary_installed_and_responsive_no_errors(host): ''' confirms FTL binary is copied and functional in installed location ''' - installed_binary = Pihole.run(''' + installed_binary = host.run(''' source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -901,11 +902,11 @@ def test_FTL_binary_installed_and_responsive_no_errors(Pihole): assert expected_stdout in installed_binary.stdout -# def test_FTL_support_files_installed(Pihole): +# def test_FTL_support_files_installed(host): # ''' # confirms FTL support files are installed # ''' -# support_files = Pihole.run(''' +# support_files = host.run(''' # source /opt/pihole/basic-install.sh # FTLdetect # stat -c '%a %n' /var/log/pihole-FTL.log @@ -918,7 +919,7 @@ def test_FTL_binary_installed_and_responsive_no_errors(Pihole): # assert '644 /var/log/pihole-FTL.log' in support_files.stdout -def test_IPv6_only_link_local(Pihole): +def test_IPv6_only_link_local(host): ''' confirms IPv6 blocking is disabled for Link-local address ''' @@ -931,9 +932,9 @@ def test_IPv6_only_link_local(Pihole): '0' ) }, - Pihole + host ) - detectPlatform = Pihole.run(''' + detectPlatform = host.run(''' source /opt/pihole/basic-install.sh find_IPv6_information ''') @@ -941,7 +942,7 @@ def test_IPv6_only_link_local(Pihole): assert expected_stdout in detectPlatform.stdout -def test_IPv6_only_ULA(Pihole): +def test_IPv6_only_ULA(host): ''' confirms IPv6 blocking is enabled for ULA addresses ''' @@ -954,9 +955,9 @@ def test_IPv6_only_ULA(Pihole): '0' ) }, - Pihole + host ) - detectPlatform = Pihole.run(''' + detectPlatform = host.run(''' source /opt/pihole/basic-install.sh find_IPv6_information ''') @@ -964,7 +965,7 @@ def test_IPv6_only_ULA(Pihole): assert expected_stdout in detectPlatform.stdout -def test_IPv6_only_GUA(Pihole): +def test_IPv6_only_GUA(host): ''' confirms IPv6 blocking is enabled for GUA addresses ''' @@ -977,9 +978,9 @@ def test_IPv6_only_GUA(Pihole): '0' ) }, - Pihole + host ) - detectPlatform = Pihole.run(''' + detectPlatform = host.run(''' source /opt/pihole/basic-install.sh find_IPv6_information ''') @@ -987,7 +988,7 @@ def test_IPv6_only_GUA(Pihole): assert expected_stdout in detectPlatform.stdout -def test_IPv6_GUA_ULA_test(Pihole): +def test_IPv6_GUA_ULA_test(host): ''' confirms IPv6 blocking is enabled for GUA and ULA addresses ''' @@ -1001,9 +1002,9 @@ def test_IPv6_GUA_ULA_test(Pihole): '0' ) }, - Pihole + host ) - detectPlatform = Pihole.run(''' + detectPlatform = host.run(''' source /opt/pihole/basic-install.sh find_IPv6_information ''') @@ -1011,7 +1012,7 @@ def test_IPv6_GUA_ULA_test(Pihole): assert expected_stdout in detectPlatform.stdout -def test_IPv6_ULA_GUA_test(Pihole): +def test_IPv6_ULA_GUA_test(host): ''' confirms IPv6 blocking is enabled for GUA and ULA addresses ''' @@ -1025,9 +1026,9 @@ def test_IPv6_ULA_GUA_test(Pihole): '0' ) }, - Pihole + host ) - detectPlatform = Pihole.run(''' + detectPlatform = host.run(''' source /opt/pihole/basic-install.sh find_IPv6_information ''') @@ -1035,13 +1036,13 @@ def test_IPv6_ULA_GUA_test(Pihole): assert expected_stdout in detectPlatform.stdout -def test_validate_ip(Pihole): +def test_validate_ip(host): ''' Tests valid_ip for various IP addresses ''' def test_address(addr, success=True): - output = Pihole.run(''' + output = host.run(''' source /opt/pihole/basic-install.sh valid_ip "{addr}" '''.format(addr=addr)) @@ -1076,9 +1077,9 @@ def test_address(addr, success=True): test_address('0.0.0.0#00001', False) -def test_os_check_fails(Pihole): +def test_os_check_fails(host): ''' Confirms install fails on unsupported OS ''' - Pihole.run(''' + host.run(''' source /opt/pihole/basic-install.sh package_manager_detect install_dependent_packages ${OS_CHECK_DEPS[@]} @@ -1088,7 +1089,7 @@ def test_os_check_fails(Pihole): VERSION_ID="2" EOT ''') - detectOS = Pihole.run('''t + detectOS = host.run('''t source /opt/pihole/basic-install.sh os_check ''') @@ -1096,15 +1097,15 @@ def test_os_check_fails(Pihole): assert expected_stdout in detectOS.stdout -def test_os_check_passes(Pihole): +def test_os_check_passes(host): ''' Confirms OS meets the requirements ''' - Pihole.run(''' + host.run(''' source /opt/pihole/basic-install.sh package_manager_detect install_dependent_packages ${OS_CHECK_DEPS[@]} install_dependent_packages ${INSTALLER_DEPS[@]} ''') - detectOS = Pihole.run(''' + detectOS = host.run(''' source /opt/pihole/basic-install.sh os_check ''') @@ -1112,10 +1113,10 @@ def test_os_check_passes(Pihole): assert expected_stdout in detectOS.stdout -def test_package_manager_has_installer_deps(Pihole): +def test_package_manager_has_installer_deps(host): ''' Confirms OS is able to install the required packages for the installer''' - mock_command('whiptail', {'*': ('', '0')}, Pihole) - output = Pihole.run(''' + mock_command('whiptail', {'*': ('', '0')}, host) + output = host.run(''' source /opt/pihole/basic-install.sh package_manager_detect install_dependent_packages ${INSTALLER_DEPS[@]} @@ -1125,10 +1126,10 @@ def test_package_manager_has_installer_deps(Pihole): assert output.rc == 0 -def test_package_manager_has_pihole_deps(Pihole): +def test_package_manager_has_pihole_deps(host): ''' Confirms OS is able to install the required packages for Pi-hole ''' - mock_command('whiptail', {'*': ('', '0')}, Pihole) - output = Pihole.run(''' + mock_command('whiptail', {'*': ('', '0')}, host) + output = host.run(''' source /opt/pihole/basic-install.sh package_manager_detect select_rpm_php @@ -1139,10 +1140,10 @@ def test_package_manager_has_pihole_deps(Pihole): assert output.rc == 0 -def test_package_manager_has_web_deps(Pihole): +def test_package_manager_has_web_deps(host): ''' Confirms OS is able to install the required packages for web ''' - mock_command('whiptail', {'*': ('', '0')}, Pihole) - output = Pihole.run(''' + mock_command('whiptail', {'*': ('', '0')}, host) + output = host.run(''' source /opt/pihole/basic-install.sh package_manager_detect select_rpm_php diff --git a/test/test_centos_7_support.py b/test/test_centos_7_support.py index 14f62637af..f72740a6b8 100644 --- a/test/test_centos_7_support.py +++ b/test/test_centos_7_support.py @@ -5,11 +5,11 @@ ) -def test_php_upgrade_default_optout_centos_eq_7(Pihole): +def test_php_upgrade_default_optout_centos_eq_7(host): ''' confirms the default behavior to opt-out of installing PHP7 from REMI ''' - package_manager_detect = Pihole.run(''' + package_manager_detect = host.run(''' source /opt/pihole/basic-install.sh package_manager_detect select_rpm_php @@ -17,18 +17,18 @@ def test_php_upgrade_default_optout_centos_eq_7(Pihole): expected_stdout = info_box + (' User opt-out of PHP 7 upgrade on CentOS. ' 'Deprecated PHP may be in use.') assert expected_stdout in package_manager_detect.stdout - remi_package = Pihole.package('remi-release') + remi_package = host.package('remi-release') assert not remi_package.is_installed -def test_php_upgrade_user_optout_centos_eq_7(Pihole): +def test_php_upgrade_user_optout_centos_eq_7(host): ''' confirms installer behavior when user opt-out of installing PHP7 from REMI (php not currently installed) ''' # Whiptail dialog returns Cancel for user prompt - mock_command('whiptail', {'*': ('', '1')}, Pihole) - package_manager_detect = Pihole.run(''' + mock_command('whiptail', {'*': ('', '1')}, host) + package_manager_detect = host.run(''' source /opt/pihole/basic-install.sh package_manager_detect select_rpm_php @@ -36,18 +36,18 @@ def test_php_upgrade_user_optout_centos_eq_7(Pihole): expected_stdout = info_box + (' User opt-out of PHP 7 upgrade on CentOS. ' 'Deprecated PHP may be in use.') assert expected_stdout in package_manager_detect.stdout - remi_package = Pihole.package('remi-release') + remi_package = host.package('remi-release') assert not remi_package.is_installed -def test_php_upgrade_user_optin_centos_eq_7(Pihole): +def test_php_upgrade_user_optin_centos_eq_7(host): ''' confirms installer behavior when user opt-in to installing PHP7 from REMI (php not currently installed) ''' # Whiptail dialog returns Continue for user prompt - mock_command('whiptail', {'*': ('', '0')}, Pihole) - package_manager_detect = Pihole.run(''' + mock_command('whiptail', {'*': ('', '0')}, host) + package_manager_detect = host.run(''' source /opt/pihole/basic-install.sh package_manager_detect select_rpm_php @@ -59,5 +59,5 @@ def test_php_upgrade_user_optin_centos_eq_7(Pihole): expected_stdout = tick_box + (' Remi\'s RPM repository has ' 'been enabled for PHP7') assert expected_stdout in package_manager_detect.stdout - remi_package = Pihole.package('remi-release') + remi_package = host.package('remi-release') assert remi_package.is_installed diff --git a/test/test_centos_8_support.py b/test/test_centos_8_support.py index bbdbb76582..464055b412 100644 --- a/test/test_centos_8_support.py +++ b/test/test_centos_8_support.py @@ -5,12 +5,12 @@ ) -def test_php_upgrade_default_continue_centos_gte_8(Pihole): +def test_php_upgrade_default_continue_centos_gte_8(host): ''' confirms the latest version of CentOS continues / does not optout (should trigger on CentOS7 only) ''' - package_manager_detect = Pihole.run(''' + package_manager_detect = host.run(''' source /opt/pihole/basic-install.sh package_manager_detect select_rpm_php @@ -19,19 +19,19 @@ def test_php_upgrade_default_continue_centos_gte_8(Pihole): ' Deprecated PHP may be in use.') assert unexpected_stdout not in package_manager_detect.stdout # ensure remi was not installed on latest CentOS - remi_package = Pihole.package('remi-release') + remi_package = host.package('remi-release') assert not remi_package.is_installed -def test_php_upgrade_user_optout_skipped_centos_gte_8(Pihole): +def test_php_upgrade_user_optout_skipped_centos_gte_8(host): ''' confirms installer skips user opt-out of installing PHP7 from REMI on latest CentOS (should trigger on CentOS7 only) (php not currently installed) ''' # Whiptail dialog returns Cancel for user prompt - mock_command('whiptail', {'*': ('', '1')}, Pihole) - package_manager_detect = Pihole.run(''' + mock_command('whiptail', {'*': ('', '1')}, host) + package_manager_detect = host.run(''' source /opt/pihole/basic-install.sh package_manager_detect select_rpm_php @@ -40,19 +40,19 @@ def test_php_upgrade_user_optout_skipped_centos_gte_8(Pihole): ' Deprecated PHP may be in use.') assert unexpected_stdout not in package_manager_detect.stdout # ensure remi was not installed on latest CentOS - remi_package = Pihole.package('remi-release') + remi_package = host.package('remi-release') assert not remi_package.is_installed -def test_php_upgrade_user_optin_skipped_centos_gte_8(Pihole): +def test_php_upgrade_user_optin_skipped_centos_gte_8(host): ''' confirms installer skips user opt-in to installing PHP7 from REMI on latest CentOS (should trigger on CentOS7 only) (php not currently installed) ''' # Whiptail dialog returns Continue for user prompt - mock_command('whiptail', {'*': ('', '0')}, Pihole) - package_manager_detect = Pihole.run(''' + mock_command('whiptail', {'*': ('', '0')}, host) + package_manager_detect = host.run(''' source /opt/pihole/basic-install.sh package_manager_detect select_rpm_php @@ -64,5 +64,5 @@ def test_php_upgrade_user_optin_skipped_centos_gte_8(Pihole): unexpected_stdout = tick_box + (' Remi\'s RPM repository has ' 'been enabled for PHP7') assert unexpected_stdout not in package_manager_detect.stdout - remi_package = Pihole.package('remi-release') + remi_package = host.package('remi-release') assert not remi_package.is_installed diff --git a/test/test_centos_common_support.py b/test/test_centos_common_support.py index 0b36cbaef8..8903a7a07e 100644 --- a/test/test_centos_common_support.py +++ b/test/test_centos_common_support.py @@ -7,13 +7,13 @@ ) -def test_release_supported_version_check_centos(Pihole): +def test_release_supported_version_check_centos(host): ''' confirms installer exits on unsupported releases of CentOS ''' # modify /etc/redhat-release to mock an unsupported CentOS release - Pihole.run('echo "CentOS Linux release 6.9" > /etc/redhat-release') - package_manager_detect = Pihole.run(''' + host.run('echo "CentOS Linux release 6.9" > /etc/redhat-release') + package_manager_detect = host.run(''' source /opt/pihole/basic-install.sh package_manager_detect select_rpm_php @@ -24,11 +24,11 @@ def test_release_supported_version_check_centos(Pihole): assert expected_stdout in package_manager_detect.stdout -def test_enable_epel_repository_centos(Pihole): +def test_enable_epel_repository_centos(host): ''' confirms the EPEL package repository is enabled when installed on CentOS ''' - package_manager_detect = Pihole.run(''' + package_manager_detect = host.run(''' source /opt/pihole/basic-install.sh package_manager_detect select_rpm_php @@ -38,22 +38,22 @@ def test_enable_epel_repository_centos(Pihole): assert expected_stdout in package_manager_detect.stdout expected_stdout = tick_box + ' Installed epel-release' assert expected_stdout in package_manager_detect.stdout - epel_package = Pihole.package('epel-release') + epel_package = host.package('epel-release') assert epel_package.is_installed -def test_php_version_lt_7_detected_upgrade_default_optout_centos(Pihole): +def test_php_version_lt_7_detected_upgrade_default_optout_centos(host): ''' confirms the default behavior to opt-out of upgrading to PHP7 from REMI ''' # first we will install the default php version to test installer behavior - php_install = Pihole.run('yum install -y php') + php_install = host.run('yum install -y php') assert php_install.rc == 0 - php_package = Pihole.package('php') + php_package = host.package('php') default_centos_php_version = php_package.version.split('.')[0] if int(default_centos_php_version) >= 7: # PHP7 is supported/recommended pytest.skip("Test deprecated . Detected default PHP version >= 7") - package_manager_detect = Pihole.run(''' + package_manager_detect = host.run(''' source /opt/pihole/basic-install.sh package_manager_detect select_rpm_php @@ -61,24 +61,24 @@ def test_php_version_lt_7_detected_upgrade_default_optout_centos(Pihole): expected_stdout = info_box + (' User opt-out of PHP 7 upgrade on CentOS. ' 'Deprecated PHP may be in use.') assert expected_stdout in package_manager_detect.stdout - remi_package = Pihole.package('remi-release') + remi_package = host.package('remi-release') assert not remi_package.is_installed -def test_php_version_lt_7_detected_upgrade_user_optout_centos(Pihole): +def test_php_version_lt_7_detected_upgrade_user_optout_centos(host): ''' confirms installer behavior when user opt-out to upgrade to PHP7 via REMI ''' # first we will install the default php version to test installer behavior - php_install = Pihole.run('yum install -y php') + php_install = host.run('yum install -y php') assert php_install.rc == 0 - php_package = Pihole.package('php') + php_package = host.package('php') default_centos_php_version = php_package.version.split('.')[0] if int(default_centos_php_version) >= 7: # PHP7 is supported/recommended pytest.skip("Test deprecated . Detected default PHP version >= 7") # Whiptail dialog returns Cancel for user prompt - mock_command('whiptail', {'*': ('', '1')}, Pihole) - package_manager_detect = Pihole.run(''' + mock_command('whiptail', {'*': ('', '1')}, host) + package_manager_detect = host.run(''' source /opt/pihole/basic-install.sh package_manager_detect select_rpm_php @@ -86,24 +86,24 @@ def test_php_version_lt_7_detected_upgrade_user_optout_centos(Pihole): expected_stdout = info_box + (' User opt-out of PHP 7 upgrade on CentOS. ' 'Deprecated PHP may be in use.') assert expected_stdout in package_manager_detect.stdout - remi_package = Pihole.package('remi-release') + remi_package = host.package('remi-release') assert not remi_package.is_installed -def test_php_version_lt_7_detected_upgrade_user_optin_centos(Pihole): +def test_php_version_lt_7_detected_upgrade_user_optin_centos(host): ''' confirms installer behavior when user opt-in to upgrade to PHP7 via REMI ''' # first we will install the default php version to test installer behavior - php_install = Pihole.run('yum install -y php') + php_install = host.run('yum install -y php') assert php_install.rc == 0 - php_package = Pihole.package('php') + php_package = host.package('php') default_centos_php_version = php_package.version.split('.')[0] if int(default_centos_php_version) >= 7: # PHP7 is supported/recommended pytest.skip("Test deprecated . Detected default PHP version >= 7") # Whiptail dialog returns Continue for user prompt - mock_command('whiptail', {'*': ('', '0')}, Pihole) - package_manager_detect = Pihole.run(''' + mock_command('whiptail', {'*': ('', '0')}, host) + package_manager_detect = host.run(''' source /opt/pihole/basic-install.sh package_manager_detect select_rpm_php @@ -118,8 +118,8 @@ def test_php_version_lt_7_detected_upgrade_user_optin_centos(Pihole): expected_stdout = tick_box + (' Remi\'s RPM repository has ' 'been enabled for PHP7') assert expected_stdout in package_manager_detect.stdout - remi_package = Pihole.package('remi-release') + remi_package = host.package('remi-release') assert remi_package.is_installed - updated_php_package = Pihole.package('php') + updated_php_package = host.package('php') updated_php_version = updated_php_package.version.split('.')[0] assert int(updated_php_version) == 7 diff --git a/test/test_centos_fedora_common_support.py b/test/test_centos_fedora_common_support.py index 21ae6d1dd0..a2a13048fc 100644 --- a/test/test_centos_fedora_common_support.py +++ b/test/test_centos_fedora_common_support.py @@ -5,7 +5,7 @@ ) -def mock_selinux_config(state, Pihole): +def mock_selinux_config(state, host): ''' Creates a mock SELinux config file with expected content ''' @@ -13,20 +13,20 @@ def mock_selinux_config(state, Pihole): valid_states = ['enforcing', 'permissive', 'disabled'] assert state in valid_states # getenforce returns the running state of SELinux - mock_command('getenforce', {'*': (state.capitalize(), '0')}, Pihole) + mock_command('getenforce', {'*': (state.capitalize(), '0')}, host) # create mock configuration with desired content - Pihole.run(''' + host.run(''' mkdir /etc/selinux echo "SELINUX={state}" > /etc/selinux/config '''.format(state=state.lower())) -def test_selinux_enforcing_exit(Pihole): +def test_selinux_enforcing_exit(host): ''' confirms installer prompts to exit when SELinux is Enforcing by default ''' - mock_selinux_config("enforcing", Pihole) - check_selinux = Pihole.run(''' + mock_selinux_config("enforcing", host) + check_selinux = host.run(''' source /opt/pihole/basic-install.sh checkSelinux ''') @@ -37,12 +37,12 @@ def test_selinux_enforcing_exit(Pihole): assert check_selinux.rc == 1 -def test_selinux_permissive(Pihole): +def test_selinux_permissive(host): ''' confirms installer continues when SELinux is Permissive ''' - mock_selinux_config("permissive", Pihole) - check_selinux = Pihole.run(''' + mock_selinux_config("permissive", host) + check_selinux = host.run(''' source /opt/pihole/basic-install.sh checkSelinux ''') @@ -51,12 +51,12 @@ def test_selinux_permissive(Pihole): assert check_selinux.rc == 0 -def test_selinux_disabled(Pihole): +def test_selinux_disabled(host): ''' confirms installer continues when SELinux is Disabled ''' - mock_selinux_config("disabled", Pihole) - check_selinux = Pihole.run(''' + mock_selinux_config("disabled", host) + check_selinux = host.run(''' source /opt/pihole/basic-install.sh checkSelinux ''') diff --git a/test/test_fedora_support.py b/test/test_fedora_support.py index 3ad84be537..63fde90e0d 100644 --- a/test/test_fedora_support.py +++ b/test/test_fedora_support.py @@ -1,16 +1,16 @@ -def test_epel_and_remi_not_installed_fedora(Pihole): +def test_epel_and_remi_not_installed_fedora(host): ''' confirms installer does not attempt to install EPEL/REMI repositories on Fedora ''' - package_manager_detect = Pihole.run(''' + package_manager_detect = host.run(''' source /opt/pihole/basic-install.sh package_manager_detect select_rpm_php ''') assert package_manager_detect.stdout == '' - epel_package = Pihole.package('epel-release') + epel_package = host.package('epel-release') assert not epel_package.is_installed - remi_package = Pihole.package('remi-release') + remi_package = host.package('remi-release') assert not remi_package.is_installed diff --git a/test/tox.centos_7.ini b/test/tox.centos_7.ini index c27526986b..88940fddf2 100644 --- a/test/tox.centos_7.ini +++ b/test/tox.centos_7.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37 +envlist = py38 [testenv] whitelist_externals = docker diff --git a/test/tox.centos_8.ini b/test/tox.centos_8.ini index c8852bae60..5088da16f7 100644 --- a/test/tox.centos_8.ini +++ b/test/tox.centos_8.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37 +envlist = py38 [testenv] whitelist_externals = docker diff --git a/test/tox.debian_10.ini b/test/tox.debian_10.ini index e8c5d74627..9c2a05d171 100644 --- a/test/tox.debian_10.ini +++ b/test/tox.debian_10.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37 +envlist = py38 [testenv] whitelist_externals = docker diff --git a/test/tox.debian_11.ini b/test/tox.debian_11.ini index af3c6e366b..f3cdbe84e7 100644 --- a/test/tox.debian_11.ini +++ b/test/tox.debian_11.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37 +envlist = py38 [testenv] whitelist_externals = docker diff --git a/test/tox.debian_9.ini b/test/tox.debian_9.ini index b33e9927ab..b46e0a493b 100644 --- a/test/tox.debian_9.ini +++ b/test/tox.debian_9.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37 +envlist = py38 [testenv] whitelist_externals = docker diff --git a/test/tox.fedora_33.ini b/test/tox.fedora_33.ini index 00ea732ae3..d33fbf5375 100644 --- a/test/tox.fedora_33.ini +++ b/test/tox.fedora_33.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37 +envlist = py38 [testenv] whitelist_externals = docker diff --git a/test/tox.fedora_34.ini b/test/tox.fedora_34.ini index 154662cff4..819291fac0 100644 --- a/test/tox.fedora_34.ini +++ b/test/tox.fedora_34.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37 +envlist = py38 [testenv] whitelist_externals = docker diff --git a/test/tox.ubuntu_16.ini b/test/tox.ubuntu_16.ini index 6f484b6b9d..bce948a2bf 100644 --- a/test/tox.ubuntu_16.ini +++ b/test/tox.ubuntu_16.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37 +envlist = py38 [testenv] whitelist_externals = docker diff --git a/test/tox.ubuntu_18.ini b/test/tox.ubuntu_18.ini index 10d4ac0951..cf7a364232 100644 --- a/test/tox.ubuntu_18.ini +++ b/test/tox.ubuntu_18.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37 +envlist = py38 [testenv] whitelist_externals = docker diff --git a/test/tox.ubuntu_20.ini b/test/tox.ubuntu_20.ini index 8b405b28eb..03b605ce98 100644 --- a/test/tox.ubuntu_20.ini +++ b/test/tox.ubuntu_20.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37 +envlist = py38 [testenv] whitelist_externals = docker diff --git a/test/tox.ubuntu_21.ini b/test/tox.ubuntu_21.ini index 651f933b73..12b1ac0b27 100644 --- a/test/tox.ubuntu_21.ini +++ b/test/tox.ubuntu_21.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37 +envlist = py38 [testenv] whitelist_externals = docker From 996a2c74fa0004fdd40b172cd9eec695a2d31768 Mon Sep 17 00:00:00 2001 From: MichaIng Date: Mon, 22 Nov 2021 13:25:13 +0100 Subject: [PATCH 13/99] Use a fixed list height for network interface selection This solves the issue reported here: https://github.com/pi-hole/pi-hole/issues/4196 It replaces the other suggested solution here: https://github.com/pi-hole/pi-hole/pull/4197 The benefit of using a fixed/limited list height, compared to allowing larger whiptail/dialogue dimension, is that it works on small screens as well, where the screen or console size itself is too small to hold the interface list + text above + whiptail frame. It the amount of list elements exceeds the defined list height, automatically a visual scroll bar is added and the list can be scrolled with up/down pageup/pagedown buttons, hence it is generally not required to adjust a list height based on the amount of elements. The fixed height of "6" is chosen since all other "--radiolist" calls use this fixed height as well, it fits and looks good within a 20 rows high whiptail dialogue, and in the common Pi-hole use cases there are no more than 6 network interfaces. Signed-off-by: MichaIng --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index c806a6aea8..5c317edc57 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -674,7 +674,7 @@ chooseInterface() { # Feed the available interfaces into this while loop done <<< "${availableInterfaces}" # The whiptail command that will be run, stored in a variable - chooseInterfaceCmd=(whiptail --separate-output --radiolist "Choose An Interface (press space to toggle selection)" "${r}" "${c}" "${interfaceCount}") + chooseInterfaceCmd=(whiptail --separate-output --radiolist "Choose An Interface (press space to toggle selection)" "${r}" "${c}" 6) # Now run the command using the interfaces saved into the array chooseInterfaceOptions=$("${chooseInterfaceCmd[@]}" "${interfacesArray[@]}" 2>&1 >/dev/tty) || \ # If the user chooses Cancel, exit From bc8150adfa12b00b7f04ea0c15270e19de84b381 Mon Sep 17 00:00:00 2001 From: jbzdarkid Date: Wed, 24 Nov 2021 22:41:40 -0800 Subject: [PATCH 14/99] Clean up bash script formatting Done with the help of beautysh (a python-based bash formatter) Signed-off-by: jbzdarkid --- advanced/Scripts/chronometer.sh | 8 +- .../Scripts/database_migration/gravity-db.sh | 10 +- advanced/Scripts/list.sh | 42 +- advanced/Scripts/piholeDebug.sh | 2 +- advanced/Scripts/query.sh | 28 +- advanced/Scripts/update.sh | 2 +- advanced/Scripts/version.sh | 8 +- advanced/Scripts/webpage.sh | 154 +++---- automated install/basic-install.sh | 380 +++++++++--------- gravity.sh | 52 +-- 10 files changed, 343 insertions(+), 343 deletions(-) diff --git a/advanced/Scripts/chronometer.sh b/advanced/Scripts/chronometer.sh index 3f85bdfc6c..312c484f6c 100755 --- a/advanced/Scripts/chronometer.sh +++ b/advanced/Scripts/chronometer.sh @@ -329,8 +329,8 @@ get_sys_stats() { *) cpu_col="$COL_URG_RED";; esac - # $COL_NC$COL_DARK_GRAY is needed for $COL_URG_RED - cpu_temp_str=" @ $cpu_col$cpu_temp$COL_NC$COL_DARK_GRAY" + # $COL_NC$COL_DARK_GRAY is needed for $COL_URG_RED + cpu_temp_str=" @ $cpu_col$cpu_temp$COL_NC$COL_DARK_GRAY" elif [[ "$temp_unit" == "F" ]]; then cpu_temp=$(printf "%.0ff\\n" "$(calcFunc "($(< $temp_file) / 1000) * 9 / 5 + 32")") @@ -445,7 +445,7 @@ get_strings() { lan_info="Gateway: $net_gateway" dhcp_info="$leased_str$ph_dhcp_num of $ph_dhcp_max" - ads_info="$total_str$ads_blocked_today of $dns_queries_today" + ads_info="$total_str$ads_blocked_today of $dns_queries_today" dns_info="$dns_count DNS servers" [[ "$recent_blocked" == "0" ]] && recent_blocked="${COL_LIGHT_RED}FTL offline${COL_NC}" @@ -488,7 +488,7 @@ chronoFunc() { ${COL_LIGHT_RED}Press Ctrl-C to exit${COL_NC} ${COL_DARK_GRAY}$scr_line_str${COL_NC}" else - echo -e "|¯¯¯(¯)_|¯|_ ___|¯|___$phc_ver_str\\n| ¯_/¯|_| ' \\/ _ \\ / -_)$lte_ver_str\\n|_| |_| |_||_\\___/_\\___|$ftl_ver_str\\n ${COL_DARK_GRAY}$scr_line_str${COL_NC}" + echo -e "|¯¯¯(¯)_|¯|_ ___|¯|___$phc_ver_str\\n| ¯_/¯|_| ' \\/ _ \\ / -_)$lte_ver_str\\n|_| |_| |_||_\\___/_\\___|$ftl_ver_str\\n ${COL_DARK_GRAY}$scr_line_str${COL_NC}" fi printFunc " Hostname: " "$sys_name" "$host_info" diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 0fecf34a18..09dc17275b 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -123,9 +123,9 @@ upgrade_gravityDB(){ version=14 fi if [[ "$version" == "14" ]]; then - # Changes the vw_adlist created in 5_to_6 - echo -e " ${INFO} Upgrading gravity database from version 14 to 15" - sqlite3 "${database}" < "${scriptPath}/14_to_15.sql" - version=15 -fi + # Changes the vw_adlist created in 5_to_6 + echo -e " ${INFO} Upgrading gravity database from version 14 to 15" + sqlite3 "${database}" < "${scriptPath}/14_to_15.sql" + version=15 + fi } diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 5bd42d55a9..bc254515aa 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -16,7 +16,7 @@ GRAVITYDB="${piholeDir}/gravity.db" # Source pihole-FTL from install script pihole_FTL="${piholeDir}/pihole-FTL.conf" if [[ -f "${pihole_FTL}" ]]; then - source "${pihole_FTL}" + source "${pihole_FTL}" fi # Set this only after sourcing pihole-FTL.conf as the gravity database path may @@ -133,7 +133,7 @@ ProcessDomainList() { else RemoveDomain "${dom}" fi - done + done } AddDomain() { @@ -145,19 +145,19 @@ AddDomain() { requestedListname="$(GetListnameFromTypeId "${typeId}")" if [[ "${num}" -ne 0 ]]; then - existingTypeId="$(sqlite3 "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")" - if [[ "${existingTypeId}" == "${typeId}" ]]; then - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} already exists in ${requestedListname}, no need to add!" - fi - else - existingListname="$(GetListnameFromTypeId "${existingTypeId}")" - sqlite3 "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';" - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} already exists in ${existingListname}, it has been moved to ${requestedListname}!" + existingTypeId="$(sqlite3 "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")" + if [[ "${existingTypeId}" == "${typeId}" ]]; then + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} ${1} already exists in ${requestedListname}, no need to add!" + fi + else + existingListname="$(GetListnameFromTypeId "${existingTypeId}")" + sqlite3 "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';" + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} ${1} already exists in ${existingListname}, it has been moved to ${requestedListname}!" + fi fi - fi - return + return fi # Domain not found in the table, add it! @@ -185,10 +185,10 @@ RemoveDomain() { requestedListname="$(GetListnameFromTypeId "${typeId}")" if [[ "${num}" -eq 0 ]]; then - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${domain} does not exist in ${requestedListname}, no need to remove!" - fi - return + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} ${domain} does not exist in ${requestedListname}, no need to remove!" + fi + return fi # Domain found in the table, remove it! @@ -256,8 +256,8 @@ NukeList() { GetComment() { comment="$1" if [[ "${comment}" =~ [^a-zA-Z0-9_\#:/\.,\ -] ]]; then - echo " ${CROSS} Found invalid characters in domain comment!" - exit + echo " ${CROSS} Found invalid characters in domain comment!" + exit fi } @@ -292,7 +292,7 @@ ProcessDomainList # Used on web interface if $web; then -echo "DONE" + echo "DONE" fi if [[ ${reload} == true && ${noReloadRequested} == false ]]; then diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index cd61582551..ad58d6c736 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -27,7 +27,7 @@ PIHOLE_COLTABLE_FILE="${PIHOLE_SCRIPTS_DIRECTORY}/COL_TABLE" # These provide the colors we need for making the log more readable if [[ -f ${PIHOLE_COLTABLE_FILE} ]]; then - source ${PIHOLE_COLTABLE_FILE} + source ${PIHOLE_COLTABLE_FILE} else COL_NC='\e[0m' # No Color COL_RED='\e[1;91m' diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 26b4508e5b..0fd9871a67 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -21,7 +21,7 @@ matchType="match" # Source pihole-FTL from install script pihole_FTL="${piholeDir}/pihole-FTL.conf" if [[ -f "${pihole_FTL}" ]]; then - source "${pihole_FTL}" + source "${pihole_FTL}" fi # Set this only after sourcing pihole-FTL.conf as the gravity database path may @@ -48,7 +48,7 @@ scanList(){ # Iterate through each regexp and check whether it matches the domainQuery # If it does, print the matching regexp and continue looping # Input 1 - regexps | Input 2 - domainQuery - "regex" ) + "regex" ) for list in ${lists}; do if [[ "${domain}" =~ ${list} ]]; then printf "%b\n" "${list}"; @@ -109,15 +109,15 @@ scanDatabaseTable() { # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores. if [[ "${table}" == "gravity" ]]; then - case "${exact}" in - "exact" ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain = '${domain}'";; - * ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; - esac + case "${exact}" in + "exact" ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain = '${domain}'";; + * ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; + esac else - case "${exact}" in - "exact" ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${type}' AND domain = '${domain}'";; - * ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${type}' AND domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; - esac + case "${exact}" in + "exact" ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${type}' AND domain = '${domain}'";; + * ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${type}' AND domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; + esac fi # Send prepared query to gravity database @@ -128,8 +128,8 @@ scanDatabaseTable() { fi if [[ "${table}" == "gravity" ]]; then - echo "${result}" - return + echo "${result}" + return fi # Mark domain as having been white-/blacklist matched (global variable) @@ -233,9 +233,9 @@ for result in "${results[@]}"; do adlistAddress="${extra/|*/}" extra="${extra#*|}" if [[ "${extra}" == "0" ]]; then - extra="(disabled)" + extra="(disabled)" else - extra="" + extra="" fi if [[ -n "${blockpage}" ]]; then diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index dae0486168..3fab9a9535 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -200,7 +200,7 @@ main() { if [[ "${FTL_update}" == true || "${core_update}" == true ]]; then ${PI_HOLE_FILES_DIR}/automated\ install/basic-install.sh --reconfigure --unattended || \ - echo -e "${basicError}" && exit 1 + echo -e "${basicError}" && exit 1 fi if [[ "${FTL_update}" == true || "${core_update}" == true || "${web_update}" == true ]]; then diff --git a/advanced/Scripts/version.sh b/advanced/Scripts/version.sh index 86ac45bcf3..2cb0a2c9ce 100755 --- a/advanced/Scripts/version.sh +++ b/advanced/Scripts/version.sh @@ -92,9 +92,9 @@ getRemoteVersion(){ if [[ -f "$cachedVersions" ]]; then IFS=' ' read -r -a arrCache < "$cachedVersions" case $daemon in - "pi-hole" ) echo "${arrCache[0]}";; - "AdminLTE" ) echo "${arrCache[1]}";; - "FTL" ) echo "${arrCache[2]}";; + "pi-hole" ) echo "${arrCache[0]}";; + "AdminLTE" ) echo "${arrCache[1]}";; + "FTL" ) echo "${arrCache[2]}";; esac return 0 @@ -117,7 +117,7 @@ getLocalBranch(){ local directory="${1}" local branch - # Local FTL btranch is stored in /etc/pihole/ftlbranch + # Local FTL btranch is stored in /etc/pihole/ftlbranch if [[ "$1" == "FTL" ]]; then branch="$(pihole-FTL branch)" else diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 2a4065651b..ec9b2cac53 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -122,14 +122,14 @@ SetWebPassword() { read -s -r -p "Enter New Password (Blank for no password): " PASSWORD echo "" - if [ "${PASSWORD}" == "" ]; then - change_setting "WEBPASSWORD" "" - echo -e " ${TICK} Password Removed" - exit 0 - fi + if [ "${PASSWORD}" == "" ]; then + change_setting "WEBPASSWORD" "" + echo -e " ${TICK} Password Removed" + exit 0 + fi - read -s -r -p "Confirm Password: " CONFIRM - echo "" + read -s -r -p "Confirm Password: " CONFIRM + echo "" fi if [ "${PASSWORD}" == "${CONFIRM}" ] ; then @@ -247,8 +247,8 @@ trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC68345710423 3 ) REV_SERVER_CIDR="${arrRev[0]}.0.0.0/8";; esac else - # Set REV_SERVER_CIDR to whatever value it was set to - REV_SERVER_CIDR="${CONDITIONAL_FORWARDING_REVERSE}" + # Set REV_SERVER_CIDR to whatever value it was set to + REV_SERVER_CIDR="${CONDITIONAL_FORWARDING_REVERSE}" fi # If REV_SERVER_CIDR is not converted by the above, then use the REV_SERVER_TARGET variable to derive it @@ -371,34 +371,34 @@ ProcessDHCPSettings() { source "${setupVars}" if [[ "${DHCP_ACTIVE}" == "true" ]]; then - interface="${PIHOLE_INTERFACE}" + interface="${PIHOLE_INTERFACE}" - # Use eth0 as fallback interface - if [ -z ${interface} ]; then - interface="eth0" - fi + # Use eth0 as fallback interface + if [ -z ${interface} ]; then + interface="eth0" + fi - if [[ "${PIHOLE_DOMAIN}" == "" ]]; then - PIHOLE_DOMAIN="lan" - change_setting "PIHOLE_DOMAIN" "${PIHOLE_DOMAIN}" - fi + if [[ "${PIHOLE_DOMAIN}" == "" ]]; then + PIHOLE_DOMAIN="lan" + change_setting "PIHOLE_DOMAIN" "${PIHOLE_DOMAIN}" + fi - if [[ "${DHCP_LEASETIME}" == "0" ]]; then - leasetime="infinite" - elif [[ "${DHCP_LEASETIME}" == "" ]]; then - leasetime="24" - change_setting "DHCP_LEASETIME" "${leasetime}" - elif [[ "${DHCP_LEASETIME}" == "24h" ]]; then - #Installation is affected by known bug, introduced in a previous version. - #This will automatically clean up setupVars.conf and remove the unnecessary "h" - leasetime="24" - change_setting "DHCP_LEASETIME" "${leasetime}" - else - leasetime="${DHCP_LEASETIME}h" - fi + if [[ "${DHCP_LEASETIME}" == "0" ]]; then + leasetime="infinite" + elif [[ "${DHCP_LEASETIME}" == "" ]]; then + leasetime="24" + change_setting "DHCP_LEASETIME" "${leasetime}" + elif [[ "${DHCP_LEASETIME}" == "24h" ]]; then + #Installation is affected by known bug, introduced in a previous version. + #This will automatically clean up setupVars.conf and remove the unnecessary "h" + leasetime="24" + change_setting "DHCP_LEASETIME" "${leasetime}" + else + leasetime="${DHCP_LEASETIME}h" + fi - # Write settings to file - echo "############################################################################### + # Write settings to file + echo "############################################################################### # DHCP SERVER CONFIG FILE AUTOMATICALLY POPULATED BY PI-HOLE WEB INTERFACE. # # ANY CHANGES MADE TO THIS FILE WILL BE LOST ON CHANGE # ############################################################################### @@ -408,34 +408,34 @@ dhcp-option=option:router,${DHCP_ROUTER} dhcp-leasefile=/etc/pihole/dhcp.leases #quiet-dhcp " > "${dhcpconfig}" - chmod 644 "${dhcpconfig}" - - if [[ "${PIHOLE_DOMAIN}" != "none" ]]; then - echo "domain=${PIHOLE_DOMAIN}" >> "${dhcpconfig}" - - # When there is a Pi-hole domain set and "Never forward non-FQDNs" is - # ticked, we add `local=/domain/` to tell FTL that this domain is purely - # local and FTL may answer queries from /etc/hosts or DHCP but should - # never forward queries on that domain to any upstream servers - if [[ "${DNS_FQDN_REQUIRED}" == true ]]; then - echo "local=/${PIHOLE_DOMAIN}/" >> "${dhcpconfig}" + chmod 644 "${dhcpconfig}" + + if [[ "${PIHOLE_DOMAIN}" != "none" ]]; then + echo "domain=${PIHOLE_DOMAIN}" >> "${dhcpconfig}" + + # When there is a Pi-hole domain set and "Never forward non-FQDNs" is + # ticked, we add `local=/domain/` to tell FTL that this domain is purely + # local and FTL may answer queries from /etc/hosts or DHCP but should + # never forward queries on that domain to any upstream servers + if [[ "${DNS_FQDN_REQUIRED}" == true ]]; then + echo "local=/${PIHOLE_DOMAIN}/" >> "${dhcpconfig}" + fi fi - fi - # Sourced from setupVars - # shellcheck disable=SC2154 - if [[ "${DHCP_rapid_commit}" == "true" ]]; then - echo "dhcp-rapid-commit" >> "${dhcpconfig}" - fi + # Sourced from setupVars + # shellcheck disable=SC2154 + if [[ "${DHCP_rapid_commit}" == "true" ]]; then + echo "dhcp-rapid-commit" >> "${dhcpconfig}" + fi - if [[ "${DHCP_IPv6}" == "true" ]]; then - echo "#quiet-dhcp6 + if [[ "${DHCP_IPv6}" == "true" ]]; then + echo "#quiet-dhcp6 #enable-ra dhcp-option=option6:dns-server,[::] dhcp-range=::100,::1ff,constructor:${interface},ra-names,slaac,64,3600 ra-param=*,0,0 " >> "${dhcpconfig}" - fi + fi else if [[ -f "${dhcpconfig}" ]]; then @@ -605,7 +605,7 @@ Interfaces: single Listen only on ${PIHOLE_INTERFACE} interface all Listen on all interfaces, permit all origins" exit 0 - fi + fi if [[ "${args[2]}" == "all" ]]; then echo -e " ${INFO} Listening on all interfaces, permitting all origins. Please use a firewall!" @@ -654,18 +654,18 @@ addAudit() domains="" for domain in "$@" do - # Check domain to be added. Only continue if it is valid - validDomain="$(checkDomain "${domain}")" - if [[ -n "${validDomain}" ]]; then - # Put comma in between domains when there is - # more than one domains to be added - # SQL INSERT allows adding multiple rows at once using the format - ## INSERT INTO table (domain) VALUES ('abc.de'),('fgh.ij'),('klm.no'),('pqr.st'); - if [[ -n "${domains}" ]]; then - domains="${domains}," + # Check domain to be added. Only continue if it is valid + validDomain="$(checkDomain "${domain}")" + if [[ -n "${validDomain}" ]]; then + # Put comma in between domains when there is + # more than one domains to be added + # SQL INSERT allows adding multiple rows at once using the format + ## INSERT INTO table (domain) VALUES ('abc.de'),('fgh.ij'),('klm.no'),('pqr.st'); + if [[ -n "${domains}" ]]; then + domains="${domains}," + fi + domains="${domains}('${domain}')" fi - domains="${domains}('${domain}')" - fi done # Insert only the domain here. The date_added field will be # filled with its default value (date_added = current timestamp) @@ -707,7 +707,7 @@ AddCustomDNSAddress() { # Restart dnsmasq to load new custom DNS entries only if $reload not false if [[ ! $reload == "false" ]]; then - RestartDNS + RestartDNS fi } @@ -726,14 +726,14 @@ RemoveCustomDNSAddress() { echo -e " ${CROSS} Invalid IP has been passed" exit 1 fi - else - echo " ${CROSS} Invalid Domain passed!" - exit 1 + else + echo " ${CROSS} Invalid Domain passed!" + exit 1 fi # Restart dnsmasq to load new custom DNS entries only if reload is not false if [[ ! $reload == "false" ]]; then - RestartDNS + RestartDNS fi } @@ -748,10 +748,10 @@ AddCustomCNAMERecord() { if [[ -n "${validDomain}" ]]; then validTarget="$(checkDomain "${target}")" if [[ -n "${validTarget}" ]]; then - echo "cname=${validDomain},${validTarget}" >> "${dnscustomcnamefile}" + echo "cname=${validDomain},${validTarget}" >> "${dnscustomcnamefile}" else - echo " ${CROSS} Invalid Target Passed!" - exit 1 + echo " ${CROSS} Invalid Target Passed!" + exit 1 fi else echo " ${CROSS} Invalid Domain passed!" @@ -759,7 +759,7 @@ AddCustomCNAMERecord() { fi # Restart dnsmasq to load new custom CNAME records only if reload is not false if [[ ! $reload == "false" ]]; then - RestartDNS + RestartDNS fi } @@ -776,8 +776,8 @@ RemoveCustomCNAMERecord() { if [[ -n "${validTarget}" ]]; then sed -i "/cname=${validDomain},${validTarget}$/d" "${dnscustomcnamefile}" else - echo " ${CROSS} Invalid Target Passed!" - exit 1 + echo " ${CROSS} Invalid Target Passed!" + exit 1 fi else echo " ${CROSS} Invalid Domain passed!" @@ -786,7 +786,7 @@ RemoveCustomCNAMERecord() { # Restart dnsmasq to update removed custom CNAME records only if $reload not false if [[ ! $reload == "false" ]]; then - RestartDNS + RestartDNS fi } diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 5c317edc57..cbb8ebae3e 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -76,7 +76,7 @@ PI_HOLE_CONFIG_DIR="/etc/pihole" PI_HOLE_BIN_DIR="/usr/local/bin" PI_HOLE_BLOCKPAGE_DIR="${webroot}/pihole" if [ -z "$useUpdateVars" ]; then - useUpdateVars=false + useUpdateVars=false fi adlistFile="/etc/pihole/adlists.list" @@ -90,7 +90,7 @@ PRIVACY_LEVEL=0 CACHE_SIZE=10000 if [ -z "${USER}" ]; then - USER="$(id -un)" + USER="$(id -un)" fi # whiptail dialog dimensions: 20 rows and 70 chars width assures to fit on small screens and is known to hold all content. @@ -133,7 +133,7 @@ fi # A simple function that just echoes out our logo in ASCII format # This lets users know that it is a Pi-hole, LLC product show_ascii_berry() { - echo -e " + echo -e " ${COL_LIGHT_GREEN}.;;,. .ccccc:,. :cccclll:. ..,, @@ -261,174 +261,174 @@ os_check() { # Compatibility package_manager_detect() { -# First check to see if apt-get is installed. -if is_command apt-get ; then - # Set some global variables here - # We don't set them earlier since the installed package manager might be rpm, so these values would be different - PKG_MANAGER="apt-get" - # A variable to store the command used to update the package cache - UPDATE_PKG_CACHE="${PKG_MANAGER} update" - # The command we will use to actually install packages - PKG_INSTALL=("${PKG_MANAGER}" -qq --no-install-recommends install) - # grep -c will return 1 if there are no matches. This is an acceptable condition, so we OR TRUE to prevent set -e exiting the script. - PKG_COUNT="${PKG_MANAGER} -s -o Debug::NoLocking=true upgrade | grep -c ^Inst || true" - # Update package cache - update_package_cache || exit 1 - # Check for and determine version number (major and minor) of current php install - local phpVer="php" - if is_command php ; then - printf " %b Existing PHP installation detected : PHP version %s\\n" "${INFO}" "$(php <<< "")" - printf -v phpInsMajor "%d" "$(php <<< "")" - printf -v phpInsMinor "%d" "$(php <<< "")" - phpVer="php$phpInsMajor.$phpInsMinor" - fi - # Packages required to perfom the os_check (stored as an array) - OS_CHECK_DEPS=(grep dnsutils) - # Packages required to run this install script (stored as an array) - INSTALLER_DEPS=(git iproute2 whiptail ca-certificates) - # Packages required to run Pi-hole (stored as an array) - PIHOLE_DEPS=(cron curl iputils-ping lsof psmisc sudo unzip idn2 sqlite3 libcap2-bin dns-root-data libcap2) - # Packages required for the Web admin interface (stored as an array) - # It's useful to separate this from Pi-hole, since the two repos are also setup separately - PIHOLE_WEB_DEPS=(lighttpd "${phpVer}-common" "${phpVer}-cgi" "${phpVer}-sqlite3" "${phpVer}-xml" "${phpVer}-intl") - # Prior to PHP8.0, JSON functionality is provided as dedicated module, required by Pi-hole AdminLTE: https://www.php.net/manual/json.installation.php - if [[ -z "${phpInsMajor}" || "${phpInsMajor}" -lt 8 ]]; then - PIHOLE_WEB_DEPS+=("${phpVer}-json") - fi - # The Web server user, - LIGHTTPD_USER="www-data" - # group, - LIGHTTPD_GROUP="www-data" - # and config file - LIGHTTPD_CFG="lighttpd.conf.debian" - - # This function waits for dpkg to unlock, which signals that the previous apt-get command has finished. - test_dpkg_lock() { - i=0 - # fuser is a program to show which processes use the named files, sockets, or filesystems - # So while the lock is held, - while fuser /var/lib/dpkg/lock >/dev/null 2>&1 - do - # we wait half a second, - sleep 0.5 - # increase the iterator, - ((i=i+1)) - done - # and then report success once dpkg is unlocked. - return 0 - } + # First check to see if apt-get is installed. + if is_command apt-get ; then + # Set some global variables here + # We don't set them earlier since the installed package manager might be rpm, so these values would be different + PKG_MANAGER="apt-get" + # A variable to store the command used to update the package cache + UPDATE_PKG_CACHE="${PKG_MANAGER} update" + # The command we will use to actually install packages + PKG_INSTALL=("${PKG_MANAGER}" -qq --no-install-recommends install) + # grep -c will return 1 if there are no matches. This is an acceptable condition, so we OR TRUE to prevent set -e exiting the script. + PKG_COUNT="${PKG_MANAGER} -s -o Debug::NoLocking=true upgrade | grep -c ^Inst || true" + # Update package cache + update_package_cache || exit 1 + # Check for and determine version number (major and minor) of current php install + local phpVer="php" + if is_command php ; then + printf " %b Existing PHP installation detected : PHP version %s\\n" "${INFO}" "$(php <<< "")" + printf -v phpInsMajor "%d" "$(php <<< "")" + printf -v phpInsMinor "%d" "$(php <<< "")" + phpVer="php$phpInsMajor.$phpInsMinor" + fi + # Packages required to perfom the os_check (stored as an array) + OS_CHECK_DEPS=(grep dnsutils) + # Packages required to run this install script (stored as an array) + INSTALLER_DEPS=(git iproute2 whiptail ca-certificates) + # Packages required to run Pi-hole (stored as an array) + PIHOLE_DEPS=(cron curl iputils-ping lsof psmisc sudo unzip idn2 sqlite3 libcap2-bin dns-root-data libcap2) + # Packages required for the Web admin interface (stored as an array) + # It's useful to separate this from Pi-hole, since the two repos are also setup separately + PIHOLE_WEB_DEPS=(lighttpd "${phpVer}-common" "${phpVer}-cgi" "${phpVer}-sqlite3" "${phpVer}-xml" "${phpVer}-intl") + # Prior to PHP8.0, JSON functionality is provided as dedicated module, required by Pi-hole AdminLTE: https://www.php.net/manual/json.installation.php + if [[ -z "${phpInsMajor}" || "${phpInsMajor}" -lt 8 ]]; then + PIHOLE_WEB_DEPS+=("${phpVer}-json") + fi + # The Web server user, + LIGHTTPD_USER="www-data" + # group, + LIGHTTPD_GROUP="www-data" + # and config file + LIGHTTPD_CFG="lighttpd.conf.debian" + + # This function waits for dpkg to unlock, which signals that the previous apt-get command has finished. + test_dpkg_lock() { + i=0 + # fuser is a program to show which processes use the named files, sockets, or filesystems + # So while the lock is held, + while fuser /var/lib/dpkg/lock >/dev/null 2>&1 + do + # we wait half a second, + sleep 0.5 + # increase the iterator, + ((i=i+1)) + done + # and then report success once dpkg is unlocked. + return 0 + } + + # If apt-get is not found, check for rpm. + elif is_command rpm ; then + # Then check if dnf or yum is the package manager + if is_command dnf ; then + PKG_MANAGER="dnf" + else + PKG_MANAGER="yum" + fi -# If apt-get is not found, check for rpm. -elif is_command rpm ; then - # Then check if dnf or yum is the package manager - if is_command dnf ; then - PKG_MANAGER="dnf" + # These variable names match the ones for apt-get. See above for an explanation of what they are for. + PKG_INSTALL=("${PKG_MANAGER}" install -y) + PKG_COUNT="${PKG_MANAGER} check-update | egrep '(.i686|.x86|.noarch|.arm|.src)' | wc -l" + OS_CHECK_DEPS=(grep bind-utils) + INSTALLER_DEPS=(git iproute newt procps-ng which chkconfig ca-certificates) + PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc sqlite libcap lsof) + PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php-common php-cli php-pdo php-xml php-json php-intl) + LIGHTTPD_USER="lighttpd" + LIGHTTPD_GROUP="lighttpd" + LIGHTTPD_CFG="lighttpd.conf.fedora" + + # If neither apt-get or yum/dnf package managers were found else - PKG_MANAGER="yum" + # we cannot install required packages + printf " %b No supported package manager found\\n" "${CROSS}" + # so exit the installer + exit fi - - # These variable names match the ones for apt-get. See above for an explanation of what they are for. - PKG_INSTALL=("${PKG_MANAGER}" install -y) - PKG_COUNT="${PKG_MANAGER} check-update | egrep '(.i686|.x86|.noarch|.arm|.src)' | wc -l" - OS_CHECK_DEPS=(grep bind-utils) - INSTALLER_DEPS=(git iproute newt procps-ng which chkconfig ca-certificates) - PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc sqlite libcap lsof) - PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php-common php-cli php-pdo php-xml php-json php-intl) - LIGHTTPD_USER="lighttpd" - LIGHTTPD_GROUP="lighttpd" - LIGHTTPD_CFG="lighttpd.conf.fedora" - -# If neither apt-get or yum/dnf package managers were found -else - # we cannot install required packages - printf " %b No supported package manager found\\n" "${CROSS}" - # so exit the installer - exit -fi } select_rpm_php(){ -# If the host OS is Fedora, -if grep -qiE 'fedora|fedberry' /etc/redhat-release; then - # all required packages should be available by default with the latest fedora release - : # continue -# or if host OS is CentOS, -elif grep -qiE 'centos|scientific' /etc/redhat-release; then - # Pi-Hole currently supports CentOS 7+ with PHP7+ - SUPPORTED_CENTOS_VERSION=7 - SUPPORTED_CENTOS_PHP_VERSION=7 - # Check current CentOS major release version - CURRENT_CENTOS_VERSION=$(grep -oP '(?<= )[0-9]+(?=\.?)' /etc/redhat-release) - # Check if CentOS version is supported - if [[ $CURRENT_CENTOS_VERSION -lt $SUPPORTED_CENTOS_VERSION ]]; then - printf " %b CentOS %s is not supported.\\n" "${CROSS}" "${CURRENT_CENTOS_VERSION}" - printf " Please update to CentOS release %s or later.\\n" "${SUPPORTED_CENTOS_VERSION}" - # exit the installer - exit - fi - # php-json is not required on CentOS 7 as it is already compiled into php - # verifiy via `php -m | grep json` - if [[ $CURRENT_CENTOS_VERSION -eq 7 ]]; then - # create a temporary array as arrays are not designed for use as mutable data structures - CENTOS7_PIHOLE_WEB_DEPS=() - for i in "${!PIHOLE_WEB_DEPS[@]}"; do - if [[ ${PIHOLE_WEB_DEPS[i]} != "php-json" ]]; then - CENTOS7_PIHOLE_WEB_DEPS+=( "${PIHOLE_WEB_DEPS[i]}" ) - fi - done - # re-assign the clean dependency array back to PIHOLE_WEB_DEPS - PIHOLE_WEB_DEPS=("${CENTOS7_PIHOLE_WEB_DEPS[@]}") - unset CENTOS7_PIHOLE_WEB_DEPS - fi - # CentOS requires the EPEL repository to gain access to Fedora packages - EPEL_PKG="epel-release" - rpm -q ${EPEL_PKG} &> /dev/null || rc=$? - if [[ $rc -ne 0 ]]; then - printf " %b Enabling EPEL package repository (https://fedoraproject.org/wiki/EPEL)\\n" "${INFO}" - "${PKG_INSTALL[@]}" ${EPEL_PKG} &> /dev/null - printf " %b Installed %s\\n" "${TICK}" "${EPEL_PKG}" - fi - - # The default php on CentOS 7.x is 5.4 which is EOL - # Check if the version of PHP available via installed repositories is >= to PHP 7 - AVAILABLE_PHP_VERSION=$("${PKG_MANAGER}" info php | grep -i version | grep -o '[0-9]\+' | head -1) - if [[ $AVAILABLE_PHP_VERSION -ge $SUPPORTED_CENTOS_PHP_VERSION ]]; then - # Since PHP 7 is available by default, install via default PHP package names - : # do nothing as PHP is current - else - REMI_PKG="remi-release" - REMI_REPO="remi-php72" - rpm -q ${REMI_PKG} &> /dev/null || rc=$? - if [[ $rc -ne 0 ]]; then - # The PHP version available via default repositories is older than version 7 - if ! whiptail --defaultno --title "PHP 7 Update (recommended)" --yesno "PHP 7.x is recommended for both security and language features.\\nWould you like to install PHP7 via Remi's RPM repository?\\n\\nSee: https://rpms.remirepo.net for more information" "${r}" "${c}"; then - # User decided to NOT update PHP from REMI, attempt to install the default available PHP version - printf " %b User opt-out of PHP 7 upgrade on CentOS. Deprecated PHP may be in use.\\n" "${INFO}" - : # continue with unsupported php version + # If the host OS is Fedora, + if grep -qiE 'fedora|fedberry' /etc/redhat-release; then + # all required packages should be available by default with the latest fedora release + : # continue + # or if host OS is CentOS, + elif grep -qiE 'centos|scientific' /etc/redhat-release; then + # Pi-Hole currently supports CentOS 7+ with PHP7+ + SUPPORTED_CENTOS_VERSION=7 + SUPPORTED_CENTOS_PHP_VERSION=7 + # Check current CentOS major release version + CURRENT_CENTOS_VERSION=$(grep -oP '(?<= )[0-9]+(?=\.?)' /etc/redhat-release) + # Check if CentOS version is supported + if [[ $CURRENT_CENTOS_VERSION -lt $SUPPORTED_CENTOS_VERSION ]]; then + printf " %b CentOS %s is not supported.\\n" "${CROSS}" "${CURRENT_CENTOS_VERSION}" + printf " Please update to CentOS release %s or later.\\n" "${SUPPORTED_CENTOS_VERSION}" + # exit the installer + exit + fi + # php-json is not required on CentOS 7 as it is already compiled into php + # verifiy via `php -m | grep json` + if [[ $CURRENT_CENTOS_VERSION -eq 7 ]]; then + # create a temporary array as arrays are not designed for use as mutable data structures + CENTOS7_PIHOLE_WEB_DEPS=() + for i in "${!PIHOLE_WEB_DEPS[@]}"; do + if [[ ${PIHOLE_WEB_DEPS[i]} != "php-json" ]]; then + CENTOS7_PIHOLE_WEB_DEPS+=( "${PIHOLE_WEB_DEPS[i]}" ) + fi + done + # re-assign the clean dependency array back to PIHOLE_WEB_DEPS + PIHOLE_WEB_DEPS=("${CENTOS7_PIHOLE_WEB_DEPS[@]}") + unset CENTOS7_PIHOLE_WEB_DEPS + fi + # CentOS requires the EPEL repository to gain access to Fedora packages + EPEL_PKG="epel-release" + rpm -q ${EPEL_PKG} &> /dev/null || rc=$? + if [[ $rc -ne 0 ]]; then + printf " %b Enabling EPEL package repository (https://fedoraproject.org/wiki/EPEL)\\n" "${INFO}" + "${PKG_INSTALL[@]}" ${EPEL_PKG} &> /dev/null + printf " %b Installed %s\\n" "${TICK}" "${EPEL_PKG}" + fi + + # The default php on CentOS 7.x is 5.4 which is EOL + # Check if the version of PHP available via installed repositories is >= to PHP 7 + AVAILABLE_PHP_VERSION=$("${PKG_MANAGER}" info php | grep -i version | grep -o '[0-9]\+' | head -1) + if [[ $AVAILABLE_PHP_VERSION -ge $SUPPORTED_CENTOS_PHP_VERSION ]]; then + # Since PHP 7 is available by default, install via default PHP package names + : # do nothing as PHP is current else - printf " %b Enabling Remi's RPM repository (https://rpms.remirepo.net)\\n" "${INFO}" - "${PKG_INSTALL[@]}" "https://rpms.remirepo.net/enterprise/${REMI_PKG}-$(rpm -E '%{rhel}').rpm" &> /dev/null - # enable the PHP 7 repository via yum-config-manager (provided by yum-utils) - "${PKG_INSTALL[@]}" "yum-utils" &> /dev/null - yum-config-manager --enable ${REMI_REPO} &> /dev/null - printf " %b Remi's RPM repository has been enabled for PHP7\\n" "${TICK}" - # trigger an install/update of PHP to ensure previous version of PHP is updated from REMI - if "${PKG_INSTALL[@]}" "php-cli" &> /dev/null; then - printf " %b PHP7 installed/updated via Remi's RPM repository\\n" "${TICK}" + REMI_PKG="remi-release" + REMI_REPO="remi-php72" + rpm -q ${REMI_PKG} &> /dev/null || rc=$? + if [[ $rc -ne 0 ]]; then + # The PHP version available via default repositories is older than version 7 + if ! whiptail --defaultno --title "PHP 7 Update (recommended)" --yesno "PHP 7.x is recommended for both security and language features.\\nWould you like to install PHP7 via Remi's RPM repository?\\n\\nSee: https://rpms.remirepo.net for more information" "${r}" "${c}"; then + # User decided to NOT update PHP from REMI, attempt to install the default available PHP version + printf " %b User opt-out of PHP 7 upgrade on CentOS. Deprecated PHP may be in use.\\n" "${INFO}" + : # continue with unsupported php version + else + printf " %b Enabling Remi's RPM repository (https://rpms.remirepo.net)\\n" "${INFO}" + "${PKG_INSTALL[@]}" "https://rpms.remirepo.net/enterprise/${REMI_PKG}-$(rpm -E '%{rhel}').rpm" &> /dev/null + # enable the PHP 7 repository via yum-config-manager (provided by yum-utils) + "${PKG_INSTALL[@]}" "yum-utils" &> /dev/null + yum-config-manager --enable ${REMI_REPO} &> /dev/null + printf " %b Remi's RPM repository has been enabled for PHP7\\n" "${TICK}" + # trigger an install/update of PHP to ensure previous version of PHP is updated from REMI + if "${PKG_INSTALL[@]}" "php-cli" &> /dev/null; then + printf " %b PHP7 installed/updated via Remi's RPM repository\\n" "${TICK}" + else + printf " %b There was a problem updating to PHP7 via Remi's RPM repository\\n" "${CROSS}" + exit 1 + fi + fi + fi # Warn user of unsupported version of Fedora or CentOS + if ! whiptail --defaultno --title "Unsupported RPM based distribution" --yesno "Would you like to continue installation on an unsupported RPM based distribution?\\n\\nPlease ensure the following packages have been installed manually:\\n\\n- lighttpd\\n- lighttpd-fastcgi\\n- PHP version 7+" "${r}" "${c}"; then + printf " %b Aborting installation due to unsupported RPM based distribution\\n" "${CROSS}" + exit else - printf " %b There was a problem updating to PHP7 via Remi's RPM repository\\n" "${CROSS}" - exit 1 + printf " %b Continuing installation with unsupported RPM based distribution\\n" "${INFO}" fi fi - fi # Warn user of unsupported version of Fedora or CentOS - if ! whiptail --defaultno --title "Unsupported RPM based distribution" --yesno "Would you like to continue installation on an unsupported RPM based distribution?\\n\\nPlease ensure the following packages have been installed manually:\\n\\n- lighttpd\\n- lighttpd-fastcgi\\n- PHP version 7+" "${r}" "${c}"; then - printf " %b Aborting installation due to unsupported RPM based distribution\\n" "${CROSS}" - exit - else - printf " %b Continuing installation with unsupported RPM based distribution\\n" "${INFO}" fi -fi -fi } # A function for checking if a directory is a git repository @@ -519,7 +519,7 @@ update_repo() { # In case extra commits have been added after tagging/release (i.e in case of metadata updates/README.MD tweaks) curBranch=$(git rev-parse --abbrev-ref HEAD) if [[ "${curBranch}" == "master" ]]; then - git reset --hard "$(git describe --abbrev=0 --tags)" || return $? + git reset --hard "$(git describe --abbrev=0 --tags)" || return $? fi # Show a completion message printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" @@ -629,12 +629,12 @@ welcomeDialogs() { IMPORTANT: If you have not already done so, you must ensure that this device has a static IP. Either through DHCP reservation, or by manually assigning one. Depending on your operating system, there are many ways to achieve this. Choose yes to indicate that you have understood this message, and wish to continue" "${r}" "${c}"; then -#Nothing to do, continue - echo -else - printf " %b Installer exited at static IP message.\\n" "${INFO}" - exit 1 -fi + #Nothing to do, continue + echo + else + printf " %b Installer exited at static IP message.\\n" "${INFO}" + exit 1 + fi } # A function that lets the user pick an interface to use with Pi-hole @@ -759,8 +759,8 @@ collect_v4andv6_information() { printf " %b IPv4 address: %s\\n" "${INFO}" "${IPV4_ADDRESS}" # if `dhcpcd` is used offer to set this as static IP for the device if [[ -f "/etc/dhcpcd.conf" ]]; then - # configure networking via dhcpcd - getStaticIPv4Settings + # configure networking via dhcpcd + getStaticIPv4Settings fi find_IPv6_information printf " %b IPv6 address: %s\\n" "${INFO}" "${IPV6_ADDRESS}" @@ -913,8 +913,8 @@ setDNS() { IFS=${OIFS} # In a whiptail dialog, show the options DNSchoices=$(whiptail --separate-output --menu "Select Upstream DNS Provider. To use your own, select Custom." "${r}" "${c}" 7 \ - "${DNSChooseOptions[@]}" 2>&1 >/dev/tty) || \ - # Exit if the user selects "Cancel" + "${DNSChooseOptions[@]}" 2>&1 >/dev/tty) || \ + # Exit if the user selects "Cancel" { printf " %bCancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}"; exit 1; } # Depending on the user's choice, set the GLOBAL variables to the IP of the respective provider @@ -1180,8 +1180,8 @@ version_check_dnsmasq() { install -D -m 644 -T "${dnsmasq_original_config}" "${dnsmasq_conf}" printf "%b %b Restoring default dnsmasq.conf...\\n" "${OVER}" "${TICK}" else - # Otherwise, don't to anything - printf " it is not a Pi-hole file, leaving alone!\\n" + # Otherwise, don't to anything + printf " it is not a Pi-hole file, leaving alone!\\n" fi else # If a file cannot be found, @@ -1216,8 +1216,8 @@ version_check_dnsmasq() { sed -i '/^server=@DNS2@/d' "${dnsmasq_pihole_01_target}" fi - # Set the cache size - sed -i "s/@CACHE_SIZE@/$CACHE_SIZE/" "${dnsmasq_pihole_01_target}" + # Set the cache size + sed -i "s/@CACHE_SIZE@/$CACHE_SIZE/" "${dnsmasq_pihole_01_target}" sed -i 's/^#conf-dir=\/etc\/dnsmasq.d$/conf-dir=\/etc\/dnsmasq.d/' "${dnsmasq_conf}" @@ -1565,7 +1565,7 @@ install_dependent_packages() { # Install Fedora/CentOS packages for i in "$@"; do - # For each package, check if it's already installed (and if so, don't add it to the installArray) + # For each package, check if it's already installed (and if so, don't add it to the installArray) printf " %b Checking for %s..." "${INFO}" "${i}" if "${PKG_MANAGER}" -q list installed "${i}" &> /dev/null; then printf "%b %b Checking for %s\\n" "${OVER}" "${TICK}" "${i}" @@ -1735,18 +1735,18 @@ finalExports() { fi # echo the information to the user { - echo "PIHOLE_INTERFACE=${PIHOLE_INTERFACE}" - echo "IPV4_ADDRESS=${IPV4_ADDRESS}" - echo "IPV6_ADDRESS=${IPV6_ADDRESS}" - echo "PIHOLE_DNS_1=${PIHOLE_DNS_1}" - echo "PIHOLE_DNS_2=${PIHOLE_DNS_2}" - echo "QUERY_LOGGING=${QUERY_LOGGING}" - echo "INSTALL_WEB_SERVER=${INSTALL_WEB_SERVER}" - echo "INSTALL_WEB_INTERFACE=${INSTALL_WEB_INTERFACE}" - echo "LIGHTTPD_ENABLED=${LIGHTTPD_ENABLED}" - echo "CACHE_SIZE=${CACHE_SIZE}" - echo "DNS_FQDN_REQUIRED=${DNS_FQDN_REQUIRED:-true}" - echo "DNS_BOGUS_PRIV=${DNS_BOGUS_PRIV:-true}" + echo "PIHOLE_INTERFACE=${PIHOLE_INTERFACE}" + echo "IPV4_ADDRESS=${IPV4_ADDRESS}" + echo "IPV6_ADDRESS=${IPV6_ADDRESS}" + echo "PIHOLE_DNS_1=${PIHOLE_DNS_1}" + echo "PIHOLE_DNS_2=${PIHOLE_DNS_2}" + echo "QUERY_LOGGING=${QUERY_LOGGING}" + echo "INSTALL_WEB_SERVER=${INSTALL_WEB_SERVER}" + echo "INSTALL_WEB_INTERFACE=${INSTALL_WEB_INTERFACE}" + echo "LIGHTTPD_ENABLED=${LIGHTTPD_ENABLED}" + echo "CACHE_SIZE=${CACHE_SIZE}" + echo "DNS_FQDN_REQUIRED=${DNS_FQDN_REQUIRED:-true}" + echo "DNS_BOGUS_PRIV=${DNS_BOGUS_PRIV:-true}" }>> "${setupVars}" chmod 644 "${setupVars}" @@ -1921,7 +1921,7 @@ displayFinalMessage() { additional="View the web interface at http://pi.hole/admin or http://${IPV4_ADDRESS%/*}/admin Your Admin Webpage login password is ${pwstring}" - fi + fi # Final completion message to user whiptail --msgbox --backtitle "Make it so." --title "Installation Complete!" "Configure your devices to use the Pi-hole as their DNS server using: @@ -2456,7 +2456,7 @@ main() { #In case of RPM based distro, select the proper PHP version if [[ "$PKG_MANAGER" == "yum" || "$PKG_MANAGER" == "dnf" ]] ; then - select_rpm_php + select_rpm_php fi # Check if SELinux is Enforcing diff --git a/gravity.sh b/gravity.sh index dfaf4feaf1..bb4f4e4bd4 100755 --- a/gravity.sh +++ b/gravity.sh @@ -213,7 +213,7 @@ database_table_from_file() { # Move source file to backup directory, create directory if not existing mkdir -p "${backup_path}" mv "${source}" "${backup_file}" 2> /dev/null || \ - echo -e " ${CROSS} Unable to backup ${source} to ${backup_path}" + echo -e " ${CROSS} Unable to backup ${source} to ${backup_path}" # Delete tmpFile rm "${tmpFile}" > /dev/null 2>&1 || \ @@ -430,9 +430,9 @@ gravity_DownloadBlocklists() { compression="--compressed" echo -e " ${INFO} Using libz compression\n" else - compression="" - echo -e " ${INFO} Libz compression not available\n" - fi + compression="" + echo -e " ${INFO} Libz compression not available\n" + fi # Loop through $sources and download each one for ((i = 0; i < "${#sources[@]}"; i++)); do url="${sources[$i]}" @@ -462,9 +462,9 @@ gravity_DownloadBlocklists() { check_url="$( sed -re 's#([^:/]*://)?([^/]+)@#\1\2#' <<< "$url" )" if [[ "${check_url}" =~ ${regex} ]]; then - echo -e " ${CROSS} Invalid Target" + echo -e " ${CROSS} Invalid Target" else - gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" + gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" fi echo "" done @@ -583,28 +583,28 @@ gravity_DownloadBlocklistFromUrl() { blocked=false case $BLOCKINGMODE in "IP-NODATA-AAAA"|"IP") - # Get IP address of this domain - ip="$(dig "${domain}" +short)" - # Check if this IP matches any IP of the system - if [[ -n "${ip}" && $(grep -Ec "inet(|6) ${ip}" <<< "$(ip a)") -gt 0 ]]; then - blocked=true - fi;; + # Get IP address of this domain + ip="$(dig "${domain}" +short)" + # Check if this IP matches any IP of the system + if [[ -n "${ip}" && $(grep -Ec "inet(|6) ${ip}" <<< "$(ip a)") -gt 0 ]]; then + blocked=true + fi;; "NXDOMAIN") - if [[ $(dig "${domain}" | grep "NXDOMAIN" -c) -ge 1 ]]; then - blocked=true - fi;; + if [[ $(dig "${domain}" | grep "NXDOMAIN" -c) -ge 1 ]]; then + blocked=true + fi;; "NULL"|*) - if [[ $(dig "${domain}" +short | grep "0.0.0.0" -c) -ge 1 ]]; then - blocked=true - fi;; - esac + if [[ $(dig "${domain}" +short | grep "0.0.0.0" -c) -ge 1 ]]; then + blocked=true + fi;; + esac if [[ "${blocked}" == true ]]; then printf -v ip_addr "%s" "${PIHOLE_DNS_1%#*}" if [[ ${PIHOLE_DNS_1} != *"#"* ]]; then - port=53 + port=53 else - printf -v port "%s" "${PIHOLE_DNS_1#*#}" + printf -v port "%s" "${PIHOLE_DNS_1#*#}" fi ip=$(dig "@${ip_addr}" -p "${port}" +short "${domain}" | tail -1) if [[ $(echo "${url}" | awk -F '://' '{print $1}') = "https" ]]; then @@ -623,11 +623,11 @@ gravity_DownloadBlocklistFromUrl() { case $url in # Did we "download" a local file? "file"*) - if [[ -s "${patternBuffer}" ]]; then - echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success=true - else - echo -e "${OVER} ${CROSS} ${str} Not found / empty list" - fi;; + if [[ -s "${patternBuffer}" ]]; then + echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success=true + else + echo -e "${OVER} ${CROSS} ${str} Not found / empty list" + fi;; # Did we "download" a remote file? *) # Determine "Status:" output based on HTTP response From 14a379d4485328b85c08ff6cee30b137210b23d5 Mon Sep 17 00:00:00 2001 From: MichaIng Date: Tue, 30 Nov 2021 02:57:44 +0100 Subject: [PATCH 15/99] Fix generated /etc/os-release file in OS check test Signed-off-by: MichaIng --- test/test_automated_install.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_automated_install.py b/test/test_automated_install.py index 911407fc7d..7959e1007a 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -1085,9 +1085,9 @@ def test_os_check_fails(host): install_dependent_packages ${OS_CHECK_DEPS[@]} install_dependent_packages ${INSTALLER_DEPS[@]} cat < /etc/os-release - ID=UnsupportedOS - VERSION_ID="2" - EOT +ID=UnsupportedOS +VERSION_ID="2" +EOT ''') detectOS = host.run('''t source /opt/pihole/basic-install.sh From bfda52ed79c5648c8956d530f4c20393e7377b6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 1 Dec 2021 11:07:17 +0100 Subject: [PATCH 16/99] Fix number of invalid domains MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: abesnier Signed-off-by: Christian König --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index bb4f4e4bd4..d66b3b8300 100755 --- a/gravity.sh +++ b/gravity.sh @@ -516,7 +516,7 @@ parseList() { # Find (up to) five domains containing invalid characters (see above) incorrect_lines="$(sed -e "/[^a-zA-Z0-9.\_-]/!d" "${src}" | head -n 5)" - local num_target_lines num_correct_lines num_invalid + local num_target_lines num_correct_lines # Get number of lines in source file num_lines="$(grep -c "^" "${src}")" # Get number of lines in destination file From e485a7b9bb3da8e0ab7ccb5b16caac07ba70782d Mon Sep 17 00:00:00 2001 From: Chiller Dragon Date: Thu, 2 Dec 2021 14:44:50 +0100 Subject: [PATCH 17/99] Some shellchecks in basic-install.sh (#4088) * Some shellchecks in basic-install.sh Signed-off-by: ChillerDragon * Use more explicit grep (thanks to @MichaIng) Signed-off-by: ChillerDragon --- automated install/basic-install.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index cbb8ebae3e..bd2bf4c34e 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2,7 +2,7 @@ # shellcheck disable=SC1090 # Pi-hole: A black hole for Internet advertisements -# (c) 2017-2018 Pi-hole, LLC (https://pi-hole.net) +# (c) 2017-2021 Pi-hole, LLC (https://pi-hole.net) # Network-wide ad blocking via your own hardware. # # Installs and Updates Pi-hole @@ -172,7 +172,7 @@ os_check() { local remote_os_domain valid_os valid_version valid_response detected_os detected_version display_warning cmdResult digReturnCode response remote_os_domain=${OS_CHECK_DOMAIN_NAME:-"versions.pi-hole.net"} - detected_os=$(grep "\bID\b" /etc/os-release | cut -d '=' -f2 | tr -d '"') + detected_os=$(grep '^ID=' /etc/os-release | cut -d '=' -f2 | tr -d '"') detected_version=$(grep VERSION_ID /etc/os-release | cut -d '=' -f2 | tr -d '"') cmdResult="$(dig +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1; echo $?)" @@ -856,7 +856,7 @@ valid_ip() { # Regex matching an optional port (starting with '#') range of 1-65536 local portelem="(#(6553[0-5]|655[0-2][0-9]|65[0-4][0-9]{2}|6[0-4][0-9]{3}|[1-5][0-9]{4}|[1-9][0-9]{0,3}|0))?"; # Build a full IPv4 regex from the above subexpressions - local regex="^${ipv4elem}\.${ipv4elem}\.${ipv4elem}\.${ipv4elem}${portelem}$" + local regex="^${ipv4elem}\\.${ipv4elem}\\.${ipv4elem}\\.${ipv4elem}${portelem}$" # Evaluate the regex, and return the result [[ $ip =~ ${regex} ]] @@ -1862,7 +1862,7 @@ checkSelinux() { local CURRENT_SELINUX local SELINUX_ENFORCING=0 # Check for SELinux configuration file and getenforce command - if [[ -f /etc/selinux/config ]] && command -v getenforce &> /dev/null; then + if [[ -f /etc/selinux/config ]] && is_command getenforce; then # Check the default SELinux mode DEFAULT_SELINUX=$(awk -F= '/^SELINUX=/ {print $2}' /etc/selinux/config) case "${DEFAULT_SELINUX,,}" in @@ -2094,7 +2094,6 @@ clone_or_update_repos() { # shellcheck disable=SC2120 FTLinstall() { # Local, named variables - local latesttag local str="Downloading and Installing FTL" printf " %b %s..." "${INFO}" "${str}" @@ -2165,7 +2164,7 @@ FTLinstall() { disable_dnsmasq() { # dnsmasq can now be stopped and disabled if it exists - if which dnsmasq &> /dev/null; then + if is_command dnsmasq; then if check_service_active "dnsmasq";then printf " %b FTL can now resolve DNS Queries without dnsmasq running separately\\n" "${INFO}" stop_service dnsmasq @@ -2295,7 +2294,7 @@ FTLcheckUpdate() { local localSha1 # if dnsmasq exists and is running at this point, force reinstall of FTL Binary - if which dnsmasq &> /dev/null; then + if is_command dnsmasq; then if check_service_active "dnsmasq";then return 0 fi From ba6d700e7e46dce79f44fea3339193ee80164f1a Mon Sep 17 00:00:00 2001 From: Subhaditya Nath <56714626+subnut@users.noreply.github.com> Date: Fri, 3 Dec 2021 02:16:11 +0530 Subject: [PATCH 18/99] Fix `pihole -v` output if WebAdmin not installed (#4370) * Fix https://github.com/pi-hole/pi-hole/issues/4279 Signed-off-by: Subhaditya Nath * Don't ignore exit code of version.sh If it exits with a non-zero return code, that means some error occurred, and so it shouldn't be ignored. Signed-off-by: Subhaditya Nath * Implement changes suggested by @Michalng Signed-off-by: Subhaditya Nath * Implement changes suggested by @PromoFaux Signed-off-by: Subhaditya Nath * Always source /etc/pihole/setupVars.conf https://github.com/pi-hole/pi-hole/pull/4370#issuecomment-978149567 Co-authored-by: Adam Warner --- advanced/Scripts/version.sh | 19 +++++++++++++------ pihole | 3 +-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/advanced/Scripts/version.sh b/advanced/Scripts/version.sh index 2cb0a2c9ce..0b7b29dd60 100755 --- a/advanced/Scripts/version.sh +++ b/advanced/Scripts/version.sh @@ -13,6 +13,10 @@ DEFAULT="-1" COREGITDIR="/etc/.pihole/" WEBGITDIR="/var/www/html/admin/" +# Source the setupvars config file +# shellcheck disable=SC1091 +source /etc/pihole/setupVars.conf + getLocalVersion() { # FTL requires a different method if [[ "$1" == "FTL" ]]; then @@ -91,10 +95,11 @@ getRemoteVersion(){ #If the above file exists, then we can read from that. Prevents overuse of GitHub API if [[ -f "$cachedVersions" ]]; then IFS=' ' read -r -a arrCache < "$cachedVersions" + case $daemon in "pi-hole" ) echo "${arrCache[0]}";; - "AdminLTE" ) echo "${arrCache[1]}";; - "FTL" ) echo "${arrCache[2]}";; + "AdminLTE" ) [[ "${INSTALL_WEB_INTERFACE}" == true ]] && echo "${arrCache[1]}";; + "FTL" ) [[ "${INSTALL_WEB_INTERFACE}" == true ]] && echo "${arrCache[2]}" || echo "${arrCache[1]}";; esac return 0 @@ -140,6 +145,11 @@ getLocalBranch(){ } versionOutput() { + if [[ "$1" == "AdminLTE" && "${INSTALL_WEB_INTERFACE}" != true ]]; then + echo " WebAdmin not installed" + return 1 + fi + [[ "$1" == "pi-hole" ]] && GITDIR=$COREGITDIR [[ "$1" == "AdminLTE" ]] && GITDIR=$WEBGITDIR [[ "$1" == "FTL" ]] && GITDIR="FTL" @@ -166,6 +176,7 @@ versionOutput() { output="Latest ${1^} hash is $latHash" else errorOutput + return 1 fi [[ -n "$output" ]] && echo " $output" @@ -177,10 +188,6 @@ errorOutput() { } defaultOutput() { - # Source the setupvars config file - # shellcheck disable=SC1091 - source /etc/pihole/setupVars.conf - versionOutput "pi-hole" "$@" if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then diff --git a/pihole b/pihole index 3135667182..4f1b95eb6c 100755 --- a/pihole +++ b/pihole @@ -95,8 +95,7 @@ uninstallFunc() { versionFunc() { shift - "${PI_HOLE_SCRIPT_DIR}"/version.sh "$@" - exit 0 + exec "${PI_HOLE_SCRIPT_DIR}"/version.sh "$@" } # Get PID of main pihole-FTL process From 647ba6ec9d92bde1955cc8474aec565b433ca088 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 2 Dec 2021 23:03:46 +0100 Subject: [PATCH 19/99] Rename variables to improve comprehensibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/gravity.sh b/gravity.sh index d66b3b8300..b2cd27b940 100755 --- a/gravity.sh +++ b/gravity.sh @@ -248,7 +248,7 @@ database_adlist_number() { return; fi - output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${num_lines}" "${num_invalid}" "${1}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${num_source_lines}" "${num_invalid}" "${1}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -503,8 +503,9 @@ gravity_DownloadBlocklists() { gravity_Blackbody=true } -total_num=0 -num_lines=0 +# num_target_lines does increase for every correctly added domain in pareseList() +num_target_lines=0 +num_source_lines=0 num_invalid=0 parseList() { local adlistID="${1}" src="${2}" target="${3}" incorrect_lines @@ -516,18 +517,20 @@ parseList() { # Find (up to) five domains containing invalid characters (see above) incorrect_lines="$(sed -e "/[^a-zA-Z0-9.\_-]/!d" "${src}" | head -n 5)" - local num_target_lines num_correct_lines + local num_target_lines_new num_correct_lines # Get number of lines in source file - num_lines="$(grep -c "^" "${src}")" - # Get number of lines in destination file - num_target_lines="$(grep -c "^" "${target}")" - num_correct_lines="$(( num_target_lines-total_num ))" - total_num="$num_target_lines" - num_invalid="$(( num_lines-num_correct_lines ))" + num_source_lines="$(grep -c "^" "${src}")" + # Get the new number of lines in destination file + num_target_lines_new="$(grep -c "^" "${target}")" + # Number of new correctly added lines + num_correct_lines="$(( num_target_lines_new-num_target_lines ))" + # Upate number of lines in target file + num_target_lines="$num_target_lines_new" + num_invalid="$(( num_source_lines-num_correct_lines ))" if [[ "${num_invalid}" -eq 0 ]]; then - echo " ${INFO} Analyzed ${num_lines} domains" + echo " ${INFO} Analyzed ${num_source_lines} domains" else - echo " ${INFO} Analyzed ${num_lines} domains, ${num_invalid} domains invalid!" + echo " ${INFO} Analyzed ${num_source_lines} domains, ${num_invalid} domains invalid!" fi # Display sample of invalid lines if we found some @@ -686,7 +689,7 @@ gravity_DownloadBlocklistFromUrl() { else echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}" # Manually reset these two numbers because we do not call parseList here - num_lines=0 + num_source_lines=0 num_invalid=0 database_adlist_number "${adlistID}" database_adlist_status "${adlistID}" "4" From e243c562c21e1c3d03c87812b45ac07c6f7194c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 3 Dec 2021 09:17:19 +0100 Subject: [PATCH 20/99] Unblock adlist domain during gravity run in NODATA mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gravity.sh b/gravity.sh index b2cd27b940..2ac9028a09 100755 --- a/gravity.sh +++ b/gravity.sh @@ -596,6 +596,10 @@ gravity_DownloadBlocklistFromUrl() { if [[ $(dig "${domain}" | grep "NXDOMAIN" -c) -ge 1 ]]; then blocked=true fi;; + "NODATA") + if [[ $(dig "${domain}" | grep "NOERROR" -c) -ge 1 ]] && [[ -z $(dig +noall +answer "${domain}" |awk '{print $5}') ]]; then + blocked=true + fi;; "NULL"|*) if [[ $(dig "${domain}" +short | grep "0.0.0.0" -c) -ge 1 ]]; then blocked=true From ae39e338fef3054e95eb6910b15e7d585195d4e2 Mon Sep 17 00:00:00 2001 From: yubiuser Date: Sat, 4 Dec 2021 10:50:21 +0100 Subject: [PATCH 21/99] Use exec to run gravity script (#4449) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- pihole | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pihole b/pihole index 4f1b95eb6c..055bd70276 100755 --- a/pihole +++ b/pihole @@ -71,8 +71,7 @@ reconfigurePiholeFunc() { } updateGravityFunc() { - "${PI_HOLE_SCRIPT_DIR}"/gravity.sh "$@" - exit $? + exec "${PI_HOLE_SCRIPT_DIR}"/gravity.sh "$@" } queryFunc() { From b52a3a021daa77f0605316c8e07f744514d2731d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 6 Dec 2021 20:27:13 +0100 Subject: [PATCH 22/99] Add comment help text to list function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/list.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index bc254515aa..fda8603cd8 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -91,7 +91,8 @@ Options: -q, --quiet Make output less verbose -h, --help Show this help dialog -l, --list Display all your ${listname}listed domains - --nuke Removes all entries in a list" + --nuke Removes all entries in a list + --comment Add a comment to the domain. If adding multiple domains the same comment will be used for all" exit 0 } From 8d6ce78c655e9a5d0bbf57f2cbf92b594473ca90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 10 Dec 2021 07:09:42 +0100 Subject: [PATCH 23/99] Allow qr code iframe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/lighttpd.conf.debian | 5 +++++ advanced/lighttpd.conf.fedora | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/advanced/lighttpd.conf.debian b/advanced/lighttpd.conf.debian index a58b5a88f0..8966dc32c3 100644 --- a/advanced/lighttpd.conf.debian +++ b/advanced/lighttpd.conf.debian @@ -92,5 +92,10 @@ $HTTP["url"] =~ "/teleporter\.php$" { } } +# allow API qr code iframe on settings page +$HTTP["url"] =~ "/admin/settings\.php$" { + setenv.add-response-header = ( "X-Frame-Options" => "SAMEORIGIN" ) +} + # Default expire header expire.url = ( "" => "access plus 0 seconds" ) diff --git a/advanced/lighttpd.conf.fedora b/advanced/lighttpd.conf.fedora index ad336a93b5..6bf9e68394 100644 --- a/advanced/lighttpd.conf.fedora +++ b/advanced/lighttpd.conf.fedora @@ -100,5 +100,10 @@ $HTTP["url"] =~ "/teleporter\.php$" { } } +# allow API qr code iframe on settings page +$HTTP["url"] =~ "/admin/settings\.php$" { + setenv.add-response-header = ( "X-Frame-Options" => "SAMEORIGIN" ) +} + # Default expire header expire.url = ( "" => "access plus 0 seconds" ) From 2eff53b2bbc19f899f206ccaf1cbf3d6acb6f57b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 10 Dec 2021 07:17:13 +0100 Subject: [PATCH 24/99] Allow qr code iframe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/lighttpd.conf.debian | 6 ++++-- advanced/lighttpd.conf.fedora | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/advanced/lighttpd.conf.debian b/advanced/lighttpd.conf.debian index 8966dc32c3..37099ad71d 100644 --- a/advanced/lighttpd.conf.debian +++ b/advanced/lighttpd.conf.debian @@ -93,8 +93,10 @@ $HTTP["url"] =~ "/teleporter\.php$" { } # allow API qr code iframe on settings page -$HTTP["url"] =~ "/admin/settings\.php$" { - setenv.add-response-header = ( "X-Frame-Options" => "SAMEORIGIN" ) +$HTTP["url"] =~ "/api_token\.php$" { + $HTTP["referer"] =~ "/admin/settings\.php" { + setenv.add-response-header = ( "X-Frame-Options" => "SAMEORIGIN" ) + } } # Default expire header diff --git a/advanced/lighttpd.conf.fedora b/advanced/lighttpd.conf.fedora index 6bf9e68394..f4916422e2 100644 --- a/advanced/lighttpd.conf.fedora +++ b/advanced/lighttpd.conf.fedora @@ -101,8 +101,10 @@ $HTTP["url"] =~ "/teleporter\.php$" { } # allow API qr code iframe on settings page -$HTTP["url"] =~ "/admin/settings\.php$" { - setenv.add-response-header = ( "X-Frame-Options" => "SAMEORIGIN" ) +$HTTP["url"] =~ "/api_token\.php$" { + $HTTP["referer"] =~ "/admin/settings\.php" { + setenv.add-response-header = ( "X-Frame-Options" => "SAMEORIGIN" ) + } } # Default expire header From a3cc5df317ffeec2b6bf78d37e075f33aeb0f79c Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Thu, 16 Dec 2021 11:19:11 -0800 Subject: [PATCH 25/99] Configure stale action (#4269) * Configure stale action * [skip ci] Update .github/workflows/stale.yml * Update .github/workflows/stale.yml --- .github/workflows/stale.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/stale.yml diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000000..1c1c47a1ae --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,24 @@ +name: Mark stale issues + +on: + schedule: + - cron: '30 * * * *' + +jobs: + stale: + + runs-on: ubuntu-latest + permissions: + issues: write + + steps: + - uses: actions/stale@v4 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + debug-only: true + days-before-stale: 30 + days-before-close: 5 + stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Please comment or update this issue or it will be closed in 5 days.' + stale-issue-label: 'Submitter Attention Required' + exempt-issue-labels: 'pinned, Fixed in next release, Bug: Confirmed' + exempt-all-issue-assignees: true From 28085cf7d8386608470d66ec59d3947b34c7970f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 17 Dec 2021 10:08:16 +0100 Subject: [PATCH 26/99] Merge iFrame exceptions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/lighttpd.conf.debian | 11 ++--------- advanced/lighttpd.conf.fedora | 11 ++--------- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/advanced/lighttpd.conf.debian b/advanced/lighttpd.conf.debian index 37099ad71d..cf728e1980 100644 --- a/advanced/lighttpd.conf.debian +++ b/advanced/lighttpd.conf.debian @@ -85,15 +85,8 @@ $HTTP["url"] =~ "^/admin/\.(.*)" { url.access-deny = ("") } -# allow teleporter iframe on settings page -$HTTP["url"] =~ "/teleporter\.php$" { - $HTTP["referer"] =~ "/admin/settings\.php" { - setenv.add-response-header = ( "X-Frame-Options" => "SAMEORIGIN" ) - } -} - -# allow API qr code iframe on settings page -$HTTP["url"] =~ "/api_token\.php$" { +# allow teleporter and API qr code iframe on settings page +$HTTP["url"] =~ "/(teleporter|api_token)\.php$" { $HTTP["referer"] =~ "/admin/settings\.php" { setenv.add-response-header = ( "X-Frame-Options" => "SAMEORIGIN" ) } diff --git a/advanced/lighttpd.conf.fedora b/advanced/lighttpd.conf.fedora index f4916422e2..626a3d8dc0 100644 --- a/advanced/lighttpd.conf.fedora +++ b/advanced/lighttpd.conf.fedora @@ -93,15 +93,8 @@ $HTTP["url"] =~ "^/admin/\.(.*)" { url.access-deny = ("") } -# allow teleporter iframe on settings page -$HTTP["url"] =~ "/teleporter\.php$" { - $HTTP["referer"] =~ "/admin/settings\.php" { - setenv.add-response-header = ( "X-Frame-Options" => "SAMEORIGIN" ) - } -} - -# allow API qr code iframe on settings page -$HTTP["url"] =~ "/api_token\.php$" { +# allow teleporter and API qr code iframe on settings page +$HTTP["url"] =~ "/(teleporter|api_token)\.php$" { $HTTP["referer"] =~ "/admin/settings\.php" { setenv.add-response-header = ( "X-Frame-Options" => "SAMEORIGIN" ) } From a780fc59e28b5cd9be7e60fa9c530db16fffff96 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Mon, 20 Dec 2021 10:56:42 -0800 Subject: [PATCH 27/99] Set DBFile permissions on creation. Signed-off-by: Dan Schaper --- gravity.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index b2cd27b940..81bca09d6e 100755 --- a/gravity.sh +++ b/gravity.sh @@ -75,7 +75,9 @@ fi # Generate new sqlite3 file from schema template generate_gravity_database() { - sqlite3 "${1}" < "${gravityDBschema}" + sqlite3 "${gravityDBFile}" < "${gravityDBschema}" + chown pihole:pihole "${gravityDBfile}" + chmod g+w "${piholeDir}" "${gravityDBfile}" } # Copy data from old to new database file and swap them @@ -279,7 +281,7 @@ migrate_to_database() { if [ ! -e "${gravityDBfile}" ]; then # Create new database file - note that this will be created in version 1 echo -e " ${INFO} Creating new gravity database" - generate_gravity_database "${gravityDBfile}" + generate_gravity_database # Check if gravity database needs to be updated upgrade_gravityDB "${gravityDBfile}" "${piholeDir}" From 76ae75689c9c2a20266cd4dc4d2ec098a08d215a Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Mon, 20 Dec 2021 11:09:11 -0800 Subject: [PATCH 28/99] Check for DNS before run. Signed-off-by: Dan Schaper --- gravity.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 81bca09d6e..f3754a9f7f 100755 --- a/gravity.sh +++ b/gravity.sh @@ -898,7 +898,11 @@ if [[ "${forceDelete:-}" == true ]]; then fi # Gravity downloads blocklists next -gravity_CheckDNSResolutionAvailable +if [[ ! gravity_CheckDNSResolutionAvailable ]]; then + echo -e " ${CROSS} Can not complete gravity update, no DNS is available. Please contact support." + exit 1 +fi + gravity_DownloadBlocklists # Create local.list From 533a77d6d5a4fad51f243f844c5c82b7b50b4476 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Mon, 20 Dec 2021 11:36:55 -0800 Subject: [PATCH 29/99] Add database function failure guards. Signed-off-by: Dan Schaper --- gravity.sh | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/gravity.sh b/gravity.sh index f3754a9f7f..ec6e149ab8 100755 --- a/gravity.sh +++ b/gravity.sh @@ -75,7 +75,10 @@ fi # Generate new sqlite3 file from schema template generate_gravity_database() { - sqlite3 "${gravityDBFile}" < "${gravityDBschema}" + if ! sqlite3 "${gravityDBfile}" < "${gravityDBschema}"; then + echo -e " ${CROSS} Unable to create ${gravityDBfile}" + return 1 + fi chown pihole:pihole "${gravityDBfile}" chmod g+w "${piholeDir}" "${gravityDBfile}" } @@ -281,7 +284,10 @@ migrate_to_database() { if [ ! -e "${gravityDBfile}" ]; then # Create new database file - note that this will be created in version 1 echo -e " ${INFO} Creating new gravity database" - generate_gravity_database + if ! generate_gravity_database; then + echo -e " ${CROSS} Error creating new gravity database. Please contact support." + return 1 + fi # Check if gravity database needs to be updated upgrade_gravityDB "${gravityDBfile}" "${piholeDir}" @@ -887,7 +893,10 @@ if [[ "${recreate_database:-}" == true ]]; then fi # Move possibly existing legacy files to the gravity database -migrate_to_database +if ! migrate_to_database; then + echo -e " ${CROSS} Unable to migrate to database. Please contact support." + exit 1 +fi if [[ "${forceDelete:-}" == true ]]; then str="Deleting existing list cache" @@ -898,7 +907,7 @@ if [[ "${forceDelete:-}" == true ]]; then fi # Gravity downloads blocklists next -if [[ ! gravity_CheckDNSResolutionAvailable ]]; then +if ! gravity_CheckDNSResolutionAvailable; then echo -e " ${CROSS} Can not complete gravity update, no DNS is available. Please contact support." exit 1 fi @@ -909,7 +918,10 @@ gravity_DownloadBlocklists gravity_generateLocalList # Migrate rest of the data from old to new database -gravity_swap_databases +if ! gravity_swap_databases; then + echo -e " ${CROSS} Unable to create database. Please contact support." + exit 1 +fi # Update gravity timestamp update_gravity_timestamp From c0f454ddfa7230944e894e8ddab01d15704caa6e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 Dec 2021 21:36:19 +0100 Subject: [PATCH 30/99] Add new interface listening option "bind" (#4476) Signed-off-by: DL6ER --- advanced/Scripts/webpage.sh | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index ec9b2cac53..4f44eca864 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -199,6 +199,8 @@ trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC68345710423 # Setup interface listening behavior of dnsmasq delete_dnsmasq_setting "interface" delete_dnsmasq_setting "local-service" + delete_dnsmasq_setting "except-interface" + delete_dnsmasq_setting "bind-interfaces" if [[ "${DNSMASQ_LISTENING}" == "all" ]]; then # Listen on all interfaces, permit all origins @@ -207,6 +209,7 @@ trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC68345710423 # Listen only on all interfaces, but only local subnets add_dnsmasq_setting "local-service" else + # Options "bind" and "single" # Listen only on one interface # Use eth0 as fallback interface if interface is missing in setupVars.conf if [ -z "${PIHOLE_INTERFACE}" ]; then @@ -214,6 +217,11 @@ trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC68345710423 fi add_dnsmasq_setting "interface" "${PIHOLE_INTERFACE}" + + if [[ "${DNSMASQ_LISTENING}" == "bind" ]]; then + # Really bind to interface + add_dnsmasq_setting "bind-interfaces" + fi fi if [[ "${CONDITIONAL_FORWARDING}" == true ]]; then @@ -600,9 +608,10 @@ Example: 'pihole -a -i local' Specify dnsmasq's network interface listening behavior Interfaces: - local Listen on all interfaces, but only allow queries from - devices that are at most one hop away (local devices) - single Listen only on ${PIHOLE_INTERFACE} interface + local Only respond to queries from devices that + are at most one hop away (local devices) + single Respond only on interface ${PIHOLE_INTERFACE} + bind Bind only on interface ${PIHOLE_INTERFACE} all Listen on all interfaces, permit all origins" exit 0 fi @@ -613,6 +622,9 @@ Interfaces: elif [[ "${args[2]}" == "local" ]]; then echo -e " ${INFO} Listening on all interfaces, permitting origins from one hop away (LAN)" change_setting "DNSMASQ_LISTENING" "local" + elif [[ "${args[2]}" == "bind" ]]; then + echo -e " ${INFO} Binding on interface ${PIHOLE_INTERFACE}" + change_setting "DNSMASQ_LISTENING" "bind" else echo -e " ${INFO} Listening only on interface ${PIHOLE_INTERFACE}" change_setting "DNSMASQ_LISTENING" "single" From 669f1b0f4ab1f27ed5c7b29e363ef1beb395f4fb Mon Sep 17 00:00:00 2001 From: yubiuser Date: Tue, 21 Dec 2021 12:58:39 +0100 Subject: [PATCH 31/99] Address reviewer's comment Co-authored-by: DL6ER --- advanced/Scripts/list.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index fda8603cd8..ffde075a91 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -92,7 +92,7 @@ Options: -h, --help Show this help dialog -l, --list Display all your ${listname}listed domains --nuke Removes all entries in a list - --comment Add a comment to the domain. If adding multiple domains the same comment will be used for all" + --comment "text" Add a comment to the domain. If adding multiple domains the same comment will be used for all" exit 0 } From 190ab796063a53ac9843fd97cc8f978d10bd4a56 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 20 Aug 2021 20:48:57 +0200 Subject: [PATCH 32/99] Implement fully-automated gravity database recovery method. Signed-off-by: DL6ER --- gravity.sh | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index b2cd27b940..911242f705 100755 --- a/gravity.sh +++ b/gravity.sh @@ -848,6 +848,42 @@ gravity_Cleanup() { fi } +database_recovery() { + str="Checking integrity of existing gravity database" + echo -ne " ${INFO} ${str}..." + if result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA integrity_check" 2>&1)"; then + echo -e "${OVER} ${TICK} ${str} - no errors found" + else + echo -e "${OVER} ${CROSS} ${str} - errors found:" + while IFS= read -r line ; do echo " - $line"; done <<< "$result" + fi + + str="Checking foreign keys of existing gravity database" + echo -ne " ${INFO} ${str}..." + if result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)"; then + echo -e "${OVER} ${TICK} ${str} - no errors found" + else + echo -e "${OVER} ${CROSS} ${str} - errors found:" + while IFS= read -r line ; do echo " - $line"; done <<< "$result" + fi + + str="Trying to recover existing gravity database" + echo -ne " ${INFO} ${str}..." + # We have to remove any possibly existing recovery database or this will fail + rm -f "${gravityDBfile}.recovered" > /dev/null 2>&1 + if result="$(pihole-FTL sqlite3 "${gravityDBfile}" ".recover" | pihole-FTL sqlite3 "${gravityDBfile}.recovered" 2>&1)"; then + echo -e "${OVER} ${TICK} ${str} - success" + mv "${gravityDBfile}" "${gravityDBfile}.old" + mv "${gravityDBfile}.recovered" "${gravityDBfile}" + else + echo -e "${OVER} ${CROSS} ${str} - the following errors happened:" + while IFS= read -r line ; do echo " - $line"; done <<< "$result" + echo -e " ${CROSS} Recovery failed. Try \"pihole -r recreate\" instead." + exit 1 + fi + echo "" +} + helpFunc() { echo "Usage: pihole -g Update domains from blocklists specified in adlists.list @@ -858,10 +894,30 @@ Options: exit 0 } +repairSelector() { + case "$1" in + "recover") recover_database=true;; + "recreate") recreate_database=true;; + *) echo "Usage: pihole -g -r {recover,recreate} +Attempt to repair gravity database + +Available options: + pihole -g -r recover Try to recover a damaged gravity database file. + Pi-hole tries to restore as much as possible + from a corrupted gravity database. + pihole -g -r recreate Create a new gravity database file from scratch. + This will remove your existing gravity database + and create a new file from scratch. If you still + have the migration backup created when migrating + to Pi-hole v5.0, Pi-hole will import these files." + exit 0;; + esac +} + for var in "$@"; do case "${var}" in "-f" | "--force" ) forceDelete=true;; - "-r" | "--recreate" ) recreate_database=true;; + "-r" | "--repair" ) repairSelector "$3";; "-h" | "--help" ) helpFunc;; esac done @@ -875,7 +931,7 @@ fi gravity_Trap if [[ "${recreate_database:-}" == true ]]; then - str="Restoring from migration backup" + str="Recreating gravity database from migration backup" echo -ne "${INFO} ${str}..." rm "${gravityDBfile}" pushd "${piholeDir}" > /dev/null || exit @@ -884,6 +940,10 @@ if [[ "${recreate_database:-}" == true ]]; then echo -e "${OVER} ${TICK} ${str}" fi +if [[ "${recover_database:-}" == true ]]; then + database_recovery +fi + # Move possibly existing legacy files to the gravity database migrate_to_database From 469c179b328f93130fb7ccb5334585b15b94dc55 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 11 Sep 2021 21:54:42 +0200 Subject: [PATCH 33/99] Return early from recovery routine when integrity checks didn't show any database errors. Signed-off-by: DL6ER --- gravity.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/gravity.sh b/gravity.sh index 911242f705..2d9b5dee33 100755 --- a/gravity.sh +++ b/gravity.sh @@ -853,15 +853,16 @@ database_recovery() { echo -ne " ${INFO} ${str}..." if result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA integrity_check" 2>&1)"; then echo -e "${OVER} ${TICK} ${str} - no errors found" - else - echo -e "${OVER} ${CROSS} ${str} - errors found:" - while IFS= read -r line ; do echo " - $line"; done <<< "$result" - fi - str="Checking foreign keys of existing gravity database" - echo -ne " ${INFO} ${str}..." - if result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)"; then - echo -e "${OVER} ${TICK} ${str} - no errors found" + str="Checking foreign keys of existing gravity database" + echo -ne " ${INFO} ${str}..." + if result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)"; then + echo -e "${OVER} ${TICK} ${str} - no errors found" + return + else + echo -e "${OVER} ${CROSS} ${str} - errors found:" + while IFS= read -r line ; do echo " - $line"; done <<< "$result" + fi else echo -e "${OVER} ${CROSS} ${str} - errors found:" while IFS= read -r line ; do echo " - $line"; done <<< "$result" From ab4bce4787093428305647ad547ed61c8d4809fd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 11 Sep 2021 21:56:44 +0200 Subject: [PATCH 34/99] Allow users to force recovery even when checks are okay using "pihole -g -r recover force" Signed-off-by: DL6ER --- gravity.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index 2d9b5dee33..c2033fcf64 100755 --- a/gravity.sh +++ b/gravity.sh @@ -858,7 +858,9 @@ database_recovery() { echo -ne " ${INFO} ${str}..." if result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)"; then echo -e "${OVER} ${TICK} ${str} - no errors found" - return + if [[ $1 != "force" ]]; then + return + fi else echo -e "${OVER} ${CROSS} ${str} - errors found:" while IFS= read -r line ; do echo " - $line"; done <<< "$result" @@ -942,7 +944,7 @@ if [[ "${recreate_database:-}" == true ]]; then fi if [[ "${recover_database:-}" == true ]]; then - database_recovery + database_recovery "$4" fi # Move possibly existing legacy files to the gravity database From b06efb6ab7c3400b99cd7e4f8509cf52092d3671 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Dec 2021 14:00:46 +0100 Subject: [PATCH 35/99] Declare variables local Signed-off-by: DL6ER --- gravity.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index c2033fcf64..bd2ae909a7 100755 --- a/gravity.sh +++ b/gravity.sh @@ -849,7 +849,8 @@ gravity_Cleanup() { } database_recovery() { - str="Checking integrity of existing gravity database" + local result + local str="Checking integrity of existing gravity database" echo -ne " ${INFO} ${str}..." if result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA integrity_check" 2>&1)"; then echo -e "${OVER} ${TICK} ${str} - no errors found" From 39a66b608b70e25e8f8218ce68fc15bd009a3ffa Mon Sep 17 00:00:00 2001 From: yubiuser Date: Tue, 21 Dec 2021 14:00:47 +0100 Subject: [PATCH 36/99] Replace Contributing Guide by link to docs.pi-hole.net (#4433) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- CONTRIBUTING.md | 107 +----------------------------------------------- 1 file changed, 1 insertion(+), 106 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0dd22b4266..018b8c5f51 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,111 +2,6 @@ Please read and understand the contribution guide before creating an issue or pull request. -## Etiquette +The guide can be found here: [https://docs.pi-hole.net/guides/github/contributing/](https://docs.pi-hole.net/guides/github/contributing/) -- Our goal for Pi-hole is **stability before features**. This means we focus on squashing critical bugs before adding new features. Often, we can do both in tandem, but bugs will take priority over a new feature. -- Pi-hole is open source and [powered by donations](https://pi-hole.net/donate/), and as such, we give our **free time** to build, maintain, and **provide user support** for this project. It would be extremely unfair for us to suffer abuse or anger for our hard work, so please take a moment to consider that. -- Please be considerate towards the developers and other users when raising issues or presenting pull requests. -- Respect our decision(s), and do not be upset or abusive if your submission is not used. -## Viability - -When requesting or submitting new features, first consider whether it might be useful to others. Open source projects are used by many people, who may have entirely different needs to your own. Think about whether or not your feature is likely to be used by other users of the project. - -## Procedure - -**Before filing an issue:** - -- Attempt to replicate and **document** the problem, to ensure that it wasn't a coincidental incident. -- Check to make sure your feature suggestion isn't already present within the project. -- Check the pull requests tab to ensure that the bug doesn't have a fix in progress. -- Check the pull requests tab to ensure that the feature isn't already in progress. - -**Before submitting a pull request:** - -- Check the codebase to ensure that your feature doesn't already exist. -- Check the pull requests to ensure that another person hasn't already submitted the feature or fix. -- Read and understand the [DCO guidelines](https://docs.pi-hole.net/guides/github/contributing/) for the project. - -## Technical Requirements - -- Submit Pull Requests to the **development branch only**. -- Before Submitting your Pull Request, merge `development` with your new branch and fix any conflicts. (Make sure you don't break anything in development!) -- Please use the [Google Style Guide for Shell](https://google.github.io/styleguide/shell.xml) for your code submission styles. -- Commit Unix line endings. -- Please use the Pi-hole brand: **Pi-hole** (Take a special look at the capitalized 'P' and a low 'h' with a hyphen) -- (Optional fun) keep to the theme of Star Trek/black holes/gravity. - -## Forking and Cloning from GitHub to GitHub - -1. Fork to a repo under a namespace you control, or have permission to use, for example: `https://github.com///`. You can do this from the github.com website. -2. Clone `https://github.com///` with the tool of you choice. -3. To keep your fork in sync with our repo, add an upstream remote for pi-hole/pi-hole to your repo. - - ```bash - git remote add upstream https://github.com/pi-hole/pi-hole.git - ``` - -4. Checkout the `development` branch from your fork `https://github.com///`. -5. Create a topic/branch, based on the `development` branch code. *Bonus fun to keep to the theme of Star Trek/black holes/gravity.* -6. Make your changes and commit to your topic branch in your repo. -7. Rebase your commits and squash any insignificant commits. See the notes below for an example. -8. Merge `development` your branch and fix any conflicts. -9. Open a Pull Request to merge your topic branch into our repo's `development` branch. - -- Keep in mind the technical requirements from above. - -## Forking and Cloning from GitHub to other code hosting sites - -- Forking is a GitHub concept and cannot be done from GitHub to other git-based code hosting sites. However, those sites may be able to mirror a GitHub repo. - -1. To contribute from another code hosting site, you must first complete the steps above to fork our repo to a GitHub namespace you have permission to use, for example: `https://github.com///`. -2. Create a repo in your code hosting site, for example: `https://gitlab.com///` -3. Follow the instructions from your code hosting site to create a mirror between `https://github.com///` and `https://gitlab.com///`. -4. When you are ready to create a Pull Request (PR), follow the steps `(starting at step #6)` from [Forking and Cloning from GitHub to GitHub](#forking-and-cloning-from-github-to-github) and create the PR from `https://github.com///`. - -## Notes for squashing commits with rebase - -- To rebase your commits and squash previous commits, you can use: - - ```bash - git rebase -i your_topic_branch~(number of commits to combine) - ``` - -- For more details visit [gitready.com](http://gitready.com/advanced/2009/02/10/squashing-commits-with-rebase.html) - -1. The following would combine the last four commits in the branch `mytopic`. - - ```bash - git rebase -i mytopic~4 - ``` - -2. An editor window opens with the most recent commits indicated: (edit the commands to the left of the commit ID) - - ```gitattributes - pick 9dff55b2 existing commit comments - squash ebb1a730 existing commit comments - squash 07cc5b50 existing commit comments - reword 9dff55b2 existing commit comments - ``` - -3. Save and close the editor. The next editor window opens: (edit the new commit message). *If you select reword for a commit, an additional editor window will open for you to edit the comment.* - - ```bash - new commit comments - Signed-off-by: yourname - ``` - -4. Save and close the editor for the rebase process to execute. The terminal output should say something like the following: - - ```bash - Successfully rebased and updated refs/heads/mytopic. - ``` - -5. Once you have a successful rebase, and before you sync your local clone, you have to force push origin to update your repo: - - ```bash - git push -f origin - ``` - -6. Continue on from step #7 from [Forking and Cloning from GitHub to GitHub](#forking-and-cloning-from-github-to-github) From 7167e6d5e42869f8ab370565ae4517cef3b6821e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Dec 2021 16:20:02 +0100 Subject: [PATCH 37/99] Apply suggestions from code review Co-authored-by: Dan Schaper --- gravity.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index bd2ae909a7..b1ecdc08aa 100755 --- a/gravity.sh +++ b/gravity.sh @@ -851,6 +851,7 @@ gravity_Cleanup() { database_recovery() { local result local str="Checking integrity of existing gravity database" + local option="${1}" echo -ne " ${INFO} ${str}..." if result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA integrity_check" 2>&1)"; then echo -e "${OVER} ${TICK} ${str} - no errors found" @@ -859,7 +860,7 @@ database_recovery() { echo -ne " ${INFO} ${str}..." if result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)"; then echo -e "${OVER} ${TICK} ${str} - no errors found" - if [[ $1 != "force" ]]; then + if [[ "${option}" != "force" ]]; then return fi else From db5e94b14a4e5a0208cd3515cfc4e25356335120 Mon Sep 17 00:00:00 2001 From: yubiuser Date: Tue, 21 Dec 2021 22:01:34 +0100 Subject: [PATCH 38/99] use +short and omit obsolet awk Co-authored-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 2ac9028a09..8fd3a47b59 100755 --- a/gravity.sh +++ b/gravity.sh @@ -597,7 +597,7 @@ gravity_DownloadBlocklistFromUrl() { blocked=true fi;; "NODATA") - if [[ $(dig "${domain}" | grep "NOERROR" -c) -ge 1 ]] && [[ -z $(dig +noall +answer "${domain}" |awk '{print $5}') ]]; then + if [[ $(dig "${domain}" | grep "NOERROR" -c) -ge 1 ]] && [[ -z $(dig +short "${domain}") ]]; then blocked=true fi;; "NULL"|*) From ff4487ff749dcaa7eaf2b72ecaeb068209831fcd Mon Sep 17 00:00:00 2001 From: yubiuser Date: Tue, 21 Dec 2021 22:10:56 +0100 Subject: [PATCH 39/99] Escape quotes Co-authored-by: DL6ER --- advanced/Scripts/list.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index ffde075a91..8945047e9c 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -92,7 +92,7 @@ Options: -h, --help Show this help dialog -l, --list Display all your ${listname}listed domains --nuke Removes all entries in a list - --comment "text" Add a comment to the domain. If adding multiple domains the same comment will be used for all" + --comment \"text\" Add a comment to the domain. If adding multiple domains the same comment will be used for all" exit 0 } From 920cf6de144b02950277b3b509a71c0adbad93a0 Mon Sep 17 00:00:00 2001 From: yubiuser Date: Wed, 22 Dec 2021 19:21:44 +0100 Subject: [PATCH 40/99] Check for updates on master based on tags not commits (#4475) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Check for updates on master based on tags not commits Signed-off-by: Christian König * Fix stickler Signed-off-by: Christian König * Address reviewer's comments Signed-off-by: Christian König * Fix stickler again Signed-off-by: Christian König * Use local git instead of relying on github Signed-off-by: Christian König * Add --tags Co-authored-by: DL6ER Co-authored-by: DL6ER --- advanced/Scripts/update.sh | 40 +++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index 3fab9a9535..d18d2e786a 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -35,25 +35,37 @@ source "/opt/pihole/COL_TABLE" GitCheckUpdateAvail() { local directory + local curBranch directory="${1}" curdir=$PWD cd "${directory}" || return # Fetch latest changes in this repo - git fetch --quiet origin - - # @ alone is a shortcut for HEAD. Older versions of git - # need @{0} - LOCAL="$(git rev-parse "@{0}")" - - # The suffix @{upstream} to a branchname - # (short form @{u}) refers - # to the branch that the branch specified - # by branchname is set to build on top of# - # (configured with branch..remote and - # branch..merge). A missing branchname - # defaults to the current one. - REMOTE="$(git rev-parse "@{upstream}")" + git fetch --tags --quiet origin + + # Check current branch. If it is master, then check for the latest available tag instead of latest commit. + curBranch=$(git rev-parse --abbrev-ref HEAD) + if [[ "${curBranch}" == "master" ]]; then + # get the latest local tag + LOCAL=$(git describe --abbrev=0 --tags master) + # get the latest tag from remote + REMOTE=$(git describe --abbrev=0 --tags origin/master) + + else + # @ alone is a shortcut for HEAD. Older versions of git + # need @{0} + LOCAL="$(git rev-parse "@{0}")" + + # The suffix @{upstream} to a branchname + # (short form @{u}) refers + # to the branch that the branch specified + # by branchname is set to build on top of# + # (configured with branch..remote and + # branch..merge). A missing branchname + # defaults to the current one. + REMOTE="$(git rev-parse "@{upstream}")" + fi + if [[ "${#LOCAL}" == 0 ]]; then echo -e "\\n ${COL_LIGHT_RED}Error: Local revision could not be obtained, please contact Pi-hole Support" From 2e1ce7fc87a5934c9316db1ce1824ac0e3636dcd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Dec 2021 19:52:08 +0100 Subject: [PATCH 41/99] Apply suggestions from code review Co-authored-by: yubiuser --- gravity.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gravity.sh b/gravity.sh index b1ecdc08aa..385ec3de77 100755 --- a/gravity.sh +++ b/gravity.sh @@ -880,6 +880,8 @@ database_recovery() { echo -e "${OVER} ${TICK} ${str} - success" mv "${gravityDBfile}" "${gravityDBfile}.old" mv "${gravityDBfile}.recovered" "${gravityDBfile}" + echo -ne " ${INFO} ${gravityDBfile} has been recovered" + echo -ne " ${INFO} The old ${gravityDBfile} has been moved to ${gravityDBfile}.old" else echo -e "${OVER} ${CROSS} ${str} - the following errors happened:" while IFS= read -r line ; do echo " - $line"; done <<< "$result" From d2a98ae95498d6f72d0b1ceee3d46ec2e7ecfe4c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Dec 2021 19:53:52 +0100 Subject: [PATCH 42/99] Document -r recover force case Signed-off-by: DL6ER --- gravity.sh | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/gravity.sh b/gravity.sh index 385ec3de77..4fcc281a5d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -909,14 +909,21 @@ repairSelector() { Attempt to repair gravity database Available options: - pihole -g -r recover Try to recover a damaged gravity database file. - Pi-hole tries to restore as much as possible - from a corrupted gravity database. - pihole -g -r recreate Create a new gravity database file from scratch. - This will remove your existing gravity database - and create a new file from scratch. If you still - have the migration backup created when migrating - to Pi-hole v5.0, Pi-hole will import these files." + pihole -g -r recover Try to recover a damaged gravity database file. + Pi-hole tries to restore as much as possible + from a corrupted gravity database. + + pihole -g -r recover force Pi-hole will run the recovery process even when + no damage is detected. This option is meant to be + a last resort. Recovery is a fragile task + consuming a lot of resources and shouldn't be + performed unnecessarily. + + pihole -g -r recreate Create a new gravity database file from scratch. + This will remove your existing gravity database + and create a new file from scratch. If you still + have the migration backup created when migrating + to Pi-hole v5.0, Pi-hole will import these files." exit 0;; esac } From ef30a85afb1653df4d3ce393dd6b8d7ecbb29ab0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 26 Dec 2021 17:10:48 +0100 Subject: [PATCH 43/99] Include port in status function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- pihole | 81 ++++++++++++++++++++++++++++++++++------------------------ 1 file changed, 47 insertions(+), 34 deletions(-) diff --git a/pihole b/pihole index 055bd70276..faa9fc2e34 100755 --- a/pihole +++ b/pihole @@ -312,42 +312,55 @@ analyze_ports() { } statusFunc() { - # Determine if there is a pihole service is listening on port 53 - local listening - listening="$(lsof -Pni:53)" - if grep -q "pihole" <<< "${listening}"; then - if [[ "${1}" != "web" ]]; then - analyze_ports "${listening}" - fi + # Determine if there is pihole-FTL service is listening on any UDP port + local listening pid port + + pid="$(getFTLPID)" + if [[ "$pid" -eq "-1" ]]; then + case "${1}" in + "web") echo "-1";; + *) echo -e " ${CROSS} DNS service is NOT running";; + esac + return 0 else - case "${1}" in - "web") echo "-1";; - *) echo -e " ${CROSS} DNS service is NOT listening";; - esac - return 0 - fi + #get the port pihole-FTL is listening on + port="$(lsof -Pni UDP -p ${pid} -a | grep -m1 : | awk -F ":" '{print $2}')" + listening="$(lsof -Pni:53)" + if [[ ! -z "$port" ]]; then + if [[ "${1}" != "web" ]]; then + analyze_ports "${listening}" + fi + else + case "${1}" in + "web") echo "-1";; + *) echo -e " ${CROSS} DNS service is NOT listening";; + esac + return 0 + fi + + # Determine if Pi-hole's blocking is enabled + if grep -q "BLOCKING_ENABLED=false" /etc/pihole/setupVars.conf; then + # A config is commented out + case "${1}" in + "web") echo 0;; + *) echo -e " ${CROSS} Pi-hole blocking is disabled";; + esac + elif grep -q "BLOCKING_ENABLED=true" /etc/pihole/setupVars.conf; then + # Configs are set + case "${1}" in + "web") echo "$port";; + *) echo -e " ${TICK} Pi-hole blocking is enabled";; + esac + else + # No configs were found + case "${1}" in + "web") echo -2;; + *) echo -e " ${INFO} Pi-hole blocking will be enabled";; + esac + # Enable blocking + "${PI_HOLE_BIN_DIR}"/pihole enable + fi - # Determine if Pi-hole's blocking is enabled - if grep -q "BLOCKING_ENABLED=false" /etc/pihole/setupVars.conf; then - # A config is commented out - case "${1}" in - "web") echo 0;; - *) echo -e " ${CROSS} Pi-hole blocking is disabled";; - esac - elif grep -q "BLOCKING_ENABLED=true" /etc/pihole/setupVars.conf; then - # Configs are set - case "${1}" in - "web") echo 1;; - *) echo -e " ${TICK} Pi-hole blocking is enabled";; - esac - else - # No configs were found - case "${1}" in - "web") echo 99;; - *) echo -e " ${INFO} Pi-hole blocking will be enabled";; - esac - # Enable blocking - "${PI_HOLE_BIN_DIR}"/pihole enable fi } From 9f0e0dbd378bc3989c63e7fd98e2ef34c2a541f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 26 Dec 2021 18:10:36 +0100 Subject: [PATCH 44/99] Fix analyse ports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- pihole | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pihole b/pihole index faa9fc2e34..c16f46e8a0 100755 --- a/pihole +++ b/pihole @@ -327,7 +327,7 @@ statusFunc() { port="$(lsof -Pni UDP -p ${pid} -a | grep -m1 : | awk -F ":" '{print $2}')" listening="$(lsof -Pni:53)" if [[ ! -z "$port" ]]; then - if [[ "${1}" != "web" ]]; then + if [[ "${1}" != "web" ]] && [[ "$port" -eq 53 ]]; then analyze_ports "${listening}" fi else From 71ebd64f4e9745ca5ea021d2bfb2035bd5c2c64b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 26 Dec 2021 18:13:14 +0100 Subject: [PATCH 45/99] mend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- pihole | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pihole b/pihole index c16f46e8a0..b0f3b02ce3 100755 --- a/pihole +++ b/pihole @@ -285,7 +285,7 @@ Options: analyze_ports() { # FTL is listening at least on at least one port when this # function is getting called - echo -e " ${TICK} DNS service is listening" + echo -e " ${TICK} DNS service is listening on port 53" # Check individual address family/protocol combinations # For a healthy Pi-hole, they should all be up (nothing printed) if grep -q "IPv4.*UDP" <<< "${1}"; then From 5ef731fc576825d2dbb9d59aff937e1838aa5c06 Mon Sep 17 00:00:00 2001 From: yubiuser Date: Tue, 28 Dec 2021 12:11:26 +0100 Subject: [PATCH 46/99] Fix indention Co-authored-by: DL6ER --- pihole | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pihole b/pihole index b0f3b02ce3..01bbaaed75 100755 --- a/pihole +++ b/pihole @@ -317,11 +317,11 @@ statusFunc() { pid="$(getFTLPID)" if [[ "$pid" -eq "-1" ]]; then - case "${1}" in - "web") echo "-1";; - *) echo -e " ${CROSS} DNS service is NOT running";; - esac - return 0 + case "${1}" in + "web") echo "-1";; + *) echo -e " ${CROSS} DNS service is NOT running";; + esac + return 0 else #get the port pihole-FTL is listening on port="$(lsof -Pni UDP -p ${pid} -a | grep -m1 : | awk -F ":" '{print $2}')" From 4a2f4c1bcea5ba554a2e3831431bef043fd89761 Mon Sep 17 00:00:00 2001 From: yubiuser Date: Tue, 28 Dec 2021 12:11:46 +0100 Subject: [PATCH 47/99] Fix indention_2 Co-authored-by: DL6ER --- pihole | 70 +++++++++++++++++++++++++++++----------------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/pihole b/pihole index 01bbaaed75..c3bab4f14d 100755 --- a/pihole +++ b/pihole @@ -323,43 +323,43 @@ statusFunc() { esac return 0 else - #get the port pihole-FTL is listening on - port="$(lsof -Pni UDP -p ${pid} -a | grep -m1 : | awk -F ":" '{print $2}')" - listening="$(lsof -Pni:53)" - if [[ ! -z "$port" ]]; then - if [[ "${1}" != "web" ]] && [[ "$port" -eq 53 ]]; then - analyze_ports "${listening}" - fi - else - case "${1}" in - "web") echo "-1";; - *) echo -e " ${CROSS} DNS service is NOT listening";; - esac - return 0 + #get the port pihole-FTL is listening on + port="$(lsof -Pni UDP -p ${pid} -a | grep -m1 : | awk -F ":" '{print $2}')" + listening="$(lsof -Pni:53)" + if [[ ! -z "$port" ]]; then + if [[ "${1}" != "web" ]] && [[ "$port" -eq 53 ]]; then + analyze_ports "${listening}" fi + else + case "${1}" in + "web") echo "-1";; + *) echo -e " ${CROSS} DNS service is NOT listening";; + esac + return 0 + fi - # Determine if Pi-hole's blocking is enabled - if grep -q "BLOCKING_ENABLED=false" /etc/pihole/setupVars.conf; then - # A config is commented out - case "${1}" in - "web") echo 0;; - *) echo -e " ${CROSS} Pi-hole blocking is disabled";; - esac - elif grep -q "BLOCKING_ENABLED=true" /etc/pihole/setupVars.conf; then - # Configs are set - case "${1}" in - "web") echo "$port";; - *) echo -e " ${TICK} Pi-hole blocking is enabled";; - esac - else - # No configs were found - case "${1}" in - "web") echo -2;; - *) echo -e " ${INFO} Pi-hole blocking will be enabled";; - esac - # Enable blocking - "${PI_HOLE_BIN_DIR}"/pihole enable - fi + # Determine if Pi-hole's blocking is enabled + if grep -q "BLOCKING_ENABLED=false" /etc/pihole/setupVars.conf; then + # A config is commented out + case "${1}" in + "web") echo 0;; + *) echo -e " ${CROSS} Pi-hole blocking is disabled";; + esac + elif grep -q "BLOCKING_ENABLED=true" /etc/pihole/setupVars.conf; then + # Configs are set + case "${1}" in + "web") echo "$port";; + *) echo -e " ${TICK} Pi-hole blocking is enabled";; + esac + else + # No configs were found + case "${1}" in + "web") echo -2;; + *) echo -e " ${INFO} Pi-hole blocking will be enabled";; + esac + # Enable blocking + "${PI_HOLE_BIN_DIR}"/pihole enable + fi fi } From 2a869419b4d7b958f98f685be8a96f4da7c594e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 28 Dec 2021 12:18:39 +0100 Subject: [PATCH 48/99] Add netcat to dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index bd2bf4c34e..6a9f94911c 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -287,7 +287,7 @@ package_manager_detect() { # Packages required to run this install script (stored as an array) INSTALLER_DEPS=(git iproute2 whiptail ca-certificates) # Packages required to run Pi-hole (stored as an array) - PIHOLE_DEPS=(cron curl iputils-ping lsof psmisc sudo unzip idn2 sqlite3 libcap2-bin dns-root-data libcap2) + PIHOLE_DEPS=(cron curl iputils-ping lsof psmisc sudo unzip idn2 sqlite3 libcap2-bin dns-root-data libcap2 netcat) # Packages required for the Web admin interface (stored as an array) # It's useful to separate this from Pi-hole, since the two repos are also setup separately PIHOLE_WEB_DEPS=(lighttpd "${phpVer}-common" "${phpVer}-cgi" "${phpVer}-sqlite3" "${phpVer}-xml" "${phpVer}-intl") @@ -332,7 +332,7 @@ package_manager_detect() { PKG_COUNT="${PKG_MANAGER} check-update | egrep '(.i686|.x86|.noarch|.arm|.src)' | wc -l" OS_CHECK_DEPS=(grep bind-utils) INSTALLER_DEPS=(git iproute newt procps-ng which chkconfig ca-certificates) - PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc sqlite libcap lsof) + PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc sqlite libcap lsof nmap-ncat) PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php-common php-cli php-pdo php-xml php-json php-intl) LIGHTTPD_USER="lighttpd" LIGHTTPD_GROUP="lighttpd" From 5729f64ddc82d3ef477293c6d404a05d1a2ac82b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 28 Dec 2021 12:21:31 +0100 Subject: [PATCH 49/99] Fix missing fi MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- pihole | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pihole b/pihole index c3bab4f14d..5b062267ed 100755 --- a/pihole +++ b/pihole @@ -337,7 +337,7 @@ statusFunc() { esac return 0 fi - + fi # Determine if Pi-hole's blocking is enabled if grep -q "BLOCKING_ENABLED=false" /etc/pihole/setupVars.conf; then # A config is commented out From f45248df805c426f9d9e654c0f71eb1fd8b795e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 28 Dec 2021 13:39:45 +0100 Subject: [PATCH 50/99] Use FTL's new dns-port API endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- pihole | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pihole b/pihole index 5b062267ed..d9f8b5fa1e 100755 --- a/pihole +++ b/pihole @@ -312,7 +312,7 @@ analyze_ports() { } statusFunc() { - # Determine if there is pihole-FTL service is listening on any UDP port + # Determine if there is pihole-FTL service is listening local listening pid port pid="$(getFTLPID)" @@ -323,21 +323,22 @@ statusFunc() { esac return 0 else - #get the port pihole-FTL is listening on - port="$(lsof -Pni UDP -p ${pid} -a | grep -m1 : | awk -F ":" '{print $2}')" + #get the port pihole-FTL is listening on by using FTL's telnet API + port="$(echo ">dns-port >quit" | nc 127.0.0.1 4711)" listening="$(lsof -Pni:53)" - if [[ ! -z "$port" ]]; then - if [[ "${1}" != "web" ]] && [[ "$port" -eq 53 ]]; then - analyze_ports "${listening}" - fi - else + if [[ "${port}" == "0" ]]; then case "${1}" in "web") echo "-1";; *) echo -e " ${CROSS} DNS service is NOT listening";; esac return 0 + else + if [[ "${1}" != "web" ]] && [[ "$port" -eq 53 ]]; then + analyze_ports "${listening}" + fi fi fi + # Determine if Pi-hole's blocking is enabled if grep -q "BLOCKING_ENABLED=false" /etc/pihole/setupVars.conf; then # A config is commented out @@ -361,7 +362,6 @@ statusFunc() { "${PI_HOLE_BIN_DIR}"/pihole enable fi - fi } tailFunc() { From 71ed842dfd782d2b20315493e9684a02297ca9f0 Mon Sep 17 00:00:00 2001 From: Matthew Nickson Date: Tue, 28 Dec 2021 18:32:06 +0000 Subject: [PATCH 51/99] Fixed path to 404 file when using custom.php (#4488) Signed-off-by: Computroniks --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index bd2bf4c34e..9a975f4b40 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1334,7 +1334,7 @@ installConfigs() { chmod 644 /etc/lighttpd/external.conf # If there is a custom block page in the html/pihole directory, replace 404 handler in lighttpd config if [[ -f "${PI_HOLE_BLOCKPAGE_DIR}/custom.php" ]]; then - sed -i 's/^\(server\.error-handler-404\s*=\s*\).*$/\1"pihole\/custom\.php"/' "${lighttpdConfig}" + sed -i 's/^\(server\.error-handler-404\s*=\s*\).*$/\1"\/pihole\/custom\.php"/' "${lighttpdConfig}" fi # Make the directories if they do not exist and set the owners mkdir -p /run/lighttpd From 2b52f9264769e9b1254da9552bd65df46f198cdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 28 Dec 2021 19:36:32 +0100 Subject: [PATCH 52/99] Inlcude port also in cli output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- pihole | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pihole b/pihole index d9f8b5fa1e..f93da52ca2 100755 --- a/pihole +++ b/pihole @@ -350,7 +350,8 @@ statusFunc() { # Configs are set case "${1}" in "web") echo "$port";; - *) echo -e " ${TICK} Pi-hole blocking is enabled";; + *) echo -e " ${TICK} Pi-hole blocking is enabled. FTL is listening on port ${port}" + ;; esac else # No configs were found From bcb59159ed659c7c8e2ccec8447285aede7f6845 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 28 Dec 2021 19:52:11 +0100 Subject: [PATCH 53/99] Analyse port also on ports other than 53 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- pihole | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pihole b/pihole index f93da52ca2..a8f20b69dc 100755 --- a/pihole +++ b/pihole @@ -325,7 +325,7 @@ statusFunc() { else #get the port pihole-FTL is listening on by using FTL's telnet API port="$(echo ">dns-port >quit" | nc 127.0.0.1 4711)" - listening="$(lsof -Pni:53)" + listening="$(lsof -Pni:${port})" if [[ "${port}" == "0" ]]; then case "${1}" in "web") echo "-1";; @@ -333,7 +333,8 @@ statusFunc() { esac return 0 else - if [[ "${1}" != "web" ]] && [[ "$port" -eq 53 ]]; then + if [[ "${1}" != "web" ]]; then + echo -e " ${TICK} FTL is listening on port ${port}" analyze_ports "${listening}" fi fi @@ -350,8 +351,7 @@ statusFunc() { # Configs are set case "${1}" in "web") echo "$port";; - *) echo -e " ${TICK} Pi-hole blocking is enabled. FTL is listening on port ${port}" - ;; + *) echo -e " ${TICK} Pi-hole blocking is enabled";; esac else # No configs were found From 3989cc19e9a50a84fc996c870a94d044d1d152f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 28 Dec 2021 19:55:42 +0100 Subject: [PATCH 54/99] Remove double text output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- pihole | 1 - 1 file changed, 1 deletion(-) diff --git a/pihole b/pihole index a8f20b69dc..cd18b27386 100755 --- a/pihole +++ b/pihole @@ -285,7 +285,6 @@ Options: analyze_ports() { # FTL is listening at least on at least one port when this # function is getting called - echo -e " ${TICK} DNS service is listening on port 53" # Check individual address family/protocol combinations # For a healthy Pi-hole, they should all be up (nothing printed) if grep -q "IPv4.*UDP" <<< "${1}"; then From 5bd7cc9c9d9be16bc6cd499635a19a042088c97e Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Sun, 2 Jan 2022 05:02:20 +1100 Subject: [PATCH 55/99] Replace `which` with `command -v` (#4499) Signed-off-by: WaLLy3K WaLLy3K@users.noreply.github.com --- automated install/basic-install.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 5b8de0223f..e3a9764f06 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2277,7 +2277,7 @@ FTLcheckUpdate() { printf " %b Checking for existing FTL binary...\\n" "${INFO}" local ftlLoc - ftlLoc=$(which pihole-FTL 2>/dev/null) + ftlLoc=$(command -v pihole-FTL 2>/dev/null) local ftlBranch @@ -2315,7 +2315,7 @@ FTLcheckUpdate() { # We already have a pihole-FTL binary downloaded. # Alt branches don't have a tagged version against them, so just confirm the checksum of the local vs remote to decide whether we download or not remoteSha1=$(curl -sSL --fail "https://ftl.pi-hole.net/${ftlBranch}/${binary}.sha1" | cut -d ' ' -f 1) - localSha1=$(sha1sum "$(which pihole-FTL)" | cut -d ' ' -f 1) + localSha1=$(sha1sum "$(command -v pihole-FTL)" | cut -d ' ' -f 1) if [[ "${remoteSha1}" != "${localSha1}" ]]; then printf " %b Checksums do not match, downloading from ftl.pi-hole.net.\\n" "${INFO}" @@ -2345,7 +2345,7 @@ FTLcheckUpdate() { printf " %b Latest FTL Binary already installed (%s). Confirming Checksum...\\n" "${INFO}" "${FTLlatesttag}" remoteSha1=$(curl -sSL --fail "https://github.com/pi-hole/FTL/releases/download/${FTLversion%$'\r'}/${binary}.sha1" | cut -d ' ' -f 1) - localSha1=$(sha1sum "$(which pihole-FTL)" | cut -d ' ' -f 1) + localSha1=$(sha1sum "$(command -v pihole-FTL)" | cut -d ' ' -f 1) if [[ "${remoteSha1}" != "${localSha1}" ]]; then printf " %b Corruption detected...\\n" "${INFO}" From 0e359a6321ff77581795a1fb76e20773ca30d81d Mon Sep 17 00:00:00 2001 From: yubiuser Date: Tue, 4 Jan 2022 09:40:07 +0100 Subject: [PATCH 56/99] Set dnsmasq interface listening by default to local (#4509) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index e3a9764f06..5f69eb734d 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1731,7 +1731,7 @@ finalExports() { # If the setup variable file exists, if [[ -e "${setupVars}" ]]; then # update the variables in the file - sed -i.update.bak '/PIHOLE_INTERFACE/d;/IPV4_ADDRESS/d;/IPV6_ADDRESS/d;/PIHOLE_DNS_1\b/d;/PIHOLE_DNS_2\b/d;/QUERY_LOGGING/d;/INSTALL_WEB_SERVER/d;/INSTALL_WEB_INTERFACE/d;/LIGHTTPD_ENABLED/d;/CACHE_SIZE/d;/DNS_FQDN_REQUIRED/d;/DNS_BOGUS_PRIV/d;' "${setupVars}" + sed -i.update.bak '/PIHOLE_INTERFACE/d;/IPV4_ADDRESS/d;/IPV6_ADDRESS/d;/PIHOLE_DNS_1\b/d;/PIHOLE_DNS_2\b/d;/QUERY_LOGGING/d;/INSTALL_WEB_SERVER/d;/INSTALL_WEB_INTERFACE/d;/LIGHTTPD_ENABLED/d;/CACHE_SIZE/d;/DNS_FQDN_REQUIRED/d;/DNS_BOGUS_PRIV/d;/DNSMASQ_LISTENING/d;' "${setupVars}" fi # echo the information to the user { @@ -1747,6 +1747,7 @@ finalExports() { echo "CACHE_SIZE=${CACHE_SIZE}" echo "DNS_FQDN_REQUIRED=${DNS_FQDN_REQUIRED:-true}" echo "DNS_BOGUS_PRIV=${DNS_BOGUS_PRIV:-true}" + echo "DNSMASQ_LISTENING=${DNSMASQ_LISTENING:-local}" }>> "${setupVars}" chmod 644 "${setupVars}" From 241e53ed454ab6a1ee02b071da848390d2b8c55e Mon Sep 17 00:00:00 2001 From: yubiuser Date: Tue, 4 Jan 2022 20:06:41 +0100 Subject: [PATCH 57/99] Skip debug upload question if called from web interface (#4494) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Skip debug upload question if called from web interface Signed-off-by: Christian König * Suppress upload error if users opt-out from uploading from web interface Signed-off-by: Christian König * Fix and reverse logic Signed-off-by: Christian König * Remove addtional space Signed-off-by: Christian König * Include reviewer's comment :D Co-authored-by: Adam Warner Co-authored-by: Adam Warner --- advanced/Scripts/piholeDebug.sh | 36 ++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 3ac63e80aa..62e2732dd4 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1386,9 +1386,9 @@ upload_to_tricorder() { log_write "${TICK} ${COL_GREEN}** FINISHED DEBUGGING! **${COL_NC}\\n" # Provide information on what they should do with their token - log_write " * The debug log can be uploaded to tricorder.pi-hole.net for sharing with developers only." + log_write " * The debug log can be uploaded to tricorder.pi-hole.net for sharing with developers only." - # If pihole -d is running automatically (usually through the dashboard) + # If pihole -d is running automatically if [[ "${AUTOMATED}" ]]; then # let the user know log_write "${INFO} Debug script running in automated mode" @@ -1396,16 +1396,19 @@ upload_to_tricorder() { curl_to_tricorder # If we're not running in automated mode, else - echo "" - # give the user a choice of uploading it or not - # Users can review the log file locally (or the output of the script since they are the same) and try to self-diagnose their problem - read -r -p "[?] Would you like to upload the log? [y/N] " response - case ${response} in - # If they say yes, run our function for uploading the log - [yY][eE][sS]|[yY]) curl_to_tricorder;; - # If they choose no, just exit out of the script - *) log_write " * Log will ${COL_GREEN}NOT${COL_NC} be uploaded to tricorder.\\n * A local copy of the debug log can be found at: ${COL_CYAN}${PIHOLE_DEBUG_LOG}${COL_NC}\\n";exit; - esac + # if not being called from the web interface + if [[ ! "${WEBCALL}" ]]; then + echo "" + # give the user a choice of uploading it or not + # Users can review the log file locally (or the output of the script since they are the same) and try to self-diagnose their problem + read -r -p "[?] Would you like to upload the log? [y/N] " response + case ${response} in + # If they say yes, run our function for uploading the log + [yY][eE][sS]|[yY]) curl_to_tricorder;; + # If they choose no, just exit out of the script + *) log_write " * Log will ${COL_GREEN}NOT${COL_NC} be uploaded to tricorder.\\n * A local copy of the debug log can be found at: ${COL_CYAN}${PIHOLE_DEBUG_LOG}${COL_NC}\\n";exit; + esac + fi fi # Check if tricorder.pi-hole.net is reachable and provide token # along with some additional useful information @@ -1425,8 +1428,13 @@ upload_to_tricorder() { # If no token was generated else # Show an error and some help instructions - log_write "${CROSS} ${COL_RED}There was an error uploading your debug log.${COL_NC}" - log_write " * Please try again or contact the Pi-hole team for assistance." + # Skip this if being called from web interface and autmatic mode was not chosen (users opt-out to upload) + if [[ "${WEBCALL}" ]] && [[ ! "${AUTOMATED}" ]]; then + : + else + log_write "${CROSS} ${COL_RED}There was an error uploading your debug log.${COL_NC}" + log_write " * Please try again or contact the Pi-hole team for assistance." + fi fi # Finally, show where the log file is no matter the outcome of the function so users can look at it log_write " * A local copy of the debug log can be found at: ${COL_CYAN}${PIHOLE_DEBUG_LOG}${COL_NC}\\n" From c6a2a6f7398cd35e89f0d444a047644e7688aa73 Mon Sep 17 00:00:00 2001 From: yubiuser Date: Tue, 4 Jan 2022 20:09:30 +0100 Subject: [PATCH 58/99] Install pihole-FTL.conf template on fresh installation (#4496) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL.conf | 2 ++ automated install/basic-install.sh | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 advanced/Templates/pihole-FTL.conf diff --git a/advanced/Templates/pihole-FTL.conf b/advanced/Templates/pihole-FTL.conf new file mode 100644 index 0000000000..269fcf9d47 --- /dev/null +++ b/advanced/Templates/pihole-FTL.conf @@ -0,0 +1,2 @@ +#; Pi-hole FTL config file +#; Comments should start with #; to avoid issues with PHP and bash reading this file diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 5f69eb734d..465c8cc119 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1299,10 +1299,10 @@ installConfigs() { echo "${DNS_SERVERS}" > "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" chmod 644 "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" - # Install empty file if it does not exist + # Install template file if it does not exist if [[ ! -r "${PI_HOLE_CONFIG_DIR}/pihole-FTL.conf" ]]; then install -d -m 0755 ${PI_HOLE_CONFIG_DIR} - if ! install -o pihole -m 664 /dev/null "${PI_HOLE_CONFIG_DIR}/pihole-FTL.conf" &>/dev/null; then + if ! install -T -o pihole -m 664 "${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole-FTL.conf" "${PI_HOLE_CONFIG_DIR}/pihole-FTL.conf" &>/dev/null; then printf " %bError: Unable to initialize configuration file %s/pihole-FTL.conf\\n" "${COL_LIGHT_RED}" "${PI_HOLE_CONFIG_DIR}" return 1 fi From 7807a93e10f114982a31d8d224827b8e14846dec Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Tue, 4 Jan 2022 21:46:06 +0000 Subject: [PATCH 59/99] If PIHOLE_DOCKER_TAG is set, then include that info in the debug run (#4515) Signed-off-by: Adam Warner --- advanced/Scripts/piholeDebug.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 62e2732dd4..ef85ed10ee 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -467,6 +467,9 @@ diagnose_operating_system() { # Display the current test that is running echo_current_diagnostic "Operating system" + # If the PIHOLE_DOCKER_TAG variable is set, include this information in the debug output + [ -n "${PIHOLE_DOCKER_TAG}" ] && log_write "${INFO} Pi-hole Docker Container: ${PIHOLE_DOCKER_TAG}" + # If there is a /etc/*release file, it's probably a supported operating system, so we can if ls /etc/*release 1> /dev/null 2>&1; then # display the attributes to the user from the function made earlier From 5823f5e254e33c3f6b5d1dcd1a94564c67fe3f0d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 5 Jan 2022 17:41:46 +0100 Subject: [PATCH 60/99] Use ss instead of lsof (#4518) * Use ss instead of lsof for pihole status checks Signed-off-by: DL6ER * Use ss FILTER instead of piping into bash Signed-off-by: DL6ER * Use ss in debug log generation Signed-off-by: DL6ER * Remove lsof from dependencies Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 26 +++++++++++--------------- automated install/basic-install.sh | 4 ++-- pihole | 14 ++++++++------ 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index ef85ed10ee..dc4a27295a 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -733,11 +733,11 @@ compare_port_to_service_assigned() { # If the service is a Pi-hole service, highlight it in green if [[ "${service_name}" == "${expected_service}" ]]; then - log_write "[${COL_GREEN}${port}${COL_NC}] is in use by ${COL_GREEN}${service_name}${COL_NC}" + log_write "${TICK} ${COL_GREEN}${port}${COL_NC} is in use by ${COL_GREEN}${service_name}${COL_NC}" # Otherwise, else # Show the service name in red since it's non-standard - log_write "[${COL_RED}${port}${COL_NC}] is in use by ${COL_RED}${service_name}${COL_NC} (${FAQ_HARDWARE_REQUIREMENTS_PORTS})" + log_write "${CROSS} ${COL_RED}${port}${COL_NC} is in use by ${COL_RED}${service_name}${COL_NC} (${FAQ_HARDWARE_REQUIREMENTS_PORTS})" fi } @@ -753,32 +753,28 @@ check_required_ports() { # Sort the addresses and remove duplicates while IFS= read -r line; do ports_in_use+=( "$line" ) - done < <( lsof -iTCP -sTCP:LISTEN -P -n +c 10 ) + done < <( ss --listening --numeric --tcp --udp --processes --oneline --no-header ) # Now that we have the values stored, for i in "${!ports_in_use[@]}"; do # loop through them and assign some local variables local service_name - service_name=$(echo "${ports_in_use[$i]}" | awk '{print $1}') + service_name=$(echo "${ports_in_use[$i]}" | awk '{gsub(/users:\(\("/,"",$7);gsub(/".*/,"",$7);print $7}') local protocol_type - protocol_type=$(echo "${ports_in_use[$i]}" | awk '{print $5}') + protocol_type=$(echo "${ports_in_use[$i]}" | awk '{print $1}') local port_number - port_number="$(echo "${ports_in_use[$i]}" | awk '{print $9}')" + port_number="$(echo "${ports_in_use[$i]}" | awk '{print $5}')" # | awk '{gsub(/^.*:/,"",$5);print $5}') - # Skip the line if it's the titles of the columns the lsof command produces - if [[ "${service_name}" == COMMAND ]]; then - continue - fi # Use a case statement to determine if the right services are using the right ports - case "$(echo "$port_number" | rev | cut -d: -f1 | rev)" in - 53) compare_port_to_service_assigned "${resolver}" "${service_name}" 53 + case "$(echo "${port_number}" | rev | cut -d: -f1 | rev)" in + 53) compare_port_to_service_assigned "${resolver}" "${service_name}" "${protocol_type}:${port_number}" ;; - 80) compare_port_to_service_assigned "${web_server}" "${service_name}" 80 + 80) compare_port_to_service_assigned "${web_server}" "${service_name}" "${protocol_type}:${port_number}" ;; - 4711) compare_port_to_service_assigned "${ftl}" "${service_name}" 4711 + 4711) compare_port_to_service_assigned "${ftl}" "${service_name}" "${protocol_type}:${port_number}" ;; # If it's not a default port that Pi-hole needs, just print it out for the user to see - *) log_write "${port_number} ${service_name} (${protocol_type})"; + *) log_write " ${protocol_type}:${port_number} is in use by ${service_name:=}"; esac done } diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 465c8cc119..3780f7b078 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -287,7 +287,7 @@ package_manager_detect() { # Packages required to run this install script (stored as an array) INSTALLER_DEPS=(git iproute2 whiptail ca-certificates) # Packages required to run Pi-hole (stored as an array) - PIHOLE_DEPS=(cron curl iputils-ping lsof psmisc sudo unzip idn2 sqlite3 libcap2-bin dns-root-data libcap2 netcat) + PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip idn2 sqlite3 libcap2-bin dns-root-data libcap2 netcat) # Packages required for the Web admin interface (stored as an array) # It's useful to separate this from Pi-hole, since the two repos are also setup separately PIHOLE_WEB_DEPS=(lighttpd "${phpVer}-common" "${phpVer}-cgi" "${phpVer}-sqlite3" "${phpVer}-xml" "${phpVer}-intl") @@ -332,7 +332,7 @@ package_manager_detect() { PKG_COUNT="${PKG_MANAGER} check-update | egrep '(.i686|.x86|.noarch|.arm|.src)' | wc -l" OS_CHECK_DEPS=(grep bind-utils) INSTALLER_DEPS=(git iproute newt procps-ng which chkconfig ca-certificates) - PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc sqlite libcap lsof nmap-ncat) + PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc sqlite libcap nmap-ncat) PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php-common php-cli php-pdo php-xml php-json php-intl) LIGHTTPD_USER="lighttpd" LIGHTTPD_GROUP="lighttpd" diff --git a/pihole b/pihole index cd18b27386..8af47dc8f7 100755 --- a/pihole +++ b/pihole @@ -283,26 +283,29 @@ Options: } analyze_ports() { + local lv4 lv6 port=${1} # FTL is listening at least on at least one port when this # function is getting called # Check individual address family/protocol combinations # For a healthy Pi-hole, they should all be up (nothing printed) - if grep -q "IPv4.*UDP" <<< "${1}"; then + lv4="$(ss --ipv4 --listening --numeric --tcp --udp src :${port})" + if grep -q "udp " <<< "${lv4}"; then echo -e " ${TICK} UDP (IPv4)" else echo -e " ${CROSS} UDP (IPv4)" fi - if grep -q "IPv4.*TCP" <<< "${1}"; then + if grep -q "tcp " <<< "${lv4}"; then echo -e " ${TICK} TCP (IPv4)" else echo -e " ${CROSS} TCP (IPv4)" fi - if grep -q "IPv6.*UDP" <<< "${1}"; then + lv6="$(ss --ipv6 --listening --numeric --tcp --udp src :${port})" + if grep -q "udp " <<< "${lv6}"; then echo -e " ${TICK} UDP (IPv6)" else echo -e " ${CROSS} UDP (IPv6)" fi - if grep -q "IPv6.*TCP" <<< "${1}"; then + if grep -q "tcp " <<< "${lv6}"; then echo -e " ${TICK} TCP (IPv6)" else echo -e " ${CROSS} TCP (IPv6)" @@ -324,7 +327,6 @@ statusFunc() { else #get the port pihole-FTL is listening on by using FTL's telnet API port="$(echo ">dns-port >quit" | nc 127.0.0.1 4711)" - listening="$(lsof -Pni:${port})" if [[ "${port}" == "0" ]]; then case "${1}" in "web") echo "-1";; @@ -334,7 +336,7 @@ statusFunc() { else if [[ "${1}" != "web" ]]; then echo -e " ${TICK} FTL is listening on port ${port}" - analyze_ports "${listening}" + analyze_ports "${port}" fi fi fi From 81698ef1eda51533366161fa23b23cac638b1b0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 5 Jan 2022 21:09:57 +0100 Subject: [PATCH 61/99] Fix Pi-hole status in chronometer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/chronometer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/chronometer.sh b/advanced/Scripts/chronometer.sh index 312c484f6c..fddb393677 100755 --- a/advanced/Scripts/chronometer.sh +++ b/advanced/Scripts/chronometer.sh @@ -357,7 +357,7 @@ get_sys_stats() { ram_used="${ram_raw[1]}" ram_total="${ram_raw[2]}" - if [[ "$(pihole status web 2> /dev/null)" == "1" ]]; then + if [[ "$(pihole status web 2> /dev/null)" -ge "1" ]]; then ph_status="${COL_LIGHT_GREEN}Active" else ph_status="${COL_LIGHT_RED}Offline" From c2080324b75487cb1130dec5f3450f57c3d62ad7 Mon Sep 17 00:00:00 2001 From: MichaIng Date: Fri, 7 Jan 2022 18:55:15 +0100 Subject: [PATCH 62/99] Install netcat-openbsd as dependency explicitly Since Debian Stretch and Ubuntu Bionic, the "netcat" package is a transitional dummy package which pulls in "netcat-traditional" on Debian Stretch+Buster and Ubuntu Bionic, and "netcat-openbsd" on Debian Bullseye, Ubuntu Focal and up. On Debian Bookworm (testing), however, the "netcat" package has been removed during the last 3 days at time or writing, so that it fails do be installed. While "netcat-traditional" and "netcat-openbsd" both "Provides: netcat", since it's two alternatives, APT does not automatically pick one but aborts, and the only solution is to install one explicitly. While this is likely a temporary state of the Debian testing suite, having a closer look at the two alternatives shows that "netcat-openbsd" is a much more actively maintained newer version with additional support for IPv6, proxies, and UNIX sockets, which is likely the reason for the gradual transition via meta package from "netcat-traditional" to "netcat-openbsd". This commit hence consequently follows this aim by skipping the transitional dummy package and installing "netcat-openbsd" explicitly as dependency, to avoid any possible errors like the one which occurs currently on Bookworm. Both packages can be installed concurrently and do no conflict, but are managed via dpkg's "update-alternatives". For reference: - https://packages.debian.org/netcat - https://packages.ubuntu.com/netcat Signed-off-by: MichaIng --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 3780f7b078..6591634ec8 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -287,7 +287,7 @@ package_manager_detect() { # Packages required to run this install script (stored as an array) INSTALLER_DEPS=(git iproute2 whiptail ca-certificates) # Packages required to run Pi-hole (stored as an array) - PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip idn2 sqlite3 libcap2-bin dns-root-data libcap2 netcat) + PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip idn2 sqlite3 libcap2-bin dns-root-data libcap2 netcat-openbsd) # Packages required for the Web admin interface (stored as an array) # It's useful to separate this from Pi-hole, since the two repos are also setup separately PIHOLE_WEB_DEPS=(lighttpd "${phpVer}-common" "${phpVer}-cgi" "${phpVer}-sqlite3" "${phpVer}-xml" "${phpVer}-intl") From bfd9fe80ef91237726e99d1462f98e68148f5b3b Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Sat, 8 Jan 2022 01:42:35 -0800 Subject: [PATCH 63/99] Remove debug from Stale Put Stale in to action. --- .github/workflows/stale.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 1c1c47a1ae..f98dcc2e18 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -2,7 +2,8 @@ name: Mark stale issues on: schedule: - - cron: '30 * * * *' + - cron: '0 * * * *' + workflow_dispatch: jobs: stale: @@ -15,7 +16,6 @@ jobs: - uses: actions/stale@v4 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - debug-only: true days-before-stale: 30 days-before-close: 5 stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Please comment or update this issue or it will be closed in 5 days.' From 3097c8fbdc431aed44e83f5e8116f2bc9242ecfd Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 8 Jan 2022 12:57:49 +0000 Subject: [PATCH 64/99] Skip the required ports check if installed in docker container. Unpriv'ed containers do not have access to the information required to resolve the service name listening - and the container should not start if there was a port conflict anyway (#4536) Signed-off-by: Adam Warner --- advanced/Scripts/piholeDebug.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index dc4a27295a..77e348c939 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -787,7 +787,9 @@ check_networking() { detect_ip_addresses "6" ping_gateway "4" ping_gateway "6" - check_required_ports + # Skip the following check if installed in docker container. Unpriv'ed containers do not have access to the information required + # to resolve the service name listening - and the container should not start if there was a port conflict anyway + [ -z "${PIHOLE_DOCKER_TAG}" ] && check_required_ports } check_x_headers() { From ce86157067b4de06462f6d220aba03f7b93adaa6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 8 Jan 2022 14:15:26 +0100 Subject: [PATCH 65/99] Fix gravity in case there are no adlists at all or all are disabled (#4535) Signed-off-by: DL6ER --- gravity.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/gravity.sh b/gravity.sh index a6ab3c8641..2f24fbdb6f 100755 --- a/gravity.sh +++ b/gravity.sh @@ -402,14 +402,12 @@ gravity_DownloadBlocklists() { )" local str="Pulling blocklist source list into range" + echo -e "${OVER} ${TICK} ${str}" - if [[ -n "${sources[*]}" ]] && [[ -n "${sourceDomains[*]}" ]]; then - echo -e "${OVER} ${TICK} ${str}" - else - echo -e "${OVER} ${CROSS} ${str}" + if [[ -z "${sources[*]}" ]] || [[ -z "${sourceDomains[*]}" ]]; then echo -e " ${INFO} No source list found, or it is empty" echo "" - return 1 + unset sources fi local url domain agent cmd_ext str target compression From a65a841c56ecce666499b20ad557c330e1c5b89b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 9 Jan 2022 07:13:51 +0100 Subject: [PATCH 66/99] Remove oneline from ss call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 77e348c939..ac4d45e2bb 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -753,7 +753,7 @@ check_required_ports() { # Sort the addresses and remove duplicates while IFS= read -r line; do ports_in_use+=( "$line" ) - done < <( ss --listening --numeric --tcp --udp --processes --oneline --no-header ) + done < <( ss --listening --numeric --tcp --udp --processes --no-header ) # Now that we have the values stored, for i in "${!ports_in_use[@]}"; do From b20b38d44fcabf685a8cd78dfee181c97e1eb6ce Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 9 Jan 2022 11:31:47 +0100 Subject: [PATCH 67/99] Include ip addr show and ip route show for us to help with local-service issues (where hops-away is measured) Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 77e348c939..51220833eb 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -779,6 +779,21 @@ check_required_ports() { done } +ip_command() { + # Obtain and log information from "ip XYZ show" commands + echo_current_diagnostic "${2}" + local entries=() + mapfile -t entries < <(ip "${1}" show) + for line in "${entries[@]}"; do + log_write " ${line}" + done +} + +check_ip_command() { + ip_command "addr" "Network interfaces and addresses" + ip_command "route" "Network routing table" +} + check_networking() { # Runs through several of the functions made earlier; we just clump them # together since they are all related to the networking aspect of things @@ -1454,6 +1469,7 @@ check_selinux check_firewalld processor_check disk_usage +check_ip_command check_networking check_name_resolution check_dhcp_servers From f1245685dc394f4bea75a7d4a14ce55f17fbb834 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 10 Jan 2022 20:07:44 +0000 Subject: [PATCH 68/99] Add action to automatically sync master to dev when code is pushed to master Add in a release.yml to ignore github-actions author when auto-generating release notes Signed-off-by: Adam Warner --- .github/release.yml | 7 +++++++ .github/workflows/sync-back-to-dev.yml | 28 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 .github/release.yml create mode 100644 .github/workflows/sync-back-to-dev.yml diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 0000000000..2e8776e999 --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,7 @@ +changelog: + exclude: + labels: + - internal + authors: + - dependabot + - github-actions diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml new file mode 100644 index 0000000000..819e9d24dc --- /dev/null +++ b/.github/workflows/sync-back-to-dev.yml @@ -0,0 +1,28 @@ +name: Sync Back to Development + +on: + push: + branches: + - master + +jobs: + sync-branches: + runs-on: ubuntu-latest + name: Syncing branches + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Opening pull request + id: pull + uses: tretuna/sync-branches@1.4.0 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + FROM_BRANCH: 'master' + TO_BRANCH: 'development' + CONTENT_COMPARISON: true + - name: Label the pull request to ignore for release note generation + uses: actions-ecosystem/action-add-labels@v1 + with: + labels: internal + repo: ${{ github.repository }} + number: ${{ steps.pull.outputs.PULL_REQUEST_NUMBER }} From 3260cb40b569d84f25a269a8b53bd1c52d6ba963 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Tue, 11 Jan 2022 19:15:30 +0000 Subject: [PATCH 69/99] ops per run -> 300 for stale Signed-off-by: Adam Warner --- .github/workflows/stale.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index f98dcc2e18..5a13b7f58c 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -22,3 +22,4 @@ jobs: stale-issue-label: 'Submitter Attention Required' exempt-issue-labels: 'pinned, Fixed in next release, Bug: Confirmed' exempt-all-issue-assignees: true + operations-per-run: 300 From ed6b85241bb820644c1c7c922a587fd3a0b3f89d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Schl=C3=B6tterer?= <80917404+lschloetterer@users.noreply.github.com> Date: Wed, 12 Jan 2022 09:23:13 +0100 Subject: [PATCH 70/99] use sed substitute instead of delete and append (#4555) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * use sed substitute instead of delete and append doesn't move the line to the end of the file, instead keeps the order of the lines in setupVars.conf intact Signed-off-by: Lukas Schlötterer <80917404+lschloetterer@users.noreply.github.com> * Match start of line as suggested in the review Signed-off-by: Lukas Schlötterer <80917404+lschloetterer@users.noreply.github.com> Co-authored-by: yubiuser Co-authored-by: yubiuser --- pihole | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pihole b/pihole index 8af47dc8f7..ddb8c70750 100755 --- a/pihole +++ b/pihole @@ -223,8 +223,7 @@ Time: fi local str="Pi-hole Disabled" - sed -i "/BLOCKING_ENABLED=/d" "${setupVars}" - echo "BLOCKING_ENABLED=false" >> "${setupVars}" + sed -i "s/^BLOCKING_ENABLED=true/BLOCKING_ENABLED=false/" "${setupVars}" fi else # Enable Pi-hole @@ -236,8 +235,7 @@ Time: echo -e " ${INFO} Enabling blocking" local str="Pi-hole Enabled" - sed -i "/BLOCKING_ENABLED=/d" "${setupVars}" - echo "BLOCKING_ENABLED=true" >> "${setupVars}" + sed -i "s/^BLOCKING_ENABLED=false/BLOCKING_ENABLED=true/" "${setupVars}" fi restartDNS reload-lists From 57ba60ce54991c3be9ce745f781eddb2de417b40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 13 Jan 2022 09:13:40 +0100 Subject: [PATCH 71/99] Change the exemption issue label pinned to internal for stale issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 5a13b7f58c..c2b19cf3c5 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -20,6 +20,6 @@ jobs: days-before-close: 5 stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Please comment or update this issue or it will be closed in 5 days.' stale-issue-label: 'Submitter Attention Required' - exempt-issue-labels: 'pinned, Fixed in next release, Bug: Confirmed' + exempt-issue-labels: 'internal, Fixed in next release, Bug: Confirmed' exempt-all-issue-assignees: true operations-per-run: 300 From cdde832ed373dc418e95823f65a5eba2233d93af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 13 Jan 2022 09:16:31 +0100 Subject: [PATCH 72/99] Some use uppercase some don't... MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index c2b19cf3c5..fc78e82054 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -20,6 +20,6 @@ jobs: days-before-close: 5 stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Please comment or update this issue or it will be closed in 5 days.' stale-issue-label: 'Submitter Attention Required' - exempt-issue-labels: 'internal, Fixed in next release, Bug: Confirmed' + exempt-issue-labels: 'Internal, Fixed in next release, Bug: Confirmed' exempt-all-issue-assignees: true operations-per-run: 300 From 6ead24b3157ae379e018d7a5e893ebc757b458a8 Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Fri, 14 Jan 2022 13:00:34 -0300 Subject: [PATCH 73/99] Move space into variable (#4562) Signed-off-by: rdwebdesign --- advanced/Scripts/query.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 0fd9871a67..9ddfdc62c2 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -233,7 +233,7 @@ for result in "${results[@]}"; do adlistAddress="${extra/|*/}" extra="${extra#*|}" if [[ "${extra}" == "0" ]]; then - extra="(disabled)" + extra=" (disabled)" else extra="" fi @@ -241,7 +241,7 @@ for result in "${results[@]}"; do if [[ -n "${blockpage}" ]]; then echo "0 ${adlistAddress}" elif [[ -n "${exact}" ]]; then - echo " - ${adlistAddress} ${extra}" + echo " - ${adlistAddress}${extra}" else if [[ ! "${adlistAddress}" == "${adlistAddress_prev:-}" ]]; then count="" @@ -256,7 +256,7 @@ for result in "${results[@]}"; do [[ "${count}" -gt "${max_count}" ]] && continue echo " ${COL_GRAY}Over ${count} results found, skipping rest of file${COL_NC}" else - echo " ${match} ${extra}" + echo " ${match}${extra}" fi fi done From 7aa28e4a3aa27a3cfdb5e495969fb4c9f923af5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 22 Jan 2022 22:09:15 +0100 Subject: [PATCH 74/99] Do a full fetch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/update.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index d18d2e786a..9da85c89d7 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -41,7 +41,7 @@ GitCheckUpdateAvail() { cd "${directory}" || return # Fetch latest changes in this repo - git fetch --tags --quiet origin + git fetch --quiet origin # Check current branch. If it is master, then check for the latest available tag instead of latest commit. curBranch=$(git rev-parse --abbrev-ref HEAD) From bad6d8a59e6617ab0fa1fab4606579324fd64475 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Schl=C3=B6tterer?= <80917404+lschloetterer@users.noreply.github.com> Date: Fri, 28 Jan 2022 16:26:57 +0100 Subject: [PATCH 75/99] add parameter to set filename for teleporter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make it possible to write pihole -a -t myname.tar.gz to configure the filename however you want Signed-off-by: Lukas Schlötterer <80917404+lschloetterer@users.noreply.github.com> --- advanced/Scripts/webpage.sh | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 4f44eca864..d823a7c1dc 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -640,12 +640,17 @@ Interfaces: } Teleporter() { - local datetimestamp - local host - datetimestamp=$(date "+%Y-%m-%d_%H-%M-%S") - host=$(hostname) - host="${host//./_}" - php /var/www/html/admin/scripts/pi-hole/php/teleporter.php > "pi-hole-${host:-noname}-teleporter_${datetimestamp}.tar.gz" + local filename + filename="${args[2]}" + if [[ -z "${filename}" ]]; then + local datetimestamp + local host + datetimestamp=$(date "+%Y-%m-%d_%H-%M-%S") + host=$(hostname) + host="${host//./_}" + filename="pi-hole-${host:-noname}-teleporter_${datetimestamp}.tar.gz" + fi + php /var/www/html/admin/scripts/pi-hole/php/teleporter.php > "${filename}" } checkDomain() From f0f5cc52d9bd7bb542a9735bdcb489152da05826 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 29 Jan 2022 22:39:45 +0100 Subject: [PATCH 76/99] Use internal SQLite3 engine in more places in gravity.sh Signed-off-by: DL6ER --- gravity.sh | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/gravity.sh b/gravity.sh index 2f24fbdb6f..ad0ba9a0cb 100755 --- a/gravity.sh +++ b/gravity.sh @@ -73,9 +73,9 @@ if [[ -r "${piholeDir}/pihole.conf" ]]; then echo -e " ${COL_LIGHT_RED}Ignoring overrides specified within pihole.conf! ${COL_NC}" fi -# Generate new sqlite3 file from schema template +# Generate new SQLite3 file from schema template generate_gravity_database() { - if ! sqlite3 "${gravityDBfile}" < "${gravityDBschema}"; then + if ! pihole-FTL sqlite3 "${gravityDBfile}" < "${gravityDBschema}"; then echo -e " ${CROSS} Unable to create ${gravityDBfile}" return 1 fi @@ -90,7 +90,7 @@ gravity_swap_databases() { echo -ne " ${INFO} ${str}..." # The index is intentionally not UNIQUE as poor quality adlists may contain domains more than once - output=$( { sqlite3 "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 ) + output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -136,7 +136,7 @@ gravity_swap_databases() { # Update timestamp when the gravity table was last updated successfully update_gravity_timestamp() { - output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -177,7 +177,7 @@ database_table_from_file() { # Get MAX(id) from domainlist when INSERTing into this table if [[ "${table}" == "domainlist" ]]; then - rowid="$(sqlite3 "${gravityDBfile}" "SELECT MAX(id) FROM domainlist;")" + rowid="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT MAX(id) FROM domainlist;")" if [[ -z "$rowid" ]]; then rowid=0 fi @@ -207,7 +207,7 @@ database_table_from_file() { # Store domains in database table specified by ${table} # Use printf as .mode and .import need to be on separate lines # see https://unix.stackexchange.com/a/445615/83260 - output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" "${tmpFile}" "${table}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" "${tmpFile}" "${table}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -227,7 +227,7 @@ database_table_from_file() { # Update timestamp of last update of this list. We store this in the "old" database as all values in the new database will later be overwritten database_adlist_updated() { - output=$( { printf ".timeout 30000\\nUPDATE adlist SET date_updated = (cast(strftime('%%s', 'now') as int)) WHERE id = %i;\\n" "${1}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nUPDATE adlist SET date_updated = (cast(strftime('%%s', 'now') as int)) WHERE id = %i;\\n" "${1}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -238,7 +238,7 @@ database_adlist_updated() { # Check if a column with name ${2} exists in gravity table with name ${1} gravity_column_exists() { - output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) if [[ "${output}" == "1" ]]; then return 0 # Bash 0 is success fi @@ -253,7 +253,7 @@ database_adlist_number() { return; fi - output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${num_source_lines}" "${num_invalid}" "${1}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${num_source_lines}" "${num_invalid}" "${1}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -269,7 +269,7 @@ database_adlist_status() { return; fi - output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -386,9 +386,9 @@ gravity_DownloadBlocklists() { fi # Retrieve source URLs from gravity database - # We source only enabled adlists, sqlite3 stores boolean values as 0 (false) or 1 (true) - mapfile -t sources <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" - mapfile -t sourceIDs <<< "$(sqlite3 "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)" + # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true) + mapfile -t sources <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" + mapfile -t sourceIDs <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)" # Parse source domains from $sources mapfile -t sourceDomains <<< "$( @@ -417,7 +417,7 @@ gravity_DownloadBlocklists() { str="Preparing new gravity database" echo -ne " ${INFO} ${str}..." rm "${gravityTEMPfile}" > /dev/null 2>&1 - output=$( { sqlite3 "${gravityTEMPfile}" < "${gravityDBschema}"; } 2>&1 ) + output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" < "${gravityDBschema}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -782,12 +782,12 @@ gravity_Table_Count() { local table="${1}" local str="${2}" local num - num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table};")" + num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table};")" if [[ "${table}" == "vw_gravity" ]]; then local unique - unique="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(DISTINCT domain) FROM ${table};")" + unique="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(DISTINCT domain) FROM ${table};")" echo -e " ${INFO} Number of ${str}: ${num} (${COL_BOLD}${unique} unique domains${COL_NC})" - sqlite3 "${gravityDBfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" + pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" else echo -e " ${INFO} Number of ${str}: ${num}" fi From 534f9a63bf0db0493d929d3737f48712ef99a495 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 30 Jan 2022 10:36:20 +0100 Subject: [PATCH 77/99] Copy database tables earlier into the new gravity database to avoid foreign key contraint violations when adding gravity entries refering to an empty adlist table Signed-off-by: DL6ER --- gravity.sh | 47 ++++++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/gravity.sh b/gravity.sh index ad0ba9a0cb..ac2fca1b53 100755 --- a/gravity.sh +++ b/gravity.sh @@ -85,7 +85,7 @@ generate_gravity_database() { # Copy data from old to new database file and swap them gravity_swap_databases() { - local str copyGravity + local str copyGravity oldAvail str="Building tree" echo -ne " ${INFO} ${str}..." @@ -102,22 +102,6 @@ gravity_swap_databases() { str="Swapping databases" echo -ne " ${INFO} ${str}..." - # Gravity copying SQL script - copyGravity="$(cat "${gravityDBcopy}")" - if [[ "${gravityDBfile}" != "${gravityDBfile_default}" ]]; then - # Replace default gravity script location by custom location - copyGravity="${copyGravity//"${gravityDBfile_default}"/"${gravityDBfile}"}" - fi - - output=$( { sqlite3 "${gravityTEMPfile}" <<< "${copyGravity}"; } 2>&1 ) - status="$?" - - if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n ${output}" - return 1 - fi - echo -e "${OVER} ${TICK} ${str}" - # Swap databases and remove or conditionally rename old database # Number of available blocks on disk availableBlocks=$(stat -f --format "%a" "${gravityDIR}") @@ -125,13 +109,19 @@ gravity_swap_databases() { gravityBlocks=$(stat --format "%b" ${gravityDBfile}) # Only keep the old database if available disk space is at least twice the size of the existing gravity.db. # Better be safe than sorry... + oldAvail=false if [ "${availableBlocks}" -gt "$((gravityBlocks * 2))" ] && [ -f "${gravityDBfile}" ]; then - echo -e " ${TICK} The old database remains available." + oldAvail=true mv "${gravityDBfile}" "${gravityOLDfile}" else rm "${gravityDBfile}" fi mv "${gravityTEMPfile}" "${gravityDBfile}" + echo -e "${OVER} ${TICK} ${str}" + + if [ oldAvail ]; then + echo -e " ${TICK} The old database remains available." + fi } # Update timestamp when the gravity table was last updated successfully @@ -475,9 +465,28 @@ gravity_DownloadBlocklists() { echo "" done + str="Creating new gravity databases" + echo -ne " ${INFO} ${str}..." + + # Gravity copying SQL script + copyGravity="$(cat "${gravityDBcopy}")" + if [[ "${gravityDBfile}" != "${gravityDBfile_default}" ]]; then + # Replace default gravity script location by custom location + copyGravity="${copyGravity//"${gravityDBfile_default}"/"${gravityDBfile}"}" + fi + + output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" <<< "${copyGravity}"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n ${output}" + return 1 + fi + echo -e "${OVER} ${TICK} ${str}" + str="Storing downloaded domains in new gravity database" echo -ne " ${INFO} ${str}..." - output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" gravity\\n" "${target}" | sqlite3 "${gravityTEMPfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" gravity\\n" "${target}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then From 5bb79de70b73662e03772564f5f242b216f85bf5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 30 Jan 2022 10:38:24 +0100 Subject: [PATCH 78/99] Clean possible leftovers in domainlist_by_group, adlist_by_group, and client_by_group before copying from database base to avoid foreign key violations. Signed-off-by: DL6ER --- advanced/Templates/gravity_copy.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/advanced/Templates/gravity_copy.sql b/advanced/Templates/gravity_copy.sql index 4a2a9b22e9..a927dd8d94 100644 --- a/advanced/Templates/gravity_copy.sql +++ b/advanced/Templates/gravity_copy.sql @@ -12,14 +12,17 @@ INSERT OR REPLACE INTO "group" SELECT * FROM OLD."group"; INSERT OR REPLACE INTO domain_audit SELECT * FROM OLD.domain_audit; INSERT OR REPLACE INTO domainlist SELECT * FROM OLD.domainlist; +DELETE FROM domainlist_by_group WHERE domainlist_id NOT IN (SELECT id FROM domainlist); INSERT OR REPLACE INTO domainlist_by_group SELECT * FROM OLD.domainlist_by_group; INSERT OR REPLACE INTO adlist SELECT * FROM OLD.adlist; +DELETE FROM adlist_by_group WHERE adlist_id NOT IN (SELECT id FROM adlist); INSERT OR REPLACE INTO adlist_by_group SELECT * FROM OLD.adlist_by_group; INSERT OR REPLACE INTO info SELECT * FROM OLD.info; INSERT OR REPLACE INTO client SELECT * FROM OLD.client; +DELETE FROM client_by_group WHERE client_id NOT IN (SELECT id FROM client); INSERT OR REPLACE INTO client_by_group SELECT * FROM OLD.client_by_group; From 8cbffa179d589cd3b6d5501733d2a634ff83cad1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 30 Jan 2022 10:42:13 +0100 Subject: [PATCH 79/99] Replace remaining sqlite3 calls by calls to our embedded pihole-FTL sqlite3 engine and remove sqlite3 as dependency in the installer. Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 30 +++++++++---------- advanced/Scripts/list.sh | 20 ++++++------- advanced/Scripts/piholeARPTable.sh | 4 +-- advanced/Scripts/piholeDebug.sh | 10 +++---- advanced/Scripts/piholeLogFlush.sh | 2 +- advanced/Scripts/query.sh | 4 +-- advanced/Scripts/webpage.sh | 12 ++++---- automated install/basic-install.sh | 2 +- gravity.sh | 2 +- 9 files changed, 43 insertions(+), 43 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 09dc17275b..a7ba60a919 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -19,13 +19,13 @@ upgrade_gravityDB(){ auditFile="${piholeDir}/auditlog.list" # Get database version - version="$(sqlite3 "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" + version="$(pihole-FTL sqlite3 "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" if [[ "$version" == "1" ]]; then # This migration script upgrades the gravity.db file by # adding the domain_audit table echo -e " ${INFO} Upgrading gravity database from version 1 to 2" - sqlite3 "${database}" < "${scriptPath}/1_to_2.sql" + pihole-FTL sqlite3 "${database}" < "${scriptPath}/1_to_2.sql" version=2 # Store audit domains in database table @@ -40,28 +40,28 @@ upgrade_gravityDB(){ # renaming the regex table to regex_blacklist, and # creating a new regex_whitelist table + corresponding linking table and views echo -e " ${INFO} Upgrading gravity database from version 2 to 3" - sqlite3 "${database}" < "${scriptPath}/2_to_3.sql" + pihole-FTL sqlite3 "${database}" < "${scriptPath}/2_to_3.sql" version=3 fi if [[ "$version" == "3" ]]; then # This migration script unifies the formally separated domain # lists into a single table with a UNIQUE domain constraint echo -e " ${INFO} Upgrading gravity database from version 3 to 4" - sqlite3 "${database}" < "${scriptPath}/3_to_4.sql" + pihole-FTL sqlite3 "${database}" < "${scriptPath}/3_to_4.sql" version=4 fi if [[ "$version" == "4" ]]; then # This migration script upgrades the gravity and list views # implementing necessary changes for per-client blocking echo -e " ${INFO} Upgrading gravity database from version 4 to 5" - sqlite3 "${database}" < "${scriptPath}/4_to_5.sql" + pihole-FTL sqlite3 "${database}" < "${scriptPath}/4_to_5.sql" version=5 fi if [[ "$version" == "5" ]]; then # This migration script upgrades the adlist view # to return an ID used in gravity.sh echo -e " ${INFO} Upgrading gravity database from version 5 to 6" - sqlite3 "${database}" < "${scriptPath}/5_to_6.sql" + pihole-FTL sqlite3 "${database}" < "${scriptPath}/5_to_6.sql" version=6 fi if [[ "$version" == "6" ]]; then @@ -69,7 +69,7 @@ upgrade_gravityDB(){ # which is automatically associated to all clients not # having their own group assignments echo -e " ${INFO} Upgrading gravity database from version 6 to 7" - sqlite3 "${database}" < "${scriptPath}/6_to_7.sql" + pihole-FTL sqlite3 "${database}" < "${scriptPath}/6_to_7.sql" version=7 fi if [[ "$version" == "7" ]]; then @@ -77,21 +77,21 @@ upgrade_gravityDB(){ # to ensure uniqueness on the group name # We also add date_added and date_modified columns echo -e " ${INFO} Upgrading gravity database from version 7 to 8" - sqlite3 "${database}" < "${scriptPath}/7_to_8.sql" + pihole-FTL sqlite3 "${database}" < "${scriptPath}/7_to_8.sql" version=8 fi if [[ "$version" == "8" ]]; then # This migration fixes some issues that were introduced # in the previous migration script. echo -e " ${INFO} Upgrading gravity database from version 8 to 9" - sqlite3 "${database}" < "${scriptPath}/8_to_9.sql" + pihole-FTL sqlite3 "${database}" < "${scriptPath}/8_to_9.sql" version=9 fi if [[ "$version" == "9" ]]; then # This migration drops unused tables and creates triggers to remove # obsolete groups assignments when the linked items are deleted echo -e " ${INFO} Upgrading gravity database from version 9 to 10" - sqlite3 "${database}" < "${scriptPath}/9_to_10.sql" + pihole-FTL sqlite3 "${database}" < "${scriptPath}/9_to_10.sql" version=10 fi if [[ "$version" == "10" ]]; then @@ -101,31 +101,31 @@ upgrade_gravityDB(){ # to keep the copying process generic (needs the same columns in both the # source and the destination databases). echo -e " ${INFO} Upgrading gravity database from version 10 to 11" - sqlite3 "${database}" < "${scriptPath}/10_to_11.sql" + pihole-FTL sqlite3 "${database}" < "${scriptPath}/10_to_11.sql" version=11 fi if [[ "$version" == "11" ]]; then # Rename group 0 from "Unassociated" to "Default" echo -e " ${INFO} Upgrading gravity database from version 11 to 12" - sqlite3 "${database}" < "${scriptPath}/11_to_12.sql" + pihole-FTL sqlite3 "${database}" < "${scriptPath}/11_to_12.sql" version=12 fi if [[ "$version" == "12" ]]; then # Add column date_updated to adlist table echo -e " ${INFO} Upgrading gravity database from version 12 to 13" - sqlite3 "${database}" < "${scriptPath}/12_to_13.sql" + pihole-FTL sqlite3 "${database}" < "${scriptPath}/12_to_13.sql" version=13 fi if [[ "$version" == "13" ]]; then # Add columns number and status to adlist table echo -e " ${INFO} Upgrading gravity database from version 13 to 14" - sqlite3 "${database}" < "${scriptPath}/13_to_14.sql" + pihole-FTL sqlite3 "${database}" < "${scriptPath}/13_to_14.sql" version=14 fi if [[ "$version" == "14" ]]; then # Changes the vw_adlist created in 5_to_6 echo -e " ${INFO} Upgrading gravity database from version 14 to 15" - sqlite3 "${database}" < "${scriptPath}/14_to_15.sql" + pihole-FTL sqlite3 "${database}" < "${scriptPath}/14_to_15.sql" version=15 fi } diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 8945047e9c..f3f97da26f 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -142,18 +142,18 @@ AddDomain() { domain="$1" # Is the domain in the list we want to add it to? - num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")" + num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")" requestedListname="$(GetListnameFromTypeId "${typeId}")" if [[ "${num}" -ne 0 ]]; then - existingTypeId="$(sqlite3 "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")" + existingTypeId="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")" if [[ "${existingTypeId}" == "${typeId}" ]]; then if [[ "${verbose}" == true ]]; then echo -e " ${INFO} ${1} already exists in ${requestedListname}, no need to add!" fi else existingListname="$(GetListnameFromTypeId "${existingTypeId}")" - sqlite3 "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';" + pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';" if [[ "${verbose}" == true ]]; then echo -e " ${INFO} ${1} already exists in ${existingListname}, it has been moved to ${requestedListname}!" fi @@ -169,10 +169,10 @@ AddDomain() { # Insert only the domain here. The enabled and date_added fields will be filled # with their default values (enabled = true, date_added = current timestamp) if [[ -z "${comment}" ]]; then - sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});" + pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});" else # also add comment when variable has been set through the "--comment" option - sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type,comment) VALUES ('${domain}',${typeId},'${comment}');" + pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type,comment) VALUES ('${domain}',${typeId},'${comment}');" fi } @@ -181,7 +181,7 @@ RemoveDomain() { domain="$1" # Is the domain in the list we want to remove it from? - num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")" + num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")" requestedListname="$(GetListnameFromTypeId "${typeId}")" @@ -198,14 +198,14 @@ RemoveDomain() { fi reload=true # Remove it from the current list - sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};" + pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};" } Displaylist() { local count num_pipes domain enabled status nicedate requestedListname requestedListname="$(GetListnameFromTypeId "${typeId}")" - data="$(sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)" + data="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)" if [[ -z $data ]]; then echo -e "Not showing empty list" @@ -243,10 +243,10 @@ Displaylist() { } NukeList() { - count=$(sqlite3 "${gravityDBfile}" "SELECT COUNT(1) FROM domainlist WHERE type = ${typeId};") + count=$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(1) FROM domainlist WHERE type = ${typeId};") listname="$(GetListnameFromTypeId "${typeId}")" if [ "$count" -gt 0 ];then - sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};" + pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};" echo " ${TICK} Removed ${count} domain(s) from the ${listname}" else echo " ${INFO} ${listname} already empty. Nothing to do!" diff --git a/advanced/Scripts/piholeARPTable.sh b/advanced/Scripts/piholeARPTable.sh index 66d05bf973..5daa025d49 100755 --- a/advanced/Scripts/piholeARPTable.sh +++ b/advanced/Scripts/piholeARPTable.sh @@ -39,7 +39,7 @@ flushARP(){ # Truncate network_addresses table in pihole-FTL.db # This needs to be done before we can truncate the network table due to # foreign key constraints - if ! output=$(sqlite3 "${DBFILE}" "DELETE FROM network_addresses" 2>&1); then + if ! output=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM network_addresses" 2>&1); then echo -e "${OVER} ${CROSS} Failed to truncate network_addresses table" echo " Database location: ${DBFILE}" echo " Output: ${output}" @@ -47,7 +47,7 @@ flushARP(){ fi # Truncate network table in pihole-FTL.db - if ! output=$(sqlite3 "${DBFILE}" "DELETE FROM network" 2>&1); then + if ! output=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM network" 2>&1); then echo -e "${OVER} ${CROSS} Failed to truncate network table" echo " Database location: ${DBFILE}" echo " Output: ${output}" diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 7be4029b66..7d3e7acff0 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -888,7 +888,7 @@ dig_at() { # This helps emulate queries to different domains that a user might query # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains local random_url - random_url=$(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity ORDER BY RANDOM() LIMIT 1") + random_url=$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity ORDER BY RANDOM() LIMIT 1") # Next we need to check if Pi-hole can resolve a domain when the query is sent to it's IP address # This better emulates how clients will interact with Pi-hole as opposed to above where Pi-hole is @@ -1202,7 +1202,7 @@ show_db_entries() { IFS=$'\r\n' local entries=() mapfile -t entries < <(\ - sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" \ + pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" \ -cmd ".headers on" \ -cmd ".mode column" \ -cmd ".width ${widths}" \ @@ -1227,7 +1227,7 @@ show_FTL_db_entries() { IFS=$'\r\n' local entries=() mapfile -t entries < <(\ - sqlite3 "${PIHOLE_FTL_DB_FILE}" \ + pihole-FTL sqlite3 "${PIHOLE_FTL_DB_FILE}" \ -cmd ".headers on" \ -cmd ".mode column" \ -cmd ".width ${widths}" \ @@ -1284,7 +1284,7 @@ analyze_gravity_list() { log_write "${COL_GREEN}${gravity_permissions}${COL_NC}" show_db_entries "Info table" "SELECT property,value FROM info" "20 40" - gravity_updated_raw="$(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT value FROM info where property = 'updated'")" + gravity_updated_raw="$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT value FROM info where property = 'updated'")" gravity_updated="$(date -d @"${gravity_updated_raw}")" log_write " Last gravity run finished at: ${COL_CYAN}${gravity_updated}${COL_NC}" log_write "" @@ -1292,7 +1292,7 @@ analyze_gravity_list() { OLD_IFS="$IFS" IFS=$'\r\n' local gravity_sample=() - mapfile -t gravity_sample < <(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity LIMIT 10") + mapfile -t gravity_sample < <(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity LIMIT 10") log_write " ${COL_CYAN}----- First 10 Gravity Domains -----${COL_NC}" for line in "${gravity_sample[@]}"; do diff --git a/advanced/Scripts/piholeLogFlush.sh b/advanced/Scripts/piholeLogFlush.sh index 5c6a2c6874..7547a5fd37 100755 --- a/advanced/Scripts/piholeLogFlush.sh +++ b/advanced/Scripts/piholeLogFlush.sh @@ -63,7 +63,7 @@ else fi fi # Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history) - deleted=$(sqlite3 "${DBFILE}" "DELETE FROM queries WHERE timestamp >= strftime('%s','now')-86400; select changes() from queries limit 1") + deleted=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM queries WHERE timestamp >= strftime('%s','now')-86400; select changes() from queries limit 1") # Restart pihole-FTL to force reloading history sudo pihole restartdns diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 9ddfdc62c2..20c891bf63 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -121,7 +121,7 @@ scanDatabaseTable() { fi # Send prepared query to gravity database - result="$(sqlite3 "${gravityDBfile}" "${querystr}")" 2> /dev/null + result="$(pihole-FTL sqlite3 "${gravityDBfile}" "${querystr}")" 2> /dev/null if [[ -z "${result}" ]]; then # Return early when there are no matches in this table return @@ -164,7 +164,7 @@ scanRegexDatabaseTable() { type="${3:-}" # Query all regex from the corresponding database tables - mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM domainlist WHERE type = ${type}" 2> /dev/null) + mapfile -t regexList < <(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT domain FROM domainlist WHERE type = ${type}" 2> /dev/null) # If we have regexps to process if [[ "${#regexList[@]}" -ne 0 ]]; then diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 4f44eca864..0b9fb62b4c 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -523,13 +523,13 @@ CustomizeAdLists() { if CheckUrl "${address}"; then if [[ "${args[2]}" == "enable" ]]; then - sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 1 WHERE address = '${address}'" + pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 1 WHERE address = '${address}'" elif [[ "${args[2]}" == "disable" ]]; then - sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 0 WHERE address = '${address}'" + pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 0 WHERE address = '${address}'" elif [[ "${args[2]}" == "add" ]]; then - sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlist (address, comment) VALUES ('${address}', '${comment}')" + pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlist (address, comment) VALUES ('${address}', '${comment}')" elif [[ "${args[2]}" == "del" ]]; then - sqlite3 "${gravityDBfile}" "DELETE FROM adlist WHERE address = '${address}'" + pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM adlist WHERE address = '${address}'" else echo "Not permitted" return 1 @@ -681,12 +681,12 @@ addAudit() done # Insert only the domain here. The date_added field will be # filled with its default value (date_added = current timestamp) - sqlite3 "${gravityDBfile}" "INSERT INTO domain_audit (domain) VALUES ${domains};" + pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domain_audit (domain) VALUES ${domains};" } clearAudit() { - sqlite3 "${gravityDBfile}" "DELETE FROM domain_audit;" + pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domain_audit;" } SetPrivacyLevel() { diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 6591634ec8..a5c206925a 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -287,7 +287,7 @@ package_manager_detect() { # Packages required to run this install script (stored as an array) INSTALLER_DEPS=(git iproute2 whiptail ca-certificates) # Packages required to run Pi-hole (stored as an array) - PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip idn2 sqlite3 libcap2-bin dns-root-data libcap2 netcat-openbsd) + PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip idn2 libcap2-bin dns-root-data libcap2 netcat-openbsd) # Packages required for the Web admin interface (stored as an array) # It's useful to separate this from Pi-hole, since the two repos are also setup separately PIHOLE_WEB_DEPS=(lighttpd "${phpVer}-common" "${phpVer}-cgi" "${phpVer}-sqlite3" "${phpVer}-xml" "${phpVer}-intl") diff --git a/gravity.sh b/gravity.sh index ac2fca1b53..9c11fa988d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -119,7 +119,7 @@ gravity_swap_databases() { mv "${gravityTEMPfile}" "${gravityDBfile}" echo -e "${OVER} ${TICK} ${str}" - if [ oldAvail ]; then + if $oldAvail; then echo -e " ${TICK} The old database remains available." fi } From 1dd9d55d82501a7b05fc86eb621a2fc9a610c8b8 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 30 Jan 2022 15:53:03 +0000 Subject: [PATCH 80/99] Replace value for BLOCKING_ENABLED (and QUERY_LOGGING, for consistency) and if value that we are trying to replace does not exist, add it to the end of the file. Co-authored-by: MichaIng Signed-off-by: Adam Warner --- pihole | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/pihole b/pihole index ddb8c70750..610c9f316a 100755 --- a/pihole +++ b/pihole @@ -170,6 +170,16 @@ restartDNS() { fi } +addOrEditKeyValPair() { + local key="${1}" + local value="${2}" + if grep -q "^${key}=" "${setupVars}"; then + sed -i "/^${key}=/c\\${key}=${value}" "${setupVars}" + else + echo "${key}=${value}" >> "${setupVars}" + fi +} + piholeEnable() { if [[ "${2}" == "-h" ]] || [[ "${2}" == "--help" ]]; then echo "Usage: pihole disable [time] @@ -223,7 +233,7 @@ Time: fi local str="Pi-hole Disabled" - sed -i "s/^BLOCKING_ENABLED=true/BLOCKING_ENABLED=false/" "${setupVars}" + addOrEditKeyValPair "BLOCKING_ENABLED" "false" fi else # Enable Pi-hole @@ -235,7 +245,7 @@ Time: echo -e " ${INFO} Enabling blocking" local str="Pi-hole Enabled" - sed -i "s/^BLOCKING_ENABLED=false/BLOCKING_ENABLED=true/" "${setupVars}" + addOrEditKeyValPair "BLOCKING_ENABLED" "true" fi restartDNS reload-lists @@ -258,7 +268,7 @@ Options: elif [[ "${1}" == "off" ]]; then # Disable logging sed -i 's/^log-queries/#log-queries/' /etc/dnsmasq.d/01-pihole.conf - sed -i 's/^QUERY_LOGGING=true/QUERY_LOGGING=false/' /etc/pihole/setupVars.conf + addOrEditKeyValPair "QUERY_LOGGING" "false" if [[ "${2}" != "noflush" ]]; then # Flush logs "${PI_HOLE_BIN_DIR}"/pihole -f @@ -268,7 +278,7 @@ Options: elif [[ "${1}" == "on" ]]; then # Enable logging sed -i 's/^#log-queries/log-queries/' /etc/dnsmasq.d/01-pihole.conf - sed -i 's/^QUERY_LOGGING=false/QUERY_LOGGING=true/' /etc/pihole/setupVars.conf + addOrEditKeyValPair "QUERY_LOGGING" "true" echo -e " ${INFO} Enabling logging..." local str="Logging has been enabled!" else From 2f4c4d9176d5081e2494e13f4a5304e91b12757c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 30 Jan 2022 20:13:10 +0100 Subject: [PATCH 81/99] Fix stale label to stale MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index fc78e82054..506af4063b 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -19,7 +19,7 @@ jobs: days-before-stale: 30 days-before-close: 5 stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Please comment or update this issue or it will be closed in 5 days.' - stale-issue-label: 'Submitter Attention Required' + stale-issue-label: 'stale' exempt-issue-labels: 'Internal, Fixed in next release, Bug: Confirmed' exempt-all-issue-assignees: true operations-per-run: 300 From 74d7d10554dbd96454ffbc5ce960d608b1a4a034 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 30 Jan 2022 21:09:24 +0100 Subject: [PATCH 82/99] Orphans need to be deleted in the old database Signed-off-by: DL6ER --- advanced/Templates/gravity_copy.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/advanced/Templates/gravity_copy.sql b/advanced/Templates/gravity_copy.sql index a927dd8d94..3bea731d1b 100644 --- a/advanced/Templates/gravity_copy.sql +++ b/advanced/Templates/gravity_copy.sql @@ -12,17 +12,17 @@ INSERT OR REPLACE INTO "group" SELECT * FROM OLD."group"; INSERT OR REPLACE INTO domain_audit SELECT * FROM OLD.domain_audit; INSERT OR REPLACE INTO domainlist SELECT * FROM OLD.domainlist; -DELETE FROM domainlist_by_group WHERE domainlist_id NOT IN (SELECT id FROM domainlist); +DELETE FROM OLD.domainlist_by_group WHERE domainlist_id NOT IN (SELECT id FROM OLD.domainlist); INSERT OR REPLACE INTO domainlist_by_group SELECT * FROM OLD.domainlist_by_group; INSERT OR REPLACE INTO adlist SELECT * FROM OLD.adlist; -DELETE FROM adlist_by_group WHERE adlist_id NOT IN (SELECT id FROM adlist); +DELETE FROM OLD.adlist_by_group WHERE adlist_id NOT IN (SELECT id FROM OLD.adlist); INSERT OR REPLACE INTO adlist_by_group SELECT * FROM OLD.adlist_by_group; INSERT OR REPLACE INTO info SELECT * FROM OLD.info; INSERT OR REPLACE INTO client SELECT * FROM OLD.client; -DELETE FROM client_by_group WHERE client_id NOT IN (SELECT id FROM client); +DELETE FROM OLD.client_by_group WHERE client_id NOT IN (SELECT id FROM OLD.client); INSERT OR REPLACE INTO client_by_group SELECT * FROM OLD.client_by_group; From 77e5121d438f7895ae6c512222802a0307c63ebb Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 30 Jan 2022 23:05:28 +0000 Subject: [PATCH 83/99] Split new function out into a separte utility script and add a test for it. Can be used in future to organise re/commonly-used code Signed-off-by: Adam Warner --- advanced/Scripts/utils.sh | 35 +++++++++++++++++++ pihole | 21 ++++------- ...stall.py => test_any_automated_install.py} | 0 test/test_any_utils.py | 16 +++++++++ test/tox.centos_7.ini | 2 +- test/tox.centos_8.ini | 2 +- test/tox.debian_10.ini | 2 +- test/tox.debian_11.ini | 2 +- test/tox.debian_9.ini | 2 +- test/tox.fedora_33.ini | 2 +- test/tox.fedora_34.ini | 2 +- test/tox.ubuntu_16.ini | 2 +- test/tox.ubuntu_18.ini | 2 +- test/tox.ubuntu_20.ini | 2 +- test/tox.ubuntu_21.ini | 2 +- 15 files changed, 69 insertions(+), 25 deletions(-) create mode 100755 advanced/Scripts/utils.sh rename test/{test_automated_install.py => test_any_automated_install.py} (100%) create mode 100644 test/test_any_utils.py diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh new file mode 100755 index 0000000000..887816cc5d --- /dev/null +++ b/advanced/Scripts/utils.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Pi-hole: A black hole for Internet advertisements +# (c) 2017 Pi-hole, LLC (https://pi-hole.net) +# Network-wide ad blocking via your own hardware. +# +# Script to hold utility functions for use in other scripts +# +# This file is copyright under the latest version of the EUPL. +# Please see LICENSE file for your rights under this license. + +# Basic Housekeeping rules +# - Functions must be self contained +# - Functions must be added in alphabetical order +# - Functions must be documented +# - New functions must have a test added for them in test/test_any_utils.py + +####################### +# Takes three arguments key, value, and file. +# Checks the target file for the existence of the key +# - If it exists, it changes the value +# - If it does not exist, it adds the value +# +# Example usage: +# addOrEditKeyValuePair "BLOCKING_ENABLED" "true" "/etc/pihole/setupVars.conf" +####################### +addOrEditKeyValPair() { + local key="${1}" + local value="${2}" + local file="${3}" + if grep -q "^${key}=" "${file}"; then + sed -i "/^${key}=/c\\${key}=${value}" "${file}" + else + echo "${key}=${value}" >> "${file}" + fi +} diff --git a/pihole b/pihole index 610c9f316a..56d47eca65 100755 --- a/pihole +++ b/pihole @@ -21,6 +21,9 @@ readonly FTL_PID_FILE="/run/pihole-FTL.pid" readonly colfile="${PI_HOLE_SCRIPT_DIR}/COL_TABLE" source "${colfile}" +readonly utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" +source "${utilsfile}" + webpageFunc() { source "${PI_HOLE_SCRIPT_DIR}/webpage.sh" main "$@" @@ -170,16 +173,6 @@ restartDNS() { fi } -addOrEditKeyValPair() { - local key="${1}" - local value="${2}" - if grep -q "^${key}=" "${setupVars}"; then - sed -i "/^${key}=/c\\${key}=${value}" "${setupVars}" - else - echo "${key}=${value}" >> "${setupVars}" - fi -} - piholeEnable() { if [[ "${2}" == "-h" ]] || [[ "${2}" == "--help" ]]; then echo "Usage: pihole disable [time] @@ -233,7 +226,7 @@ Time: fi local str="Pi-hole Disabled" - addOrEditKeyValPair "BLOCKING_ENABLED" "false" + addOrEditKeyValPair "BLOCKING_ENABLED" "false" "${setupVars}" fi else # Enable Pi-hole @@ -245,7 +238,7 @@ Time: echo -e " ${INFO} Enabling blocking" local str="Pi-hole Enabled" - addOrEditKeyValPair "BLOCKING_ENABLED" "true" + addOrEditKeyValPair "BLOCKING_ENABLED" "true" "${setupVars}" fi restartDNS reload-lists @@ -268,7 +261,7 @@ Options: elif [[ "${1}" == "off" ]]; then # Disable logging sed -i 's/^log-queries/#log-queries/' /etc/dnsmasq.d/01-pihole.conf - addOrEditKeyValPair "QUERY_LOGGING" "false" + addOrEditKeyValPair "QUERY_LOGGING" "false" "${setupVars}" if [[ "${2}" != "noflush" ]]; then # Flush logs "${PI_HOLE_BIN_DIR}"/pihole -f @@ -278,7 +271,7 @@ Options: elif [[ "${1}" == "on" ]]; then # Enable logging sed -i 's/^#log-queries/log-queries/' /etc/dnsmasq.d/01-pihole.conf - addOrEditKeyValPair "QUERY_LOGGING" "true" + addOrEditKeyValPair "QUERY_LOGGING" "true" "${setupVars}" echo -e " ${INFO} Enabling logging..." local str="Logging has been enabled!" else diff --git a/test/test_automated_install.py b/test/test_any_automated_install.py similarity index 100% rename from test/test_automated_install.py rename to test/test_any_automated_install.py diff --git a/test/test_any_utils.py b/test/test_any_utils.py new file mode 100644 index 0000000000..ba9b2d23e5 --- /dev/null +++ b/test/test_any_utils.py @@ -0,0 +1,16 @@ +def test_key_val_replacement_works(host): + ''' Confirms addOrEditKeyValPair provides the expected output ''' + host.run(''' + setupvars=./testoutput + source /opt/pihole/utils.sh + addOrEditKeyValPair "KEY_ONE" "value1" "./testoutput" + addOrEditKeyValPair "KEY_TWO" "value2" "./testoutput" + addOrEditKeyValPair "KEY_ONE" "value3" "./testoutput" + addOrEditKeyValPair "KEY_FOUR" "value4" "./testoutput" + cat ./testoutput + ''') + output = host.run(''' + cat ./testoutput + ''') + expected_stdout = 'KEY_ONE=value3\nKEY_TWO=value2\nKEY_FOUR=value4\n' + assert expected_stdout == output.stdout diff --git a/test/tox.centos_7.ini b/test/tox.centos_7.ini index 88940fddf2..319465dd1a 100644 --- a/test/tox.centos_7.ini +++ b/test/tox.centos_7.ini @@ -5,4 +5,4 @@ envlist = py38 whitelist_externals = docker deps = -rrequirements.txt commands = docker build -f _centos_7.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_automated_install.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py ./test_centos_7_support.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py ./test_centos_7_support.py diff --git a/test/tox.centos_8.ini b/test/tox.centos_8.ini index 5088da16f7..c792628994 100644 --- a/test/tox.centos_8.ini +++ b/test/tox.centos_8.ini @@ -5,4 +5,4 @@ envlist = py38 whitelist_externals = docker deps = -rrequirements.txt commands = docker build -f _centos_8.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_automated_install.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py ./test_centos_8_support.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py ./test_centos_8_support.py diff --git a/test/tox.debian_10.ini b/test/tox.debian_10.ini index 9c2a05d171..3b182cdccb 100644 --- a/test/tox.debian_10.ini +++ b/test/tox.debian_10.ini @@ -5,4 +5,4 @@ envlist = py38 whitelist_externals = docker deps = -rrequirements.txt commands = docker build -f _debian_10.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_automated_install.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py diff --git a/test/tox.debian_11.ini b/test/tox.debian_11.ini index f3cdbe84e7..c7e41a91cd 100644 --- a/test/tox.debian_11.ini +++ b/test/tox.debian_11.ini @@ -5,4 +5,4 @@ envlist = py38 whitelist_externals = docker deps = -rrequirements.txt commands = docker build -f _debian_11.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_automated_install.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py diff --git a/test/tox.debian_9.ini b/test/tox.debian_9.ini index b46e0a493b..56b9d37f09 100644 --- a/test/tox.debian_9.ini +++ b/test/tox.debian_9.ini @@ -5,4 +5,4 @@ envlist = py38 whitelist_externals = docker deps = -rrequirements.txt commands = docker build -f _debian_9.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_automated_install.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py diff --git a/test/tox.fedora_33.ini b/test/tox.fedora_33.ini index d33fbf5375..b17bd56323 100644 --- a/test/tox.fedora_33.ini +++ b/test/tox.fedora_33.ini @@ -5,4 +5,4 @@ envlist = py38 whitelist_externals = docker deps = -rrequirements.txt commands = docker build -f _fedora_33.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_automated_install.py ./test_centos_fedora_common_support.py ./test_fedora_support.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py diff --git a/test/tox.fedora_34.ini b/test/tox.fedora_34.ini index 819291fac0..2685698467 100644 --- a/test/tox.fedora_34.ini +++ b/test/tox.fedora_34.ini @@ -5,4 +5,4 @@ envlist = py38 whitelist_externals = docker deps = -rrequirements.txt commands = docker build -f _fedora_34.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_automated_install.py ./test_centos_fedora_common_support.py ./test_fedora_support.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py diff --git a/test/tox.ubuntu_16.ini b/test/tox.ubuntu_16.ini index bce948a2bf..f8f6e92a73 100644 --- a/test/tox.ubuntu_16.ini +++ b/test/tox.ubuntu_16.ini @@ -5,4 +5,4 @@ envlist = py38 whitelist_externals = docker deps = -rrequirements.txt commands = docker build -f _ubuntu_16.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_automated_install.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py diff --git a/test/tox.ubuntu_18.ini b/test/tox.ubuntu_18.ini index cf7a364232..a2513dfdcd 100644 --- a/test/tox.ubuntu_18.ini +++ b/test/tox.ubuntu_18.ini @@ -5,4 +5,4 @@ envlist = py38 whitelist_externals = docker deps = -rrequirements.txt commands = docker build -f _ubuntu_18.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_automated_install.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py diff --git a/test/tox.ubuntu_20.ini b/test/tox.ubuntu_20.ini index 03b605ce98..fb3d20d7b4 100644 --- a/test/tox.ubuntu_20.ini +++ b/test/tox.ubuntu_20.ini @@ -5,4 +5,4 @@ envlist = py38 whitelist_externals = docker deps = -rrequirements.txt commands = docker build -f _ubuntu_20.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_automated_install.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py diff --git a/test/tox.ubuntu_21.ini b/test/tox.ubuntu_21.ini index 12b1ac0b27..070d3a7218 100644 --- a/test/tox.ubuntu_21.ini +++ b/test/tox.ubuntu_21.ini @@ -5,4 +5,4 @@ envlist = py38 whitelist_externals = docker deps = -rrequirements.txt commands = docker build -f _ubuntu_21.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_automated_install.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py From e09dd56807c45ab56e5285e1eea936c799052f25 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 1 Feb 2022 07:38:57 +0100 Subject: [PATCH 84/99] Remove RPM package sqlite as well Signed-off-by: DL6ER --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index a5c206925a..e3dec82d89 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -332,7 +332,7 @@ package_manager_detect() { PKG_COUNT="${PKG_MANAGER} check-update | egrep '(.i686|.x86|.noarch|.arm|.src)' | wc -l" OS_CHECK_DEPS=(grep bind-utils) INSTALLER_DEPS=(git iproute newt procps-ng which chkconfig ca-certificates) - PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc sqlite libcap nmap-ncat) + PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc libcap nmap-ncat) PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php-common php-cli php-pdo php-xml php-json php-intl) LIGHTTPD_USER="lighttpd" LIGHTTPD_GROUP="lighttpd" From 881d92632ce8ee346ad5e1224879190eeb8c6836 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Schl=C3=B6tterer?= <80917404+lschloetterer@users.noreply.github.com> Date: Tue, 1 Feb 2022 09:41:57 +0100 Subject: [PATCH 85/99] add hint for custom teleporter filename to help function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lukas Schlötterer <80917404+lschloetterer@users.noreply.github.com> --- advanced/Scripts/webpage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index d823a7c1dc..dad5380e3d 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -45,7 +45,7 @@ Options: -h, --help Show this help dialog -i, interface Specify dnsmasq's interface listening behavior -l, privacylevel Set privacy level (0 = lowest, 3 = highest) - -t, teleporter Backup configuration as an archive" + -t, teleporter Backup configuration as an archive. Optionally specify a custom filename" exit 0 } From 444526ad582818b03263dabc8a01c6fbee5018a4 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Thu, 3 Feb 2022 18:43:19 +0000 Subject: [PATCH 86/99] Switch from centos8 to centos8:stream base image for centos 8 tests --- test/_centos_8.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/_centos_8.Dockerfile b/test/_centos_8.Dockerfile index fddb3ed14c..86e5a7787d 100644 --- a/test/_centos_8.Dockerfile +++ b/test/_centos_8.Dockerfile @@ -1,4 +1,4 @@ -FROM centos:8 +FROM quay.io/centos/centos:stream8 RUN yum install -y git ENV GITDIR /etc/.pihole From 7c60ee8df11d158cca3bde3b0b89a6f05bb9f409 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 29 Dec 2021 06:52:17 +0100 Subject: [PATCH 87/99] Remove pihole-FTL.conf man page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König Remove double https:// Signed-off-by: Christian König --- automated install/basic-install.sh | 9 +- manpages/pihole-FTL.8 | 4 +- manpages/pihole-FTL.conf.5 | 313 ----------------------------- 3 files changed, 10 insertions(+), 316 deletions(-) delete mode 100644 manpages/pihole-FTL.conf.5 diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index e3dec82d89..6bf55e923e 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1371,7 +1371,12 @@ install_manpage() { # Testing complete, copy the files & update the man db install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/manpages/pihole.8 /usr/local/share/man/man8/pihole.8 install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.8 /usr/local/share/man/man8/pihole-FTL.8 - install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.conf.5 /usr/local/share/man/man5/pihole-FTL.conf.5 + + # remvoe previously installed "pihole-FTL.conf" man page + if [[ -f "/usr/local/share/man/man5/pihole-FTL.conf.5" ]]; then + rm /usr/local/share/man/man5/pihole-FTL.conf.5 + fi + if mandb -q &>/dev/null; then # Updated successfully printf "%b %b man pages installed and database updated\\n" "${OVER}" "${TICK}" @@ -1379,7 +1384,7 @@ install_manpage() { else # Something is wrong with the system's man installation, clean up # our files, (leave everything how we found it). - rm /usr/local/share/man/man8/pihole.8 /usr/local/share/man/man8/pihole-FTL.8 /usr/local/share/man/man5/pihole-FTL.conf.5 + rm /usr/local/share/man/man8/pihole.8 /usr/local/share/man/man8/pihole-FTL.8 printf "%b %b man page db not updated, man pages not installed\\n" "${OVER}" "${CROSS}" fi } diff --git a/manpages/pihole-FTL.8 b/manpages/pihole-FTL.8 index c7b69d83fa..c1b7550ffc 100644 --- a/manpages/pihole-FTL.8 +++ b/manpages/pihole-FTL.8 @@ -144,7 +144,9 @@ Command line arguments can be arbitrarily combined, e.g: Start ftl in foreground with more verbose logging, process everything and shutdown immediately .br .SH "SEE ALSO" -\fBpihole\fR(8), \fBpihole-FTL.conf\fR(5) +\fBpihole\fR(8) +.br +\fBFor FTL's config options please see https://docs.pi-hole.net/ftldns/configfile/\fR .br .SH "COLOPHON" diff --git a/manpages/pihole-FTL.conf.5 b/manpages/pihole-FTL.conf.5 deleted file mode 100644 index 4240543624..0000000000 --- a/manpages/pihole-FTL.conf.5 +++ /dev/null @@ -1,313 +0,0 @@ -.TH "pihole-FTL.conf" "5" "pihole-FTL.conf" "pihole-FTL.conf" "November 2020" -.SH "NAME" - -pihole-FTL.conf - FTL's config file -.br -.SH "DESCRIPTION" - -/etc/pihole/pihole-FTL.conf will be read by \fBpihole-FTL(8)\fR on startup. -.br -For each setting the option shown first is the default. -.br - -\fBBLOCKINGMODE=IP|IP-AAAA-NODATA|NODATA|NXDOMAIN|NULL\fR -.br - How should FTL reply to blocked queries? - - IP - Pi-hole's IPs for blocked domains - - IP-AAAA-NODATA - Pi-hole's IP + NODATA-IPv6 for blocked domains - - NODATA - Using NODATA for blocked domains - - NXDOMAIN - NXDOMAIN for blocked domains - - NULL - Null IPs for blocked domains -.br - -\fBCNAME_DEEP_INSPECT=true|false\fR -.br - Use this option to disable deep CNAME inspection. This might be beneficial for very low-end devices. -.br - -\fBBLOCK_ESNI=true|false\fR -.br - Block requests to _esni.* sub-domains. -.br - -\fBMAXLOGAGE=24.0\fR -.br - Up to how many hours of queries should be imported from the database and logs? -.br - Maximum is 744 (31 days) -.br - -\fBPRIVACYLEVEL=0|1|2|3|4\fR -.br - Privacy level used to collect Pi-hole statistics. -.br - 0 - show everything -.br - 1 - hide domains -.br - 2 - hide domains and clients -.br - 3 - anonymous mode (hide everything) -.br - 4 - disable all statistics -.br - -\fBIGNORE_LOCALHOST=no|yes\fR -.br - Should FTL ignore queries coming from the local machine? -.br - -\fBAAAA_QUERY_ANALYSIS=yes|no\fR -.br - Should FTL analyze AAAA queries? -.br - -\fBANALYZE_ONLY_A_AND_AAAA=false|true\fR -.br - Should FTL only analyze A and AAAA queries? -.br - -\fBSOCKET_LISTENING=localonly|all\fR -.br - Listen only for local socket connections on the API port or permit all connections. -.br - -\fBFTLPORT=4711\fR -.br - On which port should FTL be listening? -.br - -\fBRESOLVE_IPV6=yes|no\fR -.br - Should FTL try to resolve IPv6 addresses to hostnames? -.br - -\fBRESOLVE_IPV4=yes|no\fR -.br - Should FTL try to resolve IPv4 addresses to hostnames? -.br - -\fBDELAY_STARTUP=0\fR -.br - Time in seconds (between 0 and 300) to delay FTL startup. -.br - -\fBNICE=-10\fR -.br - Set the niceness of the Pi-hole FTL process. -.br - Can be disabled altogether by setting a value of -999. -.br - -\fBNAMES_FROM_NETDB=true|false\fR -.br - Control whether FTL should use a fallback option and try to obtain client names from checking the network table. -.br - E.g. IPv6 clients without a hostname will be compared via MAC address to known clients. -.br - -\fB\fBREFRESH_HOSTNAMES=IPV4|ALL|NONE\fR -.br - Change how (and if) hourly PTR requests are made to check for changes in client and upstream server hostnames: -.br - IPV4 - Do the hourly PTR lookups only for IPv4 addresses resolving issues in networks with many short-lived PE IPv6 addresses. -.br - ALL - Do the hourly PTR lookups for all addresses. This can create a lot of PTR queries in networks with many IPv6 addresses. -.br - NONE - Don't do hourly PTR lookups. Look up hostnames once (when first seeing a client) and never again. Future hostname changes may be missed. -.br - -\fBMAXNETAGE=365\fR -.br - IP addresses (and associated host names) older than the specified number of days are removed. -.br - This avoids dead entries in the network overview table. -.br - -\fBEDNS0_ECS=true|false\fR -.br - Should we overwrite the query source when client information is provided through EDNS0 client subnet (ECS) information? -.br - -\fBPARSE_ARP_CACHE=true|false\fR -.br - Parse ARP cache to fill network overview table. -.br - -\fBDBIMPORT=yes|no\fR -.br - Should FTL load information from the database on startup to be aware of the most recent history? -.br - -\fBMAXDBDAYS=365\fR -.br - How long should queries be stored in the database? Setting this to 0 disables the database -.br - -\fBDBINTERVAL=1.0\fR -.br - How often do we store queries in FTL's database [minutes]? -.br - Accepts value between 0.1 (6 sec) and 1440 (1 day) -.br - -\fBDBFILE=/etc/pihole/pihole-FTL.db\fR -.br - Specify path and filename of FTL's SQLite long-term database. -.br - Setting this to DBFILE= disables the database altogether -.br - -\fBLOGFILE=/var/log/pihole-FTL.log\fR -.br - The location of FTL's log file. -.br - -\fBPIDFILE=/run/pihole-FTL.pid\fR -.br - The file which contains the PID of FTL's main process. -.br - -\fBPORTFILE=/run/pihole-FTL.port\fR -.br - Specify path and filename where the FTL process will write its API port number. -.br - -\fBSOCKETFILE=/run/pihole/FTL.sock\fR -.br - The file containing the socket FTL's API is listening on. -.br - -\fBSETUPVARSFILE=/etc/pihole/setupVars.conf\fR -.br - The config file of Pi-hole containing, e.g., the current blocking status (do not change). -.br - -\fBMACVENDORDB=/etc/pihole/macvendor.db\fR -.br - The database containing MAC -> Vendor information for the network table. -.br - -\fBGRAVITYDB=/etc/pihole/gravity.db\fR -.br - Specify path and filename of FTL's SQLite3 gravity database. This database contains all domains relevant for Pi-hole's DNS blocking. -.br - -\fBDEBUG_ALL=false|true\fR -.br - Enable all debug flags. If this is set to true, all other debug config options are ignored. -.br - -\fBDEBUG_DATABASE=false|true\fR -.br - Print debugging information about database actions such as SQL statements and performance. -.br - -\fBDEBUG_NETWORKING=false|true\fR -.br - Prints a list of the detected network interfaces on the startup of FTL. -.br - -\fBDEBUG_LOCKS=false|true\fR -.br - Print information about shared memory locks. -.br - Messages will be generated when waiting, obtaining, and releasing a lock. -.br - -\fBDEBUG_QUERIES=false|true\fR -.br - Print extensive DNS query information (domains, types, replies, etc.). -.br - -\fBDEBUG_FLAGS=false|true\fR -.br - Print flags of queries received by the DNS hooks. -.br - Only effective when \fBDEBUG_QUERIES\fR is enabled as well. - -\fBDEBUG_SHMEM=false|true\fR -.br - Print information about shared memory buffers. -.br - Messages are either about creating or enlarging shmem objects or string injections. -.br - -\fBDEBUG_GC=false|true\fR -.br - Print information about garbage collection (GC): -.br - What is to be removed, how many have been removed and how long did GC take. -.br - -\fBDEBUG_ARP=false|true\fR -.br - Print information about ARP table processing: -.br - How long did parsing take, whether read MAC addresses are valid, and if the macvendor.db file exists. -.br - -\fBDEBUG_REGEX=false|true\fR -.br - Controls if FTL should print extended details about regex matching. -.br - -\fBDEBUG_API=false|true\fR -.br - Print extra debugging information during telnet API calls. -.br - Currently only used to send extra information when getting all queries. -.br - -\fBDEBUG_OVERTIME=false|true\fR -.br - Print information about overTime memory operations, such as initializing or moving overTime slots. -.br - -\fBDEBUG_EXTBLOCKED=false|true\fR -.br - Print information about why FTL decided that certain queries were recognized as being externally blocked. -.br - -\fBDEBUG_CAPS=false|true\fR -.br - Print information about POSIX capabilities granted to the FTL process. -.br - The current capabilities are printed on receipt of SIGHUP i.e. after executing `killall -HUP pihole-FTL`. -.br - -\fBDEBUG_DNSMASQ_LINES=false|true\fR -.br - Print file and line causing a dnsmasq event into FTL's log files. -.br - This is handy to implement additional hooks missing from FTL. -.br - -\fBDEBUG_VECTORS=false|true\fR -.br - FTL uses dynamically allocated vectors for various tasks. -.br - This config option enables extensive debugging information such as information about allocation, referencing, deletion, and appending. -.br - -\fBDEBUG_RESOLVER=false|true\fR -.br - Extensive information about hostname resolution like which DNS servers are used in the first and second hostname resolving tries. -.br - -.SH "SEE ALSO" - -\fBpihole\fR(8), \fBpihole-FTL\fR(8) -.br -.SH "COLOPHON" - -Pi-hole : The Faster-Than-Light (FTL) Engine is a lightweight, purpose-built daemon used to provide statistics needed for the Pi-hole Web Interface, and its API can be easily integrated into your own projects. Although it is an optional component of the Pi-hole ecosystem, it will be installed by default to provide statistics. As the name implies, FTL does its work \fIvery quickly\fR! -.br - -Get sucked into the latest news and community activity by entering Pi-hole's orbit. Information about Pi-hole, and the latest version of the software can be found at https://pi-hole.net -.br From d7d8e9730b385342a79bbac368243f98cade359f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 30 Dec 2021 06:49:03 +0100 Subject: [PATCH 88/99] Remove pihole-FTL.conf.5 from automated tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/test_any_automated_install.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 7959e1007a..b7b4ccd84b 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -351,10 +351,6 @@ def test_installPihole_fresh_install_readableFiles(host): 'r', '/usr/local/share/man/man8/pihole-FTL.8', piholeuser) actual_rc = host.run(check_man).rc assert exit_status_success == actual_rc - check_man = test_cmd.format( - 'r', '/usr/local/share/man/man5/pihole-FTL.conf.5', piholeuser) - actual_rc = host.run(check_man).rc - assert exit_status_success == actual_rc # check not readable sudoers file check_sudo = test_cmd.format( 'r', '/etc/sudoers.d/pihole', piholeuser) From c3c5342b48b226f482bc0d44ca8b5bed13b60a08 Mon Sep 17 00:00:00 2001 From: yubiuser Date: Fri, 4 Feb 2022 21:11:54 +0100 Subject: [PATCH 89/99] Fix reviewer's comment Co-authored-by: DL6ER --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 6bf55e923e..1e004b8b27 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1372,7 +1372,7 @@ install_manpage() { install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/manpages/pihole.8 /usr/local/share/man/man8/pihole.8 install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.8 /usr/local/share/man/man8/pihole-FTL.8 - # remvoe previously installed "pihole-FTL.conf" man page + # remove previously installed "pihole-FTL.conf.5" man page if [[ -f "/usr/local/share/man/man5/pihole-FTL.conf.5" ]]; then rm /usr/local/share/man/man5/pihole-FTL.conf.5 fi From 2a0bb5b9ee12d33f35f39c035ef931f48d6370cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Schl=C3=B6tterer?= <80917404+lschloetterer@users.noreply.github.com> Date: Fri, 4 Feb 2022 21:19:09 +0100 Subject: [PATCH 90/99] Create second entry for teleporter and adjust spacing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lukas Schlötterer <80917404+lschloetterer@users.noreply.github.com> --- advanced/Scripts/webpage.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index dad5380e3d..c80934d998 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -37,15 +37,16 @@ Example: pihole -a -p password Set options for the Admin Console Options: - -p, password Set Admin Console password - -c, celsius Set Celsius as preferred temperature unit - -f, fahrenheit Set Fahrenheit as preferred temperature unit - -k, kelvin Set Kelvin as preferred temperature unit - -e, email Set an administrative contact address for the Block Page - -h, --help Show this help dialog - -i, interface Specify dnsmasq's interface listening behavior - -l, privacylevel Set privacy level (0 = lowest, 3 = highest) - -t, teleporter Backup configuration as an archive. Optionally specify a custom filename" + -p, password Set Admin Console password + -c, celsius Set Celsius as preferred temperature unit + -f, fahrenheit Set Fahrenheit as preferred temperature unit + -k, kelvin Set Kelvin as preferred temperature unit + -e, email Set an administrative contact address for the Block Page + -h, --help Show this help dialog + -i, interface Specify dnsmasq's interface listening behavior + -l, privacylevel Set privacy level (0 = lowest, 3 = highest) + -t, teleporter Backup configuration as an archive + -t, teleporter myname.tar.gz Backup configuration to archive with name myname.tar.gz as specified" exit 0 } From 08999bf315daf7f2d65bca06e205e9147f6375be Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 4 Feb 2022 21:16:02 +0000 Subject: [PATCH 91/99] Use case insensitive deletion when removing custom CNAME/DNS records in case of manual entries with mixed case having been added Signed-off-by: Adam Warner --- advanced/Scripts/webpage.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 23b4f40279..aa4795dde5 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -739,7 +739,7 @@ RemoveCustomDNSAddress() { validHost="$(checkDomain "${host}")" if [[ -n "${validHost}" ]]; then if valid_ip "${ip}" || valid_ip6 "${ip}" ; then - sed -i "/^${ip} ${validHost}$/d" "${dnscustomfile}" + sed -i "/^${ip} ${validHost}$/Id" "${dnscustomfile}" else echo -e " ${CROSS} Invalid IP has been passed" exit 1 @@ -792,7 +792,7 @@ RemoveCustomCNAMERecord() { if [[ -n "${validDomain}" ]]; then validTarget="$(checkDomain "${target}")" if [[ -n "${validTarget}" ]]; then - sed -i "/cname=${validDomain},${validTarget}$/d" "${dnscustomcnamefile}" + sed -i "/cname=${validDomain},${validTarget}$/Id" "${dnscustomcnamefile}" else echo " ${CROSS} Invalid Target Passed!" exit 1 From 9be5199f7c7648aea75c7d902adca89b4a37685f Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 20 Feb 2022 12:39:58 +0000 Subject: [PATCH 92/99] remove the CONTENT_COMPARISON setting (defaults to false) --- .github/workflows/sync-back-to-dev.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 819e9d24dc..5b9fa570e3 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -19,7 +19,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} FROM_BRANCH: 'master' TO_BRANCH: 'development' - CONTENT_COMPARISON: true - name: Label the pull request to ignore for release note generation uses: actions-ecosystem/action-add-labels@v1 with: From 899cac0aac8e86f36f5b9aaac67c58922be8c79e Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 5 Mar 2022 15:49:54 +0000 Subject: [PATCH 93/99] Ignore Documentation Needed label --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 506af4063b..783f141967 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -20,6 +20,6 @@ jobs: days-before-close: 5 stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Please comment or update this issue or it will be closed in 5 days.' stale-issue-label: 'stale' - exempt-issue-labels: 'Internal, Fixed in next release, Bug: Confirmed' + exempt-issue-labels: 'Internal, Fixed in next release, Bug: Confirmed, Documentation Needed' exempt-all-issue-assignees: true operations-per-run: 300 From 2f384525652e3e617bfd5e13e9ed09b0165a9176 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Thu, 31 Mar 2022 12:03:17 -0700 Subject: [PATCH 94/99] Wrap touch calls with if/then guards for Buster docker. Signed-off-by: Dan Schaper --- automated install/basic-install.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 1e004b8b27..99e4dc47d4 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1128,8 +1128,10 @@ chooseBlocklists() { appendToListsFile "${choice}" done # Create an empty adList file with appropriate permissions. - touch "${adlistFile}" - chmod 644 "${adlistFile}" + if [ ! -f "${adlistFile}" ]; then + touch "${adlistFile}" + chmod 644 "${adlistFile}" + fi } # Accept a string parameter, it must be one of the default lists @@ -1330,8 +1332,10 @@ installConfigs() { # and copy in the config file Pi-hole needs install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/${LIGHTTPD_CFG} "${lighttpdConfig}" # Make sure the external.conf file exists, as lighttpd v1.4.50 crashes without it - touch /etc/lighttpd/external.conf - chmod 644 /etc/lighttpd/external.conf + if [ ! -f /etc/lighttpd/external.conf ]; then + touch /etc/lighttpd/external.conf + chmod 644 /etc/lighttpd/external.conf + fi # If there is a custom block page in the html/pihole directory, replace 404 handler in lighttpd config if [[ -f "${PI_HOLE_BLOCKPAGE_DIR}/custom.php" ]]; then sed -i 's/^\(server\.error-handler-404\s*=\s*\).*$/\1"\/pihole\/custom\.php"/' "${lighttpdConfig}" From c2384ecc6f5bc55e4d00c37e56666e891c0d8f46 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Thu, 31 Mar 2022 14:23:39 -0700 Subject: [PATCH 95/99] Change touch that would always fire to install. Signed-off-by: Dan Schaper --- automated install/basic-install.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 99e4dc47d4..4c173d05e0 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1333,8 +1333,7 @@ installConfigs() { install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/${LIGHTTPD_CFG} "${lighttpdConfig}" # Make sure the external.conf file exists, as lighttpd v1.4.50 crashes without it if [ ! -f /etc/lighttpd/external.conf ]; then - touch /etc/lighttpd/external.conf - chmod 644 /etc/lighttpd/external.conf + install -m 644 /dev/null /etc/lighttpd/external.com fi # If there is a custom block page in the html/pihole directory, replace 404 handler in lighttpd config if [[ -f "${PI_HOLE_BLOCKPAGE_DIR}/custom.php" ]]; then From d45c9fc52293d907d6871cc7f3cf701c9d88c376 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Fri, 1 Apr 2022 11:08:26 -0700 Subject: [PATCH 96/99] Final touch to install fix. Signed-off-by: Dan Schaper --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 4c173d05e0..f2720d5d63 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1129,7 +1129,7 @@ chooseBlocklists() { done # Create an empty adList file with appropriate permissions. if [ ! -f "${adlistFile}" ]; then - touch "${adlistFile}" + install /dev/null "${adlistFile}" chmod 644 "${adlistFile}" fi } From 8a5c7dec719bb29f4717b0e679afc585d915c45e Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Thu, 31 Mar 2022 14:32:07 -0700 Subject: [PATCH 97/99] Ensure existing files are proper owner and mode. Signed-off-by: Dan Schaper co-authored-by: RD WebDesign --- advanced/Templates/pihole-FTL.service | 11 +++++++---- automated install/basic-install.sh | 5 +++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/advanced/Templates/pihole-FTL.service b/advanced/Templates/pihole-FTL.service index 865e2cd93f..23f4f12550 100644 --- a/advanced/Templates/pihole-FTL.service +++ b/advanced/Templates/pihole-FTL.service @@ -21,12 +21,15 @@ start() { else # Touch files to ensure they exist (create if non-existing, preserve if existing) mkdir -pm 0755 /run/pihole - touch /run/pihole-FTL.pid /run/pihole-FTL.port /var/log/pihole-FTL.log /var/log/pihole.log /etc/pihole/dhcp.leases + [ ! -f /run/pihole-FTL.pid ] && install -m 644 -o pihole -g pihole dev/null /run/pihole-FTL.pid + [ ! -f /run/pihole-FTL.port ] && install -m 644 -o pihole -g pihole dev/null /run/pihole-FTL.port + [ ! -f /var/log/pihole-FTL.log ] && install -m 644 -o pihole -g pihole dev/null /var/log/pihole.log + [ ! -f /var/log/pihole.log ] && install -m 644 -o pihole -g pihole dev/null /var/log/pihole.log + [ ! -f /etc/pihole/dhcp.leases ] && install -m 644 -o pihole -g pihole dev/null /etc/pihole/dhcp.leases # Ensure that permissions are set so that pihole-FTL can edit all necessary files - chown pihole:pihole /run/pihole-FTL.pid /run/pihole-FTL.port /var/log/pihole-FTL.log /var/log/pihole.log /etc/pihole/dhcp.leases /run/pihole /etc/pihole - chmod 0644 /run/pihole-FTL.pid /run/pihole-FTL.port /var/log/pihole-FTL.log /var/log/pihole.log /etc/pihole/dhcp.leases + chown pihole:pihole /run/pihole /etc/pihole /var/log/pihole.log /var/log/pihole.log /etc/pihole/dhcp.leases # Ensure that permissions are set so that pihole-FTL can edit the files. We ignore errors as the file may not (yet) exist - chmod -f 0644 /etc/pihole/macvendor.db + chmod -f 0644 /etc/pihole/macvendor.db /etc/pihole/dhcp.leases /var/log/pihole-FTL.log /var/log/pihole.log # Chown database files to the user FTL runs as. We ignore errors as the files may not (yet) exist chown -f pihole:pihole /etc/pihole/pihole-FTL.db /etc/pihole/gravity.db /etc/pihole/macvendor.db # Chown database file permissions so that the pihole group (web interface) can edit the file. We ignore errors as the files may not (yet) exist diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index f2720d5d63..62366f8ae3 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1129,7 +1129,8 @@ chooseBlocklists() { done # Create an empty adList file with appropriate permissions. if [ ! -f "${adlistFile}" ]; then - install /dev/null "${adlistFile}" + install -m 644 /dev/null "${adlistFile}" + else chmod 644 "${adlistFile}" fi } @@ -1333,7 +1334,7 @@ installConfigs() { install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/${LIGHTTPD_CFG} "${lighttpdConfig}" # Make sure the external.conf file exists, as lighttpd v1.4.50 crashes without it if [ ! -f /etc/lighttpd/external.conf ]; then - install -m 644 /dev/null /etc/lighttpd/external.com + install -m 644 /dev/null /etc/lighttpd/external.conf fi # If there is a custom block page in the html/pihole directory, replace 404 handler in lighttpd config if [[ -f "${PI_HOLE_BLOCKPAGE_DIR}/custom.php" ]]; then From 0f192998eb52c46e20cd422513d8ae5a9a0edf8f Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Fri, 1 Apr 2022 14:17:57 -0700 Subject: [PATCH 98/99] Create empty files. Signed-off-by: Dan Schaper --- advanced/Templates/pihole-FTL.service | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/advanced/Templates/pihole-FTL.service b/advanced/Templates/pihole-FTL.service index 23f4f12550..d4a057ef0e 100644 --- a/advanced/Templates/pihole-FTL.service +++ b/advanced/Templates/pihole-FTL.service @@ -21,11 +21,11 @@ start() { else # Touch files to ensure they exist (create if non-existing, preserve if existing) mkdir -pm 0755 /run/pihole - [ ! -f /run/pihole-FTL.pid ] && install -m 644 -o pihole -g pihole dev/null /run/pihole-FTL.pid - [ ! -f /run/pihole-FTL.port ] && install -m 644 -o pihole -g pihole dev/null /run/pihole-FTL.port - [ ! -f /var/log/pihole-FTL.log ] && install -m 644 -o pihole -g pihole dev/null /var/log/pihole.log - [ ! -f /var/log/pihole.log ] && install -m 644 -o pihole -g pihole dev/null /var/log/pihole.log - [ ! -f /etc/pihole/dhcp.leases ] && install -m 644 -o pihole -g pihole dev/null /etc/pihole/dhcp.leases + [ ! -f /run/pihole-FTL.pid ] && install -m 644 -o pihole -g pihole /dev/null /run/pihole-FTL.pid + [ ! -f /run/pihole-FTL.port ] && install -m 644 -o pihole -g pihole /dev/null /run/pihole-FTL.port + [ ! -f /var/log/pihole-FTL.log ] && install -m 644 -o pihole -g pihole /dev/null /var/log/pihole.log + [ ! -f /var/log/pihole.log ] && install -m 644 -o pihole -g pihole /dev/null /var/log/pihole.log + [ ! -f /etc/pihole/dhcp.leases ] && install -m 644 -o pihole -g pihole /dev/null /etc/pihole/dhcp.leases # Ensure that permissions are set so that pihole-FTL can edit all necessary files chown pihole:pihole /run/pihole /etc/pihole /var/log/pihole.log /var/log/pihole.log /etc/pihole/dhcp.leases # Ensure that permissions are set so that pihole-FTL can edit the files. We ignore errors as the file may not (yet) exist From b714c4598a615ae384e8f884936071d8b04e84f6 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Fri, 1 Apr 2022 14:49:30 -0700 Subject: [PATCH 99/99] Found it. Signed-off-by: Dan Schaper --- advanced/Templates/pihole-FTL.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Templates/pihole-FTL.service b/advanced/Templates/pihole-FTL.service index d4a057ef0e..41ab801811 100644 --- a/advanced/Templates/pihole-FTL.service +++ b/advanced/Templates/pihole-FTL.service @@ -23,7 +23,7 @@ start() { mkdir -pm 0755 /run/pihole [ ! -f /run/pihole-FTL.pid ] && install -m 644 -o pihole -g pihole /dev/null /run/pihole-FTL.pid [ ! -f /run/pihole-FTL.port ] && install -m 644 -o pihole -g pihole /dev/null /run/pihole-FTL.port - [ ! -f /var/log/pihole-FTL.log ] && install -m 644 -o pihole -g pihole /dev/null /var/log/pihole.log + [ ! -f /var/log/pihole-FTL.log ] && install -m 644 -o pihole -g pihole /dev/null /var/log/pihole-FTL.log [ ! -f /var/log/pihole.log ] && install -m 644 -o pihole -g pihole /dev/null /var/log/pihole.log [ ! -f /etc/pihole/dhcp.leases ] && install -m 644 -o pihole -g pihole /dev/null /etc/pihole/dhcp.leases # Ensure that permissions are set so that pihole-FTL can edit all necessary files