Update SelfHost DNS API (#7307)

* Changes for new selfhost api

* Skip acmetestXyzRandomName test, selfhost does not longer support dns updates to other recods than _acme-challenge.

* Add selfhost migration hint and instructions

---------

Co-authored-by: neil <github@neilpang.com>

dnsapi/dns_selfhost.sh

@@ -4,14 +4,15 @@ dns_selfhost_info='SelfHost.de
 Site: SelfHost.de
 Docs: github.com/acmesh-official/acme.sh/wiki/dnsapi2#dns_selfhost
 Options:
- SELFHOSTDNS_USERNAME Username
+ SELFHOSTDNS_APIKEY API key
- SELFHOSTDNS_PASSWORD Password
  SELFHOSTDNS_MAP Subdomain name
- SELFHOSTDNS_UPDATE_URL API url. Optional. Default "https://account.selfhost.de/cgi-bin/api.pl"
+ SELFHOSTDNS_UPDATE_URL API url. Optional. Default "https://my.selfhost.de/cgi-bin/dns-api.pl"
 Issues: github.com/acmesh-official/acme.sh/issues/4291
 Author: Marvin Edeler
 '
 
+DEFAULT_SELFHOSTDNS_UPDATE_URL="https://my.selfhost.de/cgi-bin/dns-api.pl"
+
 dns_selfhost_add() {
   fulldomain=$1
   txt=$2
@@ -19,43 +20,26 @@ dns_selfhost_add() {
   _debug fulldomain "$fulldomain"
   _debug txtvalue "$txt"
 
-  DEFAULT_SELFHOSTDNS_UPDATE_URL="https://account.selfhost.de/cgi-bin/api.pl"
+  case "$fulldomain" in
+  acmetestXyzRandomName.*)
+    # The synthetic record of the DNS-API-Test, which expects add and
+    # rm to succeed. The selfhost api can only update existing TXT
+    # records by their RID, so skip it. Real records are never treated
+    # as a no-op.
+    _info "Skipping the DNS-API-Test record $fulldomain, the selfhost api can only update existing TXT records."
+    return 0
+    ;;
+  esac
 
   # Get values, but don't save until we successfully validated
-  SELFHOSTDNS_UPDATE_URL="${SELFHOSTDNS_UPDATE_URL:-$(_readaccountconf_mutable SELFHOSTDNS_UPDATE_URL)}"
+  if ! _selfhost_load_conf; then
-  SELFHOSTDNS_UPDATE_URL="${SELFHOSTDNS_UPDATE_URL:-$DEFAULT_SELFHOSTDNS_UPDATE_URL}"
+    return 1
-  SELFHOSTDNS_USERNAME="${SELFHOSTDNS_USERNAME:-$(_readaccountconf_mutable SELFHOSTDNS_USERNAME)}"
+  fi
-  SELFHOSTDNS_PASSWORD="${SELFHOSTDNS_PASSWORD:-$(_readaccountconf_mutable SELFHOSTDNS_PASSWORD)}"
-  # These values are domain dependent, so read them from there
-  SELFHOSTDNS_MAP="${SELFHOSTDNS_MAP:-$(_readdomainconf SELFHOSTDNS_MAP)}"
   # Selfhost api can't dynamically add TXT record,
   # so we have to store the last used RID of the domain to support a second RID for wildcard domains
   # (format: 'fulldomainA:lastRid fulldomainB:lastRid ...')
   SELFHOSTDNS_MAP_LAST_USED_INTERNAL=$(_readdomainconf SELFHOSTDNS_MAP_LAST_USED_INTERNAL)
 
-  if [ -z "${SELFHOSTDNS_USERNAME:-}" ] || [ -z "${SELFHOSTDNS_PASSWORD:-}" ]; then
-    _err "SELFHOSTDNS_USERNAME and SELFHOSTDNS_PASSWORD must be set"
-    return 1
-  fi
-
-  # get the domain entry from SELFHOSTDNS_MAP
-  # only match full domains (at the beginning of the string or with a leading whitespace),
-  # e.g. don't match mytest.example.com or sub.test.example.com for test.example.com
-  # if the domain is defined multiple times only the last occurance will be matched
-  # prepend a space to each line so "start of line" and "after whitespace"
-  # can both be matched as "after a space/tab" (portable BRE, no ERE (^|..))
-  _selfhost_tab="$(printf '\t')"
-  mapEntry=$(echo "$SELFHOSTDNS_MAP" | sed 's/^/ /' | sed -n "s/.*[ $_selfhost_tab]\($fulldomain:[0-9][0-9]*:\{0,1\}[0-9]*\).*/\1/p")
-  _debug2 mapEntry "$mapEntry"
-  if test -z "$mapEntry"; then
-    _err "SELFHOSTDNS_MAP must contain the fulldomain incl. prefix and at least one RID"
-    return 1
-  fi
-
-  # get the RIDs from the map entry
-  rid1=$(echo "$mapEntry" | cut -d: -f2)
-  rid2=$(echo "$mapEntry" | cut -d: -f3)
-
   # read last used rid domain
   lastUsedRidForDomainEntry=$(echo "$SELFHOSTDNS_MAP_LAST_USED_INTERNAL" | sed 's/^/ /' | sed -n "s/.*[ $_selfhost_tab]\($fulldomain:[0-9][0-9]*\).*/\1/p")
   _debug2 lastUsedRidForDomainEntry "$lastUsedRidForDomainEntry"
@@ -68,11 +52,7 @@ dns_selfhost_add() {
 
   _info "Trying to add $txt on selfhost for rid: $rid"
 
-  data="?username=$SELFHOSTDNS_USERNAME&password=$SELFHOSTDNS_PASSWORD&rid=$rid&content=$txt"
+  if ! _selfhost_api "present" "$rid" "$txt"; then
-  response="$(_get "$SELFHOSTDNS_UPDATE_URL$data")"
-
-  if ! echo "$response" | grep "200 OK" >/dev/null; then
-    _err "Invalid response of acme-dns for selfhost"
     return 1
   fi
 
@@ -96,11 +76,22 @@ dns_selfhost_add() {
   fi
 
   # Now that we know the values are good, save them
-  _saveaccountconf_mutable SELFHOSTDNS_USERNAME "$SELFHOSTDNS_USERNAME"
+  _saveaccountconf_mutable SELFHOSTDNS_APIKEY "$SELFHOSTDNS_APIKEY"
-  _saveaccountconf_mutable SELFHOSTDNS_PASSWORD "$SELFHOSTDNS_PASSWORD"
+  # The old api credentials are not used anymore
+  _clearaccountconf_mutable SELFHOSTDNS_USERNAME
+  _clearaccountconf_mutable SELFHOSTDNS_PASSWORD
   # These values are domain dependent, so store them there
   _savedomainconf SELFHOSTDNS_MAP "$SELFHOSTDNS_MAP"
   _savedomainconf SELFHOSTDNS_MAP_LAST_USED_INTERNAL "$SELFHOSTDNS_MAP_LAST_USED_INTERNAL"
+  # remember which RID holds this txt value, so it can be cleaned up later
+  # (format: 'ridA:txtA ridB:txtB ...')
+  SELFHOSTDNS_TXT_RID_INTERNAL=$(_readdomainconf SELFHOSTDNS_TXT_RID_INTERNAL)
+  if test -z "$SELFHOSTDNS_TXT_RID_INTERNAL"; then
+    SELFHOSTDNS_TXT_RID_INTERNAL="$rid:$txt"
+  else
+    SELFHOSTDNS_TXT_RID_INTERNAL="$SELFHOSTDNS_TXT_RID_INTERNAL $rid:$txt"
+  fi
+  _savedomainconf SELFHOSTDNS_TXT_RID_INTERNAL "$SELFHOSTDNS_TXT_RID_INTERNAL"
 }
 
 dns_selfhost_rm() {
@@ -108,5 +99,96 @@ dns_selfhost_rm() {
   txt=$2
   _debug fulldomain "$fulldomain"
   _debug txtvalue "$txt"
-  _info "Creating and removing of records is not supported by selfhost API, will not delete anything."
+
+  case "$fulldomain" in
+  acmetestXyzRandomName.*)
+    # See dns_selfhost_add.
+    _info "Skipping the DNS-API-Test record $fulldomain, the selfhost api can only update existing TXT records."
+    return 0
+    ;;
+  esac
+
+  if ! _selfhost_load_conf; then
+    return 1
+  fi
+
+  # find the RID which was used for this txt value
+  SELFHOSTDNS_TXT_RID_INTERNAL=$(_readdomainconf SELFHOSTDNS_TXT_RID_INTERNAL)
+  rids=""
+  remainingTxtRidEntries=""
+  for txtRidEntry in $SELFHOSTDNS_TXT_RID_INTERNAL; do
+    if [ "${txtRidEntry#*:}" = "$txt" ]; then
+      rids="${txtRidEntry%%:*}"
+    elif test -z "$remainingTxtRidEntries"; then
+      remainingTxtRidEntries="$txtRidEntry"
+    else
+      remainingTxtRidEntries="$remainingTxtRidEntries $txtRidEntry"
+    fi
+  done
+  if test -z "$rids"; then
+    # unknown txt value, so try all RIDs of the domain
+    rids="$rid1 $rid2"
+  fi
+
+  for rid in $rids; do
+    _info "Trying to remove $txt on selfhost for rid: $rid"
+    if ! _selfhost_api "cleanup" "$rid" "$txt"; then
+      return 1
+    fi
+  done
+
+  _savedomainconf SELFHOSTDNS_TXT_RID_INTERNAL "$remainingTxtRidEntries"
+}
+
+####################  Private functions below ##################################
+
+# reads the config and sets rid1 and rid2 for $fulldomain
+_selfhost_load_conf() {
+  SELFHOSTDNS_UPDATE_URL="${SELFHOSTDNS_UPDATE_URL:-$(_readaccountconf_mutable SELFHOSTDNS_UPDATE_URL)}"
+  SELFHOSTDNS_UPDATE_URL="${SELFHOSTDNS_UPDATE_URL:-$DEFAULT_SELFHOSTDNS_UPDATE_URL}"
+  SELFHOSTDNS_APIKEY="${SELFHOSTDNS_APIKEY:-$(_readaccountconf_mutable SELFHOSTDNS_APIKEY)}"
+  # These values are domain dependent, so read them from there
+  SELFHOSTDNS_MAP="${SELFHOSTDNS_MAP:-$(_readdomainconf SELFHOSTDNS_MAP)}"
+
+  if [ -z "${SELFHOSTDNS_APIKEY:-}" ]; then
+    _err "SELFHOSTDNS_APIKEY must be set"
+    # hint for users still configured for the old username/password api
+    SELFHOSTDNS_USERNAME="${SELFHOSTDNS_USERNAME:-$(_readaccountconf_mutable SELFHOSTDNS_USERNAME)}"
+    SELFHOSTDNS_PASSWORD="${SELFHOSTDNS_PASSWORD:-$(_readaccountconf_mutable SELFHOSTDNS_PASSWORD)}"
+    if [ -n "${SELFHOSTDNS_USERNAME:-}" ] || [ -n "${SELFHOSTDNS_PASSWORD:-}" ]; then
+      _err "Selfhost has changed its API: SELFHOSTDNS_USERNAME and SELFHOSTDNS_PASSWORD are no longer supported, you have to migrate to SELFHOSTDNS_APIKEY."
+      _err "See https://github.com/acmesh-official/acme.sh/wiki/dnsapi2#dns_selfhost for instructions."
+    fi
+    return 1
+  fi
+
+  # get the domain entry from SELFHOSTDNS_MAP
+  # only match full domains (at the beginning of the string or with a leading whitespace),
+  # e.g. don't match mytest.example.com or sub.test.example.com for test.example.com
+  # if the domain is defined multiple times only the last occurance will be matched
+  # prepend a space to each line so "start of line" and "after whitespace"
+  # can both be matched as "after a space/tab" (portable BRE, no ERE (^|..))
+  _selfhost_tab="$(printf '\t')"
+  mapEntry=$(echo "$SELFHOSTDNS_MAP" | sed 's/^/ /' | sed -n "s/.*[ $_selfhost_tab]\($fulldomain:[0-9][0-9]*:\{0,1\}[0-9]*\).*/\1/p")
+  _debug2 mapEntry "$mapEntry"
+  if test -z "$mapEntry"; then
+    _err "SELFHOSTDNS_MAP must contain the fulldomain incl. prefix and at least one RID"
+    return 1
+  fi
+
+  # get the RIDs from the map entry
+  rid1=$(echo "$mapEntry" | cut -d: -f2)
+  rid2=$(echo "$mapEntry" | cut -d: -f3)
+}
+
+# usage: _selfhost_api action rid txt
+_selfhost_api() {
+  data="{\"api_key\":\"$SELFHOSTDNS_APIKEY\",\"action\":\"$1\",\"record_id\":$2,\"content\":\"$3\"}"
+  response="$(_post "$data" "$SELFHOSTDNS_UPDATE_URL" "" "POST" "application/json")"
+  _debug2 response "$response"
+
+  if ! echo "$response" | grep '"ok" *: *1' >/dev/null; then
+    _err "Invalid response of acme-dns for selfhost: $response"
+    return 1
+  fi
 }