diff --git a/.github/workflows/almalinux-8-build.yml b/.github/workflows/almalinux-8-build.yml index 48de328a3..65cd6e374 100644 --- a/.github/workflows/almalinux-8-build.yml +++ b/.github/workflows/almalinux-8-build.yml @@ -48,7 +48,8 @@ jobs: attr libattr-devel acl libacl-devel \ zstd libzstd-devel \ lz4 lz4-devel \ - xxhash xxhash-devel + xxhash xxhash-devel \ + libidn2 libidn2-devel alternatives --set python3 /usr/bin/python3.9 pip3 install commonmark - name: configure @@ -62,7 +63,7 @@ jobs: # crtimes-not-supported skip matches the other Linux jobs; # daemon-chroot-acl and proxy-response-line-too-long skip because # the default (secure) transport opens no listening socket. - run: RSYNC_EXPECT_SKIPPED=crtimes,daemon-access-ip,daemon-chroot-acl,proxy-response-line-too-long,recv-discard-nullderef make check + run: RSYNC_EXPECT_SKIPPED=crtimes,daemon-access-idn,daemon-access-ip,daemon-chroot-acl,proxy-response-line-too-long,recv-discard-nullderef make check - name: check (TCP daemon transport) # Second run exercising the real loopback-TCP daemon path. run: ./runtests.py --rsync-bin="$PWD/rsync" --use-tcp -j 8 diff --git a/.github/workflows/android-static-build.yml b/.github/workflows/android-static-build.yml index e91542550..237ee33e6 100644 --- a/.github/workflows/android-static-build.yml +++ b/.github/workflows/android-static-build.yml @@ -76,7 +76,7 @@ jobs: # checksums and its bundled zlib. ./configure --host=${{ matrix.triple }} --build=x86_64-pc-linux-gnu \ --enable-ipv6 \ - --disable-zstd --disable-lz4 --disable-xxhash --disable-openssl \ + --disable-zstd --disable-lz4 --disable-xxhash --disable-openssl --disable-idn \ --disable-iconv --disable-iconv-open \ --disable-acl-support --disable-xattr-support \ --disable-md2man --disable-roll-simd \ diff --git a/.github/workflows/asan-build.yml b/.github/workflows/asan-build.yml index 9acc8d8f6..584f03553 100644 --- a/.github/workflows/asan-build.yml +++ b/.github/workflows/asan-build.yml @@ -41,7 +41,7 @@ jobs: - name: prep run: | sudo apt-get update - sudo apt-get install -y clang acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev openssl + sudo apt-get install -y clang acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev libidn2-dev openssl echo "/usr/local/bin" >>"$GITHUB_PATH" - name: configure # -DNDEBUG builds as a shipped release does (assert() compiled out), so diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 078fda4ff..77e648fbb 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -26,7 +26,7 @@ jobs: - name: prep run: | sudo apt-get update - sudo apt-get install -y acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev python3-cmarkgfm openssl gcovr + sudo apt-get install -y acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev libidn2-dev python3-cmarkgfm openssl gcovr echo "/usr/local/bin" >>"$GITHUB_PATH" - name: configure run: ./configure --enable-coverage --with-rrsync diff --git a/.github/workflows/cygwin-build.yml b/.github/workflows/cygwin-build.yml index f8feebcd6..0281bffa5 100644 --- a/.github/workflows/cygwin-build.yml +++ b/.github/workflows/cygwin-build.yml @@ -26,7 +26,7 @@ jobs: run: choco install -y --no-progress cygwin cyg-get - name: prep run: | - cyg-get make autoconf automake gcc-core attr libattr-devel python39 python39-pip libzstd-devel liblz4-devel libssl-devel libxxhash0 libxxhash-devel + cyg-get make autoconf automake gcc-core attr libattr-devel python39 python39-pip libzstd-devel liblz4-devel libssl-devel libxxhash0 libxxhash-devel libidn2-devel echo "C:/tools/cygwin/bin" >>$Env:GITHUB_PATH - name: commonmark run: bash -c 'python3 -mpip install --user commonmark' @@ -46,7 +46,7 @@ jobs: # RESOLVE_BENEATH symlink-race tests. symlink-dirlink-basis also now # RUNS (the #915 non-daemon basis open uses a plain do_open, restoring # following an in-tree dir-symlink basis without RESOLVE_BENEATH). - run: bash -c 'RSYNC_EXPECT_SKIPPED=acls-default,acls-depth,acls,bare-do-open-symlink-race,chdir-symlink-race,chown,daemon-access-ip,daemon-chroot-acl,devices,dir-sgid,open-noatime,protected-regular,proxy-response-line-too-long,recv-discard-nullderef,sender-flist-symlink-leak,simd-checksum make check' + run: bash -c 'RSYNC_EXPECT_SKIPPED=acls-default,acls-depth,acls,bare-do-open-symlink-race,chdir-symlink-race,chown,daemon-access-idn,daemon-access-ip,daemon-chroot-acl,devices,dir-sgid,open-noatime,protected-regular,proxy-response-line-too-long,recv-discard-nullderef,sender-flist-symlink-leak,simd-checksum make check' - name: check (TCP daemon transport) # Second run with daemon tests over a real loopback rsyncd; the default # 'make check' above uses the secure stdio-pipe transport. diff --git a/.github/workflows/fleettest.yml b/.github/workflows/fleettest.yml index c3f66c771..3a8986158 100644 --- a/.github/workflows/fleettest.yml +++ b/.github/workflows/fleettest.yml @@ -33,7 +33,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y gcc g++ gawk autoconf automake \ - acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev \ + acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev libidn2-dev \ python3-cmarkgfm openssl rsync openssh-server - name: set up ssh to localhost run: | diff --git a/.github/workflows/freebsd-build.yml b/.github/workflows/freebsd-build.yml index 4012366d7..d52cf07a8 100644 --- a/.github/workflows/freebsd-build.yml +++ b/.github/workflows/freebsd-build.yml @@ -31,7 +31,7 @@ jobs: pkg install -y bash autotools m4 devel/xxhash zstd liblz4 python3 archivers/liblz4 git run: | freebsd-version - ./configure --with-rrsync -disable-zstd --disable-md2man --disable-xxhash --disable-lz4 + ./configure --with-rrsync -disable-zstd --disable-md2man --disable-xxhash --disable-lz4 --disable-idn make ./rsync --version make check diff --git a/.github/workflows/macos-build.yml b/.github/workflows/macos-build.yml index 697cc86af..179a21536 100644 --- a/.github/workflows/macos-build.yml +++ b/.github/workflows/macos-build.yml @@ -24,7 +24,7 @@ jobs: fetch-depth: 0 - name: prep run: | - brew install automake openssl xxhash zstd lz4 + brew install automake openssl xxhash zstd lz4 libidn2 pip3 install --user --break-system-packages commonmark echo "$(brew --prefix)/bin" >>"$GITHUB_PATH" - name: configure @@ -44,7 +44,7 @@ jobs: # chown-fake / devices-fake / xattrs / xattrs-hlink now RUN on macOS # (rsyncfns.py drives xattrs via the `xattr` command), verified on a # real macOS host, so they're no longer in the skip set. - run: sudo RSYNC_EXPECT_SKIPPED=acls-default,acls-depth,chmod-temp-dir,daemon-access-ip,daemon-chroot-acl,dir-sgid,open-noatime,preallocate,protected-regular,proxy-response-line-too-long,recv-discard-nullderef,simd-checksum,sparse make check + run: sudo RSYNC_EXPECT_SKIPPED=acls-default,acls-depth,chmod-temp-dir,daemon-access-idn,daemon-access-ip,daemon-chroot-acl,dir-sgid,open-noatime,preallocate,protected-regular,proxy-response-line-too-long,recv-discard-nullderef,simd-checksum,sparse make check - name: check (TCP daemon transport) # Second run with daemon tests over a real loopback rsyncd; the default # 'make check' above uses the secure stdio-pipe transport. diff --git a/.github/workflows/netbsd-build.yml b/.github/workflows/netbsd-build.yml index a80fb11b4..00da9440e 100644 --- a/.github/workflows/netbsd-build.yml +++ b/.github/workflows/netbsd-build.yml @@ -32,7 +32,7 @@ jobs: ln -sf /usr/pkg/bin/python3.12 /usr/pkg/bin/python3 run: | uname -a - ./configure --with-rrsync --disable-zstd --disable-md2man --disable-xxhash --disable-lz4 + ./configure --with-rrsync --disable-zstd --disable-md2man --disable-xxhash --disable-lz4 --disable-idn make ./rsync --version make check diff --git a/.github/workflows/openbsd-build.yml b/.github/workflows/openbsd-build.yml index ec38eac5e..21cd2885e 100644 --- a/.github/workflows/openbsd-build.yml +++ b/.github/workflows/openbsd-build.yml @@ -33,7 +33,7 @@ jobs: uname -a export AUTOCONF_VERSION=2.71 export AUTOMAKE_VERSION=1.16 - ./configure --with-rrsync --disable-zstd --disable-md2man --disable-xxhash --disable-lz4 + ./configure --with-rrsync --disable-zstd --disable-md2man --disable-xxhash --disable-lz4 --disable-idn make ./rsync --version make check diff --git a/.github/workflows/scan-build.yml b/.github/workflows/scan-build.yml index 42cb07b83..8803ae9f4 100644 --- a/.github/workflows/scan-build.yml +++ b/.github/workflows/scan-build.yml @@ -30,7 +30,7 @@ jobs: - name: prep run: | sudo apt-get update - sudo apt-get install -y clang-18 clang-tools-18 acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev libpopt-dev openssl + sudo apt-get install -y clang-18 clang-tools-18 acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev libidn2-dev libpopt-dev openssl - name: configure (under scan-build) # Run configure under scan-build so its analyzer compiler-wrapper is baked # into the Makefile's $(CC); --disable-md2man avoids the doc toolchain. diff --git a/.github/workflows/solaris-build.yml b/.github/workflows/solaris-build.yml index 82aa12692..0ccaeac7c 100644 --- a/.github/workflows/solaris-build.yml +++ b/.github/workflows/solaris-build.yml @@ -31,7 +31,7 @@ jobs: pkg install bash automake gnu-m4 pkg://solaris/runtime/python-35 autoconf gcc git run: | uname -a - ./configure --with-rrsync -disable-zstd --disable-md2man --disable-xxhash --disable-lz4 + ./configure --with-rrsync -disable-zstd --disable-md2man --disable-xxhash --disable-lz4 --disable-idn make ./rsync --version make check diff --git a/.github/workflows/ubuntu-22.04-build.yml b/.github/workflows/ubuntu-22.04-build.yml index 5546ff496..f9f95d52f 100644 --- a/.github/workflows/ubuntu-22.04-build.yml +++ b/.github/workflows/ubuntu-22.04-build.yml @@ -28,7 +28,7 @@ jobs: fetch-depth: 0 - name: prep run: | - sudo apt-get install acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev python3-cmarkgfm openssl + sudo apt-get install acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev libidn2-dev python3-cmarkgfm openssl echo "/usr/local/bin" >>"$GITHUB_PATH" - name: configure run: ./configure --with-rrsync @@ -39,11 +39,11 @@ jobs: - name: info run: rsync --version - name: check - run: sudo RSYNC_EXPECT_SKIPPED=crtimes,daemon-access-ip,daemon-chroot-acl,proxy-response-line-too-long,recv-discard-nullderef make check + run: sudo RSYNC_EXPECT_SKIPPED=crtimes,daemon-access-idn,daemon-access-ip,daemon-chroot-acl,proxy-response-line-too-long,recv-discard-nullderef make check - name: check30 - run: sudo RSYNC_EXPECT_SKIPPED=crtimes,daemon-access-ip,daemon-chroot-acl,proxy-response-line-too-long,recv-discard-nullderef make check30 + run: sudo RSYNC_EXPECT_SKIPPED=crtimes,daemon-access-idn,daemon-access-ip,daemon-chroot-acl,proxy-response-line-too-long,recv-discard-nullderef make check30 - name: check29 - run: sudo RSYNC_EXPECT_SKIPPED=crtimes,daemon-access-ip,daemon-chroot-acl,proxy-response-line-too-long,recv-discard-nullderef make check29 + run: sudo RSYNC_EXPECT_SKIPPED=crtimes,daemon-access-idn,daemon-access-ip,daemon-chroot-acl,proxy-response-line-too-long,recv-discard-nullderef make check29 - name: check (TCP daemon transport) # Second run with daemon tests over a real loopback rsyncd; the default # 'make check' above uses the secure stdio-pipe transport. diff --git a/.github/workflows/ubuntu-build.yml b/.github/workflows/ubuntu-build.yml index 1cec98942..25f1a9ba1 100644 --- a/.github/workflows/ubuntu-build.yml +++ b/.github/workflows/ubuntu-build.yml @@ -24,7 +24,7 @@ jobs: fetch-depth: 0 - name: prep run: | - sudo apt-get install acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev python3-cmarkgfm openssl + sudo apt-get install acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev libidn2-dev python3-cmarkgfm openssl echo "/usr/local/bin" >>"$GITHUB_PATH" - name: configure run: ./configure --with-rrsync @@ -63,11 +63,11 @@ jobs: - name: info run: rsync --version - name: check - run: sudo RSYNC_EXPECT_SKIPPED=crtimes,daemon-access-ip,daemon-chroot-acl,proxy-response-line-too-long,recv-discard-nullderef make check + run: sudo RSYNC_EXPECT_SKIPPED=crtimes,daemon-access-idn,daemon-access-ip,daemon-chroot-acl,proxy-response-line-too-long,recv-discard-nullderef make check - name: check30 - run: sudo RSYNC_EXPECT_SKIPPED=crtimes,daemon-access-ip,daemon-chroot-acl,proxy-response-line-too-long,recv-discard-nullderef make check30 + run: sudo RSYNC_EXPECT_SKIPPED=crtimes,daemon-access-idn,daemon-access-ip,daemon-chroot-acl,proxy-response-line-too-long,recv-discard-nullderef make check30 - name: check29 - run: sudo RSYNC_EXPECT_SKIPPED=crtimes,daemon-access-ip,daemon-chroot-acl,proxy-response-line-too-long,recv-discard-nullderef make check29 + run: sudo RSYNC_EXPECT_SKIPPED=crtimes,daemon-access-idn,daemon-access-ip,daemon-chroot-acl,proxy-response-line-too-long,recv-discard-nullderef make check29 - name: check (TCP daemon transport) # Second run with daemon tests over a real loopback rsyncd. The default # 'make check' above uses the secure stdio-pipe transport (no listening diff --git a/.github/workflows/ubuntu-version-mix.yml b/.github/workflows/ubuntu-version-mix.yml index 16fd32884..a080a92f1 100644 --- a/.github/workflows/ubuntu-version-mix.yml +++ b/.github/workflows/ubuntu-version-mix.yml @@ -45,7 +45,7 @@ jobs: fetch-depth: 0 - name: prep run: | - sudo apt-get install acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev python3-cmarkgfm openssl + sudo apt-get install acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev libidn2-dev python3-cmarkgfm openssl echo "/usr/local/bin" >>"$GITHUB_PATH" - name: configure run: ./configure --with-rrsync diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index c1ffaebc4..e9d37f993 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -33,7 +33,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y valgrind acl libacl1-dev attr libattr1-dev \ - liblz4-dev libzstd-dev libxxhash-dev python3-cmarkgfm openssl + liblz4-dev libzstd-dev libxxhash-dev libidn2-dev python3-cmarkgfm openssl echo "/usr/local/bin" >>"$GITHUB_PATH" - name: configure run: ./configure --with-rrsync --enable-debug diff --git a/INSTALL.md b/INSTALL.md index 8beb68c0d..e9b70aa10 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -114,6 +114,16 @@ checksums. [4]: https://www.openssl.org/docs/man1.0.2/man3/crypto.html +## libidn2 + +The [libidn2 library][5] converts an internationalized domain name into the +IDNA A-label ("Punycode") form that a resolver understands. Installing this +development library lets rsync connect to a daemon whose name has non-ASCII +characters in it, and lets a daemon's "hosts allow" & "hosts deny" settings be +written the same way. + +[5]: https://www.gnu.org/software/libidn/#libidn2 + ## Package summary To help you get the libraries installed, here are some package install commands @@ -131,6 +141,7 @@ like. > sudo apt install -y libzstd-dev > sudo apt install -y liblz4-dev > sudo apt install -y libssl-dev + > sudo apt install -y libidn2-dev Or run support/install_deps_ubuntu.sh @@ -144,6 +155,7 @@ Or run support/install_deps_ubuntu.sh > sudo yum -y install libzstd-devel > sudo yum -y install lz4-devel > sudo yum -y install openssl-devel + > sudo yum -y install libidn2-devel > python3 -mpip install --user commonmark - For Fedora 33: @@ -154,6 +166,7 @@ Or run support/install_deps_ubuntu.sh > sudo dnf -y install libzstd-devel > sudo dnf -y install lz4-devel > sudo dnf -y install openssl-devel + > sudo dnf -y install libidn2-devel - For FreeBSD (this assumes that the python3 version is 3.7): @@ -161,6 +174,7 @@ Or run support/install_deps_ubuntu.sh > sudo pkg install -y xxhash > sudo pkg install -y zstd > sudo pkg install -y liblz4 + > sudo pkg install -y libidn2 - For macOS: @@ -169,6 +183,7 @@ Or run support/install_deps_ubuntu.sh > brew install zstd > brew install lz4 > brew install openssl + > brew install libidn2 - For Cygwin (with all cygwin programs stopped, run the appropriate setup program from a cmd shell): @@ -177,6 +192,7 @@ Or run support/install_deps_ubuntu.sh > setup-x86_64 --quiet-mode -P libzstd-devel > setup-x86_64 --quiet-mode -P liblz4-devel > setup-x86_64 --quiet-mode -P libssl-devel + > setup-x86_64 --quiet-mode -P libidn2-devel Sometimes cygwin has commonmark packaged and sometimes it doesn't. Now that its python38 has stabilized, you could install python38-commonmark. Or just diff --git a/access.c b/access.c index b924e0a34..30c4e0683 100644 --- a/access.c +++ b/access.c @@ -33,6 +33,9 @@ static int match_hostname(const char **host_ptr, const char *addr, const char *t struct hostent *hp; unsigned int i; const char *host = *host_ptr; +#ifdef SUPPORT_IDN + char idn_tok[1024]; +#endif if (!host || !*host) return 0; @@ -42,6 +45,14 @@ static int match_hostname(const char **host_ptr, const char *addr, const char *t return innetgr(tok + 1, host, NULL, NULL); #endif +#ifdef SUPPORT_IDN + /* A hostname reaches us from DNS as ASCII, so fold an IDN token to its + * A-label form before comparing. An all-ASCII token, and a token we + * can't fold, are both left as they are. */ + if (idn_to_ascii(tok, 0, idn_tok, sizeof idn_tok)) + tok = idn_tok; +#endif + /* First check if the reverse-DNS-determined hostname matches. */ if (iwildmatch(tok, host)) return 1; diff --git a/configure.ac b/configure.ac index cda60405b..276954fad 100644 --- a/configure.ac +++ b/configure.ac @@ -13,7 +13,7 @@ AC_CHECK_HEADERS(sys/fcntl.h sys/select.h fcntl.h sys/time.h sys/unistd.h \ sys/acl.h acl/libacl.h attr/xattr.h sys/xattr.h sys/extattr.h dl.h \ popt.h popt/popt.h linux/falloc.h netinet/in_systm.h netgroup.h \ zlib.h xxhash.h openssl/md4.h openssl/md5.h zstd.h lz4.h sys/file.h \ - bsd/string.h) + bsd/string.h idn2.h) AC_CHECK_HEADERS([netinet/ip.h], [], [], [[#include ]]) AC_HEADER_MAJOR_FIXED @@ -626,6 +626,27 @@ else AC_MSG_RESULT(no) fi +AC_MSG_CHECKING([whether to enable IDN support]) +AC_ARG_ENABLE([idn], + AS_HELP_STRING([--disable-idn], [disable to omit IDN (Internationalized Domain Name) support])) +AH_TEMPLATE([SUPPORT_IDN], +[Undefine if you do not want IDN support. By default this is defined.]) +if test x"$enable_idn" != x"no"; then + if test x"$ac_cv_header_idn2_h" = x"yes"; then + AC_MSG_RESULT(yes) + AC_SEARCH_LIBS(idn2_lookup_ul, idn2, + [AC_DEFINE(SUPPORT_IDN)], + [err_msg="$err_msg$nl- Failed to find idn2_lookup_ul function in idn2 lib."; + no_lib="$no_lib idn"]) + else + AC_MSG_RESULT(no) + err_msg="$err_msg$nl- Failed to find idn2.h for IDN support." + no_lib="$no_lib idn" + fi +else + AC_MSG_RESULT(no) +fi + if test x"$no_lib" != x; then echo "" echo "Configure found the following issues:" diff --git a/main.c b/main.c index 9b52bbe6a..ee84a7730 100644 --- a/main.c +++ b/main.c @@ -517,6 +517,16 @@ static pid_t do_cmd(char *cmd, char *machine, char *user, char **remote_argv, in char *args[MAX_ARGS], *need_to_free = NULL; pid_t pid; int dash_l_set = 0; +#ifdef SUPPORT_IDN + char idn_machine[1024]; + + /* A daemon-over-remote-shell host is ours to resolve, so give the helper + * the A-label form. A "host:path" transfer is left alone because that + * name belongs to the user's ssh, which may be matching it against an + * ssh_config Host pattern. */ + if (machine && daemon_connection > 0 && idn_to_ascii(machine, 1, idn_machine, sizeof idn_machine)) + machine = idn_machine; +#endif if (!read_batch && !local_server) { char *t, *f, in_quote = '\0'; diff --git a/rsync.1.md b/rsync.1.md index 2d38efe30..c4c4b9681 100644 --- a/rsync.1.md +++ b/rsync.1.md @@ -321,6 +321,19 @@ when scripting rsync. WARNING: On some systems environment variables are visible to all users. On those systems using [`--password-file`](#opt) is recommended. +If rsync was built with IDN support (look for "IDN" in `rsync --version`), the +daemon host may contain non-ASCII characters: those labels are converted to +their IDNA A-label ("Punycode") form before the name is looked up. The name is +read using your locale's character encoding, so be sure your locale is set +correctly. Only the non-ASCII labels change, so an address literal, a name you +punycoded yourself, and a name that is not a valid IDN are all looked up just +as you typed them. + +This applies to the host of a daemon connection only. The host of a plain +remote-shell transfer (the single-colon syntax) is passed to your remote-shell +program unchanged, since that name may well be an ssh_config "Host" alias +rather than a DNS name. + You may establish the connection via a web proxy by setting the environment variable [`RSYNC_PROXY`](#) to a hostname:port pair pointing to your web proxy. Note that your web proxy's configuration must support proxy connections to port diff --git a/rsyncd.conf.5.md b/rsyncd.conf.5.md index 8cd810f60..75003196c 100644 --- a/rsyncd.conf.5.md +++ b/rsyncd.conf.5.md @@ -764,6 +764,18 @@ in the values of parameters. See that section for details. - an '@' followed by a netgroup name, which will match if the reverse DNS of the connecting IP is in the specified netgroup. + If rsync was built with IDN support (look for "IDN" in `rsync --version`), + a hostname pattern may contain non-ASCII characters: this file is read as + UTF-8, and each non-ASCII label is converted to its IDNA A-label + ("Punycode") form before matching, since the name the daemon has for a + client always reaches it from DNS as ASCII. Only those labels change, so + an address, a mask, an already-punycoded name, and the wildcard characters + are all matched exactly as written. A pattern that cannot be converted + that way is matched as it stands, and thus matches nothing. That includes + a pattern whose conversion would have introduced a character it was not + written with, such as the U+FF0A FULLWIDTH ASTERISK that the IDNA mapping + turns into a "*". + Note IPv6 link-local addresses can have a scope in the address specification: diff --git a/socket.c b/socket.c index 4ac79ec76..72ea346c3 100644 --- a/socket.c +++ b/socket.c @@ -196,6 +196,14 @@ int open_socket_out(char *host, int port, const char *bind_addr, int af_hint) int proxied = 0; char buffer[1024]; char *proxy_user = NULL, *proxy_pass = NULL; +#ifdef SUPPORT_IDN + char idn_host[1024]; + + /* The resolver only speaks ASCII, so an IDN host goes out as A-labels. + * An all-ASCII host is passed along untouched. */ + if (idn_to_ascii(host, 1, idn_host, sizeof idn_host)) + host = idn_host; +#endif /* if we have a RSYNC_PROXY env variable then redirect our * connection via a web proxy at the given address. */ diff --git a/support/install_deps_ubuntu.sh b/support/install_deps_ubuntu.sh index ac49055bb..619e3d78e 100755 --- a/support/install_deps_ubuntu.sh +++ b/support/install_deps_ubuntu.sh @@ -9,3 +9,4 @@ sudo apt install -y libxxhash-dev sudo apt install -y libzstd-dev sudo apt install -y liblz4-dev sudo apt install -y libssl-dev +sudo apt install -y libidn2-dev diff --git a/testsuite/daemon-access-idn_test.py b/testsuite/daemon-access-idn_test.py new file mode 100755 index 000000000..a265e5ac4 --- /dev/null +++ b/testsuite/daemon-access-idn_test.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Daemon coverage: IDN hosts allow / hosts deny matching (access.c). + +A daemon gets its peer's name from DNS as ASCII, so an rsyncd.conf entry +written with non-ASCII characters is folded to its IDNA A-label form before +being matched. The IDNA mapping also folds some non-ASCII characters onto +ASCII ones (U+FF0A FULLWIDTH ASTERISK becomes '*'), so the checks below cover +both directions: a Unicode name that has to match, and the tokens that must +stay denied rather than turn into a wildcard or an address/mask. Punycode, +mixed case and an unconvertible name are covered too. + +The peer name isn't assumed: a throwaway daemon comes up first to log the name +this host's resolver gives it, and the real config is written around that. +"forward lookup" is off throughout, which pins the match on the reverse-DNS +name and keeps a denied module from waiting out a resolver timeout on a name +that deliberately doesn't exist. + +Like daemon-access-ip_test.py this needs a real TCP peer (--use-tcp), and the +config sets no global hosts allow so each module's own patterns decide. +""" + +import re +import subprocess + +from rsyncfns import ( + FROMDIR, SCRATCHDIR, + claim_ports, make_tree, require_tcp, rmtree, rsync_argv, start_rsyncd, + start_test_daemon, test_fail, test_skipped, +) + +PROBE_PORT = 12896 +DAEMON_PORT = 12898 +require_tcp("hosts allow/deny hostname matching needs a real TCP peer") + +if '"IDN": true' not in subprocess.run(rsync_argv('-VV'), capture_output=True, + text=True).stdout: + test_skipped("rsync built without IDN support") + +src = FROMDIR +rmtree(src) +make_tree(src, depth=2) + + +def write_conf(path, modules, log, pidfile): + lines = [ + '# autogenerated by daemon-access-idn_test.py', + f'pid file = {pidfile}', + 'use chroot = no', + 'forward lookup = no', + f'log file = {log}', + '', + ] + for mod, params in modules: + lines.append(f'[{mod}]') + lines.append(f'\tpath = {src}') + lines.append('\tread only = yes') + lines += [f'\t{k} = {v}' for k, v in params.items()] + lines.append('') + # rsyncd.conf is read as UTF-8 by the daemon whatever the test's locale is. + path.write_text('\n'.join(lines) + '\n', encoding='utf-8') + return path + + +# --- find out what this host's resolver calls the loopback peer ------------- +# A throwaway daemon with one wide-open module: connect once, read the name it +# logged for us, then shut it down before the real config goes up. +probe_log = SCRATCHDIR / 'rsyncd-idn-probe.log' +probe_conf = write_conf(SCRATCHDIR / 'access-idn-probe.conf', [('probe', {})], + probe_log, SCRATCHDIR / 'rsyncd-idn-probe.pid') +claim_ports(PROBE_PORT) +probe = start_rsyncd(probe_conf, PROBE_PORT) +try: + subprocess.run(rsync_argv('-r', f'rsync://localhost:{PROBE_PORT}/probe/'), + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) +finally: + probe.terminate() + probe.wait(timeout=5) + +m = re.search(r'connect from (\S+) \(', probe_log.read_text(errors='replace')) +if not m: + test_fail(f"no 'connect from' line in the probe daemon log {probe_log}") +peer = m.group(1) +print(f"daemon sees its peer as {peer!r}") + + +def fullwidth(name): + """The fullwidth-forms spelling of an ASCII name. + + IDNA (UTS #46) maps U+FF01..U+FF5E back onto ASCII, so this gives us a + genuinely non-ASCII name that folds to a peer name we can predict without + controlling DNS. Label separators stay ASCII dots. + """ + return ''.join(chr(ord(c) - 0x21 + 0xFF01) if '!' <= c <= '~' and c != '.' + else c for c in name) + + +if fullwidth(peer) == peer: + test_skipped(f"peer name {peer!r} has no ASCII to respell in fullwidth") + +ZWSP = '​' # maps to nothing, leaving no label at all +FW_STAR = '*' # FULLWIDTH ASTERISK, which IDNA maps to '*' +FW_SLASH = '/' # FULLWIDTH SOLIDUS, which IDNA maps to '/' + +modules = [ + ('ascii-name', {'hosts allow': peer}), + ('ascii-upper', {'hosts allow': peer.upper()}), + ('ascii-wild', {'hosts allow': peer[:1] + '*'}), + # The same name in fullwidth forms, which only matches once the token has + # been folded to A-labels. + ('idn-name', {'hosts allow': fullwidth(peer)}), + ('idn-mixedcase', {'hosts allow': fullwidth(peer.upper())}), + ('idn-deny', {'hosts deny': fullwidth(peer)}), + # A real IDN that is not the peer, plus its punycode spelling: both must + # stay denied, and neither may be mistaken for a wildcard. + ('idn-other', {'hosts allow': 'čičku.example'}), + ('idn-puny', {'hosts allow': 'xn--iku-eqab.example'}), + # Tokens whose IDNA mapping yields ASCII the author never wrote. Left + # unconverted they match nothing; converted blindly, the first two would + # allow every host and the third would read as an address/mask. + ('wide-star', {'hosts allow': FW_STAR}), + ('wide-star-dom', {'hosts allow': FW_STAR + '.example'}), + ('wide-mask', {'hosts allow': '127.0.0.0' + FW_SLASH + '8'}), + # An IDN that can't be converted at all (its label maps to nothing). + ('bad-idn', {'hosts allow': ZWSP + '.example'}), +] + +conf = write_conf(SCRATCHDIR / 'access-idn.conf', modules, + SCRATCHDIR / 'rsyncd.log', SCRATCHDIR / 'rsyncd.pid') +url = start_test_daemon(conf, DAEMON_PORT) + + +def connect(mod): + """Return rsync's exit code for listing the module over the daemon.""" + return subprocess.run(rsync_argv('-r', f'{url}{mod}/'), + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, + text=True).returncode + + +def allowed(mod, why): + if connect(mod) != 0: + test_fail(f"connection to {mod} should be ALLOWED ({why}) but was refused") + + +def denied(mod, why): + if connect(mod) == 0: + test_fail(f"connection to {mod} should be DENIED ({why}) but succeeded") + + +allowed('ascii-name', "the peer's own name in a hosts allow") +allowed('ascii-upper', "hostname matching is case-insensitive") +allowed('ascii-wild', "an ASCII wildcard still matches") +allowed('idn-name', f"fullwidth {peer!r} folds to the peer's name") +allowed('idn-mixedcase', "IDNA case-folds the token") +denied('idn-deny', "hosts deny sees the folded token too") +denied('idn-other', "a different IDN must not match the peer") +denied('idn-puny', "an A-label for a different host must not match the peer") +denied('wide-star', "U+FF0A must not become a '*' that allows every host") +denied('wide-star-dom', "U+FF0A must not become a wildcard label") +denied('wide-mask', "U+FF0F must not become an address/mask separator") +denied('bad-idn', "an unconvertible IDN must not match anything") + +print("daemon-access-idn: IDN hosts allow/deny matching + no wildcard widening") diff --git a/testsuite/idn_test.py b/testsuite/idn_test.py new file mode 100644 index 000000000..326deae30 --- /dev/null +++ b/testsuite/idn_test.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +# Verify that rsync converts an IDN (internationalized domain name) host to +# its IDNA A-label (Punycode) form, and that it leaves an ASCII host name +# alone. Only the labels that are not ASCII get rewritten, so an address +# literal, an already-punycoded name, and a name that isn't a valid IDN all +# reach the resolver as typed. +# +# Two daemon connection methods carry the host name out of rsync, so both are +# checked: +# * daemon over a remote shell (what rsync-ssl does): the host is handed to +# the --rsh helper. +# * direct daemon socket: observed through a dummy HTTP proxy (RSYNC_PROXY) on +# loopback, so this part only runs under --use-tcp. +# A plain remote-shell transfer (host:path) is intentionally left alone, since +# that name belongs to the user's ssh. +# +# The daemon side of IDN -- hosts allow/deny matching -- is daemon-access-idn. + +import os +import shlex +import socket +import subprocess +import sys +import threading + +from rsyncfns import ( + RSYNC, SCRATCHDIR, USE_TCP, claim_ports, run_rsync, + test_fail, test_skipped, +) + + +if '"IDN": true' not in run_rsync('-VV', check=True, capture_output=True).stdout: + test_skipped("rsync built without IDN support") + + +def find_utf8_locale(): + try: + out = subprocess.check_output(['locale', '-a'], text=True, + stderr=subprocess.DEVNULL) + except (OSError, subprocess.CalledProcessError): + return None + avail = out.split() + for want in ('C.UTF-8', 'C.utf8', 'en_US.UTF-8', 'en_US.utf8'): + if want in avail: + return want + for loc in avail: + if loc.lower().replace('-', '').endswith('utf8'): + return loc + return None + + +utf8_locale = find_utf8_locale() +if not utf8_locale: + test_skipped("no UTF-8 locale available to encode the IDN host") + +idn_host = "\u010ci\u010dku.example" +ascii_host = "xn--iku-eqab.example" + +env = os.environ.copy() +env['LC_ALL'] = utf8_locale +out_dir = (str(SCRATCHDIR / 'out') + '/').encode() + + +def run_idn(url, *extra, extra_env=None): + # A bytes argv keeps the UTF-8 host intact regardless of Python's + # filesystem encoding. + e = dict(env) + if extra_env: + e.update(extra_env) + argv = [a.encode() for a in shlex.split(RSYNC)] + argv += [a.encode() for a in extra] + argv += [url.encode('utf-8'), out_dir] + return subprocess.run(argv, capture_output=True, env=e, timeout=30) + + +# --- daemon over a remote shell (the rsync-ssl mechanism) ------------------ +helper = SCRATCHDIR / 'idn-rsh.sh' +helper.write_text('#!/bin/sh\nprintf %s "$1" > "$IDN_RSH_OUT"\nexit 1\n') +helper.chmod(0o755) + +hostfile = SCRATCHDIR / 'idn-rsh-host' + + +def rsh_host(url_host): + """The host name rsync hands the --rsh helper for rsync:///.""" + if hostfile.exists(): + hostfile.unlink() + run_idn(f"rsync://{url_host}/module/", f"--rsh={helper}", + extra_env={'IDN_RSH_OUT': str(hostfile)}) + if not hostfile.exists(): + test_fail(f"the --rsh helper never ran for {url_host!r}") + return hostfile.read_bytes().decode('utf-8', 'surrogateescape') + + +def check_rsh(url_host, want, what): + got = rsh_host(url_host) + if got != want: + test_fail(f"daemon-over-rsh sent host {got!r} for {what} " + f"({url_host!r}), expected {want!r}") + print(f"OK: {what} -> {got}") + + +# A U-label becomes its A-label, case-folded by the IDNA mapping. An ASCII +# label is handed on byte for byte, case included, since DNS doesn't care. +check_rsh(idn_host, ascii_host, "a Unicode host") +check_rsh("ČIČKU.Example", "xn--iku-eqab.Example", + "a mixed-case Unicode host") +check_rsh(ascii_host, ascii_host, "an already-punycoded host") +check_rsh("XN--IKU-EQAB.Example", "XN--IKU-EQAB.Example", + "a mixed-case punycoded host") +# A name that isn't a valid IDN goes out as-is instead of being rewritten into +# some other name (the U+200B one would map to ".example"), so the resolver +# fails on it just as it did before. +check_rsh("xn--0.example", "xn--0.example", "an undecodable A-label") +check_rsh("ـx.example", "ـx.example", "a label with a disallowed character") +check_rsh("​.example", "​.example", "a label that maps to nothing") + + +# --- direct daemon socket, observed via a dummy proxy ----------------------- +if not USE_TCP: + print("direct-socket proxy check needs --use-tcp; skipping that part") + sys.exit(0) + +PROXY_PORT = 13335 +claim_ports(PROXY_PORT) + +listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +listener.bind(('127.0.0.1', PROXY_PORT)) +listener.listen(1) + +captured = {} + + +def serve_one(): + conn, _ = listener.accept() + conn.settimeout(5) + data = b"" + try: + while b"\r\n\r\n" not in data and len(data) < 65536: + chunk = conn.recv(8192) + if not chunk: + break + data += chunk + except socket.timeout: + pass + captured['request'] = data + try: + conn.sendall(b"HTTP/1.0 403 Forbidden\r\n\r\n") + conn.shutdown(socket.SHUT_RDWR) + except OSError: + pass + conn.close() + + +t = threading.Thread(target=serve_one) +t.daemon = True +t.start() + +proc = run_idn(f"rsync://{idn_host}:873/whatever/", + extra_env={'RSYNC_PROXY': f'127.0.0.1:{PROXY_PORT}'}) + +t.join(timeout=15) +listener.close() + +if proc.returncode >= 128: + sys.stderr.write(proc.stderr.decode('latin1')) + test_fail(f"rsync killed by signal (status={proc.returncode})") + +request = captured.get('request', b'') +if not request: + test_fail("dummy proxy received no CONNECT request from rsync") + +if ascii_host.encode() not in request: + sys.stderr.write("proxy received: %r\n" % request.split(b"\r\n", 1)[0]) + test_fail(f"expected A-label {ascii_host} in the proxy CONNECT request") + +print(f"OK: direct-socket CONNECT host sent as {ascii_host}") diff --git a/usage.c b/usage.c index f346385f4..0b3e7fcf5 100644 --- a/usage.c +++ b/usage.c @@ -138,6 +138,11 @@ static void print_info_flags(enum logcode f) #endif "crtimes", +#ifndef SUPPORT_IDN + "no " +#endif + "IDN", + "*Optimizations", #ifndef USE_ROLL_SIMD diff --git a/util1.c b/util1.c index d9d9f4bcf..ab7360432 100644 --- a/util1.c +++ b/util1.c @@ -24,6 +24,9 @@ #include "ifuncs.h" #include "itypes.h" #include "inums.h" +#ifdef SUPPORT_IDN +#include +#endif extern int dry_run; extern int module_id; @@ -838,16 +841,106 @@ void glob_expand_module(char *base1, char *arg, char ***argv_p, int *argc_p, int /** * Convert a string to lower case + * + * Only ASCII is folded. The hosts allow/deny list that calls this can hold + * UTF-8, and a per-byte fold via the locale's ctype would mangle it (in + * ISO-8859-1 the 0xC4 lead byte of "č" is an upper-case 'Ä'). **/ void strlower(char *s) { while (*s) { - if (isUpper(s)) + if (!(*(unsigned char *)s & 0x80) && isUpper(s)) *s = toLower(s); s++; } } +#ifdef SUPPORT_IDN +/* Does this label hold nothing but the [-a-z0-9] of an A-label? */ +static int is_a_label(const char *s) +{ + if (!*s) + return 0; + + for ( ; *s; s++) { + if (!(*s >= 'a' && *s <= 'z') && !(*s >= '0' && *s <= '9') && *s != '-') + return 0; + } + + return 1; +} + +/** + * Convert the non-ASCII labels of a host name into their IDNA A-label + * (Punycode) form, putting the result in buf. Returns 1 if buf was filled in, + * or 0 to tell the caller to keep the name it has. + * + * A label that is already ASCII is copied verbatim, so an address, a mask, an + * xn-- name, and any wildmatch characters come out just as they went in. A + * converted label is only used if it comes back as a bare A-label: the IDNA + * mapping folds some non-ASCII characters onto ASCII ones (U+FF0A FULLWIDTH + * ASTERISK becomes '*'), and a hosts allow/deny entry must not pick up a + * wildcard that its author never typed. Anything else leaves the name alone, + * which fails to match instead of matching too much. + * + * Set from_locale for a name that came from the command line, which is in the + * user's locale encoding; the daemon's config file is read as UTF-8. + **/ +int idn_to_ascii(const char *name, int from_locale, char *buf, size_t buflen) +{ + const char *lab, *end; + size_t len = 0; + int converted = 0; + + for (lab = name; ; lab = end + 1) { + char label[256], *idn; + size_t lablen, alen; + int is_ascii = 1; + + for (end = lab; *end && *end != '.'; end++) { + if (*(unsigned char *)end & 0x80) + is_ascii = 0; + } + lablen = end - lab; + + if (is_ascii) { + if (len + lablen + 2 > buflen) + return 0; + memcpy(buf + len, lab, lablen); + len += lablen; + } else { + int rc; + if (lablen >= sizeof label) + return 0; + memcpy(label, lab, lablen); + label[lablen] = '\0'; + rc = from_locale + ? idn2_lookup_ul(label, &idn, IDN2_NONTRANSITIONAL) + : idn2_to_ascii_8z(label, &idn, IDN2_NFC_INPUT | IDN2_NONTRANSITIONAL); + if (rc != IDN2_OK) + return 0; + alen = strlen(idn); + if (!is_a_label(idn) || len + alen + 2 > buflen) { + idn2_free(idn); + return 0; + } + memcpy(buf + len, idn, alen); + len += alen; + idn2_free(idn); + converted = 1; + } + + if (!*end) + break; + buf[len++] = '.'; + } + + buf[len] = '\0'; + + return converted; +} +#endif + /** * Split a string into tokens based (usually) on whitespace & commas. If the * string starts with a comma (after skipping any leading whitespace), then