Use epoll() instead of select()

Signed-off-by: David Woodhouse <dwmw2@infradead.org>

configure.ac

@@ -928,6 +928,8 @@ if test "$libpcsclite_pkg" = "yes"; then
 fi
 AM_CONDITIONAL(OPENCONNECT_LIBPCSCLITE, [test "$libpcsclite_pkg" = "yes"])
 
+AC_CHECK_FUNC(epoll_create1, [AC_DEFINE(HAVE_EPOLL, 1, [Have epoll])], [])
+
 AC_ARG_WITH([libpskc],
 	AS_HELP_STRING([--without-libpskc],
 	[Build without libpskc library [default=auto]]))

dtls.c

@@ -153,10 +153,10 @@ void dtls_close(struct openconnect_info *vpninfo)
 {
 	if (vpninfo->dtls_ssl) {
 		dtls_ssl_free(vpninfo);
-		closesocket(vpninfo->dtls_fd);
 		unmonitor_read_fd(vpninfo, dtls);
 		unmonitor_write_fd(vpninfo, dtls);
 		unmonitor_except_fd(vpninfo, dtls);
+		closesocket(vpninfo->dtls_fd);
 		vpninfo->dtls_ssl = NULL;
 		vpninfo->dtls_fd = -1;
 	}

esp.c

@@ -390,10 +390,10 @@ void esp_close(struct openconnect_info *vpninfo)
 	/* We close and reopen the socket in case we roamed and our
 	   local IP address has changed. */
 	if (vpninfo->dtls_fd != -1) {
-		closesocket(vpninfo->dtls_fd);
 		unmonitor_read_fd(vpninfo, dtls);
 		unmonitor_write_fd(vpninfo, dtls);
 		unmonitor_except_fd(vpninfo, dtls);
+		closesocket(vpninfo->dtls_fd);
 		vpninfo->dtls_fd = -1;
 	}
 	if (vpninfo->dtls_state > DTLS_DISABLED)

gnutls.c

@@ -2471,10 +2471,10 @@ void openconnect_close_https(struct openconnect_info *vpninfo, int final)
 		vpninfo->https_sess = NULL;
 	}
 	if (vpninfo->ssl_fd != -1) {
-		closesocket(vpninfo->ssl_fd);
 		unmonitor_read_fd(vpninfo, ssl);
 		unmonitor_write_fd(vpninfo, ssl);
 		unmonitor_except_fd(vpninfo, ssl);
+		closesocket(vpninfo->ssl_fd);
 		vpninfo->ssl_fd = -1;
 	}
 	if (final && vpninfo->https_cred) {

library.c

@@ -95,7 +95,9 @@ struct openconnect_info *openconnect_vpninfo_new(const char *useragent,
 	vpninfo->proxy_auth[AUTH_TYPE_BASIC].state = AUTH_DEFAULT_DISABLED;
 	vpninfo->http_auth[AUTH_TYPE_BASIC].state = AUTH_DEFAULT_DISABLED;
 	openconnect_set_reported_os(vpninfo, NULL);
-
+#ifdef HAVE_EPOLL
+	vpninfo->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
+#endif
 	if (!vpninfo->localname || !vpninfo->useragent)
 		goto err;
 
@@ -688,6 +690,11 @@ void openconnect_vpninfo_free(struct openconnect_info *vpninfo)
 	inflateEnd(&vpninfo->inflate_strm);
 	deflateEnd(&vpninfo->deflate_strm);
 
+#ifdef HAVE_EPOLL
+	if (vpninfo->epoll_fd >= 0)
+		close(vpninfo->epoll_fd);
+#endif
+
 	free_pkt(vpninfo, vpninfo->deflate_pkt);
 	free_pkt(vpninfo, vpninfo->tun_pkt);
 	free_pkt(vpninfo, vpninfo->dtls_pkt);

mainloop.c

@@ -318,6 +318,49 @@ int openconnect_mainloop(struct openconnect_info *vpninfo,
 			free(errstr);
 		}
 #else
+#ifdef HAVE_EPOLL
+		if (vpninfo->epoll_fd >= 0) {
+			struct epoll_event evs[5];
+
+			/* During busy periods, monitor_read_fd() and unmonitor_read_fd()
+			 * may get called multiple times as we go round and round the
+			 * loop and queues get full then have space again. In the past
+			 * with the select() loop, that was only a bitflip in the fd_set
+			 * and didn't cost much. With epoll() it's actually a system
+			 * call, so don't do it every time. Wait until we're about to
+			 * sleep, and *then* ensure that we call epoll_ctl() to sync the
+			 * set of events that we care about, if it's changed. */
+			if (vpninfo->epoll_update) {
+				update_epoll_fd(vpninfo, tun);
+				update_epoll_fd(vpninfo, ssl);
+				update_epoll_fd(vpninfo, cmd);
+				update_epoll_fd(vpninfo, dtls);
+			}
+
+			tun_r = udp_r = tcp_r = 0;
+
+			int nfds = epoll_wait(vpninfo->epoll_fd, evs, 5, timeout);
+			if (nfds < 0) {
+				if (errno != EINTR) {
+					ret = -errno;
+					vpn_perror(vpninfo, _("Failed epoll_wait() in mainloop"));
+					break;
+				}
+				nfds = 0;
+			}
+			while (nfds--) {
+				if (evs[nfds].events & EPOLLIN) {
+					if (evs[nfds].data.fd == vpninfo->tun_fd)
+						tun_r = 1;
+					else if (evs[nfds].data.fd == vpninfo->ssl_fd)
+						tcp_r = 1;
+					else if (evs[nfds].data.fd == vpninfo->dtls_fd)
+						udp_r = 1;
+				}
+			}
+			continue;
+		}
+#endif
 		memcpy(&rfds, &vpninfo->_select_rfds, sizeof(rfds));
 		memcpy(&wfds, &vpninfo->_select_wfds, sizeof(wfds));
 		memcpy(&efds, &vpninfo->_select_efds, sizeof(efds));

openconnect-internal.h

@@ -87,6 +87,7 @@
 #include <sys/types.h>
 #include <unistd.h>
 #include <string.h>
+#include <errno.h>
 
 #ifdef LIBPROXY_HDR
 #include LIBPROXY_HDR
@@ -108,6 +109,10 @@
 #include <libp11.h>
 #endif
 
+#ifdef HAVE_EPOLL
+#include <sys/epoll.h>
+#endif
+
 #ifdef ENABLE_NLS
 #include <libintl.h>
 #define _(s) dgettext("openconnect", s)
@@ -706,6 +711,11 @@ struct openconnect_info {
 	fd_set _select_rfds;
 	fd_set _select_wfds;
 	fd_set _select_efds;
+#ifdef HAVE_EPOLL
+	int epoll_fd;
+	int epoll_update;
+	uint32_t tun_epoll, ssl_epoll, dtls_epoll, cmd_epoll;
+#endif
 #endif
 
 #ifdef __sun__
@@ -818,6 +828,12 @@ static inline void free_pkt(struct openconnect_info *vpninfo, struct pkt *pkt)
 		free(pkt);
 }
 
+#define vpn_progress(_v, lvl, ...) do {					\
+	if ((_v)->verbose >= (lvl))					\
+		(_v)->progress((_v)->cbdata, lvl, __VA_ARGS__);	\
+	} while(0)
+#define vpn_perror(vpninfo, msg) vpn_progress((vpninfo), PRG_ERR, "%s: %s\n", (msg), strerror(errno))
+
 #ifdef _WIN32
 #define monitor_read_fd(_v, _n) _v->_n##_monitored |= FD_READ
 #define monitor_write_fd(_v, _n) _v->_n##_monitored |= FD_WRITE
@@ -830,17 +846,82 @@ static inline void free_pkt(struct openconnect_info *vpninfo, struct pkt *pkt)
 #define read_fd_monitored(_v, _n) (_v->_n##_monitored & FD_READ)
 
 #else
-#define monitor_read_fd(_v, _n) FD_SET(_v-> _n##_fd, &vpninfo->_select_rfds)
-#define unmonitor_read_fd(_v, _n) FD_CLR(_v-> _n##_fd, &vpninfo->_select_rfds)
-#define monitor_write_fd(_v, _n) FD_SET(_v-> _n##_fd, &vpninfo->_select_wfds)
-#define unmonitor_write_fd(_v, _n) FD_CLR(_v-> _n##_fd, &vpninfo->_select_wfds)
-#define monitor_except_fd(_v, _n) FD_SET(_v-> _n##_fd, &vpninfo->_select_efds)
-#define unmonitor_except_fd(_v, _n) FD_CLR(_v-> _n##_fd, &vpninfo->_select_efds)
-
-#define monitor_fd_new(_v, _n) do { \
-		if (_v->_select_nfds <= vpninfo->_n##_fd) \
-			vpninfo->_select_nfds = vpninfo->_n##_fd + 1; \
-	} while (0)
+
+
+#ifdef HAVE_EPOLL
+static inline void __sync_epoll_fd(struct openconnect_info *vpninfo, int fd, uint32_t *fd_evts)
+{
+	if (vpninfo->epoll_fd >= 0 && fd >= 0) {
+		struct epoll_event ev = { 0 };
+		ev.data.fd = fd;
+		if (FD_ISSET(fd, &vpninfo->_select_rfds))
+			ev.events |= EPOLLIN;
+		if (FD_ISSET(fd, &vpninfo->_select_wfds))
+			ev.events |= EPOLLOUT;
+		if (ev.events != *fd_evts) {
+			if (epoll_ctl(vpninfo->epoll_fd, EPOLL_CTL_MOD, fd, &ev)) {
+				vpn_perror(vpninfo, "EPOLL_CTL_MOD");
+				close(vpninfo->epoll_fd);
+				vpninfo->epoll_fd = -1;
+			}
+			*fd_evts = ev.events;
+		}
+	}
+}
+#define update_epoll_fd(_v, _n) __sync_epoll_fd(_v, _v->_n##_fd, &_v->_n##_epoll)
+#endif
+
+static inline void __monitor_fd(struct openconnect_info *vpninfo,
+				int fd, fd_set *set)
+{
+	if (fd < 0 || FD_ISSET(fd, set))
+		return;
+
+	FD_SET(fd, set);
+#ifdef HAVE_EPOLL
+	vpninfo->epoll_update = 1;
+#endif
+}
+
+static inline void __unmonitor_fd(struct openconnect_info *vpninfo,
+				  int fd, fd_set *set)
+{
+	if (fd < 0 || !FD_ISSET(fd, set))
+		return;
+
+	FD_CLR(fd, set);
+#ifdef HAVE_EPOLL
+	vpninfo->epoll_update = 1;
+#endif
+}
+
+#define monitor_read_fd(_v, _n) __monitor_fd(_v, _v->_n##_fd, &_v->_select_rfds)
+#define unmonitor_read_fd(_v, _n) __unmonitor_fd(_v, _v->_n##_fd, &_v->_select_rfds)
+#define monitor_write_fd(_v, _n) __monitor_fd(_v, _v->_n##_fd, &_v->_select_wfds)
+#define unmonitor_write_fd(_v, _n) __unmonitor_fd(_v, _v->_n##_fd, &_v->_select_wfds)
+#define monitor_except_fd(_v, _n) __monitor_fd(_v, _v->_n##_fd, &_v->_select_efds)
+#define unmonitor_except_fd(_v, _n) __unmonitor_fd(_v, _v->_n##_fd, &_v->_select_efds)
+
+static inline void __monitor_fd_new(struct openconnect_info *vpninfo,
+				    int fd)
+{
+	if (vpninfo->_select_nfds <= fd)
+		vpninfo->_select_nfds = fd + 1;
+#ifdef HAVE_EPOLL
+	if (vpninfo->epoll_fd >= 0) {
+		struct epoll_event ev = { 0 };
+		ev.data.fd = fd;
+		if (epoll_ctl(vpninfo->epoll_fd, EPOLL_CTL_ADD, fd, &ev)) {
+			vpn_perror(vpninfo, "EPOLL_CTL_ADD");
+			close(vpninfo->epoll_fd);
+			vpninfo->epoll_fd = -1;
+		}
+	}
+#endif
+}
+
+
+#define monitor_fd_new(_v, _n) __monitor_fd_new(_v, _v->_n##_fd)
 
 #define read_fd_monitored(_v, _n) FD_ISSET(_v->_n##_fd, &_v->_select_rfds)
 #endif
@@ -872,12 +953,6 @@ static inline void free_pkt(struct openconnect_info *vpninfo, struct pkt *pkt)
 #define MAX_IV_SIZE		16
 #define MAX_ESP_PAD		17	/* Including the next-header field */
 
-#define vpn_progress(_v, lvl, ...) do {					\
-	if ((_v)->verbose >= (lvl))					\
-		(_v)->progress((_v)->cbdata, lvl, __VA_ARGS__);	\
-	} while(0)
-#define vpn_perror(vpninfo, msg) vpn_progress((vpninfo), PRG_ERR, "%s: %s\n", (msg), strerror(errno))
-
 /****************************************************************************/
 /* Oh Solaris how we hate thee! */
 #ifdef HAVE_SUNOS_BROKEN_TIME

openssl.c

@@ -1998,10 +1998,10 @@ void openconnect_close_https(struct openconnect_info *vpninfo, int final)
 		vpninfo->https_ssl = NULL;
 	}
 	if (vpninfo->ssl_fd != -1) {
-		closesocket(vpninfo->ssl_fd);
 		unmonitor_read_fd(vpninfo, ssl);
 		unmonitor_write_fd(vpninfo, ssl);
 		unmonitor_except_fd(vpninfo, ssl);
+		closesocket(vpninfo->ssl_fd);
 		vpninfo->ssl_fd = -1;
 	}
 	if (final) {

www/changelog.xml

@@ -15,6 +15,7 @@
 <ul>
    <li><b>OpenConnect HEAD</b>
      <ul>
+       <li>Use <tt>epoll()</tt> where available.</li>
        <li>Support non-AEAD ciphersuites in DTLSv1.2 with AnyConnect. (<a href="https://gitlab.com/openconnect/openconnect/-/issues/249">#249</a>)</li>
        <li>Make <tt>tncc-emulate.py</tt> work with Python 3.7+. (<a href="https://gitlab.com/openconnect/openconnect/-/issues/152">!152</a>, <a href="https://gitlab.com/openconnect/openconnect/merge_requests/120">!120</a>)</li>
        <li>Emulated a newer version of GlobalProtect official clients, 5.1.5-8; was 4.0.2-19 (<a href="https://gitlab.com/openconnect/openconnect/merge_requests/131">!131</a>)</li>