Skip to content

Commit

Permalink
tools: add supports for proxy protocol
Browse files Browse the repository at this point in the history 
Signed-off-by: ywc689 <ywc689@163.com>
  • Loading branch information
@ywc689
ywc689 committed Dec 8, 2023
1 parent e879335 commit 45ad38b
Show file tree
Hide file tree
Showing 4 changed files with 193 additions and 76 deletions.
1 change: 1 addition & 0 deletions include/conf/service.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ enum {
PROXY_PROTOCOL_DISABLE = 0,
PROXY_PROTOCOL_V1,
PROXY_PROTOCOL_V2,
PROXY_PROTOCOL_MAX,
};

struct dest_check_configs {
Expand Down
56 changes: 47 additions & 9 deletions tools/healthcheck/pkg/helthcheck/http_checker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,13 @@
package hc

import (
"bytes"
"context"
"crypto/tls"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"strings"
Expand All @@ -30,6 +33,14 @@ import (

var _ CheckMethod = (*HttpChecker)(nil)

var (
proxyProtoV1LocalCmd = "PROXY UNKNOWN\r\n"
proxyProtoV2LocalCmd []byte = []byte{
0x0D, 0x0A, 0x0D, 0x0A, 0x00, 0x0D, 0x0A, 0x51,
0x55, 0x49, 0x54, 0x0A, 0x20, 0x00, 0x00, 0x00,
}
)

type HttpCodeRange struct {
start int // inclusive
end int // inclusive
Expand All @@ -45,13 +56,14 @@ type HttpChecker struct {
ResponseCodes []HttpCodeRange
Response string

Secure bool
TLSVerify bool
Proxy bool
Secure bool
TLSVerify bool
Proxy bool
ProxyProto int // proxy protocol: 0 - close, 1 - version 1, 2 - version 2
}

// NewHttpChecker returns an initialised HttpChecker.
func NewHttpChecker(method, host, uri string) *HttpChecker {
func NewHttpChecker(method, host, uri string, proxyProto int) *HttpChecker {
if len(method) == 0 {
method = "GET"
}
Expand All @@ -66,6 +78,7 @@ func NewHttpChecker(method, host, uri string) *HttpChecker {
Secure: false,
TLSVerify: true,
Proxy: false,
ProxyProto: proxyProto,
}
}

Expand Down Expand Up @@ -127,12 +140,37 @@ func (hc *HttpChecker) Check(target Target, timeout time.Duration) *Result {
tlsConfig := &tls.Config{
InsecureSkipVerify: !hc.TLSVerify,
}

tr := &http.Transport{
Proxy: proxy,
TLSClientConfig: tlsConfig,
}
if hc.ProxyProto != 0 {
tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
conn, err := (&net.Dialer{}).DialContext(ctx, network, addr)
if err != nil {
return nil, err
}
// Alternatively, use the go-proxyproto package:
// https://pkg.go.dev/github.com/pires/go-proxyproto
if hc.ProxyProto == 2 {
n, err := bytes.NewReader(proxyProtoV2LocalCmd).WriteTo(conn)
if err != nil || n < 16 {
return nil, err
}
} else if hc.ProxyProto == 1 {
n, err := strings.NewReader(proxyProtoV1LocalCmd).WriteTo(conn)
if err != nil || n < int64(len(proxyProtoV1LocalCmd)) {
return nil, err
}
}
return conn, nil
}
}

client := &http.Client{
Transport: &http.Transport{
Proxy: proxy,
TLSClientConfig: tlsConfig,
},
Timeout: timeout,
Transport: tr,
Timeout: timeout,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return errors.New("redirect not permitted")
},
Expand Down
26 changes: 24 additions & 2 deletions tools/healthcheck/pkg/helthcheck/http_checker_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,11 @@ var http_targets = []Target{
{net.ParseIP("192.168.88.30"), 443, utils.IPProtoTCP},
{net.ParseIP("2001::30"), 80, utils.IPProtoTCP},
{net.ParseIP("2001::30"), 443, utils.IPProtoTCP},
{net.ParseIP("192.168.88.30"), 8002, utils.IPProtoTCP}, // control group for proxy protocol
}

var http_proxy_proto_targets = []Target{
{net.ParseIP("192.168.88.30"), 8002, utils.IPProtoTCP},
}

var http_url_targets = []string{
Expand All @@ -43,7 +48,7 @@ var http_url_targets = []string{

func TestHttpChecker(t *testing.T) {
for _, target := range http_targets {
checker := NewHttpChecker("", "", "")
checker := NewHttpChecker("", "", "", 0)
checker.Host = target.Addr()
/*
if target.Port == 443 {
Expand All @@ -57,9 +62,26 @@ func TestHttpChecker(t *testing.T) {
fmt.Printf("[ HTTP ] %s ==> %v\n", target, result)
}

for _, target := range http_proxy_proto_targets {
checker := NewHttpChecker("", "", "", 1)
checker.Host = target.Addr()
id := Id(target.String())
config := NewCheckerConfig(&id, checker, &target, StateUnknown,
0, 3*time.Second, 2*time.Second, 3)
result := checker.Check(target, config.Timeout)
fmt.Printf("[ HTTP(PPv1) ] %s ==> %v\n", target, result)
checker2 := NewHttpChecker("", "", "", 2)
checker2.Host = target.Addr()
id2 := Id(target.String())
config2 := NewCheckerConfig(&id2, checker2, &target, StateUnknown,
0, 3*time.Second, 2*time.Second, 3)
result2 := checker2.Check(target, config2.Timeout)
fmt.Printf("[ HTTP(PPv2) ] %s ==> %v\n", target, result2)
}

for _, target := range http_url_targets {
host := target[strings.Index(target, "://")+3:]
checker := NewHttpChecker("GET", target, "")
checker := NewHttpChecker("GET", target, "", 0)
checker.Host = host
checker.ResponseCodes = []HttpCodeRange{{200, 200}}
if strings.HasPrefix(target, "https") {
Expand Down
186 changes: 121 additions & 65 deletions tools/keepalived/keepalived/check/check_http.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#include <assert.h>

#ifdef _WITH_REGEX_CHECK_
#define PCRE2_CODE_UNIT_WIDTH 8
Expand Down Expand Up @@ -923,6 +924,8 @@ install_ssl_check_keyword(void)
*
* http_connect_thread (handle layer4 connect)
* v
* http_send_proxy_protocol (handle proxy protocol)
* v
* http_check_thread (handle SSL connect)
* v
* http_request_thread (send SSL GET request)
Expand Down Expand Up @@ -1541,6 +1544,45 @@ http_request_thread(thread_ref_t thread)
return 1;
}

static int http_send_proxy_protocol(thread_ref_t thread)
{
checker_t *checker = THREAD_ARG(thread);
virtual_server_t *vs = checker->vs;

unsigned int len;
char *ppbuf;
const char ppv2_local_cmd[] = {
0x0D, 0x0A, 0x0D, 0x0A, 0x00, 0x0D, 0x0A, 0x51,
0x55, 0x49, 0x54, 0x0A, 0x20, 0x00, 0x00, 0x00,
};

assert(vs->proxy_protocol >= PROXY_PROTOCOL_DISABLE
&& vs->proxy_protocol < PROXY_PROTOCOL_MAX);

ppbuf = (char *) MALLOC(16);
if (!ppbuf) {
return -1;
}
memset(ppbuf, 0, 16);

if (PROXY_PROTOCOL_V1 == vs->proxy_protocol) {
len = 15;
sprintf(ppbuf, "%s", "PROXY UNKNOWN\r\n");
} else if (PROXY_PROTOCOL_V2 == vs->proxy_protocol) {
len = 16;
memcpy(ppbuf, ppv2_local_cmd, 16);
}

if (send(thread->u.f.fd, ppbuf, len, 0) != len) {
log_message(LOG_WARNING, "Send proxy protocol data failed!");
FREE(ppbuf);
return -1;
}

FREE(ppbuf);
return 0;
}

/* WEB checkers threads */
static int
http_check_thread(thread_ref_t thread)
Expand All @@ -1551,12 +1593,84 @@ http_check_thread(thread_ref_t thread)
request_t *req = http_get_check->req;
#endif
int ret = 1;
int status;
unsigned long timeout = 0;
int ssl_err = 0;
bool new_req = false;

status = tcp_socket_state(thread, http_check_thread);
if (!http_get_check->req) {
http_get_check->req = (request_t *) MALLOC(sizeof (request_t));
new_req = true;
} else
new_req = false;

if (http_get_check->proto == PROTO_SSL) {
timeout = timer_long(thread->sands) - timer_long(time_now);
if (thread->type != THREAD_WRITE_TIMEOUT &&
thread->type != THREAD_READ_TIMEOUT)
ret = ssl_connect(thread, new_req);
else
return timeout_epilog(thread, "Timeout connecting");

if (ret == -1) {
switch ((ssl_err = SSL_get_error(http_get_check->req->ssl,
ret))) {
case SSL_ERROR_WANT_READ:
thread_add_read(thread->master,
http_check_thread,
THREAD_ARG(thread),
thread->u.f.fd, timeout, true);
thread_del_write(thread);
break;
case SSL_ERROR_WANT_WRITE:
thread_add_write(thread->master,
http_check_thread,
THREAD_ARG(thread),
thread->u.f.fd, timeout, true);
thread_del_read(thread);
break;
default:
ret = 0;
break;
}
if (ret == -1)
return 0;
} else if (ret != 1)
ret = 0;
}

if (ret) {
/* Remote WEB server is connected.
* Register the next step thread ssl_request_thread.
*/
DBG("Remote Web server %s connected.", FMT_CHK(checker));
thread_add_write(thread->master,
http_request_thread, checker,
thread->u.f.fd,
checker->co->connection_to, true);
thread_del_read(thread);
} else {
DBG("Connection trouble to: %s."
, FMT_CHK(checker));
#ifdef _DEBUG_
if (http_get_check->proto == PROTO_SSL)
ssl_printerr(SSL_get_error
(req->ssl, ret));
#endif
return timeout_epilog(thread, "SSL handshake/communication error"
" connecting to");
}

return 0;
}

static int
http_proxy_protocol_thread(thread_ref_t thread)
{
int status;
unsigned long timeout;
checker_t *checker = THREAD_ARG(thread);

status = tcp_socket_state(thread, http_proxy_protocol_thread);
switch (status) {
case connect_error:
return timeout_epilog(thread, "Error connecting");
Expand All @@ -1571,68 +1685,9 @@ http_check_thread(thread_ref_t thread)
break;

case connect_success:
if (!http_get_check->req) {
http_get_check->req = (request_t *) MALLOC(sizeof (request_t));
new_req = true;
} else
new_req = false;

if (http_get_check->proto == PROTO_SSL) {
timeout = timer_long(thread->sands) - timer_long(time_now);
if (thread->type != THREAD_WRITE_TIMEOUT &&
thread->type != THREAD_READ_TIMEOUT)
ret = ssl_connect(thread, new_req);
else
return timeout_epilog(thread, "Timeout connecting");

if (ret == -1) {
switch ((ssl_err = SSL_get_error(http_get_check->req->ssl,
ret))) {
case SSL_ERROR_WANT_READ:
thread_add_read(thread->master,
http_check_thread,
THREAD_ARG(thread),
thread->u.f.fd, timeout, true);
thread_del_write(thread);
break;
case SSL_ERROR_WANT_WRITE:
thread_add_write(thread->master,
http_check_thread,
THREAD_ARG(thread),
thread->u.f.fd, timeout, true);
thread_del_read(thread);
break;
default:
ret = 0;
break;
}
if (ret == -1)
break;
} else if (ret != 1)
ret = 0;
}

if (ret) {
/* Remote WEB server is connected.
* Register the next step thread ssl_request_thread.
*/
DBG("Remote Web server %s connected.", FMT_CHK(checker));
thread_add_write(thread->master,
http_request_thread, checker,
thread->u.f.fd,
checker->co->connection_to, true);
thread_del_read(thread);
} else {
DBG("Connection trouble to: %s."
, FMT_CHK(checker));
#ifdef _DEBUG_
if (http_get_check->proto == PROTO_SSL)
ssl_printerr(SSL_get_error
(req->ssl, ret));
#endif
return timeout_epilog(thread, "SSL handshake/communication error"
" connecting to");
}
http_send_proxy_protocol(thread);
timeout = timer_long(thread->sands) - timer_long(time_now);
thread_add_write(thread->master, http_check_thread, checker, thread->u.f.fd, timeout, true);
break;
}

Expand Down Expand Up @@ -1686,7 +1741,7 @@ http_connect_thread(thread_ref_t thread)
status = tcp_bind_connect(fd, co);

/* handle tcp connection status & register check worker thread */
if(tcp_connection_state(fd, status, thread, http_check_thread,
if(tcp_connection_state(fd, status, thread, http_proxy_protocol_thread,
co->connection_to)) {
close(fd);
if (status == connect_fail) {
Expand All @@ -1707,6 +1762,7 @@ register_check_http_addresses(void)
{
register_thread_address("http_check_thread", http_check_thread);
register_thread_address("http_connect_thread", http_connect_thread);
register_thread_address("http_proxy_protocol_thread", http_proxy_protocol_thread);
register_thread_address("http_read_thread", http_read_thread);
register_thread_address("http_request_thread", http_request_thread);
register_thread_address("http_response_thread", http_response_thread);
Expand Down

0 comments on commit 45ad38b

Please sign in to comment.