Adding timeout detection to both SSH and Telnet command as well. This will prevent a dead remote connection to block backend request processing

This commit is contained in:
NI
2019-09-19 14:54:54 +08:00
parent 6aa585c228
commit 020a1e8eaf
2 changed files with 32 additions and 16 deletions

View File

@@ -28,6 +28,7 @@ import (
"github.com/niruix/sshwifty/application/command" "github.com/niruix/sshwifty/application/command"
"github.com/niruix/sshwifty/application/log" "github.com/niruix/sshwifty/application/log"
"github.com/niruix/sshwifty/application/network"
"github.com/niruix/sshwifty/application/rw" "github.com/niruix/sshwifty/application/rw"
) )
@@ -109,6 +110,7 @@ var (
type sshRemoteConnWrapper struct { type sshRemoteConnWrapper struct {
net.Conn net.Conn
writerConn network.WriteTimeoutConn
requestTimeoutRetry func(s *sshRemoteConnWrapper) bool requestTimeoutRetry func(s *sshRemoteConnWrapper) bool
} }
@@ -140,6 +142,10 @@ func (s *sshRemoteConnWrapper) Read(b []byte) (int, error) {
} }
} }
func (s *sshRemoteConnWrapper) Write(b []byte) (int, error) {
return s.writerConn.Write(b)
}
type sshRemoteConn struct { type sshRemoteConn struct {
writer io.Writer writer io.Writer
closer func() error closer func() error
@@ -364,10 +370,10 @@ func (d *sshClient) disableRemoteReadTimeoutRetry() {
} }
func (d *sshClient) dialRemote( func (d *sshClient) dialRemote(
network, networkName,
addr string, addr string,
config *ssh.ClientConfig) (*ssh.Client, func(), error) { config *ssh.ClientConfig) (*ssh.Client, func(), error) {
conn, err := d.cfg.Dial(network, addr, config.Timeout) conn, err := d.cfg.Dial(networkName, addr, config.Timeout)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
@@ -375,6 +381,7 @@ func (d *sshClient) dialRemote(
sshConn := &sshRemoteConnWrapper{ sshConn := &sshRemoteConnWrapper{
Conn: conn, Conn: conn,
writerConn: network.NewWriteTimeoutConn(conn, d.cfg.DialTimeout),
requestTimeoutRetry: func(s *sshRemoteConnWrapper) bool { requestTimeoutRetry: func(s *sshRemoteConnWrapper) bool {
d.remoteReadTimeoutRetryLock.Lock() d.remoteReadTimeoutRetryLock.Lock()
defer d.remoteReadTimeoutRetryLock.Unlock() defer d.remoteReadTimeoutRetryLock.Unlock()
@@ -393,6 +400,9 @@ func (d *sshClient) dialRemote(
}, },
} }
// Set timeout for writer, otherwise the Timeout writer will never
// be triggered
sshConn.SetWriteDeadline(time.Now().Add(d.cfg.DialTimeout))
sshConn.SetReadDeadline(time.Now().Add(config.Timeout)) sshConn.SetReadDeadline(time.Now().Add(config.Timeout))
c, chans, reqs, err := ssh.NewClientConn(sshConn, addr, config) c, chans, reqs, err := ssh.NewClientConn(sshConn, addr, config)
@@ -527,11 +537,7 @@ func (d *sshClient) remote(
d.remoteConnReceive <- sshRemoteConn{ d.remoteConnReceive <- sshRemoteConn{
writer: in, writer: in,
closer: func() error { closer: func() error {
sErr := session.Close() session.Close()
if sErr != nil {
return sErr
}
return conn.Close() return conn.Close()
}, },
@@ -626,6 +632,8 @@ func (d *sshClient) local(
_, wErr := remote.writer.Write(rData) _, wErr := remote.writer.Write(rData)
if wErr != nil { if wErr != nil {
remote.closer()
d.l.Debug("Failed to write data to remote: %s", wErr) d.l.Debug("Failed to write data to remote: %s", wErr)
} }

View File

@@ -19,11 +19,13 @@ package commands
import ( import (
"errors" "errors"
"io" "net"
"sync" "sync"
"time"
"github.com/niruix/sshwifty/application/command" "github.com/niruix/sshwifty/application/command"
"github.com/niruix/sshwifty/application/log" "github.com/niruix/sshwifty/application/log"
"github.com/niruix/sshwifty/application/network"
"github.com/niruix/sshwifty/application/rw" "github.com/niruix/sshwifty/application/rw"
) )
@@ -49,8 +51,8 @@ type telnetClient struct {
l log.Logger l log.Logger
w command.StreamResponder w command.StreamResponder
cfg command.Configuration cfg command.Configuration
remoteChan chan io.WriteCloser remoteChan chan net.Conn
remoteConn io.WriteCloser remoteConn net.Conn
closeWait sync.WaitGroup closeWait sync.WaitGroup
} }
@@ -63,7 +65,7 @@ func newTelnet(
l: l, l: l,
w: w, w: w,
cfg: cfg, cfg: cfg,
remoteChan: make(chan io.WriteCloser, 1), remoteChan: make(chan net.Conn, 1),
remoteConn: nil, remoteConn: nil,
closeWait: sync.WaitGroup{}, closeWait: sync.WaitGroup{},
} }
@@ -79,8 +81,6 @@ func (d *telnetClient) Bootup(
addrErr, TelnetRequestErrorBadRemoteAddress) addrErr, TelnetRequestErrorBadRemoteAddress)
} }
// TODO: Test whether or not the address is allowed
d.closeWait.Add(1) d.closeWait.Add(1)
go d.remote(addr.String()) go d.remote(addr.String())
@@ -118,7 +118,13 @@ func (d *telnetClient) remote(addr string) {
return return
} }
d.remoteChan <- clientConn // Set timeout for writer, otherwise the Timeout writer will never
// be triggered
clientConn.SetWriteDeadline(time.Now().Add(d.cfg.DialTimeout))
timeoutClientConn := network.NewWriteTimeoutConn(
clientConn, d.cfg.DialTimeout)
d.remoteChan <- &timeoutClientConn
for { for {
rLen, rErr := clientConn.Read(buf[d.w.HeaderSize():]) rLen, rErr := clientConn.Read(buf[d.w.HeaderSize():])
@@ -136,7 +142,7 @@ func (d *telnetClient) remote(addr string) {
} }
} }
func (d *telnetClient) getRemote() (io.WriteCloser, error) { func (d *telnetClient) getRemote() (net.Conn, error) {
if d.remoteConn != nil { if d.remoteConn != nil {
return d.remoteConn, nil return d.remoteConn, nil
} }
@@ -176,6 +182,8 @@ func (d *telnetClient) client(
_, wErr := remoteConn.Write(rBuf) _, wErr := remoteConn.Write(rBuf)
if wErr != nil { if wErr != nil {
remoteConn.Close()
d.l.Debug("Failed to write data to remote: %s", wErr) d.l.Debug("Failed to write data to remote: %s", wErr)
} }