From 5adef3feab02cc5ce3194d9a693ff25940a03ca4 Mon Sep 17 00:00:00 2001 From: Amir Khan Date: Thu, 30 Mar 2023 12:35:24 -0400 Subject: [PATCH 1/3] Fixed typos --- psiphon/common/obfuscator/history.go | 2 +- psiphon/common/obfuscator/obfuscator.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/psiphon/common/obfuscator/history.go b/psiphon/common/obfuscator/history.go index 6390d9293..c79622479 100644 --- a/psiphon/common/obfuscator/history.go +++ b/psiphon/common/obfuscator/history.go @@ -64,7 +64,7 @@ func NewSeedHistory(config *SeedHistoryConfig) *SeedHistory { // positives. // // Limitation: As go-cache-lru does not currently support iterating over all - // items (without making a full copy of the enture cache), the client IP with + // items (without making a full copy of the entire cache), the client IP with // shorter TTL is stored in a second, smaller cache instead of the same cache // with a a pruner. This incurs some additional overhead, as the seed key is // stored twice, once in each cache. diff --git a/psiphon/common/obfuscator/obfuscator.go b/psiphon/common/obfuscator/obfuscator.go index fb1709d02..40e362e4b 100644 --- a/psiphon/common/obfuscator/obfuscator.go +++ b/psiphon/common/obfuscator/obfuscator.go @@ -310,7 +310,7 @@ func readSeedMessage( // // Another false positive case: a retired server IP may be recycled and // deployed with a new obfuscation key; legitimate clients may still attempt - // to connect using the old obfuscation key; this case is partically + // to connect using the old obfuscation key; this case is practically // mitigated by the server entry pruning mechanism. // // Network I/O failures (e.g., failure to read the expected number of seed From 087ace525dfe99044ad3a2a46098105f18b148b7 Mon Sep 17 00:00:00 2001 From: Amir Khan Date: Mon, 17 Apr 2023 16:35:39 -0400 Subject: [PATCH 2/3] Added arbitrary prefix to Obfuscated SSH --- .../common/obfuscator/obfuscatedSshConn.go | 129 +++- psiphon/common/obfuscator/obfuscator.go | 540 +++++++++++--- psiphon/common/obfuscator/obfuscator_test.go | 659 +++++++++++++++--- psiphon/common/obfuscator/skipReader.go | 114 +++ psiphon/common/obfuscator/skipReader_test.go | 175 +++++ psiphon/common/parameters/parameters.go | 21 + psiphon/common/protocol/serverEntry.go | 1 + psiphon/common/tactics/tactics.go | 2 +- psiphon/config.go | 33 + psiphon/dialParameters.go | 50 ++ psiphon/meekConn.go | 2 +- psiphon/notice.go | 6 + psiphon/server/api.go | 1 + psiphon/server/tunnelServer.go | 17 + psiphon/serverApi.go | 6 + psiphon/tunnel.go | 1 + 16 files changed, 1535 insertions(+), 222 deletions(-) create mode 100644 psiphon/common/obfuscator/skipReader.go create mode 100644 psiphon/common/obfuscator/skipReader_test.go diff --git a/psiphon/common/obfuscator/obfuscatedSshConn.go b/psiphon/common/obfuscator/obfuscatedSshConn.go index dd7dc9964..38235eb40 100644 --- a/psiphon/common/obfuscator/obfuscatedSshConn.go +++ b/psiphon/common/obfuscator/obfuscatedSshConn.go @@ -57,7 +57,6 @@ const ( // WARNING: doesn't fully conform to net.Conn concurrency semantics: there's // no synchronization of access to the read/writeBuffers, so concurrent // calls to one of Read or Write will result in undefined behavior. -// type ObfuscatedSSHConn struct { net.Conn mode ObfuscatedSSHConnMode @@ -84,7 +83,8 @@ const ( type ObfuscatedSSHReadState int const ( - OBFUSCATION_READ_STATE_IDENTIFICATION_LINES = iota + OBFUSCATION_READ_STATE_CLIENT_READ_PREFIX = iota + OBFUSCATION_READ_STATE_IDENTIFICATION_LINES OBFUSCATION_READ_STATE_KEX_PACKETS OBFUSCATION_READ_STATE_FLUSH OBFUSCATION_READ_STATE_FINISHED @@ -93,8 +93,8 @@ const ( type ObfuscatedSSHWriteState int const ( - OBFUSCATION_WRITE_STATE_CLIENT_SEND_SEED_MESSAGE = iota - OBFUSCATION_WRITE_STATE_SERVER_SEND_IDENTIFICATION_LINE_PADDING + OBFUSCATION_WRITE_STATE_CLIENT_SEND_PREAMBLE = iota + OBFUSCATION_WRITE_STATE_SERVER_SEND_PREFIX_AND_IDENTIFICATION_LINE_PADDING OBFUSCATION_WRITE_STATE_IDENTIFICATION_LINE OBFUSCATION_WRITE_STATE_KEX_PACKETS OBFUSCATION_WRITE_STATE_FINISHED @@ -127,6 +127,8 @@ func NewObfuscatedSSHConn( obfuscationKeyword string, obfuscationPaddingPRNGSeed *prng.Seed, obfuscatorSeedTransformerParameters *transforms.ObfuscatorSeedTransformerParameters, + clientPrefixSpec *OSSHPrefixSpec, + serverPrefixSepcs transforms.Specs, minPadding, maxPadding *int, seedHistory *SeedHistory, irregularLogger func( @@ -139,11 +141,16 @@ func NewObfuscatedSSHConn( var readDeobfuscate, writeObfuscate func([]byte) var writeState ObfuscatedSSHWriteState + conn = WrapConnWithSkipReader(conn) + + readState := ObfuscatedSSHReadState(OBFUSCATION_READ_STATE_IDENTIFICATION_LINES) + if mode == OBFUSCATION_CONN_MODE_CLIENT { obfuscator, err = NewClientObfuscator( &ObfuscatorConfig{ IsOSSH: true, Keyword: obfuscationKeyword, + ClientPrefixSpec: clientPrefixSpec, PaddingPRNGSeed: obfuscationPaddingPRNGSeed, MinPadding: minPadding, MaxPadding: maxPadding, @@ -154,14 +161,21 @@ func NewObfuscatedSSHConn( } readDeobfuscate = obfuscator.ObfuscateServerToClient writeObfuscate = obfuscator.ObfuscateClientToServer - writeState = OBFUSCATION_WRITE_STATE_CLIENT_SEND_SEED_MESSAGE + writeState = OBFUSCATION_WRITE_STATE_CLIENT_SEND_PREAMBLE + + if obfuscator.prefixHeader != nil { + // Client expects prefix with terminator from the server. + readState = OBFUSCATION_READ_STATE_CLIENT_READ_PREFIX + } + } else { // NewServerObfuscator reads a seed message from conn obfuscator, err = NewServerObfuscator( &ObfuscatorConfig{ - Keyword: obfuscationKeyword, - SeedHistory: seedHistory, - IrregularLogger: irregularLogger, + Keyword: obfuscationKeyword, + ServerPrefixSpecs: serverPrefixSepcs, + SeedHistory: seedHistory, + IrregularLogger: irregularLogger, }, common.IPAddressFromAddr(conn.RemoteAddr()), conn) @@ -178,7 +192,7 @@ func NewObfuscatedSSHConn( } readDeobfuscate = obfuscator.ObfuscateClientToServer writeObfuscate = obfuscator.ObfuscateServerToClient - writeState = OBFUSCATION_WRITE_STATE_SERVER_SEND_IDENTIFICATION_LINE_PADDING + writeState = OBFUSCATION_WRITE_STATE_SERVER_SEND_PREFIX_AND_IDENTIFICATION_LINE_PADDING } paddingPRNG, err := obfuscator.GetDerivedPRNG("obfuscated-ssh-padding") @@ -192,7 +206,7 @@ func NewObfuscatedSSHConn( obfuscator: obfuscator, readDeobfuscate: readDeobfuscate, writeObfuscate: writeObfuscate, - readState: OBFUSCATION_READ_STATE_IDENTIFICATION_LINES, + readState: readState, writeState: writeState, readBuffer: new(bytes.Buffer), writeBuffer: new(bytes.Buffer), @@ -209,6 +223,7 @@ func NewClientObfuscatedSSHConn( obfuscationKeyword string, obfuscationPaddingPRNGSeed *prng.Seed, obfuscatorSeedTransformerParameters *transforms.ObfuscatorSeedTransformerParameters, + prefixSpec *OSSHPrefixSpec, minPadding, maxPadding *int) (*ObfuscatedSSHConn, error) { return NewObfuscatedSSHConn( @@ -217,6 +232,8 @@ func NewClientObfuscatedSSHConn( obfuscationKeyword, obfuscationPaddingPRNGSeed, obfuscatorSeedTransformerParameters, + prefixSpec, + nil, minPadding, maxPadding, nil, nil) @@ -228,6 +245,7 @@ func NewServerObfuscatedSSHConn( conn net.Conn, obfuscationKeyword string, seedHistory *SeedHistory, + serverPrefixSpecs transforms.Specs, irregularLogger func( clientIP string, err error, @@ -238,6 +256,8 @@ func NewServerObfuscatedSSHConn( conn, obfuscationKeyword, nil, nil, + nil, + serverPrefixSpecs, nil, nil, seedHistory, irregularLogger) @@ -311,20 +331,26 @@ func (conn *ObfuscatedSSHConn) Write(buffer []byte) (int, error) { // // Psiphon's server sends extra lines before the version line, as // permitted by http://www.ietf.org/rfc/rfc4253.txt sec 4.2: -// The server MAY send other lines of data before sending the -// version string. [...] Clients MUST be able to process such lines. +// +// The server MAY send other lines of data before sending the +// version string. [...] Clients MUST be able to process such lines. // // A comment in exchangeVersions explains that the golang code doesn't // support this: -// Contrary to the RFC, we do not ignore lines that don't -// start with "SSH-2.0-" to make the library usable with -// nonconforming servers. +// +// Contrary to the RFC, we do not ignore lines that don't +// start with "SSH-2.0-" to make the library usable with +// nonconforming servers. // // In addition, Psiphon's server sends up to 512 characters per extra // line. It's not clear that the 255 max string size in sec 4.2 refers // to the extra lines as well, but in any case golang's code only // supports 255 character lines. // +// State OBFUSCATION_READ_STATE_CLIENT_READ_PREFIX: the initial +// state, when the client expects prefix with terminator before the +// rest of the tunnel. In this state, the prefix is read and discarded. +// // State OBFUSCATION_READ_STATE_IDENTIFICATION_LINES: in this // state, extra lines are read and discarded. Once the peer's // identification string line is read, it is buffered and returned @@ -339,10 +365,36 @@ func (conn *ObfuscatedSSHConn) Write(buffer []byte) (int, error) { // packet may need to be buffered due to partial reading. func (conn *ObfuscatedSSHConn) readAndTransform(buffer []byte) (int, error) { + if conn.readState == OBFUSCATION_READ_STATE_CLIENT_READ_PREFIX { + skipReader, ok := conn.Conn.(*SkipReader) + if !ok { + return 0, errors.TraceNew("expected SkipReader") + } + + preambleHeader := make([]byte, PREAMBLE_HEADER_LENGTH) + _, err := io.ReadFull(skipReader, preambleHeader) + if err != nil { + return 0, errors.Trace(err) + } + + terminator, err := makeTerminator(conn.obfuscator.keyword, + preambleHeader, OBFUSCATE_SERVER_TO_CLIENT_IV) + if err != nil { + return 0, errors.Trace(err) + } + + err = skipReader.SkipUpToToken(terminator, PREFIX_TERM_SEARCH_BUF_SIZE, PREFIX_MAX_LENGTH) + if err != nil { + return 0, errors.Trace(err) + } + conn.readState = OBFUSCATION_READ_STATE_IDENTIFICATION_LINES + } + nextState := conn.readState switch conn.readState { case OBFUSCATION_READ_STATE_IDENTIFICATION_LINES: + // TODO: only client should accept multiple lines? if conn.readBuffer.Len() == 0 { for { @@ -410,7 +462,7 @@ func (conn *ObfuscatedSSHConn) readAndTransform(buffer []byte) (int, error) { // state, when the client has not sent any data. In this state, the seed message // is injected into the client output stream. // -// State OBFUSCATION_WRITE_STATE_SERVER_SEND_IDENTIFICATION_LINE_PADDING: the +// State OBFUSCATION_WRITE_STATE_SERVER_SEND_PREFIX_AND_IDENTIFICATION_LINE_PADDING: the // initial state, when the server has not sent any data. In this state, the // additional lines of padding are injected into the server output stream. // This padding is a partial defense against traffic analysis against the @@ -426,11 +478,13 @@ func (conn *ObfuscatedSSHConn) readAndTransform(buffer []byte) (int, error) { // State OBFUSCATION_WRITE_STATE_KEX_PACKETS: follows the binary // packet protocol, parsing each packet until the first SSH_MSG_NEWKEYS. // http://www.ietf.org/rfc/rfc4253.txt sec 6: -// uint32 packet_length -// byte padding_length -// byte[n1] payload; n1 = packet_length - padding_length - 1 -// byte[n2] random padding; n2 = padding_length -// byte[m] mac (Message Authentication Code - MAC); m = mac_length +// +// uint32 packet_length +// byte padding_length +// byte[n1] payload; n1 = packet_length - padding_length - 1 +// byte[n2] random padding; n2 = padding_length +// byte[m] mac (Message Authentication Code - MAC); m = mac_length +// // m is 0 as no MAC ha yet been negotiated. // http://www.ietf.org/rfc/rfc4253.txt sec 7.3, 12: // The payload for SSH_MSG_NEWKEYS is one byte, the packet type, value 21. @@ -441,22 +495,43 @@ func (conn *ObfuscatedSSHConn) readAndTransform(buffer []byte) (int, error) { // these packets is authenticated in the "exchange hash"). func (conn *ObfuscatedSSHConn) transformAndWrite(buffer []byte) error { - // The seed message (client) and identification line padding (server) - // are injected before any standard SSH traffic. - if conn.writeState == OBFUSCATION_WRITE_STATE_CLIENT_SEND_SEED_MESSAGE { - _, err := conn.Conn.Write(conn.obfuscator.SendSeedMessage()) + // The preamble (client) and requested prefix with + // identification line padding (server) are injected before any standard SSH traffic. + if conn.writeState == OBFUSCATION_WRITE_STATE_CLIENT_SEND_PREAMBLE { + + preamble := conn.obfuscator.SendPreamble() + + _, err := conn.Conn.Write(preamble) if err != nil { return errors.Trace(err) } + conn.writeState = OBFUSCATION_WRITE_STATE_IDENTIFICATION_LINE - } else if conn.writeState == OBFUSCATION_WRITE_STATE_SERVER_SEND_IDENTIFICATION_LINE_PADDING { + + } else if conn.writeState == OBFUSCATION_WRITE_STATE_SERVER_SEND_PREFIX_AND_IDENTIFICATION_LINE_PADDING { + + var buffer bytes.Buffer + + if preamble := conn.obfuscator.SendPreamble(); preamble != nil { + _, err := buffer.Write(preamble) + if err != nil { + return errors.Trace(err) + } + } + padding := makeServerIdentificationLinePadding(conn.paddingPRNG) conn.paddingLength = len(padding) conn.writeObfuscate(padding) - _, err := conn.Conn.Write(padding) + _, err := buffer.Write(padding) if err != nil { return errors.Trace(err) } + + _, err = conn.Conn.Write(buffer.Bytes()) + if err != nil { + return errors.Trace(err) + } + conn.writeState = OBFUSCATION_WRITE_STATE_IDENTIFICATION_LINE } diff --git a/psiphon/common/obfuscator/obfuscator.go b/psiphon/common/obfuscator/obfuscator.go index 40e362e4b..75ed1270c 100644 --- a/psiphon/common/obfuscator/obfuscator.go +++ b/psiphon/common/obfuscator/obfuscator.go @@ -23,13 +23,17 @@ import ( "bytes" "crypto/rc4" "crypto/sha1" + "crypto/sha256" "encoding/binary" + "fmt" "io" "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common" "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/errors" "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/prng" + "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/regen" "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/transforms" + "golang.org/x/crypto/hkdf" ) const ( @@ -40,8 +44,30 @@ const ( OBFUSCATE_MAGIC_VALUE = 0x0BF5CA7E OBFUSCATE_CLIENT_TO_SERVER_IV = "client_to_server" OBFUSCATE_SERVER_TO_CLIENT_IV = "server_to_client" + + // Preamble header is the first 24 bytes of the connection. If no prefix is applied, + // the first 24 bytes are the Obfuscated SSH seed, magic value and padding length. + PREAMBLE_HEADER_LENGTH = OBFUSCATE_SEED_LENGTH + 8 // 4 bytes each for magic value and padding length + + PREFIX_TERMINATOR_LENGTH = 16 + PREFIX_TERM_SEARCH_BUF_SIZE = 8192 + PREFIX_MAX_LENGTH = 65536 + PREFIX_MAX_HEADER_LENGTH = 4096 ) +type OSSHPrefixSpec struct { + Name string + Spec transforms.Spec + Seed *prng.Seed +} + +// OSSHPrefixHeader is the prefix header. It is written by the client +// when a prefix is applied, and read by the server to determine the +// prefix-spec to use. +type OSSHPrefixHeader struct { + SpecName string +} + // Obfuscator implements the seed message, key derivation, and // stream ciphers for: // https://github.com/brl/obfuscated-openssh/blob/master/README.obfuscation @@ -52,7 +78,13 @@ const ( // with legacy clients. New protocols and schemes should not use this // obfuscator. type Obfuscator struct { - seedMessage []byte + preamble []byte + + // prefixHeader is the prefix header written by the client, + // or the prefix header read by the server. + prefixHeader *OSSHPrefixHeader + + keyword string paddingLength int clientToServerCipher *rc4.Cipher serverToClientCipher *rc4.Cipher @@ -64,6 +96,8 @@ type Obfuscator struct { type ObfuscatorConfig struct { IsOSSH bool Keyword string + ClientPrefixSpec *OSSHPrefixSpec + ServerPrefixSpecs transforms.Specs PaddingPRNGSeed *prng.Seed MinPadding *int MaxPadding *int @@ -138,14 +172,18 @@ func NewClientObfuscator( maxPadding = *config.MaxPadding } - seedMessage, paddingLength, err := makeSeedMessage( - paddingPRNG, minPadding, maxPadding, obfuscatorSeed, clientToServerCipher) + preamble, prefixHeader, paddingLength, err := makeClientPreamble( + config.Keyword, config.ClientPrefixSpec, + paddingPRNG, minPadding, maxPadding, obfuscatorSeed, + clientToServerCipher) if err != nil { return nil, errors.Trace(err) } return &Obfuscator{ - seedMessage: seedMessage, + preamble: preamble, + prefixHeader: prefixHeader, + keyword: config.Keyword, paddingLength: paddingLength, clientToServerCipher: clientToServerCipher, serverToClientCipher: serverToClientCipher, @@ -169,13 +207,21 @@ func NewServerObfuscator( return nil, errors.TraceNew("missing keyword") } - clientToServerCipher, serverToClientCipher, paddingPRNGSeed, err := readSeedMessage( + clientToServerCipher, serverToClientCipher, paddingPRNGSeed, prefixHeader, err := readPreamble( config, clientIP, clientReader) if err != nil { return nil, errors.Trace(err) } + preamble, err := makeServerPreamble(prefixHeader, config.ServerPrefixSpecs, config.Keyword) + if err != nil { + return nil, errors.Trace(err) + } + return &Obfuscator{ + preamble: preamble, + prefixHeader: prefixHeader, + keyword: config.Keyword, paddingLength: -1, clientToServerCipher: clientToServerCipher, serverToClientCipher: serverToClientCipher, @@ -201,12 +247,12 @@ func (obfuscator *Obfuscator) GetPaddingLength() int { return obfuscator.paddingLength } -// SendSeedMessage returns the seed message created in NewObfuscatorClient, -// removing the reference so that it may be garbage collected. -func (obfuscator *Obfuscator) SendSeedMessage() []byte { - seedMessage := obfuscator.seedMessage - obfuscator.seedMessage = nil - return seedMessage +// SendPreamble returns the preamble created in NewObfuscatorClient or +// NewServerObfuscator, removing the reference so that it may be garbage collected. +func (obfuscator *Obfuscator) SendPreamble() []byte { + msg := obfuscator.preamble + obfuscator.preamble = nil + return msg } // ObfuscateClientToServer applies the client RC4 stream to the bytes in buffer. @@ -262,158 +308,454 @@ func deriveKey(obfuscatorSeed, keyword, iv []byte) ([]byte, error) { return digest[0:OBFUSCATE_KEY_LENGTH], nil } -func makeSeedMessage( +// makeClientPreamble generates the preamble bytes for the Obfuscated SSH protocol. +// +// If a prefix is applied, preamble bytes refer to the prefix, prefix terminator, +// followed by the Obufscted SSH initial client message, followed by the +// prefix header. +// +// If a prefix is not applied, preamble bytes refer to the Obfuscated SSH +// initial client message (referred to as the "seed message" in the original spec): +// https://github.com/brl/obfuscated-openssh/blob/master/README.obfuscation +// +// Obfuscated SSH initial client message (no prefix): +// +// [ 16 byte random seed ][ OSSH magic ][ padding length ][ padding ] +// |_____________________||_________________________________________| +// +// | | +// Plaintext Encrypted with key derived from seed +// +// Prefix + Obfuscated SSH initial client message: +// +// [ 24+ byte prefix ][ terminator ][ OSSH initial client message ][ prefix header ] +// |_________________||____________________________________________________________| +// +// | | +// Plaintext Encrypted with key derived from first 24 bytes +// +// Returns the preamble, the prefix header if a prefix was generated, +// and the padding length. +func makeClientPreamble( + keyword string, + prefixSpec *OSSHPrefixSpec, paddingPRNG *prng.PRNG, minPadding, maxPadding int, obfuscatorSeed []byte, - clientToServerCipher *rc4.Cipher) ([]byte, int, error) { + clientToServerCipher *rc4.Cipher) ([]byte, *OSSHPrefixHeader, int, error) { padding := paddingPRNG.Padding(minPadding, maxPadding) buffer := new(bytes.Buffer) + magicValueStartIndex := len(obfuscatorSeed) + + if prefixSpec != nil { + // Writes the prefix and terminator to the buffer. + prefix, err := makePrefix(prefixSpec, keyword, OBFUSCATE_CLIENT_TO_SERVER_IV) + if err != nil { + return nil, nil, 0, errors.Trace(err) + } + + _, err = buffer.Write(prefix) + if err != nil { + return nil, nil, 0, errors.Trace(err) + } + + magicValueStartIndex += len(prefix) + } + err := binary.Write(buffer, binary.BigEndian, obfuscatorSeed) if err != nil { - return nil, 0, errors.Trace(err) + return nil, nil, 0, errors.Trace(err) } err = binary.Write(buffer, binary.BigEndian, uint32(OBFUSCATE_MAGIC_VALUE)) if err != nil { - return nil, 0, errors.Trace(err) + return nil, nil, 0, errors.Trace(err) } err = binary.Write(buffer, binary.BigEndian, uint32(len(padding))) if err != nil { - return nil, 0, errors.Trace(err) + return nil, nil, 0, errors.Trace(err) } err = binary.Write(buffer, binary.BigEndian, padding) if err != nil { - return nil, 0, errors.Trace(err) + return nil, nil, 0, errors.Trace(err) } - seedMessage := buffer.Bytes() - clientToServerCipher.XORKeyStream(seedMessage[len(obfuscatorSeed):], seedMessage[len(obfuscatorSeed):]) - return seedMessage, len(padding), nil + + var prefixHeader *OSSHPrefixHeader = nil + if prefixSpec != nil { + // Writes the prefix header after the padding. + err := prefixSpec.writePrefixHeader(buffer) + if err != nil { + return nil, nil, 0, errors.Trace(err) + } + + prefixHeader = &OSSHPrefixHeader{ + SpecName: prefixSpec.Name, + } + } + + preamble := buffer.Bytes() + + // Encryptes what comes after the magic value. + clientToServerCipher.XORKeyStream( + preamble[magicValueStartIndex:], + preamble[magicValueStartIndex:]) + + return preamble, prefixHeader, len(padding), nil } -func readSeedMessage( +// makeServerPreamble generates a server preamble (prefix or nil). +// If the header is nil, nil is returned. Otherwise, prefix is generated +// from serverSpecs matching the spec name in the header. +// If the spec name is not found in serverSpecs, random bytes +// of length PREAMBLE_HEADER_LENGTH are returned. +func makeServerPreamble( + header *OSSHPrefixHeader, + serverSpecs transforms.Specs, + keyword string) ([]byte, error) { + + if header == nil { + return nil, nil + } + + spec, ok := serverSpecs[header.SpecName] + if !ok { + // Generate a random prefix if the spec is not found. + spec = transforms.Spec{{"", fmt.Sprintf(`[\x00-\xff]{%d}`, PREAMBLE_HEADER_LENGTH)}} + } + + seed, err := prng.NewSeed() + if err != nil { + return nil, errors.Trace(err) + } + + prefixSpec := &OSSHPrefixSpec{ + Name: header.SpecName, + Spec: spec, + Seed: seed, + } + return makePrefix(prefixSpec, keyword, OBFUSCATE_SERVER_TO_CLIENT_IV) +} + +// readPreamble reads the preamble bytes from the client. If it does not detect +// valid magic value in the first 24 bytes, it assumes that a prefix is applied. +// If a prefix is applied, it first discard the prefix and the terminator, before +// looking for a valid Obfuscated SSH initial client message. +func readPreamble( config *ObfuscatorConfig, clientIP string, - clientReader io.Reader) (*rc4.Cipher, *rc4.Cipher, *prng.Seed, error) { + clientReader io.Reader) (*rc4.Cipher, *rc4.Cipher, *prng.Seed, *OSSHPrefixHeader, error) { + return readPreambleHelper(config, clientIP, clientReader, false) +} - seed := make([]byte, OBFUSCATE_SEED_LENGTH) - _, err := io.ReadFull(clientReader, seed) - if err != nil { - return nil, nil, nil, errors.Trace(err) - } - - // Irregular events that indicate an invalid client are logged via - // IrregularLogger. Note that event detection isn't infallible. For example, - // a man-in-the-middle may have manipulated the seed message sent by a valid - // client; or with a very small probability a valid client may generate a - // duplicate seed message. - // - // Another false positive case: a retired server IP may be recycled and - // deployed with a new obfuscation key; legitimate clients may still attempt - // to connect using the old obfuscation key; this case is practically - // mitigated by the server entry pruning mechanism. - // - // Network I/O failures (e.g., failure to read the expected number of seed - // message bytes) are not considered a reliable indicator of irregular - // events. +func readPreambleHelper( + config *ObfuscatorConfig, + clientIP string, + clientReader io.Reader, + removedPrefix bool) (*rc4.Cipher, *rc4.Cipher, *prng.Seed, *OSSHPrefixHeader, error) { // To distinguish different cases, irregular tunnel logs should indicate // which function called NewServerObfuscator. errBackTrace := "obfuscator.NewServerObfuscator" - if config.SeedHistory != nil { - ok, duplicateLogFields := config.SeedHistory.AddNew( - config.StrictHistoryMode, clientIP, "obfuscator-seed", seed) - errStr := "duplicate obfuscation seed" - if duplicateLogFields != nil { - if config.IrregularLogger != nil { - config.IrregularLogger( - clientIP, - errors.BackTraceNew(errBackTrace, errStr), - *duplicateLogFields) - } - } - if !ok { - return nil, nil, nil, errors.TraceNew(errStr) - } - } + // Since the OSSH stream might be prefixed, the seed might not be the first + // 16 bytes of the stream. The stream is read until valid magic value + // is detected, PREFIX_MAX_LENGTH is reached, or until the stream is exhausted. + // If the magic value is found, the seed is the 16 bytes before the magic value, + // and is added to and checked against the seed history. - clientToServerCipher, serverToClientCipher, err := initObfuscatorCiphers(config, seed) + preambleHeader := make([]byte, PREAMBLE_HEADER_LENGTH) + _, err := io.ReadFull(clientReader, preambleHeader) if err != nil { - return nil, nil, nil, errors.Trace(err) + return nil, nil, nil, nil, errors.Trace(err) } - fixedLengthFields := make([]byte, 8) // 4 bytes each for magic value and padding length - _, err = io.ReadFull(clientReader, fixedLengthFields) + osshSeed := preambleHeader[:OBFUSCATE_SEED_LENGTH] + + clientToServerCipher, serverToClientCipher, err := initObfuscatorCiphers( + config, osshSeed) if err != nil { - return nil, nil, nil, errors.Trace(err) + return nil, nil, nil, nil, errors.Trace(err) } - clientToServerCipher.XORKeyStream(fixedLengthFields, fixedLengthFields) - - buffer := bytes.NewReader(fixedLengthFields) + osshFixedLengthFields := make([]byte, 8) // 4 bytes each for magic value and padding length + clientToServerCipher.XORKeyStream(osshFixedLengthFields, preambleHeader[OBFUSCATE_SEED_LENGTH:]) // The magic value must be validated before acting on paddingLength as // paddingLength validation is vulnerable to a chosen ciphertext probing // attack: only a fixed number of any possible byte value for each // paddingLength is valid. + buffer := bytes.NewReader(osshFixedLengthFields) var magicValue, paddingLength int32 err = binary.Read(buffer, binary.BigEndian, &magicValue) if err != nil { - return nil, nil, nil, errors.Trace(err) + return nil, nil, nil, nil, errors.Trace(err) } err = binary.Read(buffer, binary.BigEndian, &paddingLength) if err != nil { - return nil, nil, nil, errors.Trace(err) - } - - errStr := "" - - if magicValue != OBFUSCATE_MAGIC_VALUE { - errStr = "invalid magic value" + return nil, nil, nil, nil, errors.Trace(err) } - if errStr == "" && (paddingLength < 0 || paddingLength > OBFUSCATE_MAX_PADDING) { - errStr = "invalid padding length" - } - - if errStr != "" { + if magicValue != OBFUSCATE_MAGIC_VALUE && removedPrefix { + // Prefix terminator was found, but rest of the stream is not valid + // Obfuscated SSH. + errStr := "invalid magic value" if config.IrregularLogger != nil { config.IrregularLogger( clientIP, errors.BackTraceNew(errBackTrace, errStr), nil) } - return nil, nil, nil, errors.TraceNew(errStr) + return nil, nil, nil, nil, errors.TraceNew(errStr) } - padding := make([]byte, paddingLength) - _, err = io.ReadFull(clientReader, padding) - if err != nil { - return nil, nil, nil, errors.Trace(err) + if magicValue == OBFUSCATE_MAGIC_VALUE { + + if config.SeedHistory != nil { + // Adds the seed to the seed history only if the magic value is valid. + // This is to prevent malicious clients from filling up the history cache. + ok, duplicateLogFields := config.SeedHistory.AddNew( + config.StrictHistoryMode, clientIP, "obfuscator-seed", osshSeed) + errStr := "duplicate obfuscation seed" + if duplicateLogFields != nil { + if config.IrregularLogger != nil { + config.IrregularLogger( + clientIP, + errors.BackTraceNew(errBackTrace, errStr), + *duplicateLogFields) + } + } + if !ok { + return nil, nil, nil, nil, errors.TraceNew(errStr) + } + } + + if paddingLength < 0 || paddingLength > OBFUSCATE_MAX_PADDING { + errStr := "invalid padding length" + if config.IrregularLogger != nil { + config.IrregularLogger( + clientIP, + errors.BackTraceNew(errBackTrace, errStr), + nil) + } + return nil, nil, nil, nil, errors.TraceNew(errStr) + } + + padding := make([]byte, paddingLength) + _, err = io.ReadFull(clientReader, padding) + if err != nil { + return nil, nil, nil, nil, errors.Trace(err) + } + clientToServerCipher.XORKeyStream(padding, padding) + + var prefixHeader *OSSHPrefixHeader = nil + if removedPrefix { + // This is a valid prefixed OSSH stream. + prefixHeader, err = readPrefixHeader(clientReader, clientToServerCipher) + if err != nil { + if config.IrregularLogger != nil { + config.IrregularLogger( + clientIP, + errors.BackTraceNew(errBackTrace, "invalid prefix header"), + nil) + } + return nil, nil, nil, nil, errors.Trace(err) + } + } + + // Use the first prng.SEED_LENGTH bytes of padding as a PRNG seed for + // subsequent operations. This allows the client to direct server-side + // replay of certain protocol attributes. + // + // Since legacy clients may send < prng.SEED_LENGTH bytes of padding, + // generate a new seed in that case. + + var paddingPRNGSeed *prng.Seed + + if len(padding) >= prng.SEED_LENGTH { + paddingPRNGSeed = new(prng.Seed) + copy(paddingPRNGSeed[:], padding[0:prng.SEED_LENGTH]) + } else { + paddingPRNGSeed, err = prng.NewSeed() + if err != nil { + return nil, nil, nil, nil, errors.Trace(err) + } + } + + return clientToServerCipher, serverToClientCipher, paddingPRNGSeed, prefixHeader, nil } - clientToServerCipher.XORKeyStream(padding, padding) + if !removedPrefix { + // No magic value found, could be a prefixed OSSH stream. + // Skips up to the prefix terminator, and looks for the magic value again. - // Use the first prng.SEED_LENGTH bytes of padding as a PRNG seed for - // subsequent operations. This allows the client to direct server-side - // replay of certain protocol attributes. - // - // Since legacy clients may send < prng.SEED_LENGTH bytes of padding, - // generate a new seed in that case. + clientReader, ok := clientReader.(*SkipReader) + if !ok { + return nil, nil, nil, nil, errors.TraceNew("expected SkipReader") + } - var paddingPRNGSeed *prng.Seed + terminator, err := makeTerminator(config.Keyword, preambleHeader, OBFUSCATE_CLIENT_TO_SERVER_IV) + if err != nil { + return nil, nil, nil, nil, errors.Trace(err) + } - if len(padding) >= prng.SEED_LENGTH { - paddingPRNGSeed = new(prng.Seed) - copy(paddingPRNGSeed[:], padding[0:prng.SEED_LENGTH]) - } else { - paddingPRNGSeed, err = prng.NewSeed() + err = clientReader.SkipUpToToken(terminator, PREFIX_TERM_SEARCH_BUF_SIZE, PREFIX_MAX_LENGTH) if err != nil { - return nil, nil, nil, errors.Trace(err) + // No magic value or prefix terminator found, + // log irregular tunnel and return error. + errStr := "no prefix terminator or invalid magic value" + if config.IrregularLogger != nil { + config.IrregularLogger( + clientIP, + errors.BackTraceNew(errBackTrace, errStr), + nil) + } + return nil, nil, nil, nil, errors.TraceNew(errStr) } + + // Reads OSSH initial client message followed by prefix header. + return readPreambleHelper(config, clientIP, clientReader, true) } - return clientToServerCipher, serverToClientCipher, paddingPRNGSeed, nil + // Should never reach here. + return nil, nil, nil, nil, errors.TraceNew("unexpected error") +} + +// makeTerminator generates a prefix terminator used in finding end of prefix +// placed before OSSH stream. +// prefix should be at least PREAMBLE_HEADER_LENGTH bytes and contain enough entropy. +func makeTerminator(keyword string, prefix []byte, direction string) ([]byte, error) { + + // prefix length is at least equal to obfuscator seed message. + if len(prefix) < PREAMBLE_HEADER_LENGTH { + return nil, errors.TraceNew("prefix too short") + } + + if (direction != OBFUSCATE_CLIENT_TO_SERVER_IV) && + (direction != OBFUSCATE_SERVER_TO_CLIENT_IV) { + return nil, errors.TraceNew("invalid direction") + } + + hkdf := hkdf.New(sha256.New, + []byte(keyword), + prefix[:PREAMBLE_HEADER_LENGTH], + []byte(direction)) + + terminator := make([]byte, PREFIX_TERMINATOR_LENGTH) + _, err := io.ReadFull(hkdf, terminator) + if err != nil { + return nil, errors.Trace(err) + } + + return terminator, nil +} + +// makePrefix generates a prefix followed by it's terminator using the given spec. +// If the generated prefix is shorter than PREAMBLE_HEADER_LENGTH, it is padded +// with random bytes. +func makePrefix(spec *OSSHPrefixSpec, keyword, direction string) ([]byte, error) { + + if len(spec.Spec) != 1 || len(spec.Spec[0]) != 2 || spec.Spec[0][1] == "" { + return nil, errors.TraceNew("invalid prefix spec") + } + + rng := prng.NewPRNGWithSeed(spec.Seed) + + args := ®en.GeneratorArgs{ + RngSource: rng, + ByteMode: true, + } + + gen, err := regen.NewGenerator(spec.Spec[0][1], args) + if err != nil { + return nil, errors.Trace(err) + } + + prefix, err := gen.Generate() + if err != nil { + return nil, errors.Trace(err) + } + + if len(prefix) < PREAMBLE_HEADER_LENGTH { + // Add random padding to fill up to PREAMBLE_HEADER_LENGTH. + padding := rng.Bytes(PREAMBLE_HEADER_LENGTH - len(prefix)) + prefix = append(prefix, padding...) + } + + terminator, err := makeTerminator(keyword, prefix, direction) + + if err != nil { + return nil, errors.Trace(err) + } + terminatedPrefix := append(prefix, terminator...) + + return terminatedPrefix, nil +} + +// writePrefixHeader writes the prefix header to the given writer. +// The prefix header is written in the following format: +// +// [ 2 byte version ][4 byte spec-length ][ .. prefix-spec-name ...] +func (spec *OSSHPrefixSpec) writePrefixHeader(w io.Writer) error { + if len(spec.Name) > PREFIX_MAX_HEADER_LENGTH { + return errors.TraceNew("prefix name too long") + } + err := binary.Write(w, binary.BigEndian, uint16(0x01)) + if err != nil { + return errors.Trace(err) + } + err = binary.Write(w, binary.BigEndian, uint16(len(spec.Name))) + if err != nil { + return errors.Trace(err) + } + _, err = w.Write([]byte(spec.Name)) + if err != nil { + return errors.Trace(err) + } + return nil +} + +func readPrefixHeader( + clientReader io.Reader, + cipher *rc4.Cipher) (*OSSHPrefixHeader, error) { + + fixedLengthFields := make([]byte, 4) + _, err := io.ReadFull(clientReader, fixedLengthFields) + if err != nil { + return nil, errors.Trace(err) + } + + cipher.XORKeyStream(fixedLengthFields, fixedLengthFields) + + buffer := bytes.NewBuffer(fixedLengthFields) + var version uint16 + err = binary.Read(buffer, binary.BigEndian, &version) + if err != nil { + return nil, errors.Trace(err) + } + + if version != 0x01 { + return nil, errors.TraceNew("invalid version") + } + + var specLen uint16 + err = binary.Read(buffer, binary.BigEndian, &specLen) + if err != nil { + return nil, errors.Trace(err) + } + if specLen > PREFIX_MAX_HEADER_LENGTH { + return nil, errors.TraceNew("invalid header length") + } + + // Read the spec name. + specName := make([]byte, specLen) + _, err = io.ReadFull(clientReader, specName) + if err != nil { + return nil, errors.Trace(err) + } + cipher.XORKeyStream(specName, specName) + + return &OSSHPrefixHeader{ + SpecName: string(specName), + }, nil } diff --git a/psiphon/common/obfuscator/obfuscator_test.go b/psiphon/common/obfuscator/obfuscator_test.go index fb9e8eb14..26c33c292 100644 --- a/psiphon/common/obfuscator/obfuscator_test.go +++ b/psiphon/common/obfuscator/obfuscator_test.go @@ -23,9 +23,12 @@ import ( "bytes" "crypto/rand" "crypto/rsa" + "encoding/hex" "errors" + "fmt" "math/bits" "net" + "strings" "testing" "time" @@ -76,9 +79,9 @@ func TestObfuscator(t *testing.T) { t.Fatalf("NewClientObfuscator failed: %s", err) } - seedMessage := client.SendSeedMessage() + preamble := client.SendPreamble() - server, err := NewServerObfuscator(config, "", bytes.NewReader(seedMessage)) + server, err := NewServerObfuscator(config, "", bytes.NewReader(preamble)) if err != nil { t.Fatalf("NewServerObfuscator failed: %s", err) } @@ -110,18 +113,18 @@ func TestObfuscator(t *testing.T) { t.Fatalf("NewClientObfuscator failed: %s", err) } - seedMessage = client.SendSeedMessage() + preamble = client.SendPreamble() clientIP := "192.168.0.1" - _, err = NewServerObfuscator(config, clientIP, bytes.NewReader(seedMessage)) + _, err = NewServerObfuscator(config, clientIP, bytes.NewReader(preamble)) if err != nil { t.Fatalf("NewServerObfuscator failed: %s", err) } irregularLogFields = nil - _, err = NewServerObfuscator(config, clientIP, bytes.NewReader(seedMessage)) + _, err = NewServerObfuscator(config, clientIP, bytes.NewReader(preamble)) if err != nil { t.Fatalf("NewServerObfuscator failed: %s", err) } @@ -133,7 +136,7 @@ func TestObfuscator(t *testing.T) { irregularLogFields = nil - _, err = NewServerObfuscator(config, "192.168.0.2", bytes.NewReader(seedMessage)) + _, err = NewServerObfuscator(config, "192.168.0.2", bytes.NewReader(preamble)) if err == nil { t.Fatalf("NewServerObfuscator unexpectedly succeeded") } @@ -147,7 +150,7 @@ func TestObfuscator(t *testing.T) { irregularLogFields = nil - _, err = NewServerObfuscator(config, clientIP, bytes.NewReader(seedMessage)) + _, err = NewServerObfuscator(config, clientIP, bytes.NewReader(preamble)) if err == nil { t.Fatalf("NewServerObfuscator unexpectedly succeeded") } @@ -158,8 +161,535 @@ func TestObfuscator(t *testing.T) { } } +func TestObfuscatorSeedTransformParameters(t *testing.T) { + + keyword := prng.HexString(32) + + maxPadding := 256 + + paddingPRNGSeed, err := prng.NewSeed() + if err != nil { + t.Fatalf("prng.NewSeed failed: %s", err) + } + + type test struct { + name string + transformerParamters *transforms.ObfuscatorSeedTransformerParameters + + // nil means seedMessage looks random (transformer was not applied) + expectedResult []byte + expectedResultLength int + } + + tests := []test{ + { + name: "4 byte transform", + transformerParamters: &transforms.ObfuscatorSeedTransformerParameters{ + TransformName: "four-zeros", + TransformSeed: &prng.Seed{0}, + TransformSpec: transforms.Spec{{"^.{8}", "00000000"}}, + }, + expectedResult: []byte{0, 0, 0, 0}, + expectedResultLength: 4, + }, + { + name: "invalid '%' character in the regex", + transformerParamters: &transforms.ObfuscatorSeedTransformerParameters{ + TransformName: "invalid-spec", + TransformSeed: &prng.Seed{0}, + TransformSpec: transforms.Spec{{"^.{8}", "%00000000"}}, + }, + expectedResult: nil, + expectedResultLength: 0, + }, + } + + for _, tt := range tests { + + t.Run(tt.name, func(t *testing.T) { + + config := &ObfuscatorConfig{ + IsOSSH: true, + Keyword: keyword, + MaxPadding: &maxPadding, + PaddingPRNGSeed: paddingPRNGSeed, + ObfuscatorSeedTransformerParameters: tt.transformerParamters, + } + + client, err := NewClientObfuscator(config) + if err != nil { + // if there is a expectedResult, then the error is unexpected + if tt.expectedResult != nil { + t.Fatalf("NewClientObfuscator failed: %s", err) + } + return + } + + preamble := client.SendPreamble() + + if tt.expectedResult == nil { + + // Verify that the seed message looks random. + // obfuscator seed is generated with common.MakeSecureRandomBytes, + // and is not affected by the config. + popcount := 0 + for _, b := range preamble[:tt.expectedResultLength] { + popcount += bits.OnesCount(uint(b)) + } + popcount_per_byte := float64(popcount) / float64(tt.expectedResultLength) + if popcount_per_byte < 3.6 || popcount_per_byte > 4.4 { + t.Fatalf("unexpected popcount_per_byte: %f", popcount_per_byte) + } + + } else if !bytes.Equal(preamble[:tt.expectedResultLength], tt.expectedResult) { + t.Fatalf("unexpected seed message") + } + + }) + + } + +} + +// TestClientObfuscatorPrefixGen tests the generated prefix, terminator, and +// prefix header for the client obfuscator. +func TestClientObfuscatorPrefix(t *testing.T) { + + // fix keyword and seed for reproducing the same prefix + + keyword := prng.HexString(32) + + prefixSeed, err := prng.NewSeed() + if err != nil { + t.Fatalf("prng.NewSeed failed: %s", err) + } + + generatePrefix := func(spec string) []byte { + prefixSpec := OSSHPrefixSpec{ + Spec: transforms.Spec{{"", spec}}, + Seed: prefixSeed, + } + b, _ := makePrefix(&prefixSpec, keyword, OBFUSCATE_CLIENT_TO_SERVER_IV) + // return the prefix without the terminator + return b[:len(b)-PREFIX_TERMINATOR_LENGTH] + } + + type test struct { + name string + prefixSpec transforms.Spec + expectedPrefix []byte + } + + tests := []test{ + { + name: "24 byte prefix", + prefixSpec: transforms.Spec{{"", "\\x00{24}"}}, + expectedPrefix: bytes.Repeat([]byte{0}, 24), + }, + { + name: "long prefix", + prefixSpec: transforms.Spec{{"", "\\x00{1000}\\x00{1000}\\x00{1000}\\x00{1000}"}}, + expectedPrefix: bytes.Repeat([]byte{0}, 4000), + }, + { + name: "short prefix spec", + prefixSpec: transforms.Spec{{"", "\\x00\\x00\\x00\\x00"}}, + expectedPrefix: generatePrefix("\\x00\\x00\\x00\\x00"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + paddingPRNGSeed, err := prng.NewSeed() + if err != nil { + t.Fatalf("prng.NewSeed failed: %s", err) + } + + config := &ObfuscatorConfig{ + IsOSSH: true, + Keyword: keyword, + PaddingPRNGSeed: paddingPRNGSeed, + ClientPrefixSpec: &OSSHPrefixSpec{ + Name: tt.name, + Spec: tt.prefixSpec, + Seed: prefixSeed, + }, + } + + client, err := NewClientObfuscator(config) + if err != nil { + t.Fatalf("NewClientObfuscator failed: %s", err) + } + + preamble := bytes.NewBuffer(client.SendPreamble()) + + // check prefix + prefix := preamble.Next(len(tt.expectedPrefix)) + if !bytes.Equal(prefix, tt.expectedPrefix) { + t.Fatalf("expected prefix to be all zeros") + } + + // check terminator + terminator := preamble.Next(PREFIX_TERMINATOR_LENGTH) + expectedTerminator, err := makeTerminator(keyword, tt.expectedPrefix, OBFUSCATE_CLIENT_TO_SERVER_IV) + if err != nil { + t.Fatalf("makeTerminator failed: %s", err) + } + if !bytes.Equal(terminator, expectedTerminator) { + t.Fatalf("unexpected terminator") + } + + // OSSH key derivation + seed := preamble.Next(OBFUSCATE_SEED_LENGTH) + clientToServerCipher, _, err := initObfuscatorCiphers(config, seed) + if err != nil { + t.Fatalf("initObfuscatorCiphers failed: %s", err) + } + + // skip OSSH initial client message + osshInitialClientMsg := preamble.Next(8 + client.paddingLength) // 8: 4 bytes each for magic value and padding length + clientToServerCipher.XORKeyStream(osshInitialClientMsg, osshInitialClientMsg) + + // read prefix header + prefixHeader, err := readPrefixHeader(preamble, clientToServerCipher) + if err != nil { + t.Fatalf("readPrefixHeader failed: %s", err) + } + if prefixHeader.SpecName != tt.name { + t.Fatalf("unexpected spec name") + } + }) + } + +} + +// TestServerObfuscatorPrefix tests server obfuscator reading prefixed +// stream from client obfuscator, and generating expected prefix. +func TestServerObfuscatorPrefix(t *testing.T) { + + keyword := prng.HexString(32) + + paddingPRNGSeed, err := prng.NewSeed() + if err != nil { + t.Fatalf("prng.NewSeed failed: %s", err) + } + + prefixSeed, err := prng.NewSeed() + if err != nil { + t.Fatalf("prng.NewSeed failed: %s", err) + } + + clientPrefixLen := prng.Intn(976) + PREAMBLE_HEADER_LENGTH // max 1000 + + clientPrefixSpec := &OSSHPrefixSpec{ + Name: "zero-prefix", + Spec: transforms.Spec{{"", fmt.Sprintf("\\x00{%d}", clientPrefixLen)}}, + Seed: prefixSeed, + } + + serverPrefixSpec := transforms.Spec{{"", "(SERVER){4}"}} + + expectedServerPrefix := bytes.Repeat([]byte("SERVER"), 4) + serverTermInd := 24 // index of terminator in server prefix + + config := &ObfuscatorConfig{ + IsOSSH: true, + Keyword: keyword, + PaddingPRNGSeed: paddingPRNGSeed, + ClientPrefixSpec: clientPrefixSpec, + ServerPrefixSpecs: transforms.Specs{"zero-prefix": serverPrefixSpec}, + } + + client, err := NewClientObfuscator(config) + if err != nil { + t.Fatalf("NewClientObfuscator failed: %s", err) + } + + preamble := client.SendPreamble() + reader := WrapConnWithSkipReader(newConn(preamble)) + + // test server obfuscator + server, err := NewServerObfuscator(config, "", reader) + if err != nil { + t.Fatalf("NewServerObfuscator failed: %s", err) + } + + // check server prefix reply + serverPrefix := server.SendPreamble() + if !bytes.Equal(serverPrefix[:serverTermInd], expectedServerPrefix) { + t.Fatalf("unexpected server prefix") + } + + // check server terminator after prefix + serverTerminator := serverPrefix[serverTermInd:] + expectedTerminator, err := makeTerminator(keyword, serverPrefix, OBFUSCATE_SERVER_TO_CLIENT_IV) + if err != nil { + t.Fatalf("makeTerminator failed: %s", err) + } + if !bytes.Equal(serverTerminator, expectedTerminator) { + t.Fatalf("unexpected terminator") + } + + // check client terminator doesn't match server terminator + clientTerminator := preamble[clientPrefixLen : clientPrefixLen+PREFIX_TERMINATOR_LENGTH] + if bytes.Equal(clientTerminator, serverTerminator) { + t.Fatalf("client terminator should not match server terminator") + } + + clientMessage := []byte("client hello") + + b := append([]byte(nil), clientMessage...) + client.ObfuscateClientToServer(b) + server.ObfuscateClientToServer(b) + + if !bytes.Equal(clientMessage, b) { + t.Fatalf("unexpected client message") + } + + serverMessage := []byte("server hello") + + b = append([]byte(nil), serverMessage...) + client.ObfuscateServerToClient(b) + server.ObfuscateServerToClient(b) + + if !bytes.Equal(serverMessage, b) { + t.Fatalf("unexpected client message") + } + +} + +func TestIrregularConnections(t *testing.T) { + keyword := prng.HexString(32) + + maxPadding := 256 + + paddingPRNGSeed, err := prng.NewSeed() + if err != nil { + t.Fatalf("prng.NewSeed failed: %s", err) + } + + var irregularLogFields common.LogFields + + clientPrefixSpec := &OSSHPrefixSpec{ + Name: "zeros", + Spec: transforms.Spec{{"", "CLIENT\\x00{94}"}}, // 100 byte prefix + Seed: &prng.Seed{0}, + } + + seedHistory := NewSeedHistory(&SeedHistoryConfig{ClientIPTTL: 500 * time.Millisecond}) + + makeConfig := func(clientPrefix *OSSHPrefixSpec) *ObfuscatorConfig { + return &ObfuscatorConfig{ + IsOSSH: true, + Keyword: keyword, + MaxPadding: &maxPadding, + PaddingPRNGSeed: paddingPRNGSeed, + ClientPrefixSpec: clientPrefix, + SeedHistory: seedHistory, + IrregularLogger: func(_ string, err error, logFields common.LogFields) { + if logFields == nil { + logFields = make(common.LogFields) + } + logFields["tunnel_error"] = err.Error() + irregularLogFields = logFields + t.Logf("IrregularLogger: %+v", logFields) + }, + } + } + + config := makeConfig(clientPrefixSpec) + seedInd := 100 + PREFIX_TERMINATOR_LENGTH + + // Prefixed client cases + client, err := NewClientObfuscator(config) + if err != nil { + t.Fatalf("NewClientObfuscator failed: %s", err) + } + + if client.prefixHeader == nil { + t.Fatalf("unexpected nil prefixHeader") + } + + preamble := client.SendPreamble() + seed := hex.EncodeToString(preamble[seedInd : seedInd+OBFUSCATE_SEED_LENGTH]) + + clientIP := "192.168.0.1" + + // Test: successful connection + clientReader := WrapConnWithSkipReader(newConn(preamble)) + server, err := NewServerObfuscator(config, clientIP, clientReader) + if err != nil { + t.Fatalf("NewServerObfuscator failed: %s", err) + } + if server.prefixHeader == nil { + t.Fatalf("unexpected nil prefixHeader") + } + + irregularLogFields = nil + + // Test: replayed prefixd connection with same IP + clientReader = WrapConnWithSkipReader(newConn(preamble)) + _, err = NewServerObfuscator(config, clientIP, clientReader) + if err != nil { + t.Fatalf("NewServerObfuscator failed: %s", err) + } + + duplicateClientID := irregularLogFields["duplicate_client_ip"] + if duplicateClientID != "equal" { + t.Fatalf("Unexpected duplicate_client_ip: %s", duplicateClientID) + } + + duplicateSeed := irregularLogFields["duplicate_seed"] + if duplicateSeed != seed { + t.Fatalf("Unexpected duplicate_seed: %s", duplicateSeed) + } + + irregularLogFields = nil + + // Test: replayed prefixed connection with different IP + clientReader = WrapConnWithSkipReader(newConn(preamble)) + _, err = NewServerObfuscator(config, "192.168.0.2", clientReader) + if err == nil { + t.Fatalf("NewServerObfuscator unexpectedly succeeded") + } + + duplicateClientID = irregularLogFields["duplicate_client_ip"] + if duplicateClientID != "unequal" { + t.Fatalf("Unexpected duplicate_client_ip: %s", duplicateClientID) + } + + duplicateSeed = irregularLogFields["duplicate_seed"] + if duplicateSeed != seed { + t.Fatalf("Unexpected duplicate_seed: %s", duplicateSeed) + } + + irregularLogFields = nil + + // Test: replayed prefixed connection with same IP, but TTL expired + time.Sleep(600 * time.Millisecond) + + clientReader = WrapConnWithSkipReader(newConn(preamble)) + _, err = NewServerObfuscator(config, clientIP, clientReader) + if err == nil { + t.Fatalf("NewServerObfuscator unexpectedly succeeded") + } + + duplicateClientID = irregularLogFields["duplicate_client_ip"] + if duplicateClientID != "unknown" { + t.Fatalf("Unexpected duplicate_client_ip: %s", duplicateClientID) + } + + duplicateSeed = irregularLogFields["duplicate_seed"] + if duplicateSeed != seed { + t.Fatalf("Unexpected duplicate_seed: %s", duplicateSeed) + } + + irregularLogFields = nil + + // Test: Tacked on prefix from another connection, repeated seed + previousPrefix := bytes.Repeat([]byte{1}, PREAMBLE_HEADER_LENGTH) + terminator, err := makeTerminator(keyword, previousPrefix, OBFUSCATE_CLIENT_TO_SERVER_IV) + if err != nil { + t.Fatalf("makeTerminator failed: %s", err) + } + b := append(previousPrefix, terminator...) + b = append(b, preamble[seedInd:]...) + + clientReader = WrapConnWithSkipReader(newConn(b)) + _, err = NewServerObfuscator(config, clientIP, clientReader) + + if err == nil { + t.Fatalf("NewServerObfuscator failed: %s", err) + } + + duplicateSeed = irregularLogFields["duplicate_seed"] + if duplicateSeed != seed { + t.Fatalf("Unexpected duplicate_seed: %s", duplicateSeed) + } + + irregularLogFields = nil + + // Test: irregular logging of invalid magic value + client, err = NewClientObfuscator(config) + if err != nil { + t.Fatalf("NewClientObfuscator failed: %s", err) + } + + preamble = client.SendPreamble() + seedInd = 100 + PREFIX_TERMINATOR_LENGTH + preamble[seedInd+OBFUSCATE_SEED_LENGTH] = 0x00 // mutate magic value + + clientReader = WrapConnWithSkipReader(newConn(preamble)) + server, err = NewServerObfuscator(config, clientIP, clientReader) + if server != nil || err == nil { + t.Fatalf("NewServerObfuscator unexpectedly succeeded") + } + + tunnelError := irregularLogFields["tunnel_error"].(string) + if !strings.Contains(tunnelError, "invalid magic value") { + t.Fatalf("Unexpected tunnel_error: %s", tunnelError) + } + + irregularLogFields = nil + + // Test: irregular logging of invalid padding length + client, err = NewClientObfuscator(config) + if err != nil { + t.Fatalf("NewClientObfuscator failed: %s", err) + } + + preamble = client.SendPreamble() + seedInd = 100 + PREFIX_TERMINATOR_LENGTH + preamble[seedInd+OBFUSCATE_SEED_LENGTH+4] = 0x00 // mutate padding length + + clientReader = WrapConnWithSkipReader(newConn(preamble)) + server, err = NewServerObfuscator(config, clientIP, clientReader) + if server != nil || err == nil { + t.Fatalf("NewServerObfuscator unexpectedly succeeded") + } + + tunnelError = irregularLogFields["tunnel_error"].(string) + if !strings.Contains(tunnelError, "invalid padding length") { + t.Fatalf("Unexpected tunnel_error: %s", tunnelError) + } + + irregularLogFields = nil + +} + func TestObfuscatedSSHConn(t *testing.T) { + t.Run("non-prefixed", func(t *testing.T) { + obfuscatedSSHConnTestHelper(t, nil, nil) + }) + + t.Run("prefixed", func(t *testing.T) { + // prefixed obfuscated SSH + seed, err := prng.NewSeed() + if err != nil { + t.Fatalf("prng.NewSeed failed: %s", err) + } + + clientPrefixSpec := &OSSHPrefixSpec{ + Name: "spec-name", + Spec: transforms.Spec{{"", "CLIENT"}}, + Seed: seed, + } + + serverPrefixSpecs := transforms.Specs{ + "spec-name": transforms.Spec{{"", "SERVER"}}, + } + + obfuscatedSSHConnTestHelper(t, clientPrefixSpec, serverPrefixSpecs) + }) +} + +func obfuscatedSSHConnTestHelper( + t *testing.T, clientPrefixSpec *OSSHPrefixSpec, serverPrefixSpecs transforms.Specs) { + + t.Helper() + keyword := prng.HexString(32) serverAddress := "127.0.0.1:2222" @@ -193,12 +723,14 @@ func TestObfuscatedSSHConn(t *testing.T) { go func() { conn, err := listener.Accept() + defer listener.Close() if err == nil { conn, err = NewServerObfuscatedSSHConn( conn, keyword, NewSeedHistory(nil), + serverPrefixSpecs, func(_ string, err error, logFields common.LogFields) { t.Logf("IrregularLogger: %s %+v", err, logFields) }) @@ -213,6 +745,15 @@ func TestObfuscatedSSHConn(t *testing.T) { _, _, _, err = ssh.NewServerConn(conn, config) } + obfuscatedConn := conn.(*ObfuscatedSSHConn) + if obfuscatedConn.readState != OBFUSCATION_READ_STATE_FINISHED { + result <- errors.New("server readState not finished") + } + + if obfuscatedConn.writeState != OBFUSCATION_WRITE_STATE_FINISHED { + result <- errors.New("server writeState not finished") + } + if err != nil { select { case result <- err: @@ -235,7 +776,7 @@ func TestObfuscatedSSHConn(t *testing.T) { conn, keyword, paddingPRNGSeed, - nil, nil, nil) + nil, clientPrefixSpec, nil, nil) } var KEXPRNGSeed *prng.Seed @@ -251,6 +792,15 @@ func TestObfuscatedSSHConn(t *testing.T) { _, _, _, err = ssh.NewClientConn(conn, "", config) } + obfuscatedConn := conn.(*ObfuscatedSSHConn) + if obfuscatedConn.readState != OBFUSCATION_READ_STATE_FINISHED { + result <- errors.New("client readState not finished") + } + + if obfuscatedConn.writeState != OBFUSCATION_WRITE_STATE_FINISHED { + result <- errors.New("client writeState not finished") + } + // Sends nil on success select { case result <- err: @@ -264,92 +814,13 @@ func TestObfuscatedSSHConn(t *testing.T) { } } -func TestObfuscatorSeedTransformParameters(t *testing.T) { - - keyword := prng.HexString(32) - - maxPadding := 256 - - paddingPRNGSeed, err := prng.NewSeed() - if err != nil { - t.Fatalf("prng.NewSeed failed: %s", err) - } - - type test struct { - name string - transformerParamters *transforms.ObfuscatorSeedTransformerParameters - - // nil means seedMessage looks random (transformer was not applied) - expectedResult []byte - expectedResultLength int - } - - tests := []test{ - { - name: "4 byte transform", - transformerParamters: &transforms.ObfuscatorSeedTransformerParameters{ - TransformName: "four-zeros", - TransformSeed: &prng.Seed{0}, - TransformSpec: transforms.Spec{{"^.{8}", "00000000"}}, - }, - expectedResult: []byte{0, 0, 0, 0}, - expectedResultLength: 4, - }, - { - name: "invalid '%' character in the regex", - transformerParamters: &transforms.ObfuscatorSeedTransformerParameters{ - TransformName: "invalid-spec", - TransformSeed: &prng.Seed{0}, - TransformSpec: transforms.Spec{{"^.{8}", "%00000000"}}, - }, - expectedResult: nil, - expectedResultLength: 0, - }, - } - - for _, tt := range tests { - - t.Run(tt.name, func(t *testing.T) { +func newConn(b []byte) net.Conn { + conn1, conn2 := net.Pipe() - config := &ObfuscatorConfig{ - IsOSSH: true, - Keyword: keyword, - MaxPadding: &maxPadding, - PaddingPRNGSeed: paddingPRNGSeed, - ObfuscatorSeedTransformerParameters: tt.transformerParamters, - } - - client, err := NewClientObfuscator(config) - if err != nil { - // if there is a expectedResult, then the error is unexpected - if tt.expectedResult != nil { - t.Fatalf("NewClientObfuscator failed: %s", err) - } - return - } - - seedMessage := client.SendSeedMessage() - - if tt.expectedResult == nil { - - // Verify that the seed message looks random. - // obfuscator seed is generated with common.MakeSecureRandomBytes, - // and is not affected by the config. - popcount := 0 - for _, b := range seedMessage[:tt.expectedResultLength] { - popcount += bits.OnesCount(uint(b)) - } - popcount_per_byte := float64(popcount) / float64(tt.expectedResultLength) - if popcount_per_byte < 3.6 || popcount_per_byte > 4.4 { - t.Fatalf("unexpected popcount_per_byte: %f", popcount_per_byte) - } - - } else if !bytes.Equal(seedMessage[:tt.expectedResultLength], tt.expectedResult) { - t.Fatalf("unexpected seed message") - } - - }) - - } + go func() { + defer conn2.Close() + conn2.Write(b) + }() + return conn1 } diff --git a/psiphon/common/obfuscator/skipReader.go b/psiphon/common/obfuscator/skipReader.go new file mode 100644 index 000000000..9a670ddff --- /dev/null +++ b/psiphon/common/obfuscator/skipReader.go @@ -0,0 +1,114 @@ +package obfuscator + +import ( + "bytes" + "io" + "net" + + "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/errors" +) + +type SkipReader struct { + net.Conn + offset int // buf offset for next Read + end int // buf end index for next Read + buf []byte +} + +func WrapConnWithSkipReader(conn net.Conn) net.Conn { + return &SkipReader{ + Conn: conn, + offset: 0, + end: 0, + buf: nil, + } +} + +func (sr *SkipReader) Read(b []byte) (int, error) { + + // read buffered bytes first + if sr.offset < sr.end { + n := copy(b, sr.buf[sr.offset:sr.end]) + if n == 0 { + // should never happen if len(b) > 0 + return 0, errors.TraceNew("read failed") + } + + sr.offset += n + + // clear resources if all buffered bytes are read + if sr.offset == sr.end { + sr.offset = 0 + sr.end = 0 + sr.buf = nil + } + + return n, nil + } + + return sr.Conn.Read(b) +} + +// SkipUpToToken reads from the underlying conn initially len(token) bytes, +// and then readSize bytes at a time up to maxSearchSize until token is found, +// or error. If the token is found, stream is rewound to end of the token. +// +// Note that maxSearchSize is not a strict limit on the total number of bytes read. +func (sr *SkipReader) SkipUpToToken( + token []byte, readSize, maxSearchSize int) error { + + if len(token) == 0 { + return nil + } + if readSize < 1 { + return errors.TraceNew("readSize too small") + } + if maxSearchSize < readSize { + return errors.TraceNew("maxSearchSize too small") + } + + sr.offset = 0 + sr.end = 0 + sr.buf = make([]byte, readSize+len(token)) + + // Reads at least len(token) bytes. + nTotal, err := io.ReadFull(sr.Conn, sr.buf[:len(token)]) + if err == io.ErrUnexpectedEOF { + return errors.TraceNew("token not found") + } + if err != nil { + return err + } + + if bytes.Equal(sr.buf[:len(token)], token) { + return nil + } + + for nTotal < maxSearchSize { + + // The underlying conn is read into buf[len(token):]. + // buf[:len(token)] stores bytes from the previous read. + n, err := sr.Conn.Read(sr.buf[len(token):]) + if err != nil && err != io.EOF { + return err + } + + if idx := bytes.Index(sr.buf[:n+len(token)], token); idx != -1 { + // Found match, sets offset and end for next Read to start after the token. + sr.offset = idx + len(token) + sr.end = n + len(token) + return err + } + + if err == io.EOF { + // Reached the end of stream, token not found. + return errors.TraceNew("token not found") + } + + // Copies last len(token) bytes to the beginning of the buffer. + copy(sr.buf, sr.buf[n:n+len(token)]) + nTotal += n + } + + return errors.TraceNew("exceeded max search size") +} diff --git a/psiphon/common/obfuscator/skipReader_test.go b/psiphon/common/obfuscator/skipReader_test.go new file mode 100644 index 000000000..daf485526 --- /dev/null +++ b/psiphon/common/obfuscator/skipReader_test.go @@ -0,0 +1,175 @@ +package obfuscator + +import ( + "bytes" + "io" + "strings" + "testing" +) + +func TestReadBuffer(t *testing.T) { + t.Parallel() + + type test struct { + name string + prefix []byte + terminator []byte + postfix []byte + readSize int + expectedErrStr string + } + + tests := []test{ + { + name: "1 byte terminnator at start", + prefix: []byte{}, + terminator: []byte{'a'}, + postfix: []byte("postfix"), + readSize: 1024, + }, + { + name: "no prefix", + prefix: []byte{}, + terminator: []byte("[terminator]"), + postfix: []byte("postfix"), + readSize: 1, + }, + { + name: "small prefix", + prefix: []byte("prefix"), + terminator: []byte("[terminator]"), + postfix: []byte("postfix"), + readSize: 1, + }, + { + name: "large prefix", + prefix: []byte(strings.Repeat("prefix", 1000)), + terminator: []byte("[terminator]"), + postfix: []byte("postfix"), + readSize: 1, + }, + { + name: "large read size", + prefix: []byte(strings.Repeat("prefix", 1000)), + terminator: []byte("[terminator]"), + postfix: []byte("postfix"), + readSize: 8192, + }, + { + name: "max prefix size", + prefix: bytes.Repeat([]byte{'a'}, PREFIX_MAX_LENGTH), + terminator: []byte("[terminator]"), + postfix: []byte{}, + readSize: 8192, + expectedErrStr: "", + }, + { + name: "exceed max prefix length", + prefix: bytes.Repeat([]byte{'a'}, PREFIX_MAX_LENGTH+1), + terminator: []byte("[terminator]"), + postfix: []byte{}, + readSize: 8192, + expectedErrStr: "exceeded max search size", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + conn := newConn(append(tt.prefix, append(tt.terminator, tt.postfix...)...)) + reader, _ := WrapConnWithSkipReader(conn).(*SkipReader) + defer reader.Close() + + err := reader.SkipUpToToken( + tt.terminator, tt.readSize, PREFIX_MAX_LENGTH+len(tt.terminator)) + + if tt.expectedErrStr != "" { + if err == nil { + t.Fatalf("SkipUpToToken returned nil error, expected %s", tt.expectedErrStr) + } else if !strings.Contains(err.Error(), tt.expectedErrStr) { + t.Fatalf("SkipUpToToken returned error %s, expected %s", err, tt.expectedErrStr) + } else { + return + } + } + + if err != nil { + t.Fatalf("SkipUpToToken returned unexpected error: %s", err) + } + + // read the rest one byte at a time + var buff bytes.Buffer + for { + b := make([]byte, 1) + _, err := reader.Read(b) + if err != nil { + if err == io.EOF { + break + } + t.Fatal(err) + } + buff.Write(b) + } + + if !bytes.Equal(buff.Bytes(), tt.postfix) { + t.Fatalf("Read returned %v, expected %v", buff.Bytes(), tt.postfix) + } + + }) + } + +} + +func BenchmarkBase(b *testing.B) { + + data := make([]byte, 1024*1024) + for i := 0; i < len(data); i++ { + data[i] = byte(i % 256) + } + terminator := []byte("[terminator]postfix") + copy(data[len(data)-len(terminator):], terminator) + + b.ResetTimer() + + idx := bytes.Index(data, []byte("[terminator]")) + if idx == -1 { + b.Fatal("terminator not found") + } + + if idx != len(data)-len(terminator) { + b.Fatalf("terminator not at expected position: %d", idx) + } + +} + +func BenchmarkSkipReader(b *testing.B) { + + data := make([]byte, 1024*1024) + for i := 0; i < len(data); i++ { + data[i] = byte(i % 256) + } + tail := []byte("[terminator]postfix") + copy(data[len(data)-len(tail):], tail) + + conn := newConn(data) + reader, _ := WrapConnWithSkipReader(conn).(*SkipReader) + defer reader.Close() + + b.ResetTimer() + + err := reader.SkipUpToToken([]byte("[terminator]"), 1024, 1024*1024*1024) + if err != nil { + b.Fatalf("SkipUpToToken failed: %s", err) + } + + b.StopTimer() + + // read the rest + rest, err := io.ReadAll(reader) + if err != nil { + b.Fatal(err) + } + if string(rest) != "postfix" { + b.Fatalf("Read returned %s, expected 'postfix'", rest) + } +} diff --git a/psiphon/common/parameters/parameters.go b/psiphon/common/parameters/parameters.go index 26e36ac68..c8c564752 100755 --- a/psiphon/common/parameters/parameters.go +++ b/psiphon/common/parameters/parameters.go @@ -243,6 +243,7 @@ const ( ReplayResolveParameters = "ReplayResolveParameters" ReplayHTTPTransformerParameters = "ReplayHTTPTransformerParameters" ReplayOSSHSeedTransformerParameters = "ReplayOSSHSeedTransformerParameters" + ReplayOSSHPrefix = "ReplayOSSHPrefix" APIRequestUpstreamPaddingMinBytes = "APIRequestUpstreamPaddingMinBytes" APIRequestUpstreamPaddingMaxBytes = "APIRequestUpstreamPaddingMaxBytes" APIRequestDownstreamPaddingMinBytes = "APIRequestDownstreamPaddingMinBytes" @@ -337,6 +338,10 @@ const ( ObfuscatedQUICNonceTransformSpecs = "ObfuscatedQUICNonceTransformSpecs" ObfuscatedQUICNonceTransformScopedSpecNames = "ObfuscatedQUICNonceTransformScopedSpecNames" ObfuscatedQUICNonceTransformProbability = "ObfuscatedQUICNonceTransformProbability" + OSSHPrefixSpecs = "OSSHPrefixSpecs" + OSSHPrefixScopedSpecNames = "OSSHPrefixScopedSpecNames" + OSSHPrefixProbability = "OSSHPrefixProbability" + ServerOSSHPrefixSpecs = "ServerOSSHPrefixSpecs" ) const ( @@ -595,6 +600,7 @@ var defaultParameters = map[string]struct { ReplayResolveParameters: {value: true}, ReplayHTTPTransformerParameters: {value: true}, ReplayOSSHSeedTransformerParameters: {value: true}, + ReplayOSSHPrefix: {value: true}, APIRequestUpstreamPaddingMinBytes: {value: 0, minimum: 0}, APIRequestUpstreamPaddingMaxBytes: {value: 1024, minimum: 0}, @@ -713,6 +719,12 @@ var defaultParameters = map[string]struct { ObfuscatedQUICNonceTransformSpecs: {value: transforms.Specs{}}, ObfuscatedQUICNonceTransformScopedSpecNames: {value: transforms.ScopedSpecNames{}}, ObfuscatedQUICNonceTransformProbability: {value: 0.0, minimum: 0.0}, + + OSSHPrefixSpecs: {value: transforms.Specs{}}, + OSSHPrefixScopedSpecNames: {value: transforms.ScopedSpecNames{}}, + OSSHPrefixProbability: {value: 0.0, minimum: 0.0}, + + ServerOSSHPrefixSpecs: {value: transforms.Specs{}, flags: serverSideOnly}, } // IsServerSideOnly indicates if the parameter specified by name is used @@ -926,6 +938,13 @@ func (p *Parameters) Set( obfuscatedQuicNonceTransformSpecs, _ := obfuscatedQuicNonceTransformSpecsValue.(transforms.Specs) + osshPrefixSpecsValue, err := getAppliedValue( + OSSHPrefixSpecs, parameters, applyParameters) + if err != nil { + return nil, errors.Trace(err) + } + osshPrefixSpecs, _ := osshPrefixSpecsValue.(transforms.Specs) + for i := 0; i < len(applyParameters); i++ { count := 0 @@ -1118,6 +1137,8 @@ func (p *Parameters) Set( specs = osshObfuscatorSeedTransformSpecs } else if name == ObfuscatedQUICNonceTransformScopedSpecNames { specs = obfuscatedQuicNonceTransformSpecs + } else if name == OSSHPrefixScopedSpecNames { + specs = osshPrefixSpecs } err := v.Validate(specs) diff --git a/psiphon/common/protocol/serverEntry.go b/psiphon/common/protocol/serverEntry.go index b8a5fba28..e467112d1 100644 --- a/psiphon/common/protocol/serverEntry.go +++ b/psiphon/common/protocol/serverEntry.go @@ -78,6 +78,7 @@ type ServerEntry struct { DisableHTTPTransforms bool `json:"disableHTTPTransforms"` DisableObfuscatedQUICTransforms bool `json:"disableObfuscatedQUICTransforms"` DisableOSSHTransforms bool `json:"disableOSSHTransforms"` + DisableOSSHPrefix bool `json:"disableOSSHPrefix"` // These local fields are not expected to be present in downloaded server // entries. They are added by the client to record and report stats about diff --git a/psiphon/common/tactics/tactics.go b/psiphon/common/tactics/tactics.go index 1ce88c349..de637dcfd 100755 --- a/psiphon/common/tactics/tactics.go +++ b/psiphon/common/tactics/tactics.go @@ -1782,7 +1782,7 @@ func boxPayload( return nil, errors.Trace(err) } - obfuscatedBox := obfuscator.SendSeedMessage() + obfuscatedBox := obfuscator.SendPreamble() seedLen := len(obfuscatedBox) obfuscatedBox = append(obfuscatedBox, box...) diff --git a/psiphon/config.go b/psiphon/config.go index 93cec3b33..16dbca315 100755 --- a/psiphon/config.go +++ b/psiphon/config.go @@ -840,6 +840,10 @@ type Config struct { ObfuscatedQUICNonceTransformScopedSpecNames transforms.ScopedSpecNames ObfuscatedQUICNonceTransformProbability *float64 + OSSHPrefixSpecs transforms.Specs + OSSHPrefixScopedSpecNames transforms.ScopedSpecNames + OSSHPrefixProbability *float64 + // params is the active parameters.Parameters with defaults, config values, // and, optionally, tactics applied. // @@ -1974,6 +1978,18 @@ func (config *Config) makeConfigParameters() map[string]interface{} { applyParameters[parameters.ObfuscatedQUICNonceTransformProbability] = *config.ObfuscatedQUICNonceTransformProbability } + if config.OSSHPrefixSpecs != nil { + applyParameters[parameters.OSSHPrefixSpecs] = config.OSSHPrefixSpecs + } + + if config.OSSHPrefixScopedSpecNames != nil { + applyParameters[parameters.OSSHPrefixScopedSpecNames] = config.OSSHPrefixScopedSpecNames + } + + if config.OSSHPrefixProbability != nil { + applyParameters[parameters.OSSHPrefixProbability] = *config.OSSHPrefixProbability + } + // When adding new config dial parameters that may override tactics, also // update setDialParametersHash. @@ -2458,6 +2474,23 @@ func (config *Config) setDialParametersHash() { binary.Write(hash, binary.LittleEndian, *config.ObfuscatedQUICNonceTransformProbability) } + if config.OSSHPrefixSpecs != nil { + hash.Write([]byte("OSSHPrefixSpecs")) + encodedOSSHPrefixSpecs, _ := json.Marshal(config.OSSHPrefixSpecs) + hash.Write(encodedOSSHPrefixSpecs) + } + + if config.OSSHPrefixScopedSpecNames != nil { + hash.Write([]byte("")) + encodedOSSHPrefixScopedSpecNames, _ := json.Marshal(config.OSSHPrefixScopedSpecNames) + hash.Write(encodedOSSHPrefixScopedSpecNames) + } + + if config.OSSHPrefixProbability != nil { + hash.Write([]byte("OSSHPrefixProbability")) + binary.Write(hash, binary.LittleEndian, *config.OSSHPrefixProbability) + } + config.dialParametersHash = hash.Sum(nil) } diff --git a/psiphon/dialParameters.go b/psiphon/dialParameters.go index 5eab20c49..82e1bc2f9 100644 --- a/psiphon/dialParameters.go +++ b/psiphon/dialParameters.go @@ -34,6 +34,7 @@ import ( "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common" "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/errors" "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/fragmentor" + "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/obfuscator" "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/parameters" "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/prng" "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/protocol" @@ -91,6 +92,8 @@ type DialParameters struct { ObfuscatorPaddingSeed *prng.Seed OSSHObfuscatorSeedTransformerParameters *transforms.ObfuscatorSeedTransformerParameters + OSSHPrefixSpec *obfuscator.OSSHPrefixSpec + FragmentorSeed *prng.Seed FrontingProviderID string @@ -206,6 +209,7 @@ func MakeDialParameters( replayResolveParameters := p.Bool(parameters.ReplayResolveParameters) replayHTTPTransformerParameters := p.Bool(parameters.ReplayHTTPTransformerParameters) replayOSSHSeedTransformerParameters := p.Bool(parameters.ReplayOSSHSeedTransformerParameters) + replayOSSHPrefix := p.Bool(parameters.ReplayOSSHPrefix) // Check for existing dial parameters for this server/network ID. @@ -871,6 +875,25 @@ func MakeDialParameters( dialParams.OSSHObfuscatorSeedTransformerParameters = nil } } + + if serverEntry.DisableOSSHPrefix { + dialParams.OSSHPrefixSpec = nil + } else if !isReplay || !replayOSSHPrefix { + dialPortNumber, err := serverEntry.GetDialPortNumber(dialParams.TunnelProtocol) + if err != nil { + return nil, errors.Trace(err) + } + params, err := makeOSSHPrefixSpecParameters(p, strconv.Itoa(dialPortNumber)) + if err != nil { + return nil, errors.Trace(err) + } + + if params.Spec != nil { + dialParams.OSSHPrefixSpec = params + } else { + dialParams.OSSHPrefixSpec = nil + } + } } if protocol.TunnelProtocolUsesMeekHTTP(dialParams.TunnelProtocol) { @@ -1591,3 +1614,30 @@ func makeSeedTransformerParameters(p parameters.ParametersAccessor, }, nil } } + +func makeOSSHPrefixSpecParameters( + p parameters.ParametersAccessor, dialPortNumber string) (*obfuscator.OSSHPrefixSpec, error) { + + if !p.WeightedCoinFlip(parameters.OSSHPrefixProbability) { + return &obfuscator.OSSHPrefixSpec{}, nil + } + + specs := p.ProtocolTransformSpecs(parameters.OSSHPrefixSpecs) + scopedSpecNames := p.ProtocolTransformScopedSpecNames(parameters.OSSHPrefixScopedSpecNames) + + name, spec := specs.Select(dialPortNumber, scopedSpecNames) + + if spec == nil { + return &obfuscator.OSSHPrefixSpec{}, nil + } else { + seed, err := prng.NewSeed() + if err != nil { + return nil, errors.Trace(err) + } + return &obfuscator.OSSHPrefixSpec{ + Name: name, + Spec: spec, + Seed: seed, + }, nil + } +} diff --git a/psiphon/meekConn.go b/psiphon/meekConn.go index 22063857a..a8622cd83 100644 --- a/psiphon/meekConn.go +++ b/psiphon/meekConn.go @@ -1679,7 +1679,7 @@ func makeMeekObfuscationValues( if err != nil { return nil, "", 0, 0, 0.0, errors.Trace(err) } - obfuscatedCookie := obfuscator.SendSeedMessage() + obfuscatedCookie := obfuscator.SendPreamble() seedLen := len(obfuscatedCookie) obfuscatedCookie = append(obfuscatedCookie, encryptedCookie...) obfuscator.ObfuscateClientToServer(obfuscatedCookie[seedLen:]) diff --git a/psiphon/notice.go b/psiphon/notice.go index 607ca7a76..b77057170 100644 --- a/psiphon/notice.go +++ b/psiphon/notice.go @@ -597,6 +597,12 @@ func noticeWithDialParameters(noticeType string, dialParams *DialParameters, pos } } + if dialParams.OSSHPrefixSpec != nil { + if dialParams.OSSHPrefixSpec.Spec != nil { + args = append(args, "OSSHPrefix", dialParams.OSSHPrefixSpec.Name) + } + } + if dialParams.DialConnMetrics != nil { metrics := dialParams.DialConnMetrics.GetMetrics() for name, value := range metrics { diff --git a/psiphon/server/api.go b/psiphon/server/api.go index 91d66e0a7..e639553d9 100644 --- a/psiphon/server/api.go +++ b/psiphon/server/api.go @@ -937,6 +937,7 @@ var baseDialParams = []requestParamSpec{ {"dns_attempt", isIntString, requestParamOptional | requestParamLogStringAsInt}, {"http_transform", isAnyString, requestParamOptional}, {"seed_transform", isAnyString, requestParamOptional}, + {"ossh_prefix", isAnyString, requestParamOptional}, } // baseSessionAndDialParams adds baseDialParams to baseSessionParams. diff --git a/psiphon/server/tunnelServer.go b/psiphon/server/tunnelServer.go index d6895c39c..24928222a 100644 --- a/psiphon/server/tunnelServer.go +++ b/psiphon/server/tunnelServer.go @@ -50,6 +50,7 @@ import ( "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/quic" "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/refraction" "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/tactics" + "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/transforms" "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/tun" "github.com/marusama/semaphore" cache "github.com/patrickmn/go-cache" @@ -1907,12 +1908,28 @@ func (sshClient *sshClient) run( if err == nil && protocol.TunnelProtocolUsesObfuscatedSSH(sshClient.tunnelProtocol) { + p, err := sshClient.sshServer.support.ServerTacticsParametersCache.Get(sshClient.geoIPData) + + // Log error, but continue. A default prefix spec will be used by the server. + if err != nil { + log.WithTraceFields(LogFields{"error": errors.Trace(err)}).Warning( + "ServerTacticsParametersCache.Get failed") + } + + var serverOsshPrefixSpecs transforms.Specs = nil + if !p.IsNil() { + serverOsshPrefixSpecs = p.ProtocolTransformSpecs(parameters.ServerOSSHPrefixSpecs) + // Allow garbage collection. + p.Close() + } + // Note: NewServerObfuscatedSSHConn blocks on network I/O // TODO: ensure this won't block shutdown result.obfuscatedSSHConn, err = obfuscator.NewServerObfuscatedSSHConn( conn, sshClient.sshServer.support.Config.ObfuscatedSSHKey, sshClient.sshServer.obfuscatorSeedHistory, + serverOsshPrefixSpecs, func(clientIP string, err error, logFields common.LogFields) { logIrregularTunnel( sshClient.sshServer.support, diff --git a/psiphon/serverApi.go b/psiphon/serverApi.go index 6eeb5ed67..db9105628 100644 --- a/psiphon/serverApi.go +++ b/psiphon/serverApi.go @@ -1127,6 +1127,12 @@ func getBaseAPIParameters( } } + if dialParams.OSSHPrefixSpec != nil { + if dialParams.OSSHPrefixSpec.Spec != nil { + params["ossh_prefix"] = dialParams.OSSHPrefixSpec.Name + } + } + if dialParams.DialConnMetrics != nil { metrics := dialParams.DialConnMetrics.GetMetrics() for name, value := range metrics { diff --git a/psiphon/tunnel.go b/psiphon/tunnel.go index 64b8f8933..13c4e9489 100644 --- a/psiphon/tunnel.go +++ b/psiphon/tunnel.go @@ -983,6 +983,7 @@ func dialTunnel( dialParams.ServerEntry.SshObfuscatedKey, dialParams.ObfuscatorPaddingSeed, dialParams.OSSHObfuscatorSeedTransformerParameters, + dialParams.OSSHPrefixSpec, &obfuscatedSSHMinPadding, &obfuscatedSSHMaxPadding) if err != nil { From cf5ce056e725840647a5ee908d5b323b40807269 Mon Sep 17 00:00:00 2001 From: Amir Khan Date: Mon, 12 Jun 2023 11:42:17 -0400 Subject: [PATCH 3/3] OSSH prefix overrides OSSH seed transforms --- psiphon/dialParameters.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/psiphon/dialParameters.go b/psiphon/dialParameters.go index 82e1bc2f9..773ce792c 100644 --- a/psiphon/dialParameters.go +++ b/psiphon/dialParameters.go @@ -850,8 +850,8 @@ func MakeDialParameters( } - // OSSH seed transforms are applied only to the OSSH tunnel protocol, and - // not to any other protocol layered over OSSH. + // OSSH prefix and seed transform are applied only to the OSSH tunnel protocol, + // and not to any other protocol layered over OSSH. if dialParams.TunnelProtocol == protocol.TUNNEL_PROTOCOL_OBFUSCATED_SSH { if serverEntry.DisableOSSHTransforms { @@ -894,6 +894,14 @@ func MakeDialParameters( dialParams.OSSHPrefixSpec = nil } } + + // OSSHPrefix supersedes OSSHObfuscatorSeedTransform. + // This ensures both tactics are not used simultaneously, + // until OSSHObfuscatorSeedTransform is removed. + if dialParams.OSSHPrefixSpec != nil { + dialParams.OSSHObfuscatorSeedTransformerParameters = nil + } + } if protocol.TunnelProtocolUsesMeekHTTP(dialParams.TunnelProtocol) {