Add new SSH public key crypto code in ssh_kex

The original code is too complicated and is broken under OpenSSL 1.1.0,
so I rewrite new crypto code based on the original one, and make it work
on OpenSSL 1.1.0.

Thanks libssh2 for the HAVE_OPAQUE_STRUCT macro.

6 files changed, 162 insertions, 29 deletions

diff --git a/src/protocol/CMakeLists.txt b/src/protocol/CMakeLists.txt
index cd7d2f4..40f0dba 100644
--- a/src/protocol/CMakeLists.txt
+++ b/src/protocol/CMakeLists.txt
@@ -13,6 +13,8 @@ set(export_SRCS
 )
 
 set(internal_SRCS
+  internal/ssh_pubkey_crypto.h
+  internal/ssh_pubkey_crypto.c
   internal/crc32.h
   internal/fqterm_serialization.h
   internal/fqterm_ssh_auth.h
@@ -28,7 +30,6 @@ set(internal_SRCS
   internal/fqterm_ssh_packet.h
   internal/fqterm_ssh1_packet.h
   internal/fqterm_ssh2_packet.h
-  internal/fqterm_ssh_rsa.h
   internal/fqterm_ssh_channel.h
   internal/fqterm_ssh_types.h
   internal/fqterm_ssh_pubkey.h
@@ -43,7 +44,6 @@ set(internal_SRCS
   internal/fqterm_ssh_packet.cpp
   internal/fqterm_ssh1_packet.cpp
   internal/fqterm_ssh2_packet.cpp
-  internal/fqterm_ssh_rsa.cpp
   internal/fqterm_ssh_channel.cpp
 )
 
diff --git a/src/protocol/internal/fqterm_ssh2_kex.h b/src/protocol/internal/fqterm_ssh2_kex.h
index 671e094..23beba2 100644
--- a/src/protocol/internal/fqterm_ssh2_kex.h
+++ b/src/protocol/internal/fqterm_ssh2_kex.h
@@ -73,8 +73,8 @@ private:
 
   bool is_first_kex_;
 
-  FQTermSSHRSA *host_key_;
-  FQTermSSHRSA *server_key_;
+  ssh_pubkey_t *host_key_;
+  ssh_pubkey_t *server_key_;
 
   u_char cookie_[16];
   int server_flag_, ciphers_, auth_;
diff --git a/src/protocol/internal/fqterm_ssh_kex.cpp b/src/protocol/internal/fqterm_ssh_kex.cpp
index d27bf14..eb3b012 100644
--- a/src/protocol/internal/fqterm_ssh_kex.cpp
+++ b/src/protocol/internal/fqterm_ssh_kex.cpp
@@ -82,7 +82,11 @@ void FQTermSSH1Kex::handlePacket(int type) {
 
 void FQTermSSH1Kex::makeSessionKey() {
   int i;
-  BIGNUM *key;
+  BIGNUM *key = BN_new();
+  BIGNUM *host_rsa_e = BN_new();
+  BIGNUM *host_rsa_n = BN_new();
+  BIGNUM *server_rsa_e = BN_new();
+  BIGNUM *server_rsa_n = BN_new();
   u_int32_t rand_val;
   int bits;
   int rbits;
@@ -94,12 +98,13 @@ void FQTermSSH1Kex::makeSessionKey() {
   packet_receiver_->getRawData((char*)cookie_, 8);
 
   // Get the public key.
-  server_key_ = new FQTermSSHRSA;
+  server_key_ = ssh_pubkey_new(SSH_RSA);
   bits = packet_receiver_->getInt();
-  packet_receiver_->getBN(server_key_->d_rsa->e);
-  packet_receiver_->getBN(server_key_->d_rsa->n);
+  packet_receiver_->getBN(server_rsa_e);
+  packet_receiver_->getBN(server_rsa_n);
+  ssh_pubkey_setrsa(server_key_, server_rsa_n, server_rsa_e, NULL);
+  rbits = BN_num_bits(server_rsa_n);
 
-  rbits = BN_num_bits(server_key_->d_rsa->n);
   if (bits != rbits) {
     FQ_TRACE("sshkex", 0) << "Warning: Server lies about "
                           << "size of server public key: "
@@ -110,12 +115,13 @@ void FQTermSSH1Kex::makeSessionKey() {
   }
 
   // Get the host key.
-  host_key_ = new FQTermSSHRSA;
+  host_key_ = ssh_pubkey_new(SSH_RSA);
   bits = packet_receiver_->getInt();
-  packet_receiver_->getBN(host_key_->d_rsa->e);
-  packet_receiver_->getBN(host_key_->d_rsa->n);
+  packet_receiver_->getBN(host_rsa_e);
+  packet_receiver_->getBN(host_rsa_n);
+  ssh_pubkey_setrsa(host_key_, host_rsa_n, host_rsa_e, NULL);
+  rbits = BN_num_bits(host_rsa_n);
 
-  rbits = BN_num_bits(host_key_->d_rsa->n);
   if (bits != rbits) {
     FQ_TRACE("sshkex", 0) << "Warning: Server lies about "
                           << "size of server public key: "
@@ -149,8 +155,6 @@ void FQTermSSH1Kex::makeSessionKey() {
     rand_val >>= 8;
   }
 
-  key = BN_new();
-
   BN_set_word(key, 0);
   for (i = 0; i < 32; i++) {
     BN_lshift(key, key, 8);
@@ -161,16 +165,16 @@ void FQTermSSH1Kex::makeSessionKey() {
     } 
   }
 
-  if (BN_cmp(server_key_->d_rsa->n, host_key_->d_rsa->n) < 0) {
-    server_key_->publicEncrypt(key, key);
-    host_key_->publicEncrypt(key, key);
+  if (BN_cmp(server_rsa_n, host_rsa_n) < 0) {
+	  ssh_pubkey_encrypt(server_key_, key, key);
+	  ssh_pubkey_encrypt(host_key_, key, key);
   } else {
-    host_key_->publicEncrypt(key, key);
-    server_key_->publicEncrypt(key, key);
+	  ssh_pubkey_encrypt(host_key_, key, key);
+	  ssh_pubkey_encrypt(server_key_, key, key);
   }
 
-  delete host_key_;
-  delete server_key_;
+  ssh_pubkey_free(host_key_);
+  ssh_pubkey_free(server_key_);
 
   packet_sender_->startPacket(SSH1_CMSG_SESSION_KEY);
   packet_sender_->putByte(SSH_CIPHER_3DES);
@@ -189,15 +193,21 @@ void FQTermSSH1Kex::makeSessionId() {
   u_char *p;
   FQTermSSHMD5 *md5;
   int servlen, hostlen;
+  const BIGNUM *host_n;
+  const BIGNUM *server_n;
+  const BIGNUM *e, *d;
 
   md5 = new FQTermSSHMD5;
-  servlen = BN_num_bytes(server_key_->d_rsa->n);
-  hostlen = BN_num_bytes(host_key_->d_rsa->n);
+  ssh_pubkey_getrsa(server_key_, &server_n, &e, &d);
+  ssh_pubkey_getrsa(host_key_, &host_n, &e, &d);
+  servlen = BN_num_bytes(server_n);
+  hostlen = BN_num_bytes(host_n);
 
   p = new u_char[servlen + hostlen];
 
-  BN_bn2bin(host_key_->d_rsa->n, p);
-  BN_bn2bin(server_key_->d_rsa->n, p + hostlen);
+  BN_bn2bin(host_n, p);
+  BN_bn2bin(server_n, p+hostlen);
+
   md5->update(p, servlen + hostlen);
   md5->update(cookie_, 8);
   md5->final(session_id_);
diff --git a/src/protocol/internal/fqterm_ssh_kex.h b/src/protocol/internal/fqterm_ssh_kex.h
index b2f9c56..f84c0d5 100644
--- a/src/protocol/internal/fqterm_ssh_kex.h
+++ b/src/protocol/internal/fqterm_ssh_kex.h
@@ -25,9 +25,9 @@
 #include <QObject>
 
 #include "fqterm_ssh_types.h"
-#include "fqterm_ssh_rsa.h"
 #include "fqterm_ssh_packet.h"
 #include "fqterm_ssh_const.h"
+#include "ssh_pubkey_crypto.h"
 
 namespace FQTerm {
 
@@ -75,8 +75,8 @@ private:
 
   bool is_first_kex_;
 
-  FQTermSSHRSA *host_key_;
-  FQTermSSHRSA *server_key_;
+  struct ssh_pubkey_t *host_key_;
+  struct ssh_pubkey_t *server_key_;
 
   u_char cookie_[8];
   int server_flag_, ciphers_, auth_;
diff --git a/src/protocol/internal/ssh_pubkey_crypto.c b/src/protocol/internal/ssh_pubkey_crypto.c
new file mode 100644
index 0000000..43465eb
--- /dev/null
+++ b/src/protocol/internal/ssh_pubkey_crypto.c
@@ -0,0 +1,76 @@
+#include "ssh_pubkey_crypto.h"
+#include <stdlib.h>
+
+struct ssh_pubkey_t*
+ssh_pubkey_new(enum pubkey_type t)
+{
+	struct ssh_pubkey_t *k = (struct ssh_pubkey_t*)
+		malloc(sizeof(struct ssh_pubkey_t));
+	k->key_type = t;
+	switch (t) {
+	case SSH_RSA:
+		k->key.ssh_rsa = RSA_new();
+	}
+	return k;
+}
+
+void
+ssh_pubkey_free(struct ssh_pubkey_t *k)
+{
+	switch (k->key_type) {
+	case SSH_RSA:
+		RSA_free(k->key.ssh_rsa);
+	}
+	free(k);
+}
+
+static int
+ssh_pubkey_encrypt_rsa(RSA *k, BIGNUM *out, BIGNUM *in)
+{
+	size_t len, ilen, olen;
+
+	olen = RSA_size(k);
+	ilen = BN_num_bytes(in);
+
+	unsigned char outbuf[olen], inbuf[ilen];
+
+	BN_bn2bin(in, inbuf);
+	len = RSA_public_encrypt(ilen, inbuf, outbuf, k,
+				 RSA_PKCS1_PADDING);
+	if (len <= 0) {
+		return -1;
+	}
+	BN_bin2bn(outbuf, len, out);
+	return 0;
+}
+
+int
+ssh_pubkey_encrypt(struct ssh_pubkey_t *k, BIGNUM *out, BIGNUM *in)
+{
+	switch (k->key_type) {
+	case SSH_RSA:
+		return ssh_pubkey_encrypt_rsa(k->key.ssh_rsa, out, in);
+	}
+}
+
+#ifndef HAVE_OPAQUE_STRUCTS
+int
+ssh_pubkey_setrsa(struct ssh_pubkey_t *k, BIGNUM *n, BIGNUM *e, BIGNUM *d)
+{
+	RSA *r = k->key.ssh_rsa;
+	r->n = n;
+	r->e = e;
+	r->d = d;
+	return 0;
+}
+
+int
+ssh_pubkey_getrsa(struct ssh_pubkey_t *k, const BIGNUM **n, const BIGNUM **e, const BIGNUM **d)
+{
+	RSA *r = k->key.ssh_rsa;
+	*n = r->n;
+	*e = r->e;
+	*d = r->d;
+	return 0;
+}
+#endif
diff --git a/src/protocol/internal/ssh_pubkey_crypto.h b/src/protocol/internal/ssh_pubkey_crypto.h
new file mode 100644
index 0000000..4816249
--- /dev/null
+++ b/src/protocol/internal/ssh_pubkey_crypto.h
@@ -0,0 +1,47 @@
+/* This file is part of FQTerm project
+ * written by Iru Cai <mytbk920423@gmail.com>
+ */
+
+#ifndef SSH_PUBKEY_CRYPTO_H
+#define SSH_PUBKEY_CRYPTO_H
+
+#include <openssl/rsa.h>
+
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L && \
+    !defined(LIBRESSL_VERSION_NUMBER)
+# define HAVE_OPAQUE_STRUCTS 1
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+	enum pubkey_type {
+		SSH_RSA
+	};
+
+	struct ssh_pubkey_t
+	{
+		enum pubkey_type key_type; /* now only RSA is supported */
+		union {
+			RSA *ssh_rsa;
+		} key;
+	};
+
+	struct ssh_pubkey_t *ssh_pubkey_new(enum pubkey_type);
+	void ssh_pubkey_free(struct ssh_pubkey_t*);
+	int ssh_pubkey_encrypt(struct ssh_pubkey_t *k, BIGNUM *out, BIGNUM *in);
+
+#ifdef HAVE_OPAQUE_STRUCTS
+#define ssh_pubkey_setrsa(k,n,e,d) RSA_set0_key(k->key.ssh_rsa, n, e, d)
+#define ssh_pubkey_getrsa(k,n,e,d) RSA_get0_key(k->key.ssh_rsa, n, e, d)
+#else
+	int ssh_pubkey_setrsa(struct ssh_pubkey_t *k, BIGNUM *n, BIGNUM *e, BIGNUM *d);
+	int ssh_pubkey_getrsa(struct ssh_pubkey_t *k, const BIGNUM **n, const BIGNUM **e, const BIGNUM **d);
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif