gentoo-overlay/net-libs/libtorrent/files/libtorrent-0001-Fix-the-DH-parameters-generation-with-OpenSSL-1.1.patch

From 4607bbf78040789dee29266878ce109136b984ef Mon Sep 17 00:00:00 2001
From: rakshasa <sundell.software@gmail.com>
Date: Tue, 20 Dec 2016 19:51:02 +0900
Subject: [PATCH] Added support for openssl 1.1.

---
 configure.ac                |  4 ++++
 src/utils/diffie_hellman.cc | 36 ++++++++++++++++++++++++++++++++++--
 2 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/configure.ac b/configure.ac
index 65e34872..27e33570 100644
--- a/configure.ac
+++ b/configure.ac
@@ -69,12 +69,15 @@ AC_ARG_ENABLE(openssl,
   [  --disable-openssl       Don't use OpenSSL's SHA1 implementation.],
   [
     if test "$enableval" = "yes"; then
+dnl move to scripts.
       PKG_CHECK_MODULES(OPENSSL, libcrypto,
         CXXFLAGS="$CXXFLAGS $OPENSSL_CFLAGS";
         LIBS="$LIBS $OPENSSL_LIBS")
 
       AC_DEFINE(USE_OPENSSL, 1, Using OpenSSL.)
       AC_DEFINE(USE_OPENSSL_SHA, 1, Using OpenSSL's SHA1 implementation.)
+      AC_CHECK_LIB([crypto], [DH_set0_pqg], [AC_DEFINE(USE_OPENSSL_1_1, 1, Using OpenSSL 1.1.)])
+
     else
       AC_DEFINE(USE_NSS_SHA, 1, Using Mozilla's SHA1 implementation.)
     fi
@@ -85,6 +88,7 @@ AC_ARG_ENABLE(openssl,
 
     AC_DEFINE(USE_OPENSSL, 1, Using OpenSSL.)
     AC_DEFINE(USE_OPENSSL_SHA, 1, Using OpenSSL's SHA1 implementation.)
+    AC_CHECK_LIB([crypto], [DH_set0_pqg], [AC_DEFINE(USE_OPENSSL_1_1, 1, Using OpenSSL 1.1.)])
   ]
 )
 
diff --git a/src/utils/diffie_hellman.cc b/src/utils/diffie_hellman.cc
index aa653d45..7ec13165 100644
--- a/src/utils/diffie_hellman.cc
+++ b/src/utils/diffie_hellman.cc
@@ -54,11 +54,23 @@ DiffieHellman::DiffieHellman(const unsigned char *prime, int primeLength,
   m_secret(NULL), m_size(0) {
 
 #ifdef USE_OPENSSL
+
   m_dh = DH_new();
+
+#ifdef USE_OPENSSL_1_1
+  BIGNUM * const dh_p = BN_bin2bn(prime, primeLength, NULL);
+  BIGNUM * const dh_g = BN_bin2bn(generator, generatorLength, NULL);
+
+  if (dh_p == NULL || dh_g == NULL ||
+      !DH_set0_pqg(m_dh, dh_p, NULL, dh_g))
+	  throw internal_error("Could not generate Diffie-Hellman parameters");
+#else
   m_dh->p = BN_bin2bn(prime, primeLength, NULL);
   m_dh->g = BN_bin2bn(generator, generatorLength, NULL);
+#endif
 
   DH_generate_key(m_dh);
+
 #else
   throw internal_error("Compiled without encryption support.");
 #endif
@@ -74,7 +86,19 @@ DiffieHellman::~DiffieHellman() {
 bool
 DiffieHellman::is_valid() const {
 #ifdef USE_OPENSSL
+  if (m_dh == NULL)
+    return false;
+
+#ifdef USE_OPENSSL_1_1
+  const BIGNUM *pub_key;
+
+  DH_get0_key(m_dh, &pub_key, NULL);
+
+  return pub_key != NULL;
+#else
   return m_dh != NULL && m_dh->pub_key != NULL;
+#endif
+
 #else
   return false;
 #endif
@@ -103,8 +127,16 @@ DiffieHellman::store_pub_key(unsigned char* dest, unsigned int length) {
 #ifdef USE_OPENSSL
   std::memset(dest, 0, length);
 
-  if ((int)length >= BN_num_bytes(m_dh->pub_key))
-    BN_bn2bin(m_dh->pub_key, dest + length - BN_num_bytes(m_dh->pub_key));
+  const BIGNUM *pub_key;
+
+#ifdef USE_OPENSSL_1_1
+  DH_get0_key(m_dh, &pub_key, NULL);
+#else
+  pub_key = m_dh->pub_key;
+#endif
+
+  if ((int)length >= BN_num_bytes(pub_key))
+    BN_bn2bin(pub_key, dest + length - BN_num_bytes(pub_key));
 #endif
 }