bpo-9216: [WIP] Support OpenSSL FIPS mode.

Lib/hashlib.py

@@ -24,6 +24,16 @@
 Choose your hash function wisely.  Some have known collision weaknesses.
 sha384 and sha512 will be slow on 32 bit platforms.
 
+If the underlying implementation supports "FIPS mode", and this is enabled, it
+may restrict the available hashes to only those that are compliant with FIPS
+regulations.  For example, it may deny the use of MD5, on the grounds that this
+is not secure for uses such as authentication, system integrity checking, or
+digital signatures.   If you need to use such a hash for non-security purposes
+(such as indexing into a data structure for speed), you can override the keyword
+argument "usedforsecurity" from True to False to signify that your code is not
+relying on the hash for security purposes, and this will allow the hash to be
+usable even in FIPS mode.
+
 Hash objects have these methods:
  - update(arg): Update the hash object with the bytes in arg. Repeated calls
                 are equivalent to a single call with the concatenation of all
@@ -67,6 +77,19 @@
 __all__ = __always_supported + ('new', 'algorithms_guaranteed',
                                 'algorithms_available', 'pbkdf2_hmac')
 
+import functools
+def __ignore_usedforsecurity(func):
+    """Used for sha3_* functions. Until OpenSSL implements them, we want
+    to use them from Python _sha3 module, but we want them to accept
+    usedforsecurity argument too."""
+    # TODO: remove this function when OpenSSL implements sha3
+    @functools.wraps(func)
+    def inner(*args, **kwargs):
+        if 'usedforsecurity' in kwargs:
+            kwargs.pop('usedforsecurity')
+        return func(*args, **kwargs)
+    return inner
+
 
 __builtin_constructor_cache = {}
 
@@ -121,23 +144,33 @@ def __get_openssl_constructor(name):
         f = getattr(_hashlib, 'openssl_' + name)
         # Allow the C module to raise ValueError.  The function will be
         # defined but the hash not actually available thanks to OpenSSL.
-        f()
+        # We pass "usedforsecurity=False" to disable FIPS-based restrictions:
+        # at this stage we're merely seeing if the function is callable,
+        # rather than using it for actual work.
+        f(usedforsecurity=False)
         # Use the C function directly (very fast)
         return f
     except (AttributeError, ValueError):
+        # TODO: We want to just raise here when OpenSSL implements sha3
+        # because we want to make sure that Fedora uses everything from OpenSSL
         return __get_builtin_constructor(name)
 
 
-def __py_new(name, data=b'', **kwargs):
-    """new(name, data=b'', **kwargs) - Return a new hashing object using the
-    named algorithm; optionally initialized with data (which must be bytes).
+def __py_new(name, data=b'', *, usedforsecurity=True, **kwargs):
+    """new(name, data=b'', usedforsecurity=True) - Return a new hashing object using
+    the named algorithm; optionally initialized with data (which must be bytes).
+    The 'usedforsecurity' keyword argument does nothing, and is for compatibilty
+    with the OpenSSL implementation
     """
     return __get_builtin_constructor(name)(data, **kwargs)
 
 
-def __hash_new(name, data=b'', **kwargs):
-    """new(name, data=b'') - Return a new hashing object using the named algorithm;
-    optionally initialized with data (which must be bytes).
+def __hash_new(name, data=b'', *, usedforsecurity=True, **kwargs):
+    """new(name, data=b'', usedforsecurity=True) - Return a new hashing object using
+    the named algorithm; optionally initialized with data (which must be bytes).
+
+    Override 'usedforsecurity' to False when using for non-security purposes in
+    a FIPS environment
     """
     if name in {'blake2b', 'blake2s'}:
         # Prefer our blake2 implementation.
@@ -146,12 +179,10 @@ def __hash_new(name, data=b'', **kwargs):
         # salt, personal, tree hashing or SSE.
         return __get_builtin_constructor(name)(data, **kwargs)
     try:
-        return _hashlib.new(name, data)
+        return _hashlib.new(name, data, usedforsecurity)
     except ValueError:
-        # If the _hashlib module (OpenSSL) doesn't support the named
-        # hash, try using our builtin implementations.
-        # This allows for SHA224/256 and SHA384/512 support even though
-        # the OpenSSL library prior to 0.9.8 doesn't provide them.
+        # TODO: We want to just raise here when OpenSSL implements sha3
+        # because we want to make sure that Fedora uses everything from OpenSSL
         return __get_builtin_constructor(name)(data)
 
 
@@ -162,8 +193,8 @@ def __hash_new(name, data=b'', **kwargs):
     algorithms_available = algorithms_available.union(
             _hashlib.openssl_md_meth_names)
 except ImportError:
-    new = __py_new
-    __get_hash = __get_builtin_constructor
+    # We don't build the legacy modules
+    raise
 
 try:
     # OpenSSL's PKCS5_PBKDF2_HMAC requires OpenSSL 1.0+ with HMAC and SHA
@@ -240,12 +271,16 @@ def prf(msg, inner=inner, outer=outer):
     # try them all, some may not work due to the OpenSSL
     # version not supporting that algorithm.
     try:
-        globals()[__func_name] = __get_hash(__func_name)
+        func = __get_hash(__func_name)
+        if __func_name.startswith(('sha3_', 'blake2', 'shake_')):
+            func = __ignore_usedforsecurity(func)
+        globals()[__func_name] = func
     except ValueError:
         import logging
         logging.exception('code for hash %s was not found.', __func_name)
 
 
 # Cleanup locals()
 del __always_supported, __func_name, __get_hash
-del __py_new, __hash_new, __get_openssl_constructor
+del __hash_new, __get_openssl_constructor
+del __ignore_usedforsecurity

Lib/test/test_hashlib.py

@@ -24,7 +24,22 @@
 COMPILED_WITH_PYDEBUG = hasattr(sys, 'gettotalrefcount')
 
 c_hashlib = import_fresh_module('hashlib', fresh=['_hashlib'])
-py_hashlib = import_fresh_module('hashlib', blocked=['_hashlib'])
+# skipped on Fedora, since we always use OpenSSL implementation
+# py_hashlib = import_fresh_module('hashlib', blocked=['_hashlib'])
+
+def openssl_enforces_fips():
+    # Use the "openssl" command (if present) to try to determine if the local
+    # OpenSSL is configured to enforce FIPS
+    from subprocess import Popen, PIPE
+    try:
+        p = Popen(['openssl', 'md5'],
+                  stdin=PIPE, stdout=PIPE, stderr=PIPE)
+    except OSError:
+        # "openssl" command not found
+        return False
+    stdout, stderr = p.communicate(input=b'abc')
+    return b'unknown cipher' in stderr
+OPENSSL_ENFORCES_FIPS = openssl_enforces_fips()
 
 try:
     import _blake2
@@ -68,6 +83,17 @@ def read_vectors(hash_name):
             yield parts
 
 
+# hashlib and _hashlib-based functions support a "usedforsecurity" keyword
+# argument, and FIPS mode requires that it be used overridden with a False
+# value for these selftests to work.  Other cryptographic code within Python
+# doesn't support this keyword.
+# Modify a function to one in which "usedforsecurity=False" is added to the
+# keyword arguments:
+def suppress_fips(f):
+    def g(*args, **kwargs):
+        return f(*args, usedforsecurity=False, **kwargs)
+    return g
+
 class HashLibTestCase(unittest.TestCase):
     supported_hash_names = ( 'md5', 'MD5', 'sha1', 'SHA1',
                              'sha224', 'SHA224', 'sha256', 'SHA256',
@@ -106,11 +132,11 @@ def __init__(self, *args, **kwargs):
         # For each algorithm, test the direct constructor and the use
         # of hashlib.new given the algorithm name.
         for algorithm, constructors in self.constructors_to_test.items():
-            constructors.add(getattr(hashlib, algorithm))
-            def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, **kwargs):
+            constructors.add(suppress_fips(getattr(hashlib, algorithm)))
+            def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, usedforsecurity=True, **kwargs):
                 if data is None:
-                    return hashlib.new(_alg, **kwargs)
-                return hashlib.new(_alg, data, **kwargs)
+                    return suppress_fips(hashlib.new)(_alg)
+                return suppress_fips(hashlib.new)(_alg, data)
             constructors.add(_test_algorithm_via_hashlib_new)
 
         _hashlib = self._conditional_import_module('_hashlib')
@@ -122,26 +148,12 @@ def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, **kwargs):
             for algorithm, constructors in self.constructors_to_test.items():
                 constructor = getattr(_hashlib, 'openssl_'+algorithm, None)
                 if constructor:
-                    constructors.add(constructor)
+                    constructors.add(suppress_fips(constructor))
 
         def add_builtin_constructor(name):
             constructor = getattr(hashlib, "__get_builtin_constructor")(name)
             self.constructors_to_test[name].add(constructor)
 
-        _md5 = self._conditional_import_module('_md5')
-        if _md5:
-            add_builtin_constructor('md5')
-        _sha1 = self._conditional_import_module('_sha1')
-        if _sha1:
-            add_builtin_constructor('sha1')
-        _sha256 = self._conditional_import_module('_sha256')
-        if _sha256:
-            add_builtin_constructor('sha224')
-            add_builtin_constructor('sha256')
-        _sha512 = self._conditional_import_module('_sha512')
-        if _sha512:
-            add_builtin_constructor('sha384')
-            add_builtin_constructor('sha512')
         if _blake2:
             add_builtin_constructor('blake2s')
             add_builtin_constructor('blake2b')
@@ -206,9 +218,6 @@ def test_get_builtin_constructor(self):
             else:
                 del sys.modules['_md5']
         self.assertRaises(TypeError, get_builtin_constructor, 3)
-        constructor = get_builtin_constructor('md5')
-        self.assertIs(constructor, _md5.md5)
-        self.assertEqual(sorted(builtin_constructor_cache), ['MD5', 'md5'])
 
     def test_hexdigest(self):
         for cons in self.hash_constructors:
@@ -808,6 +817,65 @@ def hash_in_chunks(chunk_size):
 
         self.assertEqual(expected_hash, hasher.hexdigest())
 
+    def test_issue9146(self):
+        # Ensure that various ways to use "MD5" from "hashlib" don't segfault:
+        m = hashlib.md5(usedforsecurity=False)
+        m.update(b'abc\n')
+        self.assertEquals(m.hexdigest(), "0bee89b07a248e27c83fc3d5951213c1")
+
+        m = hashlib.new('md5', usedforsecurity=False)
+        m.update(b'abc\n')
+        self.assertEquals(m.hexdigest(), "0bee89b07a248e27c83fc3d5951213c1")
+
+        m = hashlib.md5(b'abc\n', usedforsecurity=False)
+        self.assertEquals(m.hexdigest(), "0bee89b07a248e27c83fc3d5951213c1")
+
+        m = hashlib.new('md5', b'abc\n', usedforsecurity=False)
+        self.assertEquals(m.hexdigest(), "0bee89b07a248e27c83fc3d5951213c1")
+
+    @unittest.skipUnless(OPENSSL_ENFORCES_FIPS,
+                         'FIPS enforcement required for this test.')
+    def test_hashlib_fips_mode(self):
+        # Ensure that we raise a ValueError on vanilla attempts to use MD5
+        # in hashlib in a FIPS-enforced setting:
+        with self.assertRaisesRegexp(ValueError, '.*unknown cipher'):
+            m = hashlib.md5()
+
+        if not self._conditional_import_module('_md5'):
+            with self.assertRaisesRegexp(ValueError, '.*unknown cipher'):
+                m = hashlib.new('md5')
+
+    @unittest.skipUnless(OPENSSL_ENFORCES_FIPS,
+                         'FIPS enforcement required for this test.')
+    def test_hashopenssl_fips_mode(self):
+        # Verify the _hashlib module's handling of md5:
+        _hashlib = self._conditional_import_module('_hashlib')
+        if _hashlib:
+            assert hasattr(_hashlib, 'openssl_md5')
+
+            # Ensure that _hashlib raises a ValueError on vanilla attempts to
+            # use MD5 in a FIPS-enforced setting:
+            with self.assertRaisesRegexp(ValueError, '.*unknown cipher'):
+                m = _hashlib.openssl_md5()
+            with self.assertRaisesRegexp(ValueError, '.*unknown cipher'):
+                m = _hashlib.new('md5')
+
+            # Ensure that in such a setting we can whitelist a callsite with
+            # usedforsecurity=False and have it succeed:
+            m = _hashlib.openssl_md5(usedforsecurity=False)
+            m.update(b'abc\n')
+            self.assertEquals(m.hexdigest(), "0bee89b07a248e27c83fc3d5951213c1")
+
+            m = _hashlib.new('md5', usedforsecurity=False)
+            m.update(b'abc\n')
+            self.assertEquals(m.hexdigest(), "0bee89b07a248e27c83fc3d5951213c1")
+
+            m = _hashlib.openssl_md5(b'abc\n', usedforsecurity=False)
+            self.assertEquals(m.hexdigest(), "0bee89b07a248e27c83fc3d5951213c1")
+
+            m = _hashlib.new('md5', b'abc\n', usedforsecurity=False)
+            self.assertEquals(m.hexdigest(), "0bee89b07a248e27c83fc3d5951213c1")
+
 
 class KDFTests(unittest.TestCase):
 
@@ -898,6 +966,7 @@ def _test_pbkdf2_hmac(self, pbkdf2):
             iterations=1, dklen=None)
         self.assertEqual(out, self.pbkdf2_results['sha1'][0][0])
 
+    @unittest.skip('skipped on Fedora, as we always use OpenSSL pbkdf2_hmac')
     def test_pbkdf2_hmac_py(self):
         self._test_pbkdf2_hmac(py_hashlib.pbkdf2_hmac)