lib: Silently truncate rbytes after a maximum of 512 bits for yescrypt.

Likewise for gost-yescrypt and scrypt, as those hashing methods share
the same codebase.

Author: besser82

lib/crypt-gost-yescrypt.c

@@ -58,6 +58,10 @@ gensalt_gost_yescrypt_rn (unsigned long count,
                           const uint8_t *rbytes, size_t nrbytes,
                           uint8_t *output, size_t o_size)
 {
+  /* Up to 512 bits (64 bytes) of entropy for computing the salt portion
+     of the MCF-setting are supported.  */
+  nrbytes = (nrbytes > 64 ? 64 : nrbytes);
+
   if (o_size < 4 + 8 * 6 + BASE64_LEN (nrbytes) + 1 ||
       CRYPT_GENSALT_OUTPUT_SIZE < 4 + 8 * 6 + BASE64_LEN (nrbytes) + 1)
     {

lib/crypt-scrypt.c

@@ -165,6 +165,10 @@ gensalt_scrypt_rn (unsigned long count,
                    const uint8_t *rbytes, size_t nrbytes,
                    uint8_t *output, size_t o_size)
 {
+  /* Up to 512 bits (64 bytes) of entropy for computing the salt portion
+     of the MCF-setting are supported.  */
+  nrbytes = (nrbytes > 64 ? 64 : nrbytes);
+
   if (o_size < 3 + 1 + 5 * 2 + BASE64_LEN (nrbytes) + 1 ||
       CRYPT_GENSALT_OUTPUT_SIZE < 3 + 1 + 5 * 2 + BASE64_LEN (nrbytes) + 1)
     {

lib/crypt-yescrypt.c

@@ -106,6 +106,10 @@ gensalt_yescrypt_rn (unsigned long count,
                      const uint8_t *rbytes, size_t nrbytes,
                      uint8_t *output, size_t o_size)
 {
+  /* Up to 512 bits (64 bytes) of entropy for computing the salt portion
+     of the MCF-setting are supported.  */
+  nrbytes = (nrbytes > 64 ? 64 : nrbytes);
+
   if (o_size < 3 + 8 * 6 + 1 + BASE64_LEN (nrbytes) + 1 ||
       CRYPT_GENSALT_OUTPUT_SIZE < 3 + 8 * 6 + 1 + BASE64_LEN (nrbytes) + 1)
     {