lib/src/: a2i_strto[iu](): Check the base before strto[iu]max(3)

Some implementations of strto[iu]max(3) do modify *endptr if the base is
invalid.  Actually, glibc and musl are the weirdos that leave it
untouched, and all other systems I have checked (Bionic and the BSDs)
set it to nptr.

This means that it's impossible to portably detech an invalid base after
a call to strto[iu]max(3).  The base must be tested before the call.

Fixes: 8491813fee34 (2024-07-20, "lib/src/: a2i_strto[iu](): Pass the base to strto[iu]max(3)")
Reported-by: Robert Elz <kre@munnari.OZ.AU>
Signed-off-by: Alejandro Colomar <alx@kernel.org>

lib/src/a2i/strtoi/strtoi/strtoi.h

diff --git a/lib/src/a2i/strtoi/strtoi/strtoi.h b/lib/src/a2i/strtoi/strtoi/strtoi.h
index cc76700..2ccce36 100644
--- a/lib/src/a2i/strtoi/strtoi/strtoi.h
+++ b/lib/src/a2i/strtoi/strtoi/strtoi.h
@@ -44,27 +44,28 @@
 	errno_saved = errno;
 	errno = 0;
 
-	e = NULL;
+	if (endp == NULL)
+		endp = &e;
+	if (base < 0 || base == 1 || base > 36) {
+		n = 0;
+		st = EINVAL;
+		goto out;
+	}
 
-	n = strtoimax(s, &e, base);
+	n = strtoimax(s, endp, base);
 
-	if (endp != NULL && e != NULL)
-		*endp = e;
-
-	if (e == NULL)
-		st = errno;
-	else
 	if (errno != 0 && errno != EINVAL)
 		st = errno;
-	else if (e == s)
+	else if (*endp == s)
 		st = ECANCELED;
 	else if (n < min || n > max)
 		st = ERANGE;
-	else if (*e != '\0')
+	else if (**endp != '\0')
 		st = ENOTSUP;
 	else
 		st = 0;
 
+out:
 	if (status != NULL)
 		*status = st;
 

lib/src/a2i/strtoi/strtou/strtou.h

diff --git a/lib/src/a2i/strtoi/strtou/strtou.h b/lib/src/a2i/strtoi/strtou/strtou.h
index a7761ed..767deb2 100644
--- a/lib/src/a2i/strtoi/strtou/strtou.h
+++ b/lib/src/a2i/strtoi/strtou/strtou.h
@@ -44,27 +44,28 @@
 	errno_saved = errno;
 	errno = 0;
 
-	e = NULL;
+	if (endp == NULL)
+		endp = &e;
+	if (base < 0 || base == 1 || base > 36) {
+		n = 0;
+		st = EINVAL;
+		goto out;
+	}
 
-	n = strtoumax(s, &e, base);
+	n = strtoumax(s, endp, base);
 
-	if (endp != NULL && e != NULL)
-		*endp = e;
-
-	if (e == NULL)
-		st = errno;
-	else
 	if (errno != 0 && errno != EINVAL)
 		st = errno;
-	else if (e == s)
+	else if (*endp == s)
 		st = ECANCELED;
 	else if (n < min || n > max)
 		st = ERANGE;
-	else if (*e != '\0')
+	else if (**endp != '\0')
 		st = ENOTSUP;
 	else
 		st = 0;
 
+out:
 	if (status != NULL)
 		*status = st;