rc: avoid undefined C

There are two bugs in pdec() on INT_MIN:

* wrong output.

`n = 1-n' should be `n = -1-n' when n is INT_MIN.

* infinite loop.

gcc optimizes `if(n>=0)' into `if(true)' because `-INT_MIN' (signed integer overflow) is undefined behavior in C, and gcc assumes the negation of a negative number must be positive.  The resulting binary keeps printing '-' forever given INT_MIN.

Try the simplified pdec.c below.

$ gcc pdec.c
$ ./a.out -2147483648
--214748364*

$ gcc pdec.c -O2
$ ./a.out -2147483648
<infinite loop>

$ gcc pdec.c -O2 -D__PATCH__
$ ./a.out -2147483648
-2147483648

=== pdec.c ===

#include <stdio.h>
#include <stdlib.h>
#include <limits.h>

#define io void

void pchr(io *f, int c)
{
        putchar(c);
}

void pdec(io *f, int n)
{
        if(n<0){
#ifndef __PATCH__
                n=-n;
                if(n>=0){
                        pchr(f, '-');
                        pdec(f, n);
                        return;
                }
                /* n is two's complement minimum integer */
                n = 1-n;
#else
                if(n!=INT_MIN){
                        pchr(f, '-');
                        pdec(f, -n);
                        return;
                }
                /* n is two's complement minimum integer */
                n = -(INT_MIN+1);
#endif
                pchr(f, '-');
                pdec(f, n/10);
                pchr(f, n%10+'1');
                return;
        }
        if(n>9)
                pdec(f, n/10);
        pchr(f, n%10+'0');
}

int main(int argc, char **argv)
{
        int n = atoi(argv[1]);
        pdec(NULL, n);
        putchar('\n');
}

R=rsc
CC=plan9port.codebot
https://codereview.appspot.com/7241055

1 files changed, 4 insertions, 4 deletions

diff --git a/src/cmd/rc/io.c b/src/cmd/rc/io.c
index f2420019..bb8af4ab 100644
--- a/src/cmd/rc/io.c
+++ b/src/cmd/rc/io.c
@@ -1,3 +1,4 @@
+#include <limits.h>
 #include "rc.h"
 #include "exec.h"
 #include "io.h"
@@ -119,14 +120,13 @@ void
 pdec(io *f, int n)
 {
 	if(n<0){
-		n=-n;
-		if(n>=0){
+		if(n!=INT_MIN){
 			pchr(f, '-');
-			pdec(f, n);
+			pdec(f, -n);
 			return;
 		}
 		/* n is two's complement minimum integer */
-		n = 1-n;
+		n = -(INT_MIN+1);
 		pchr(f, '-');
 		pdec(f, n/10);
 		pchr(f, n%10+'1');