rc: avoid undefined C

There are two bugs in pdec() on INT_MIN:

* wrong output.

`n = 1-n' should be `n = -1-n' when n is INT_MIN.

* infinite loop.

gcc optimizes `if(n>=0)' into `if(true)' because `-INT_MIN' (signed integer overflow) is undefined behavior in C, and gcc assumes the negation of a negative number must be positive.  The resulting binary keeps printing '-' forever given INT_MIN.

Try the simplified pdec.c below.

$ gcc pdec.c
$ ./a.out -2147483648
--214748364*

$ gcc pdec.c -O2
$ ./a.out -2147483648
<infinite loop>

$ gcc pdec.c -O2 -D__PATCH__
$ ./a.out -2147483648
-2147483648

=== pdec.c ===

#include <stdio.h>
#include <stdlib.h>
#include <limits.h>

#define io void

void pchr(io *f, int c)
{
        putchar(c);
}

void pdec(io *f, int n)
{
        if(n<0){
#ifndef __PATCH__
                n=-n;
                if(n>=0){
                        pchr(f, '-');
                        pdec(f, n);
                        return;
                }
                /* n is two's complement minimum integer */
                n = 1-n;
#else
                if(n!=INT_MIN){
                        pchr(f, '-');
                        pdec(f, -n);
                        return;
                }
                /* n is two's complement minimum integer */
                n = -(INT_MIN+1);
#endif
                pchr(f, '-');
                pdec(f, n/10);
                pchr(f, n%10+'1');
                return;
        }
        if(n>9)
                pdec(f, n/10);
        pchr(f, n%10+'0');
}

int main(int argc, char **argv)
{
        int n = atoi(argv[1]);
        pdec(NULL, n);
        putchar('\n');
}

R=rsc
CC=plan9port.codebot
https://codereview.appspot.com/7241055

0 files changed, 0 insertions, 0 deletions