From fa721ba8c77daadfbb664679754b04fad24ad2cf Mon Sep 17 00:00:00 2001
From: Kai Engert <kaie@kuix.de>
Date: Mon, 25 Feb 2019 19:11:53 +0100
Subject: [PATCH] Bug 1529959, memory leaks in atob/btoa, r=jcj

---
 cmd/atob/atob.c | 36 +++++++++++++++++++++++++-----------
 cmd/btoa/btoa.c | 35 ++++++++++++++++++++++++-----------
 2 files changed, 49 insertions(+), 22 deletions(-)

diff --git a/cmd/atob/atob.c b/cmd/atob/atob.c
index 115b0e9a0f..3e15ceadb0 100644
--- a/cmd/atob/atob.c
+++ b/cmd/atob/atob.c
@@ -105,7 +105,6 @@ Usage(char *progName)
             "-i input");
     fprintf(stderr, "%-20s Define an output file to use (default is stdout)\n",
             "-o output");
-    exit(-1);
 }
 
 int
@@ -113,12 +112,12 @@ main(int argc, char **argv)
 {
     char *progName;
     SECStatus rv;
-    FILE *inFile, *outFile;
-    PLOptState *optstate;
+    FILE *inFile = NULL, *outFile = NULL;
+    PRBool closeIn = PR_TRUE, closeOut = PR_TRUE;
+    PLOptState *optstate = NULL;
     PLOptStatus status;
+    int exitCode = -1;
 
-    inFile = 0;
-    outFile = 0;
     progName = strrchr(argv[0], '/');
     progName = progName ? progName + 1 : argv[0];
 
@@ -129,6 +128,7 @@ main(int argc, char **argv)
             case '?':
             case 'h':
                 Usage(progName);
+                goto loser;
                 break;
 
             case 'i':
@@ -136,7 +136,7 @@ main(int argc, char **argv)
                 if (!inFile) {
                     fprintf(stderr, "%s: unable to open \"%s\" for reading\n",
                             progName, optstate->value);
-                    return -1;
+                    goto loser;
                 }
                 break;
 
@@ -145,13 +145,15 @@ main(int argc, char **argv)
                 if (!outFile) {
                     fprintf(stderr, "%s: unable to open \"%s\" for writing\n",
                             progName, optstate->value);
-                    return -1;
+                    goto loser;
                 }
                 break;
         }
     }
-    if (!inFile)
+    if (!inFile) {
         inFile = stdin;
+        closeIn = PR_FALSE;
+    }
     if (!outFile) {
 #if defined(WIN32)
         int smrv = _setmode(_fileno(stdout), _O_BINARY);
@@ -159,16 +161,28 @@ main(int argc, char **argv)
             fprintf(stderr,
                     "%s: Cannot change stdout to binary mode. Use -o option instead.\n",
                     progName);
-            return smrv;
+            goto loser;
         }
 #endif
         outFile = stdout;
+        closeOut = PR_FALSE;
     }
     rv = decode_file(outFile, inFile);
     if (rv != SECSuccess) {
         fprintf(stderr, "%s: lossage: error=%d errno=%d\n",
                 progName, PORT_GetError(), errno);
-        return -1;
+        goto loser;
+    }
+    exitCode = 0;
+loser:
+    if (optstate) {
+        PL_DestroyOptState(optstate);
+    }
+    if (inFile && closeIn) {
+        fclose(inFile);
+    }
+    if (outFile && closeOut) {
+        fclose(outFile);
     }
-    return 0;
+    return exitCode;
 }
diff --git a/cmd/btoa/btoa.c b/cmd/btoa/btoa.c
index 2a5e6d4c65..434afa81fd 100644
--- a/cmd/btoa/btoa.c
+++ b/cmd/btoa/btoa.c
@@ -99,7 +99,6 @@ Usage(char *progName)
             "-w suffix");
     fprintf(stderr, "%-20s (use \"c\" as a shortcut for suffix CERTIFICATE)\n",
             "");
-    exit(-1);
 }
 
 int
@@ -107,13 +106,13 @@ main(int argc, char **argv)
 {
     char *progName;
     SECStatus rv;
-    FILE *inFile, *outFile;
-    PLOptState *optstate;
+    FILE *inFile = NULL, *outFile = NULL;
+    PRBool closeIn = PR_TRUE, closeOut = PR_TRUE;
+    PLOptState *optstate = NULL;
     PLOptStatus status;
     char *suffix = NULL;
+    int exitCode = -1;
 
-    inFile = 0;
-    outFile = 0;
     progName = strrchr(argv[0], '/');
     if (!progName)
         progName = strrchr(argv[0], '\\');
@@ -125,6 +124,7 @@ main(int argc, char **argv)
         switch (optstate->option) {
             default:
                 Usage(progName);
+                goto loser;
                 break;
 
             case 'i':
@@ -132,7 +132,7 @@ main(int argc, char **argv)
                 if (!inFile) {
                     fprintf(stderr, "%s: unable to open \"%s\" for reading\n",
                             progName, optstate->value);
-                    return -1;
+                    goto loser;
                 }
                 break;
 
@@ -141,7 +141,7 @@ main(int argc, char **argv)
                 if (!outFile) {
                     fprintf(stderr, "%s: unable to open \"%s\" for writing\n",
                             progName, optstate->value);
-                    return -1;
+                    goto loser;
                 }
                 break;
 
@@ -166,10 +166,11 @@ main(int argc, char **argv)
             fprintf(stderr,
                     "%s: Cannot change stdin to binary mode. Use -i option instead.\n",
                     progName);
-            return smrv;
+            goto loser;
         }
 #endif
         inFile = stdin;
+        closeIn = PR_FALSE;
     }
     if (!outFile) {
 #if defined(WIN32)
@@ -182,10 +183,11 @@ main(int argc, char **argv)
             fprintf(stderr,
                     "%s: Cannot change stdout to binary mode. Use -o option instead.\n",
                     progName);
-            return smrv;
+            goto loser;
         }
 #endif
         outFile = stdout;
+        closeOut = PR_FALSE;
     }
     if (suffix) {
         fprintf(outFile, "-----BEGIN %s-----\n", suffix);
@@ -194,10 +196,21 @@ main(int argc, char **argv)
     if (rv != SECSuccess) {
         fprintf(stderr, "%s: lossage: error=%d errno=%d\n",
                 progName, PORT_GetError(), errno);
-        return -1;
+        goto loser;
     }
     if (suffix) {
         fprintf(outFile, "-----END %s-----\n", suffix);
     }
-    return 0;
+    exitCode = 0;
+loser:
+    if (optstate) {
+        PL_DestroyOptState(optstate);
+    }
+    if (inFile && closeIn) {
+        fclose(inFile);
+    }
+    if (outFile && closeOut) {
+        fclose(outFile);
+    }
+    return exitCode;
 }