Merge pull request #30 from PolarGeospatialCenter/erik-dev

Show full stack trace in error messages/logs

lib/mosaic.py

@@ -1,9 +1,15 @@
-import os, sys, shutil, math, glob, re, tarfile, logging, platform, argparse, subprocess
-from datetime import datetime, timedelta
+
+import glob
+import logging
+import math
+import os
+import shutil
+from datetime import datetime
 from xml.etree import cElementTree as ET
-from osgeo import gdal, ogr, osr, gdalconst
+
 import numpy
 from numpy import flatnonzero
+from osgeo import gdal, ogr, osr
 
 from lib import utils
 
@@ -333,6 +339,7 @@ def get_attributes_from_xml(self):
                 try:
                     metad = utils.getGEMetadataAsXml(metapath)
                 except Exception as err:
+                    logger.error(utils.capture_error_trace())
                     logger.debug("ERROR parsing metadata: %s, %s", err, metapath)
                 #### Write IK01 code 
 
@@ -373,6 +380,7 @@ def get_attributes_from_xml(self):
                                 vallist.append(val)
 
                             except Exception as e:
+                                logger.error(utils.capture_error_trace())
                                 logger.debug("Error reading metadata values: %s, %s", metapath, e)
 
                     if dTags[tag] == 'tdi' and len(taglist) > 1:    

lib/ortho_functions.py

@@ -1,11 +1,20 @@
-import os, sys, shutil, math, glob, re, tarfile, logging, platform, argparse
-from datetime import datetime, timedelta
 
-from lib import utils, taskhandler
+import argparse
+import glob
+import logging
+import math
+import os
+import platform
+import re
+import shutil
+import tarfile
+from datetime import datetime
 from xml.dom import minidom
 from xml.etree import cElementTree as ET
 
-from osgeo import gdal, ogr, osr, gdalconst
+from osgeo import gdal, gdalconst, ogr, osr
+
+from lib import taskhandler, utils
 
 #### Create Loggers
 logger = logging.getLogger("logger")
@@ -352,7 +361,8 @@ def process_image(srcfp, dstfp, args, target_extent_geom=None):
     #### Verify EPSG
     try:
         spatial_ref = utils.SpatialRef(args.epsg)
-    except RuntimeError:
+    except RuntimeError as e:
+        logger.error(utils.capture_error_trace())
         logger.error("Invalid EPSG code: %i", args.epsg)
         err = 1
     else:
@@ -617,6 +627,7 @@ def stackIkBands(dstfp, members):
     try:
         os.remove(vrt)
     except Exception as e:
+        logger.error(utils.capture_error_trace())
         logger.warning("Cannot remove file: %s, %s", vrt, e)
     return rc
 
@@ -1056,6 +1067,7 @@ def ExtractDGMetadataFile(srcfp, wd):
                         fpfh.close()
                         tf.close()
             except Exception:
+                logger.error(utils.capture_error_trace())
                 logger.error("Cannot open Tar file: %s", tarpath)
 
     if metapath and os.path.isfile(metapath):
@@ -1507,6 +1519,7 @@ def ExtractRPB(item, rpb_p):
                     tf.close()
                     # status = 0
         except Exception:
+            logger.error(utils.capture_error_trace())
             logger.error("Cannot open Tar file: %s", tar_p)
             rc = 1
     else:
@@ -1550,6 +1563,7 @@ def getDGXmlData(xmlpath, stretch):
     try:
         xmldoc = minidom.parse(xmlpath)
     except Exception:
+        logger.error(utils.capture_error_trace())
         logger.error("Cannot parse metadata file: %s", xmlpath)
         return None
     else:
@@ -1815,6 +1829,7 @@ def getGEMetadata(fp_mode, metafile):
                 try:
                     band = int(node.attrib["bandNumber"])
                 except Exception:
+                    logger.error(utils.capture_error_trace())
                     logger.error("Unable to retrieve band number in GE metadata")
                 else:
                     node = node.find(".//{}".format(key))

lib/taskhandler.py

@@ -4,9 +4,13 @@
 task handler classes and methods
 """
 
-import os, sys, shutil, signal, glob, re, logging, subprocess, platform
-import multiprocessing as mp
 import codecs
+import logging
+import multiprocessing as mp
+import os
+import platform
+import signal
+import subprocess
 
 #### Create Logger
 logger = logging.getLogger("logger")

lib/utils.py

@@ -1,9 +1,17 @@
-import os, sys, shutil, math, glob, re, tarfile, logging, platform, argparse, subprocess
-from datetime import datetime, timedelta
 
-from xml.dom import minidom
+import contextlib
+import glob
+import logging
+import math
+import os
+import re
+import sys
+import traceback
+from datetime import datetime
+from io import StringIO
 from xml.etree import cElementTree as ET
-from osgeo import gdal, ogr, osr, gdalconst
+
+from osgeo import gdal, ogr, osr
 
 gdal.SetConfigOption('GDAL_PAM_ENABLED', 'NO')
 
@@ -14,6 +22,26 @@
 package_version = '1.5.7'
 
 
+@contextlib.contextmanager
+def capture_stdout_stderr():
+    oldout, olderr = sys.stdout, sys.stderr
+    out = [StringIO(), StringIO()]
+    try:
+        sys.stdout, sys.stderr = out
+        yield out
+    finally:
+        sys.stdout, sys.stderr = oldout, olderr
+        out[0] = out[0].getvalue()
+        out[1] = out[1].getvalue()
+
+
+def capture_error_trace():
+    with capture_stdout_stderr() as out:
+        traceback.print_exc()
+    caught_out, caught_err = out
+    return caught_err
+
+
 class SpatialRef(object):
 
     def __init__(self, epsg):
@@ -189,6 +217,7 @@ def delete_temp_files(names):
                 try:
                     os.remove(f)
                 except Exception as e:
+                    logger.error(capture_error_trace())
                     logger.warning('Could not remove %s: %s', os.path.basename(f), e)
 
 
@@ -441,6 +470,10 @@ def doesCross180(geom):
     :param geom: <osgeo.ogr.Geometry>
     :return: <bool>
     """
+    if geom.GetGeometryName() == "MULTIPOLYGON":
+        err = "Function does not support testing MULTIPOLYGON geometry"
+        raise RuntimeError(err)
+
     result = False
     _mat = re.findall(r"-?\d+\.\d+", geom.ExportToWkt())
     if _mat:

pgc_mosaic.py

@@ -1,10 +1,15 @@
-import os, sys, shutil, glob, re, tarfile, logging, argparse, subprocess, math
-from datetime import datetime, date, timedelta
-from xml.etree import cElementTree as ET
-from osgeo import gdal, ogr, osr, gdalconst
-import numpy
+#!/usr/bin/env python
 
-from lib import mosaic, utils, taskhandler
+import argparse
+import logging
+import math
+import os
+import sys
+from datetime import date, datetime
+
+from osgeo import ogr, osr
+
+from lib import mosaic, taskhandler, utils
 
 ### Create Logger
 logger = logging.getLogger("logger")
@@ -208,6 +213,7 @@ def main():
             try:
                 task_handler = taskhandler.PBSTaskHandler(qsubpath, l)
             except RuntimeError as e:
+                logger.error(utils.capture_error_trace())
                 logger.error(e)
             else:
                 task_handler.run_tasks(task_queue)
@@ -216,6 +222,7 @@ def main():
             try:
                 task_handler = taskhandler.SLURMTaskHandler(qsubpath)
             except RuntimeError as e:
+                logger.error(utils.capture_error_trace())
                 logger.error(e)
             else:
                 task_handler.run_tasks(task_queue)
@@ -224,6 +231,7 @@ def main():
             try:
                 run_mosaic(tile_builder_script, inpath, mosaicname, mosaic_dir, args, pos_arg_keys)
             except RuntimeError as e:
+                logger.error(utils.capture_error_trace())
                 logger.error(e)
 
     else:
@@ -560,6 +568,7 @@ def run_mosaic(tile_builder_script, inpath, mosaicname, mosaic_dir, args, pos_ar
             try:
                 task_handler = taskhandler.ParallelTaskHandler(args.parallel_processes)
             except RuntimeError as e:
+                logger.error(utils.capture_error_trace())
                 logger.error(e)
             else:
                 if task_handler.num_processes > 1:

pgc_mosaic_build_tile.py

@@ -1,9 +1,16 @@
-import os, sys, shutil, glob, re, tarfile, logging, argparse
+#!/usr/bin/env python
+
+import argparse
+import logging
+import os
+import shutil
+import sys
 
-from lib import mosaic, utils, taskhandler
 import numpy
-from osgeo import gdal, ogr, osr, gdalconst
-
+from osgeo import gdal
+
+from lib import mosaic, taskhandler, utils
+
 logger = logging.getLogger("logger")
 logger.setLevel(logging.DEBUG)
 

pgc_mosaic_query_index.py

@@ -1,8 +1,15 @@
-import os, sys, logging, argparse, numpy, glob
-from datetime import datetime, date
-from osgeo import gdal, ogr, osr, gdalconst
+#!/usr/bin/env python
 
-from lib import ortho_functions, mosaic, utils, taskhandler
+import argparse
+import glob
+import logging
+import os
+import sys
+from datetime import date, datetime
+
+from osgeo import ogr, osr
+
+from lib import mosaic, ortho_functions, utils
 
 ### Create Logger
 logger = logging.getLogger("logger")
@@ -75,6 +82,7 @@ def main():
     try:
         dsp, lyrn = utils.get_source_names(src)
     except RuntimeError as e:
+        logger.error(utils.capture_error_trace())
         parser.error(e)
     if not os.path.isfile(csvpath):
         parser.error("Arg2 is not a valid file path: %s" %csvpath)
@@ -190,6 +198,7 @@ def main():
                     try:
                         HandleTile(t, src, dstdir, csvpath, args, exclude_list)
                     except RuntimeError as e:
+                        logger.error(utils.capture_error_trace())
                         logger.error(e)
 
     else:
@@ -201,6 +210,7 @@ def main():
                 try:
                     HandleTile(t, src, dstdir, csvpath, args, exclude_list)
                 except RuntimeError as e:
+                    logger.error(utils.capture_error_trace())
                     logger.error(e)
 
 

pgc_ndvi.py

@@ -1,8 +1,16 @@
-import os, sys, shutil, math, glob, re, tarfile, argparse, subprocess, logging
-from datetime import datetime, timedelta
-from osgeo import gdal, ogr, osr, gdalconst, numpy
+#!/usr/bin/env python
 
-from lib import ortho_functions, utils, taskhandler
+import argparse
+import logging
+import math
+import os
+import shutil
+import sys
+
+import numpy
+from osgeo import gdal
+
+from lib import ortho_functions, taskhandler, utils
 
 #### Create Loggers
 logger = logging.getLogger("logger")
@@ -131,6 +139,7 @@ def main():
             try:
                 task_handler = taskhandler.PBSTaskHandler(qsubpath, l)
             except RuntimeError as e:
+                logger.error(utils.capture_error_trace())
                 logger.error(e)
             else:
                 if not args.dryrun:
@@ -140,6 +149,7 @@ def main():
             try:
                 task_handler = taskhandler.SLURMTaskHandler(qsubpath)
             except RuntimeError as e:
+                logger.error(utils.capture_error_trace())
                 logger.error(e)
             else:
                 if not args.dryrun:
@@ -149,6 +159,7 @@ def main():
             try:
                 task_handler = taskhandler.ParallelTaskHandler(args.parallel_processes)
             except RuntimeError as e:
+                logger.error(utils.capture_error_trace())
                 logger.error(e)
             else:
                 logger.info("Number of child processes to spawn: %i", task_handler.num_processes)
@@ -408,12 +419,14 @@ def calc_ndvi(srcfp, dstfp, args):
                     try:
                         os.remove(f)
                     except Exception as e:
+                        logger.error(utils.capture_error_trace())
                         logger.warning('Could not remove %s: %s', os.path.basename(f), e)
             if wd != dstdir:
                 for f in wd_files:
                     try:
                         os.remove(f)
                     except Exception as e:
+                        logger.error(utils.capture_error_trace())
                         logger.warning('Could not remove %s: %s', os.path.basename(f), e)
         else:
             logger.error("pgc_ndvi.py: %s was not created", dstfp_local)