diff --git a/lib/mosaic.py b/lib/mosaic.py index c8f3b6f..60609e5 100755 --- a/lib/mosaic.py +++ b/lib/mosaic.py @@ -1,9 +1,15 @@ -import os, sys, shutil, math, glob, re, tarfile, logging, platform, argparse, subprocess -from datetime import datetime, timedelta + +import glob +import logging +import math +import os +import shutil +from datetime import datetime from xml.etree import cElementTree as ET -from osgeo import gdal, ogr, osr, gdalconst + import numpy from numpy import flatnonzero +from osgeo import gdal, ogr, osr from lib import utils @@ -333,6 +339,7 @@ def get_attributes_from_xml(self): try: metad = utils.getGEMetadataAsXml(metapath) except Exception as err: + logger.error(utils.capture_error_trace()) logger.debug("ERROR parsing metadata: %s, %s", err, metapath) #### Write IK01 code @@ -373,6 +380,7 @@ def get_attributes_from_xml(self): vallist.append(val) except Exception as e: + logger.error(utils.capture_error_trace()) logger.debug("Error reading metadata values: %s, %s", metapath, e) if dTags[tag] == 'tdi' and len(taglist) > 1: diff --git a/lib/ortho_functions.py b/lib/ortho_functions.py index 63f817e..a89efa8 100755 --- a/lib/ortho_functions.py +++ b/lib/ortho_functions.py @@ -1,11 +1,20 @@ -import os, sys, shutil, math, glob, re, tarfile, logging, platform, argparse -from datetime import datetime, timedelta -from lib import utils, taskhandler +import argparse +import glob +import logging +import math +import os +import platform +import re +import shutil +import tarfile +from datetime import datetime from xml.dom import minidom from xml.etree import cElementTree as ET -from osgeo import gdal, ogr, osr, gdalconst +from osgeo import gdal, gdalconst, ogr, osr + +from lib import taskhandler, utils #### Create Loggers logger = logging.getLogger("logger") @@ -352,7 +361,8 @@ def process_image(srcfp, dstfp, args, target_extent_geom=None): #### Verify EPSG try: spatial_ref = utils.SpatialRef(args.epsg) - except RuntimeError: + except RuntimeError as e: + logger.error(utils.capture_error_trace()) logger.error("Invalid EPSG code: %i", args.epsg) err = 1 else: @@ -617,6 +627,7 @@ def stackIkBands(dstfp, members): try: os.remove(vrt) except Exception as e: + logger.error(utils.capture_error_trace()) logger.warning("Cannot remove file: %s, %s", vrt, e) return rc @@ -1056,6 +1067,7 @@ def ExtractDGMetadataFile(srcfp, wd): fpfh.close() tf.close() except Exception: + logger.error(utils.capture_error_trace()) logger.error("Cannot open Tar file: %s", tarpath) if metapath and os.path.isfile(metapath): @@ -1507,6 +1519,7 @@ def ExtractRPB(item, rpb_p): tf.close() # status = 0 except Exception: + logger.error(utils.capture_error_trace()) logger.error("Cannot open Tar file: %s", tar_p) rc = 1 else: @@ -1550,6 +1563,7 @@ def getDGXmlData(xmlpath, stretch): try: xmldoc = minidom.parse(xmlpath) except Exception: + logger.error(utils.capture_error_trace()) logger.error("Cannot parse metadata file: %s", xmlpath) return None else: @@ -1815,6 +1829,7 @@ def getGEMetadata(fp_mode, metafile): try: band = int(node.attrib["bandNumber"]) except Exception: + logger.error(utils.capture_error_trace()) logger.error("Unable to retrieve band number in GE metadata") else: node = node.find(".//{}".format(key)) diff --git a/lib/taskhandler.py b/lib/taskhandler.py index 881b90b..773482d 100755 --- a/lib/taskhandler.py +++ b/lib/taskhandler.py @@ -4,9 +4,13 @@ task handler classes and methods """ -import os, sys, shutil, signal, glob, re, logging, subprocess, platform -import multiprocessing as mp import codecs +import logging +import multiprocessing as mp +import os +import platform +import signal +import subprocess #### Create Logger logger = logging.getLogger("logger") diff --git a/lib/utils.py b/lib/utils.py index f40416e..0834ec4 100755 --- a/lib/utils.py +++ b/lib/utils.py @@ -1,9 +1,17 @@ -import os, sys, shutil, math, glob, re, tarfile, logging, platform, argparse, subprocess -from datetime import datetime, timedelta -from xml.dom import minidom +import contextlib +import glob +import logging +import math +import os +import re +import sys +import traceback +from datetime import datetime +from io import StringIO from xml.etree import cElementTree as ET -from osgeo import gdal, ogr, osr, gdalconst + +from osgeo import gdal, ogr, osr gdal.SetConfigOption('GDAL_PAM_ENABLED', 'NO') @@ -14,6 +22,26 @@ package_version = '1.5.7' +@contextlib.contextmanager +def capture_stdout_stderr(): + oldout, olderr = sys.stdout, sys.stderr + out = [StringIO(), StringIO()] + try: + sys.stdout, sys.stderr = out + yield out + finally: + sys.stdout, sys.stderr = oldout, olderr + out[0] = out[0].getvalue() + out[1] = out[1].getvalue() + + +def capture_error_trace(): + with capture_stdout_stderr() as out: + traceback.print_exc() + caught_out, caught_err = out + return caught_err + + class SpatialRef(object): def __init__(self, epsg): @@ -189,6 +217,7 @@ def delete_temp_files(names): try: os.remove(f) except Exception as e: + logger.error(capture_error_trace()) logger.warning('Could not remove %s: %s', os.path.basename(f), e) @@ -441,6 +470,10 @@ def doesCross180(geom): :param geom: :return: """ + if geom.GetGeometryName() == "MULTIPOLYGON": + err = "Function does not support testing MULTIPOLYGON geometry" + raise RuntimeError(err) + result = False _mat = re.findall(r"-?\d+\.\d+", geom.ExportToWkt()) if _mat: diff --git a/pgc_mosaic.py b/pgc_mosaic.py index f78ede4..b717608 100755 --- a/pgc_mosaic.py +++ b/pgc_mosaic.py @@ -1,10 +1,15 @@ -import os, sys, shutil, glob, re, tarfile, logging, argparse, subprocess, math -from datetime import datetime, date, timedelta -from xml.etree import cElementTree as ET -from osgeo import gdal, ogr, osr, gdalconst -import numpy +#!/usr/bin/env python -from lib import mosaic, utils, taskhandler +import argparse +import logging +import math +import os +import sys +from datetime import date, datetime + +from osgeo import ogr, osr + +from lib import mosaic, taskhandler, utils ### Create Logger logger = logging.getLogger("logger") @@ -208,6 +213,7 @@ def main(): try: task_handler = taskhandler.PBSTaskHandler(qsubpath, l) except RuntimeError as e: + logger.error(utils.capture_error_trace()) logger.error(e) else: task_handler.run_tasks(task_queue) @@ -216,6 +222,7 @@ def main(): try: task_handler = taskhandler.SLURMTaskHandler(qsubpath) except RuntimeError as e: + logger.error(utils.capture_error_trace()) logger.error(e) else: task_handler.run_tasks(task_queue) @@ -224,6 +231,7 @@ def main(): try: run_mosaic(tile_builder_script, inpath, mosaicname, mosaic_dir, args, pos_arg_keys) except RuntimeError as e: + logger.error(utils.capture_error_trace()) logger.error(e) else: @@ -560,6 +568,7 @@ def run_mosaic(tile_builder_script, inpath, mosaicname, mosaic_dir, args, pos_ar try: task_handler = taskhandler.ParallelTaskHandler(args.parallel_processes) except RuntimeError as e: + logger.error(utils.capture_error_trace()) logger.error(e) else: if task_handler.num_processes > 1: diff --git a/pgc_mosaic_build_tile.py b/pgc_mosaic_build_tile.py index 51203f1..efcb562 100755 --- a/pgc_mosaic_build_tile.py +++ b/pgc_mosaic_build_tile.py @@ -1,9 +1,16 @@ -import os, sys, shutil, glob, re, tarfile, logging, argparse +#!/usr/bin/env python + +import argparse +import logging +import os +import shutil +import sys -from lib import mosaic, utils, taskhandler import numpy -from osgeo import gdal, ogr, osr, gdalconst - +from osgeo import gdal + +from lib import mosaic, taskhandler, utils + logger = logging.getLogger("logger") logger.setLevel(logging.DEBUG) diff --git a/pgc_mosaic_query_index.py b/pgc_mosaic_query_index.py index 8d7937a..21f5319 100755 --- a/pgc_mosaic_query_index.py +++ b/pgc_mosaic_query_index.py @@ -1,8 +1,15 @@ -import os, sys, logging, argparse, numpy, glob -from datetime import datetime, date -from osgeo import gdal, ogr, osr, gdalconst +#!/usr/bin/env python -from lib import ortho_functions, mosaic, utils, taskhandler +import argparse +import glob +import logging +import os +import sys +from datetime import date, datetime + +from osgeo import ogr, osr + +from lib import mosaic, ortho_functions, utils ### Create Logger logger = logging.getLogger("logger") @@ -75,6 +82,7 @@ def main(): try: dsp, lyrn = utils.get_source_names(src) except RuntimeError as e: + logger.error(utils.capture_error_trace()) parser.error(e) if not os.path.isfile(csvpath): parser.error("Arg2 is not a valid file path: %s" %csvpath) @@ -190,6 +198,7 @@ def main(): try: HandleTile(t, src, dstdir, csvpath, args, exclude_list) except RuntimeError as e: + logger.error(utils.capture_error_trace()) logger.error(e) else: @@ -201,6 +210,7 @@ def main(): try: HandleTile(t, src, dstdir, csvpath, args, exclude_list) except RuntimeError as e: + logger.error(utils.capture_error_trace()) logger.error(e) diff --git a/pgc_ndvi.py b/pgc_ndvi.py index 5dc8862..f051f25 100755 --- a/pgc_ndvi.py +++ b/pgc_ndvi.py @@ -1,8 +1,16 @@ -import os, sys, shutil, math, glob, re, tarfile, argparse, subprocess, logging -from datetime import datetime, timedelta -from osgeo import gdal, ogr, osr, gdalconst, numpy +#!/usr/bin/env python -from lib import ortho_functions, utils, taskhandler +import argparse +import logging +import math +import os +import shutil +import sys + +import numpy +from osgeo import gdal + +from lib import ortho_functions, taskhandler, utils #### Create Loggers logger = logging.getLogger("logger") @@ -131,6 +139,7 @@ def main(): try: task_handler = taskhandler.PBSTaskHandler(qsubpath, l) except RuntimeError as e: + logger.error(utils.capture_error_trace()) logger.error(e) else: if not args.dryrun: @@ -140,6 +149,7 @@ def main(): try: task_handler = taskhandler.SLURMTaskHandler(qsubpath) except RuntimeError as e: + logger.error(utils.capture_error_trace()) logger.error(e) else: if not args.dryrun: @@ -149,6 +159,7 @@ def main(): try: task_handler = taskhandler.ParallelTaskHandler(args.parallel_processes) except RuntimeError as e: + logger.error(utils.capture_error_trace()) logger.error(e) else: logger.info("Number of child processes to spawn: %i", task_handler.num_processes) @@ -408,12 +419,14 @@ def calc_ndvi(srcfp, dstfp, args): try: os.remove(f) except Exception as e: + logger.error(utils.capture_error_trace()) logger.warning('Could not remove %s: %s', os.path.basename(f), e) if wd != dstdir: for f in wd_files: try: os.remove(f) except Exception as e: + logger.error(utils.capture_error_trace()) logger.warning('Could not remove %s: %s', os.path.basename(f), e) else: logger.error("pgc_ndvi.py: %s was not created", dstfp_local) diff --git a/pgc_ortho.py b/pgc_ortho.py index 129cf11..eaf89d8 100755 --- a/pgc_ortho.py +++ b/pgc_ortho.py @@ -1,9 +1,15 @@ +#!/usr/bin/env python + from __future__ import division -import os, sys, logging, argparse, math -from datetime import datetime +import argparse +import logging +import math +import os +import sys import xml.etree.ElementTree as ET -from lib import ortho_functions, utils, taskhandler + +from lib import ortho_functions, taskhandler, utils from lib.taskhandler import argval2str #### Create Loggers @@ -242,6 +248,7 @@ def main(): try: task_handler = taskhandler.PBSTaskHandler(qsubpath, l) except RuntimeError as e: + logger.error(utils.capture_error_trace()) logger.error(e) else: if not args.dryrun: @@ -251,6 +258,7 @@ def main(): try: task_handler = taskhandler.SLURMTaskHandler(qsubpath) except RuntimeError as e: + logger.error(utils.capture_error_trace()) logger.error(e) else: if not args.dryrun: @@ -260,6 +268,7 @@ def main(): try: task_handler = taskhandler.ParallelTaskHandler(args.parallel_processes) except RuntimeError as e: + logger.error(utils.capture_error_trace()) logger.error(e) else: logger.info("Number of child processes to spawn: %i", task_handler.num_processes) @@ -283,6 +292,8 @@ def main(): if not args.dryrun: results[task.name] = task.method(src, dstfp, task_arg_obj) + else: + print(src) #### remove existing file handler logger.removeHandler(lfh) diff --git a/pgc_pansharpen.py b/pgc_pansharpen.py index 5c8946b..375fb4a 100755 --- a/pgc_pansharpen.py +++ b/pgc_pansharpen.py @@ -1,11 +1,21 @@ +#!/usr/bin/env python + from __future__ import division -import os, sys, shutil, math, glob, re, tarfile, argparse, subprocess, logging, platform -from datetime import datetime, timedelta +import argparse +import glob +import logging +import math +import os +import platform +import re +import shutil +import sys import xml.etree.ElementTree as ET -from osgeo import gdal, ogr, osr, gdalconst -from lib import ortho_functions, utils, taskhandler +from osgeo import gdal, gdalconst, ogr, osr + +from lib import ortho_functions, taskhandler, utils from lib.taskhandler import argval2str #### Create Loggers @@ -281,6 +291,7 @@ def main(): try: spatial_ref = utils.SpatialRef(args.epsg) except RuntimeError as e: + logger.error(utils.capture_error_trace()) parser.error(e) #### Verify that dem and ortho_height are not both specified @@ -348,6 +359,7 @@ def main(): try: image_pair = ImagePair(srcfp, spatial_ref) except RuntimeError as e: + logger.error(utils.capture_error_trace()) logger.error(e) else: logger.info("Image: %s, Sensor: %s", image_pair.mul_srcfn, image_pair.sensor) @@ -408,7 +420,7 @@ def main(): task = taskhandler.Task( task_item_srcfn, - 'Psh{:04g}'.format(i), + 'Psh{:04g}'.format(job_count), 'python', '{} {} {} {}'.format( argval2str(scriptpath), @@ -429,6 +441,7 @@ def main(): try: task_handler = taskhandler.PBSTaskHandler(qsubpath, l) except RuntimeError as e: + logger.error(utils.capture_error_trace()) logger.error(e) else: if not args.dryrun: @@ -438,6 +451,7 @@ def main(): try: task_handler = taskhandler.SLURMTaskHandler(qsubpath) except RuntimeError as e: + logger.error(utils.capture_error_trace()) logger.error(e) else: if not args.dryrun: @@ -447,6 +461,7 @@ def main(): try: task_handler = taskhandler.ParallelTaskHandler(args.parallel_processes) except RuntimeError as e: + logger.error(utils.capture_error_trace()) logger.error(e) else: logger.info("Number of child processes to spawn: %i", task_handler.num_processes) @@ -597,6 +612,7 @@ def exec_pansharpen(image_pair, pansh_dstfp, args): try: os.remove(f) except Exception as e: + logger.error(utils.capture_error_trace()) logger.warning('Could not remove %s: %s', os.path.basename(f), e) if os.path.isfile(pansh_dstfp):