Miscellaneous fixes

lib/taskhandler.py

@@ -55,7 +55,7 @@ def run_tasks(self, tasks, dryrun=False):
                 self.qsub_args,
                 task.abrv,
                 task.exe,
-                task.cmd,
+                escape_problem_jobsubmit_chars(task.cmd),
                 self.qsubscript
             )
             if dryrun:
@@ -90,7 +90,7 @@ def run_tasks(self, tasks):
             cmd = r'sbatch -J {} --export=p1="{} {}" "{}"'.format(
                 task.abrv,
                 task.exe,
-                task.cmd,
+                escape_problem_jobsubmit_chars(task.cmd),
                 self.qsubscript
             )
             subprocess.call(cmd, shell=True)
@@ -127,7 +127,6 @@ def _format_task(self, task):
 
 def exec_cmd_mp(job):
     job_name, cmd = job
-    cmd = codecs.decode(cmd, 'unicode-escape')
     logger.info('Running job: %s', job_name)
     logger.debug('Cmd: %s', cmd)
     if platform.system() == "Windows":
@@ -168,17 +167,24 @@ def exec_cmd(cmd):
 
 def argval2str(item):
     if type(item) is str:
-        if item.startswith("'") and item.endswith("'"):
-            item_str = r"\'{}\'".format(item[1:-1])
-        elif item.startswith('"') and item.endswith('"'):
-            item_str = r'\"{}\"'.format(item[1:-1])
+        if (   (item.startswith("'") and item.endswith("'"))
+            or (item.startswith('"') and item.endswith('"'))):
+            item_str = item
         else:
-            item_str = r'\"{}\"'.format(item)
+            item_str = '"{}"'.format(item)
     else:
         item_str = '{}'.format(item)
     return item_str
 
 
+def escape_problem_jobsubmit_chars(str_item):
+    str_item = str_item.replace("'", "\\'")
+    str_item = str_item.replace('"', '\\"')
+    # str_item = str_item.replace(',', '@COMMA@')
+    # str_item = str_item.replace(' ', '@SPACE@')
+    return str_item
+
+
 def convert_optional_args_to_string(args, positional_arg_keys, arg_keys_to_remove):
 
     args_dict = vars(args)

lib/utils.py

@@ -9,13 +9,17 @@
 import sys
 import traceback
 from datetime import datetime
-from collections.abc import Collection
 from io import StringIO
 from xml.etree import cElementTree as ET
 
 import numpy as np
 from osgeo import gdal, ogr, osr
 
+try:
+    import collections.abc as collectionsAbc
+except ImportError:
+    import collections as collectionsAbc
+
 gdal.SetConfigOption('GDAL_PAM_ENABLED', 'NO')
 
 #### Create Loggers
@@ -484,9 +488,13 @@ def doesCross180(geom):
         raise RuntimeError(err)
 
     result = False
-    _mat = re.findall(r"-?\d+(?:\.\d+)?", geom.ExportToWkt())
-    if _mat:
-        x_coords = [float(lng) for (lng, lat) in [_mat[i:i+2] for i in range(0, len(_mat), 2)]]
+
+    # Get an array of the longitudes of all the points of all the rings in the polygon
+    x_coords = []
+    for ring in geom:
+        for pt in range(0, ring.GetPointCount()):
+            x_coords.append(ring.GetX(pt))
+    if x_coords:
         result = (max(x_coords) - min(x_coords)) > 180.0
 
     return result
@@ -651,7 +659,7 @@ def yield_task_args(task_list, script_args,
         object, yielded for each task in `task_list`.
     """
     test_task = task_list[0]
-    if isinstance(test_task, Collection) and not isinstance(test_task, str):
+    if isinstance(test_task, collectionsAbc.Iterable) and not isinstance(test_task, str):
         test_task_nargs = len(test_task)
     else:
         test_task_nargs = 1
@@ -695,7 +703,7 @@ def yield_task_args(task_list, script_args,
         # or could be a 2D list or NumPy array of argument values.
         # Convert 1D single-argument task to the multiple-argument
         # structure (a list with a single argument) for code simplicity.
-        if isinstance(task, Collection) and not isinstance(task, str):
+        if isinstance(task, collectionsAbc.Iterable) and not isinstance(task, str):
             task_arg_list = task
         else:
             task_arg_list = [task]

pgc_ortho.py

@@ -233,7 +233,7 @@ def main():
             dem_filesz_gb = total_dem_filesz_gb
         else:
             dem_filesz_gb = os.path.getsize(args.dem) / 1024.0 / 1024 / 1024
-        pbs_req_mem_gb = int(min(50, max(4, math.ceil(dem_filesz_gb) + 2)))
+        pbs_req_mem_gb = int(min(50, max(8, math.ceil(dem_filesz_gb) + 2)))
         args.l = 'mem={}gb'.format(pbs_req_mem_gb)
 
     ## Group Ikonos

pgc_pansharpen.py

@@ -252,8 +252,7 @@ def main():
                 dem_filesz_gb = total_dem_filesz_gb
             else:
                 dem_filesz_gb = os.path.getsize(args.dem) / 1024.0 / 1024 / 1024
-
-            pbs_req_mem_gb = int(max(math.ceil(dem_filesz_gb) + 4, 8))
+            pbs_req_mem_gb = int(min(50, max(8, math.ceil(dem_filesz_gb) + 2)))
             args.l = 'mem={}gb'.format(pbs_req_mem_gb)
 
     ## Check GDAL version (2.1.0 minimum)

qsub_mosaic.sh

@@ -5,6 +5,7 @@
 #PBS -m n
 #PBS -k oe
 #PBS -j oe
+#PBS -q batch
 
 echo ________________________________________________________
 echo

qsub_ndvi.sh

@@ -4,6 +4,7 @@
 #PBS -m n
 #PBS -k oe
 #PBS -j oe
+#PBS -q batch
 
 echo ________________________________________________________
 echo

qsub_ortho.sh

@@ -4,6 +4,7 @@
 #PBS -m n
 #PBS -k oe
 #PBS -j oe
+#PBS -q batch
 
 echo ________________________________________________________
 echo

qsub_pansharpen.sh

@@ -4,6 +4,7 @@
 #PBS -m n
 #PBS -k oe
 #PBS -j oe
+#PBS -q batch
 
 echo ________________________________________________________
 echo