CLI

datum/cli.py

@@ -0,0 +1,39 @@
+import click
+import datum
+
+import logging
+log = logging.getLogger(__name__)
+
+@click.group()
+def cli():
+    pass
+
+@cli.command()
+@click.option('--connection', '-d', help='The database connection string', required=True)
+@click.argument('table')
+def truncate(table, connection):
+    db = datum.connect(connection)
+    db.table(table).delete()
+
+@cli.command()
+@click.option('--connection', '-d', help='The database connection string', required=True)
+@click.argument('csvfile', type=click.File('rU'))
+@click.argument('table')
+def load(csvfile, table, connection):
+    db = datum.connect(connection)
+    db.table(table).load(csvfile)
+
+@cli.command()
+@click.option('--connection', '-d', help='The database connection string', required=True)
+@click.argument('sql')
+def execute(sql, connection):
+    db = datum.connect(connection)
+    rows = db.execute(sql)
+    if datum.util.isiterable(rows):
+        import csv, sys
+        writer = csv.writer(sys.stdout)
+        writer.writerow(rows.header)
+        writer.writerows(rows)
+
+if __name__ == '__main__':
+    cli()

datum/oracle_stgeom/database.py

@@ -16,7 +16,7 @@ def __init__(self, parent):
         self.password = p['password']
         self.name = p['db_name']
         if self.name: self.name = self.name.lower()
-        
+
         dsn = '{user}/{password}@{host}'.format(**self.__dict__)
         if self.name: dsn += '/' + self.name
 
@@ -28,6 +28,13 @@ def __init__(self, parent):
 
     def execute(self, stmt):
         self._c.execute(stmt)
+
+        class SimpleView (list):
+            """
+            A trivial wrapper on `list` to allow attaching a header
+            """
+            pass
+
         try:
             rows = self._c.fetchall()
         # Return rowcount for non-SELECT operations
@@ -36,14 +43,18 @@ def execute(self, stmt):
         # Unpack single values
         if len(rows) > 0 and len(rows[0]) == 1:
             rows = [x[0] for x in rows]
+
+        rows = SimpleView(rows)
+        rows.header = [field[0] for field in self._c.description]
+
         return rows
 
     def close(self):
         self.cxn.close()
 
     # def table(self, name):
     #     # Check for a schema
-    #     if '.' in 
+    #     if '.' in
     #     return self.parent.table(name)
 
     @property
@@ -67,7 +78,7 @@ def tables(self):
 
     def _dictify(self, geom_field=None):
         '''
-        Turns query results into a list of dictionaries. This reads from the 
+        Turns query results into a list of dictionaries. This reads from the
         cursor because calling fetchall() on rows breaks the geometry LOB.
         '''
         fields = [x[0].lower() for x in self._c.description]
@@ -104,7 +115,7 @@ def read(self, table, fields, geom_field=None, dictify=True, where=None, limit=N
         fields = list(fields)  # Make a copy
         if fields != ['*']:
             if geom_field:
-                fields.append(self._wkt_getter(geom_field)) 
+                fields.append(self._wkt_getter(geom_field))
             fields_joined = ', '.join(fields)
             table = table.upper()
             stmt = "SELECT {} FROM {}".format(fields_joined, table)
@@ -147,14 +158,14 @@ def save(self):
     def bulk_insert(self, table, rows, geom_field=None, from_srid=None, \
         multi_geom=True, chunk_size=None):
         '''
-        Inserts dictionary row objects in the the database 
+        Inserts dictionary row objects in the the database
         Args: list of row dicts, table name
         '''
         fields = rows[0].keys()
         if geom_field:
             non_geom_fields = [x for x in fields if x != geom_field]
         fields_joined = ', '.join(fields)
-        stmt = "INSERT INTO {} ({}) VALUES ".format(table, fields_joined)       
+        stmt = "INSERT INTO {} ({}) VALUES ".format(table, fields_joined)
 
         len_rows = len(rows)
         if chunk_size is None or len_rows < chunk_size:

datum/oracle_stgeom/table.py

@@ -5,7 +5,7 @@
 from datum.oracle_stgeom.util import WktTransformer
 import cx_Oracle
 
-# These are strings because one type (OBJECTVAR) isn't importable from 
+# These are strings because one type (OBJECTVAR) isn't importable from
 # the cx_Oracle module.
 FIELD_TYPE_MAP = {
     'NUMBER':       'num',
@@ -98,7 +98,7 @@ def _get_geom_type(self):
                 bitand(eflags, 2),
                 bitand(eflags, 4) + bitand(eflags, 8),
                 bitand(eflags, 16),
-                bitand(eflags, 262144) 
+                bitand(eflags, 262144)
             from sde.layers
             where
                 owner = '{}' and
@@ -179,7 +179,7 @@ def non_geom_fields(self):
     def _get_wkt_selector(self, to_srid=None):
         assert self.geom_field
         geom_field_t = geom_field = self.geom_field
-        # SDE.ST_Transform doesn't work when the datums differ. Unfortunately, 
+        # SDE.ST_Transform doesn't work when the datums differ. Unfortunately,
         # 4326 <=> 2272 is one of those. Using Shapely + PyProj for now.
         # if to_srid and to_srid != self.srid:
         #     geom_field_t = "SDE.ST_Transform({}, {})"\
@@ -209,7 +209,7 @@ def count(self):
 
     def read(self, fields=None, aliases=None, geom_field=None, to_srid=None,
         return_geom=True, limit=None, where=None, sort=None):
-        # If no geom_field was specified and we're supposed to return geom, 
+        # If no geom_field was specified and we're supposed to return geom,
         # get it from the object.
         geom_field = geom_field or (self.geom_field if return_geom else None)
 
@@ -234,13 +234,13 @@ def read(self, fields=None, aliases=None, geom_field=None, to_srid=None,
             stmt += " WHERE ROWNUM <= {}".format(limit)
 
         self._c.execute(stmt)
-        
+
         # Handle aliases
         # fields = [re.sub('.+ AS ', '', x, flags=re.IGNORECASE) for x in fields]
         if aliases:
           fields = [aliases[x] if x in aliases else x for x in fields]
 
-        fields_lower = [x.lower() for x in fields] 
+        fields_lower = [x.lower() for x in fields]
         if geom_field:
             geom_field_i = fields.index(geom_field)
         rows = []
@@ -264,7 +264,7 @@ def read(self, fields=None, aliases=None, geom_field=None, to_srid=None,
                 for row in rows:
                     geom = self._remove_m_value(row[geom_field_i])
                     row[geom_field_i] = geom
-        
+
             # TODO if the WKT geom is single but the geom_type for the table
             # is multi, we may want to convert it. Seems to be working for now
             # though.
@@ -279,7 +279,7 @@ def read(self, fields=None, aliases=None, geom_field=None, to_srid=None,
             for row in rows:
                 geom = row[geom_field_l]
                 geom_t = tsf.transform(geom)
-                row[geom_field_l] = geom_t        
+                row[geom_field_l] = geom_t
 
         return rows
 
@@ -347,21 +347,27 @@ def write(self, rows, from_srid=None, chunk_size=None):
         Args: list of row dicts, table name, ordered field names
 
         Originally this formed one big INSERT statement with a chunks of x
-        rows, but it's considerably faster to use the cx_Oracle `executemany` 
+        rows, but it is considerably faster to use the cx_Oracle `executemany`
         function. See methods 1 and 2 below.
 
         TODO: it might be faster to call NEXTVAL on the DB sequence for OBJECTID
         rather than use the SDE helper function.
         """
-        if len(rows) == 0:
+        try:
+            len_rows = len(rows)
+        except TypeError:
+            rows = tuple(rows)
+            len_rows = len(rows)
+
+        if len_rows == 0:
             return
 
         # Get fields from the row because some fields from self.fields may be
         # optional, such as an autoincrementing PK.
         fields = rows[0].keys()
         # Sort so LOB fields are at the end
-        fields = sorted(fields, key=lambda x: 'lob' in self.metadata[x]['type'])
-        
+        fields = sorted(fields, key=lambda x: 'lob' in self.metadata[x.lower()]['type'])
+
         geom_field = self.geom_field
         geom_type = self.geom_type
         srid = from_srid or self.srid
@@ -412,49 +418,24 @@ def write(self, rows, from_srid=None, chunk_size=None):
         # # SELECT 1 FROM DUAL;
         # fields_joined = ', '.join(fields)
         # stmt = "INSERT ALL {} SELECT 1 FROM DUAL"
-        
+
         # # We always have to pass in a value for OBJECTID (or whatever the SDE
         # # PK field is; sometimes it's something like OBJECTID_3). Check to see
-        # # if the user passed in a value for object ID (not likely), otherwise 
+        # # if the user passed in a value for object ID (not likely), otherwise
         # # hardcode the sequence incrementor into the prepared statement.
         # if self.objectid_field in fields:
         #     into_clause = "INTO {} ({}) VALUES ({{}})".format(self.name, \
         #         fields_joined)
         # else:
         #     incrementor = "SDE.GDB_UTIL.NEXT_ROWID('{}', '{}')".format(self._owner, self.name)
         #     into_clause = "INTO {} ({}, {}) VALUES ({{}}, {})".format(self.name, fields_joined, self.objectid_field, incrementor)
-
-        # METHOD 2: executemany (not working with SDE.ST_Geometry call)
-        placeholders = []
 
-        # Create placeholders for prepared statement
-        for field in fields:
-            type_ = type_map[field]
-            if type_ == 'geom':
-                placeholders.append('SDE.ST_Geometry(:{}, {})'\
-                    .format(field, self.srid))
-            elif type_ == 'date':
-                # Insert an ISO-8601 timestring
-                placeholders.append("TO_TIMESTAMP(:{}, 'YYYY-MM-DD\"T\"HH24:MI:SS.FF\"+00:00\"')".format(field))
-            else:
-                placeholders.append(':' + field)
-
-        # Inject the object ID field if it's missing from the supplied rows
-        stmt_fields = list(fields)
-        if self.objectid_field and self.objectid_field not in fields:
-            stmt_fields.append(self.objectid_field)
-            incrementor = "SDE.GDB_UTIL.NEXT_ROWID('{}', '{}')"\
-                .format(self._owner, self.name)
-            placeholders.append(incrementor)
-        # Prepare statement
-        placeholders_joined = ', '.join(placeholders)
-        stmt_fields_joined = ', '.join(stmt_fields)
-        stmt = "INSERT INTO {} ({}) VALUES ({})".format(self.name, \
-            stmt_fields_joined, placeholders_joined)
+        # METHOD 2: executemany (not working with SDE.ST_Geometry call)
+        stmt = self.build_insert_statement(fields, type_map)
         self._c.prepare(stmt)
 
         # END OF METHODS
-        
+
         len_rows = len(rows)
         if chunk_size is None or len_rows < chunk_size:
             iterations = 1
@@ -503,6 +484,42 @@ def write(self, rows, from_srid=None, chunk_size=None):
             # print(self._c.getbatcherrors())
             self._save()
 
+    def build_insert_statement(self, input_fields, type_map):
+        placeholders = []
+
+        # Build up an exact-case mapping of field names
+        db_fields = self.fields
+        icase = {}
+        for field in input_fields:
+            db_field = [n for n in db_fields if field.lower() == n.lower()][0]
+            icase[field] = db_field
+
+        # Create placeholders for prepared statement
+        for field in input_fields:
+            type_ = type_map[field]
+            if type_ == 'geom':
+                placeholders.append('SDE.ST_Geometry(:{}, {})'\
+                    .format(field, self.srid))
+            elif type_ == 'date':
+                # Insert an ISO-8601 timestring
+                placeholders.append("TO_TIMESTAMP(:{}, 'YYYY-MM-DD\"T\"HH24:MI:SS.FF\"+00:00\"')".format(field))
+            else:
+                placeholders.append(':' + field + '_')
+
+        # Inject the object ID field if it's missing from the supplied rows
+        stmt_fields = [icase[field] for field in input_fields]
+        if self.objectid_field and self.objectid_field not in input_fields:
+            stmt_fields.append(self.objectid_field)
+            incrementor = "SDE.GDB_UTIL.NEXT_ROWID('{}', '{}')"\
+                .format(self._owner, self.name)
+            placeholders.append(incrementor)
+        # Prepare statement
+        placeholders_joined = ', '.join(placeholders)
+        stmt_fields_joined = ', '.join(stmt_fields)
+        stmt = 'INSERT INTO {} ({}) VALUES ({})'.format(self.name, \
+            stmt_fields_joined, placeholders_joined)
+        return stmt
+
     def delete(self, cascade=False):
         """Delete all rows."""
         name = self._name_p