_mysql_exceptions error(1064, "check the manual that corresponds to your MySQL server version for the right syntax to use near 'default) VALUES

Question

I am trying to import CSVs to mysql database automatically using python script. I used the script from https://bitbucket.org/richardpenman/csv2mysql to achieve this task. Below is the code:

import os
import re
import sys
import csv
import time
import argparse
import collections
import MySQLdb
import warnings 
# suppress annoying mysql warnings
warnings.filterwarnings(action='ignore', category=MySQLdb.Warning) 

def get_type(s):
    """Find type for this string
    """
    # try integer type
    try:
        v = int(s)
    except ValueError:
        pass
    else:
        if abs(v) > 2147483647:
            return 'bigint'
        else:
            return 'int'
    # try float type
    try:
        float(s)
    except ValueError:
        pass
    else:
        return 'double'

    # check for timestamp
    dt_formats = (
        ('%Y-%m-%d %H:%M:%S', 'datetime'),
        ('%Y-%m-%d %H:%M:%S.%f', 'datetime'),
        ('%Y-%m-%d', 'date'),
        ('%H:%M:%S', 'time'),
    )
    for dt_format, dt_type in dt_formats:
        try:
            time.strptime(s, dt_format)
        except ValueError:
            pass
        else:
            return dt_type

    # doesn't match any other types so assume text
    if len(s) > 255:
        return 'text'
    else:
        return 'varchar(255)'


def most_common(l, default='varchar(255)'):
    """Return most common value from list
    """
    # some formats trump others
    if l:
        for dt_type in ('text', 'bigint'):
            if dt_type in l:
                return dt_type
        return max(l, key=l.count)
    return default


def get_col_types(input_file, max_rows=1000):
    """Find the type for each CSV column
    """
    csv_types = collections.defaultdict(list)
    print (os.getcwd())
    # os.chdir("scripts/CSV")
    reader = csv.reader(open(input_file))
    # test the first few rows for their data types
    for row_i, row in enumerate(reader):
        if row_i == 0:
            header = row
        else:
            for col_i, s in enumerate(row):
                data_type = get_type(s)
                csv_types[header[col_i]].append(data_type)

        if row_i == max_rows:
            break

    # take the most common data type for each row
    return [most_common(csv_types[col]) for col in header]


def get_insert(table, header):
    """Generate the SQL for inserting rows
    """
    field_names = ', '.join(header)
    field_markers = ', '.join('%s' for col in header)
    return 'INSERT INTO %s (%s) VALUES (%s);' % \
        (table, field_names, field_markers)


def format_header(row):
    """Format column names to remove illegal characters and duplicates
    """
    safe_col = lambda s: re.sub('\W+', '_', s.lower()).strip('_')
    header = []
    counts = collections.defaultdict(int)
    for col in row:
        col = safe_col(col)
        counts[col] += 1
        if counts[col] > 1:
            col = '{}{}'.format(col, counts[col])
        header.append(col)
    return header


def main(input_file, user, password, host, table, database, max_inserts=10000):
    print ("Importing `%s' into MySQL database `%s.%s'" % (input_file, database, table))
    db = MySQLdb.connect(host=host, user=user, passwd=password, charset='utf8')
    cursor = db.cursor()
    # create database and if doesn't exist
    cursor.execute('CREATE DATABASE IF NOT EXISTS %s;' % database)
    db.select_db(database)

    # define table
    print ("Analyzing column types ...")
    col_types = get_col_types(input_file)
    print (col_types)

    header = None
    for i, row in enumerate(csv.reader(open(input_file))):
        if header:
            while len(row) < len(header):
                row.append('') # this row is missing columns so pad blank values
            cursor.execute(insert_sql, row)
            if i % max_inserts == 0:
                db.commit()
                print ("commit")
        else:
            header = format_header(row)
            print ("Inserting rows ...")
            # SQL string for inserting data
            insert_sql = get_insert(table, header)

    # commit rows to database
    print ("Committing rows to database ...")
    db.commit()
    print ("Done!")



if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Automatically insert CSV contents into MySQL')
    parser.add_argument('--table', dest='table', help='Set the name of the table. If not set the CSV filename will be used')
    parser.add_argument('--database', dest='database', default=os.environ['MYSQL_DATABASE'], help='Set the name of the database. If not set the test database will be used')
    parser.add_argument('--user', dest='user', default=os.environ['MYSQL_USER'], help='The MySQL login username')
    parser.add_argument('--password', dest='password', default=os.environ['MYSQL_PASSWORD'], help='The MySQL login password')
    parser.add_argument('--host', dest='host', default=os.environ['MYSQL_CONTAINER_NAME'], help='The MySQL host')
    parser.add_argument('input_file', help='The input CSV file')
    args = parser.parse_args(sys.argv[1:])
    if not args.table:
        # use input file name for table
        args.table = os.path.splitext(os.path.basename(args.input_file))[0]

    main(args.input_file, args.user, args.password, args.host, args.table, args.database)

    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "WebApp.settings.local")

    from django.core.management import execute_from_command_line

    execute_from_command_line(sys.argv)

Even though I am able to read my csv file and import , it is throwing error for one particular table ie

DROP TABLE IF EXISTS `param_system`;
CREATE TABLE `param_system` (
`ID` int(11) NOT NULL,
  `EXTERNAL_EDIT` int(11) DEFAULT '0',
  `INTERNAL_EDIT` int(11) DEFAULT '0',
  `FORTRAN_TYPE` varchar(50) DEFAULT NULL,
  `LABEL` varchar(255) DEFAULT NULL,
  `DESCRIPTION` varchar(255) DEFAULT NULL,
  `HELP_ID` int(11) DEFAULT '0',
  `HELP_TEXT` text DEFAULT NULL,
  `GROUPNAME` varchar(255) DEFAULT NULL,
  `ROWNUM` int(11) DEFAULT '0',
  `WIDGET` varchar(50) DEFAULT NULL,
  `OPTIONS` varchar(255) DEFAULT NULL,
  `DISABLED` int(11) DEFAULT '0',
  `READONLY` int(11) DEFAULT '0',
  `REQUIRED` int(11) DEFAULT '0',
  `UI` text DEFAULT NULL,
  `MIN_VALUE` varchar(50) DEFAULT NULL,
  `MAX_VALUE` varchar(50) DEFAULT NULL,
  `FORM_VAR_NAME` varchar(255) DEFAULT NULL,
  `PARAM` varchar(255) DEFAULT NULL,
  `VAL` varchar(255) DEFAULT NULL,
  `DEFAULT` varchar(255) DEFAULT NULL
) ENGINE=InnoDB AUTO_INCREMENT=50 DEFAULT CHARSET=utf8;

The error ie being thrown is:

_mysql_exceptions.ProgrammingError: (1064, "You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'default) VALUES ('5', '0', '0', 'integer', '', '', '1', '', 'Base Parameters', '' at line 1")

Below is the screen shot of the csv that I am trying to import:

As you can see that it is not able to read the number "1" after "Base Parameters" and throwing the error. Could someone help me with what is going wrong?

Answer 1

_mysql_exceptions.ProgrammingError: (1064, "You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'default) VALUES ('5', '0', '0', 'integer', '', '', '1', '', 'Base Parameters', '' at line 1")

What you are seeing is a fragment of an INSERT statement. It isn't showing you the whole INSERT statement, it cuts it off. You said you think it is not reading the '1' in the ROWNUM field of your input data, but you are misinterpreting the error message.

It's just a coincidence that you see two single-quotes next to each other in the error message. The error message is formatted like this:

_mysql_exceptions.ProgrammingError: (1064, "You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '...' at line 1")

Where ... will be a fragment of the long SQL statement, starting with the first token that confused the parser, and continuing in this case for 80 characters. This 80-character fragment is:

default) VALUES ('5', '0', '0', 'integer', '', '', '1', '', 'Base Parameters', '

It's purely by accident that the 80th character is a single-quote, and then the next character in the error message is also a single-quote. It is not an empty string in place of the value '1' you expected to be read from the input. In fact, I assume it is reading the data value from the input.

So the problem reported in the error is that you're using the SQL reserved word DEFAULT as a column name. This Python script is not delimiting it. So the appearance of the reserved word in the INSERT statement confuses the parser.

I believe you can fix this in the Python script by formatting the column names inside back-ticks in the INSERT statement:

def get_insert(table, header):
    """Generate the SQL for inserting rows
    """
    field_names = ', '.join('`%s`' % col for col in header)
    field_markers = ', '.join('%s' for col in header)
    return 'INSERT INTO %s (%s) VALUES (%s);' % \
        (table, field_names, field_markers)

You could alternatively edit your input CSV file to avoid using SQL reserved words in the column names defined in the header.

Answer 2

@BillKarwin, When I used Django admin page to see the same table that was loaded in mysql db (after it was modified to take DEFAULT as a field name), it was throwing the "string index out of range" error. I couldn't pinpoint to the exact location where it is throwing the error. Is it because of the len(header) code in the main function?