SELECT a row from SQL server perform a web request using the information and then UPDATE the same row

Question

I am writing a program to iterate through a database table (1.4 million rows), selecting information, making a web request to geocode from Google then I want to store the longitude and latitude in the same row I got the address from.

I think there might be a better way than I have come up with as I have two connections opened to the database and the opening and closing of the connection each time is most probably going to give me some overhead.

I am aware that I could use a JSON return instead ox XML and I might change to that but I would like to figure out the database connection issue first and then a way to make this program wait for the enforced 24 hour wait period to elapse before re-using the API keys?

So here is what I have so far, I would be very grateful for any insights any of you have.

string ApiKey = apikey[a];  //I have a couple of API keys in an array
while (a < 7)
{
    using (SqlConnection conn = new SqlConnection(connection))
    {
        for (int j = 1; j <= 1463758; j++)
        {
            if (count % 2498 == 0) //2500 requests per API per day
            {
                a = a + 1;
            }
            SqlCommand cmd = new SqlCommand("SELECT AddressToGeoCode FROM testDB WHERE id = @id AND Status = 'Not Done'", conn);
            try
            {
                conn.Open();
                cmd.Parameters.AddWithValue("@id", j);
                SqlDataReader reader = cmd.ExecuteReader();
                while (reader.Read())
                {
                    string address = reader[0].ToString();

                    var requestUri = string.Format("https://maps.googleapis.com/maps/api/geocode/xml?address={0}&key={1}", Uri.EscapeDataString(address), ApiKey);

                    var request = WebRequest.Create(requestUri);
                    var response = request.GetResponse();
                    var xdoc = XDocument.Load(response.GetResponseStream());

                    string status = xdoc.Element("GeocodeResponse").Element("status").Value;
                    var result = xdoc.Element("GeocodeResponse").Element("result");
                    var locationType = result.Element("geometry").Element("location_type").Value;
                    var locationElement = result.Element("geometry").Element("location");
                    float lat = float.Parse(locationElement.Element("lat").Value.ToString());
                    float lng = float.Parse(locationElement.Element("lng").Value.ToString());
                    Console.WriteLine("Api key number: " + a + ", " + apikey[a]);
                    Console.WriteLine("status: " + status);
                    Console.WriteLine("Coordinates: " + lat + ", " + lng);

                    using (SqlConnection conn2 = new SqlConnection(connection))
                    {
                        SqlCommand cmd2 = new SqlCommand("UPDATE testDB SET Lat = @lat, Long = @lng, LocationType = @lType WHERE id = @id", conn2);
                        try
                        {
                            conn2.Open();
                            cmd2.Parameters.AddWithValue("@id", j);
                            cmd2.Parameters.AddWithValue("@lat", lat);
                            cmd2.Parameters.AddWithValue("@lng", lng);
                            cmd2.Parameters.AddWithValue("@lType", locationType);
                            cmd2.ExecuteNonQuery();
                            conn2.Close();
                        }
                        catch (Exception ex)
                        {
                            Library.WriteErrorLog("Update exception " + ex.ToString());
                        }
                    }
                    count = count + 1;
                }
                conn.Close();
            }
            catch (Exception e)
            {
                Library.WriteErrorLog("Update exception " + .ToString());
            };
        }

        Console.Read();
    }
}

Answer 1

it's much more efficient to update all rows with one SQL Query rather than update each row in the loop.

you can insert all items that you want to update in a tempTable and than make an update join statement and execute it outside of the loop.

bulk-record-update-with-sql is an example