Connecting to HDInsight Emulator

Question

I am trying to connect with c#.

Here is the class that submits hive queries successfully to my remote HDInsight cluster. what do i need to change here to connect to the local emulator

public class HadoopImporter : IImporter
{
    public static readonly Logger log = LogManager.GetCurrentClassLogger();

    public void Import(string _query)
    {
        try
        {
            log.Warn("Inside Hive submission method");
            var store = new X509Store();
            store.Open(OpenFlags.ReadOnly);
            var cert =
                store.Certificates.Cast<X509Certificate2>()
                     .First(item => item.Thumbprint == "MYCERTTUMBPRINT");
            if (cert == null)
                log.Error("no cert found");
            log.Warn(cert.FriendlyName);
            log.Warn("got the cert with thumbprint ", cert.Thumbprint.ToString())
            ;

            log.Warn("trying to create credentials from cert");
            var creds = new JobSubmissionCertificateCredential(new Guid("MYSUBSCRIPTIONID"),
                                                               cert, "MYSTORAGECONTAINER");
            log.Warn("trying to connect with cert");
            var jobClient = JobSubmissionClientFactory.Connect(creds);


            log.Warn("Setting Hive job parameters");
            var hiveJob = new HiveJobCreateParameters()
                {
                    Query = _query,
                    StatusFolder = "/samplequeryoutput"
                };

            var jobResults = jobClient.CreateHiveJob(hiveJob);

            log.Warn("Executing wait for jhive results");
            WaitForJobCompletion(jobResults, jobClient);

            using (var stream = jobClient.GetJobOutput(jobResults.JobId))
            {
                var reader = new StreamReader(stream);
                var res = reader.ReadToEnd();
                log.Warn("trying to get the job results " + res.ToString());
            }
        }
        catch (Exception exp)
        {   
            log.Error(exp);

        }
    }


    private static void WaitForJobCompletion(JobCreationResults jobDetails, IJobSubmissionClient client)
    {
        var jobInProgress = client.GetJob(jobDetails.JobId);
        while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed)
        {
            log.Warn("Inside the while loop waiting for hive job to complete");
            jobInProgress = client.GetJob(jobInProgress.JobId);
            Thread.Sleep(TimeSpan.FromSeconds(10));
        }
        log.Trace("HIVE Job has  Imported " + jobDetails.JobId);
    }
}

Answer 1

You should be able to connect to a local one-box using the REST implementation of the client.

You're looking for the WebHCatHttpClient interface. The code below runs a basic query against my local one-box.

var httpClient = new WebHCatHttpClient(new Uri("http://localhost:50111/"), "username", "password");
string outputDir = "basichivejob";
var task = httpClient.CreateHiveJob(@"select * from iris;", null, null, outputDir, null);
task.Wait();
var response = task.Result;
var output = response.Content.ReadAsAsync<JObject>();
output.Wait();
response.EnsureSuccessStatusCode();

string id = output.Result.GetValue("id").ToString();
httpClient.WaitForJobToCompleteAsync(id).Wait();

See the SDK docs for more info.