堆棧內存溢出
  簡體   English   中英

HBase范圍如何掃描十六進制行鍵？

[英]How to do HBase range scan for Hexadecimal row key?

 原文  2019-08-07 09:25:45  3   2    java/ hex/ hbase

問題描述

當嘗試在HBase Shell上執行范圍掃描時，以下在HBase Shell中起作用。 

scan 'mytable', {STARTROW => "\x00\x00\x00\x00\x01\x8F\xF6\x83", ENDROW => "\x00\x00\x00\x00\x01\x8F\xF6\x8D"}

但是，當嘗試實現Java客戶端執行相同的操作時，它不會檢索到任何結果。 

Scan scan = new Scan(Bytes.ToBytes("\x00\x00\x00\x00\x01\x8F\xF6\x83"),Bytes.toBytes("\x00\x00\x00\x00\x01\x8F\xF6\x8D"); 
scan.setFilter(colFilter);
scan.setOtherStuff...

ResultScanner scanner = table.getScanner(scan);
for (Result result = scanner.next(); result != null; result = scanner.next()) {
    ....
}

我試圖轉義“ \\”字符並傳遞開始和結束行鍵。 但是它沒有按預期工作。

我將輸入數據作為命令行參數傳遞。 

time java -jar $ARIADNE3D_CLI PCRangeSearchTxt -table_name $TABLE_NAME -m 4 -start_key "\x00\x00\x00\x00\x01\x8F\xF6\x8D" -end_key "\x00\x00\x00\x00\x01\x8F\xF6\x8D" -o $SCRATCH/txt-1.txt

PCRangeSearchTxt的Java實現如下 

/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package umg.ariadne3d.core.query.pc;

import java.io.*;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.PosixParser;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import umg.ariadne3d.core.common.Constants;
import umg.core.common.Executable;

/**
 * Point cloud range search.
 * @author VVo
 */
public class PCRangeSearchTxt implements Executable {

    static Logger LOGGER = Logger.getLogger(PCRangeSearchTxt.class);
    public static final String NAME = "PCRANGESEARCHTXT"; //PCRangeSearchTxt

    public static void main(String[] args) {
        args = new String[]{
               // "-t", "d15-tiny-m4",
               // "-m", "4",
               // "-index", "/Users/vu/scratch/ariadne3d/pointcloud/meta/hilbert.json",
               // "-query", "/Users/vu/scratch/ariadne3d/query/q0.json",
               // "-las_meta", "/Users/vu/scratch/ariadne3d/pointcloud/meta/d15-meta.json",
               // "-o", "/Users/vu/tmp/a.las"
        };

        Executable prog = new PCRangeSearchTxt();
        int err = prog.run(args);
        System.exit(err);
    }

    @Override
    public int run(String[] args) {
        CommandLine cmd = parseArgs(args);
        String tableName = cmd.getOptionValue("t");

          String start_key = cmd.getOptionValue("start_key");
          String end_key = cmd.getOptionValue("end_key");
          final String FILENAME = cmd.getOptionValue("o");

        int modelNo = Integer.parseInt(cmd.getOptionValue("m"));

        try{
            File file = new File(FILENAME);
            // if file doesnt exists, then create it
            if (!file.exists()) {
                file.createNewFile();
            }
        }catch (IOException e) {

            e.printStackTrace();

        }

        Configuration conf = HBaseConfiguration.create();

        String[] connectionParams = null;
        if (cmd.hasOption("conn")) {
            connectionParams = cmd.getOptionValues("conn");
        }

        if (connectionParams != null) {
            conf.set(Constants.HBASE_CONFIGURATION_ZOOKEEPER_QUORUM, connectionParams[0]);
            LOGGER.debug(String.format("Set quorum string %s", conf.get(Constants.HBASE_CONFIGURATION_ZOOKEEPER_QUORUM)));
            conf.setInt(Constants.HBASE_CONFIGURATION_ZOOKEEPER_CLIENTPORT, Integer.parseInt(connectionParams[1]));
            LOGGER.debug(String.format("Set port %d", conf.getInt(Constants.HBASE_CONFIGURATION_ZOOKEEPER_CLIENTPORT, 0)));
        }

        try {

            long start = System.currentTimeMillis();

            Connection connection = ConnectionFactory.createConnection(conf);

            HBaseConfiguration.addHbaseResources(conf);

            Table table = connection.getTable(TableName.valueOf(tableName));

            byte[] keyStart = Bytes.toBytes(start_key);
            byte[] keyEnd  = Bytes.toBytes(end_key);

            Scan scan = new Scan(keyStart, keyEnd);

            ResultScanner scanner = table.getScanner(scan);
            FileWriter writer = new FileWriter(FILENAME, true);
            try{
                for (Result result = scanner.next(); result != null; result = scanner.next()) {
                    writer.write(result.toString()+"\n");


                }
            }finally {
                writer.close();
                scanner.close();
            }
            long end = System.currentTimeMillis();

            System.out.printf("Total time %d \n", end - start);

            table.close();
            connection.close();

            return 0;
        } catch (IOException ex) {
            LOGGER.error(ex);
            return 1;
        }
    }

    private static CommandLine parseArgs(String[] args) {
        Options options = new Options();

        Option o;

        // table name
        o = new Option("t","table_name", true, "HBase table name");
        options.addOption(o);

        o = new Option("m", "model_number", true, "model number");
        options.addOption(o);

        o = new Option("start_key", true, "start key for range scan");
        options.addOption(o);

        o = new Option("end_key", true, "end key for range scan");
        options.addOption(o);
        o = new Option("o", "output", true, "create output file");
        o.setRequired(false);
        options.addOption(o);

        // connection parameters
        o = new Option("conn", "connection", true, "Zookepper quorum and port");
        o.setArgs(2);
        o.setRequired(false);
        options.addOption(o);

        // debug flag
        options.addOption("d", "debug", false, "switch on DEBUG log level");

        CommandLineParser parser = new PosixParser();
        CommandLine cmd = null;

        try {
            cmd = parser.parse(options, args);
        } catch (Exception e) {
            System.err.println("ERROR: " + e.getMessage() + "\n");
            HelpFormatter formatter = new HelpFormatter();
            formatter.printHelp(NAME + " ", options, true);
            System.exit(-1);
        }

        if (cmd.hasOption("d")) {
            LOGGER.setLevel(Level.DEBUG);
            System.out.println("DEBUG ON");
        }

        return cmd;
    }

}

在十六進制的行鍵上實現HBase范圍搜索的正確方法是什么？

2 個解決方案

解決方案1
 1  2019-08-15 16:37:26

我想您知道您在HBase表中用作鍵的原因，所以我不明白為什么您不能這樣做： 

byte[]start = Hex.decodeHex("startKey".toCharArray());
byte[]end = Hex.decodeHex("endKey".toCharArray());
Scan scan = new Scan(start, end)

只是不確定為什么要嘗試以其他方式執行此操作。

否則，這里有一個問題的答案： HBase Shell RowKey中的非十六進制字符是什么？

希望有幫助:)

解決方案2
 1  2019-08-22 11:27:52

我能夠找到此問題的解決方案。 基本上，我在代碼中所做的就是將行鍵聲明為String變量。 當我將start_key和end_key作為命令行參數傳遞並且HBase在內部以字節array []序列存儲數據時，我可以簡單地以已知格式傳遞開始鍵，也就是說，我不必將值作為十六進制代碼傳遞。 例如，我可以將start_key和end_key都以其原始形式/人類可讀的形式傳遞，並且HBase會將這種形式映射為HBase內部字節array []形式。

因此，我修改了上述Java類，以將start_key和end_key接受為雙精度數據類型值。 

// key format: 392994.475499
double startTS = Double.parseDouble(cmd.getOptionValue("start_key"));
double endTS = Double.parseDouble(cmd.getOptionValue("end_key"));

在總結時，要了解HBase的內部結構並接受命令行值作為double值。

通過這樣做，我能夠按預期運行代碼。

為了讓大家全面了解我所做的事情，下面共享了該類的修改后的代碼。 

/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package umg.ariadne3d.core.query.pc;

import java.io.*;
import java.nio.file.Paths;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import umg.ariadne3d.core.store.schema.pc.Model4;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.PosixParser;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import umg.ariadne3d.core.common.Constants;
import umg.core.common.Executable;

/**
 * Point cloud range search.
 * @author 
 */
public class PCMultiClientRangeSearchSQN implements Executable {

    static Logger LOGGER = Logger.getLogger(PCMultiClientRangeSearchSQN.class);
    public static final String NAME = "PCMULTICLIENTRANGESEARCHSQN"; //PCMultiClientRangeSearchSQN

    public static void main(String[] args) {
        args = new String[]{
                // "-t", "d15-tiny-m4",
                // "-m", "4",
                // "-index", "/Users/vu/scratch/ariadne3d/pointcloud/meta/hilbert.json",
                // "-query", "/Users/vu/scratch/ariadne3d/query/q0.json",
                // "-las_meta", "/Users/vu/scratch/ariadne3d/pointcloud/meta/d15-meta.json",
                // "-o", "/Users/vu/tmp/a.las"
        };

        Executable prog = new PCMultiClientRangeSearchSQN();
        int err = prog.run(args);
        System.exit(err);
    }

    @Override
    public int run(String[] args) {
        CommandLine cmd = parseArgs(args);
        String tableName = cmd.getOptionValue("t");

        // key format: 392994.475499
        double startTS = Double.parseDouble(cmd.getOptionValue("start_key"));
        double endTS = Double.parseDouble(cmd.getOptionValue("end_key"));

        long startRowkey = Math.round((startTS - 388800) / 0.000001);
        long endRowkey = Math.round((endTS - 388800) / 0.000001);

        int numOfClients;
        if (cmd.hasOption("clients")) {
            numOfClients = Integer.parseInt(cmd.getOptionValue("clients"));
        } else {
            numOfClients = 1;//Runtime.getRuntime().availableProcessors();
        }
        //System.out.println(numOfClients);
        final String FILENAME = cmd.getOptionValue("o");
        final String FILEPATH = new File("").getAbsolutePath();

        for(int i=0; i<numOfClients; i++){
            try{
                File file = new File(FILENAME+i+".txt");
                // if file doesnt exists, then create it
                if (!file.exists()) {
                    file.createNewFile();
                }
            }catch (IOException e) {
                e.printStackTrace();
            }
        }

        Configuration conf = HBaseConfiguration.create();

        String[] connectionParams = null;
        if (cmd.hasOption("conn")) {
            connectionParams = cmd.getOptionValues("conn");
        }

        if (connectionParams != null) {
            conf.set(Constants.HBASE_CONFIGURATION_ZOOKEEPER_QUORUM, connectionParams[0]);
            LOGGER.debug(String.format("Set quorum string %s", conf.get(Constants.HBASE_CONFIGURATION_ZOOKEEPER_QUORUM)));
            conf.setInt(Constants.HBASE_CONFIGURATION_ZOOKEEPER_CLIENTPORT, Integer.parseInt(connectionParams[1]));
            LOGGER.debug(String.format("Set port %d", conf.getInt(Constants.HBASE_CONFIGURATION_ZOOKEEPER_CLIENTPORT, 0)));
        }

        try {

            Connection connection = ConnectionFactory.createConnection(conf);
            HBaseConfiguration.addHbaseResources(conf);
            Table table = connection.getTable(TableName.valueOf(tableName));

            byte[] keyStart = Bytes.toBytes(startRowkey);
            byte[] keyEnd  = Bytes.toBytes(endRowkey);

            ExecutorService executorService = Executors.newFixedThreadPool(100);

            //submit the range scan task for execution
            for(int j =0; j< numOfClients; j++){
                executorService.execute(new RangeScan(keyStart, keyEnd, table, new File(FILEPATH+"/"+FILENAME+j+".txt")));
            }


            executorService.shutdown();
            System.out.println("-----------------------");
            // wait until all tasks are finished
            try{
                executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
            }catch (Exception e){
                System.out.println("Error ");
            }

            System.out.println("All tasks are finished!");

            table.close();
            connection.close();

            return 0;
        } catch (IOException ex) {
            LOGGER.error(ex);
            return 1;
        }
    }

    class RangeScan implements Runnable{
        byte[] keyStart;
        byte[] keyEnd;
        Table table;
        File file;

        RangeScan(byte[] keyStart, byte[] keyEnd, Table table, File file){
            this.keyStart = keyStart;
            this.keyEnd = keyEnd;
            this.table = table;
            this.file = file;
        }

        @Override
        public void run() {
            long start = System.currentTimeMillis();

            Scan scan = new Scan(keyStart, keyEnd);
            try{
                ResultScanner scanner = table.getScanner(scan);
                FileWriter writer = new FileWriter(file, true);
                try{
                    for (Result result = scanner.next(); result != null; result = scanner.next()) {
                    //    System.out.println("result "+result.toString());
                        //byte[] rawPointBytes = result.getValue(Model4.RAW_SENSING_DATA_FAM, Model4.POINT_COL);

                        //LASPointProtos.LASPointP pointP = LASPointProtos.LASPointP.parseFrom(rawPointBytes);
                        writer.write(result.toString()+"\n");
                    }
                }finally {
                    writer.close();
                    scanner.close();
                }
                long end = System.currentTimeMillis();
                System.out.printf("Total time For File %s is %d \n", file.toString(),end - start);
            }
            catch (Exception ex){
                LOGGER.error(ex);
            }
            // return 0;
        }
    }

    private static CommandLine parseArgs(String[] args) {
        Options options = new Options();

        Option o;

        // table name
        o = new Option("t","table_name", true, "HBase table name");
        options.addOption(o);

        o = new Option("start_key", true, "start key for range scan");
        options.addOption(o);

        o = new Option("end_key", true, "end key for range scan");
        options.addOption(o);

        o = new Option("clients", true, "number of concurrent clients");
        options.addOption(o);

        o = new Option("o", "output", true, "create output file");
        o.setRequired(false);
        options.addOption(o);

        // connection parameters
        o = new Option("conn", "connection", true, "Zookepper quorum and port");
        o.setArgs(2);
        o.setRequired(false);
        options.addOption(o);

        // debug flag
        options.addOption("d", "debug", false, "switch on DEBUG log level");

        CommandLineParser parser = new PosixParser();
        CommandLine cmd = null;

        try {
            cmd = parser.parse(options, args);
        } catch (Exception e) {
            System.err.println("ERROR: " + e.getMessage() + "\n");
            HelpFormatter formatter = new HelpFormatter();
            formatter.printHelp(NAME + " ", options, true);
            System.exit(-1);
        }

        if (cmd.hasOption("d")) {
            LOGGER.setLevel(Level.DEBUG);
            System.out.println("DEBUG ON");
        }

        return cmd;
    }

}

暫無
暫無

聲明:本站的技術帖子網頁，遵循CC BY-SA 4.0協議，如果您需要轉載，請注明本站網址或者原文地址。任何問題請咨詢:yoyou2525@163.com.

相關問題 如何通過行鍵的一部分過濾 HBase 的掃描？ 如何掃描hbase中的數值范圍 如何從hbase掃描結果中獲取行密鑰 hbase如何使用包含&#39;\\ 1&#39;的密鑰進行掃描 如果密鑰長度不固定，如何有效掃描HBase？ 如何在HBase協處理器中進行攔截掃描 如何僅使用它們的鍵來獲取 HBase 行？ 如何在Java中獲取HBase的行鍵 使用Thrift Java API Hbase進行范圍掃描 如何在Hbase中獲得給定行鍵的一部分的所有行
相關標簽
 
粵ICP備18138465號  © 2020-2024 STACKOOM.COM