hbase分页查询实现

Swear_Max 贡献于2013-01-15

作者   创建于2012-06-06 01:30:27   修改者  修改于2012-12-19 11:09:06字数6724

文档摘要:hbase分页查询实现Hbase本身是没有分页查询的,我在网上找了很多资料来实现一个分页功能,在这里做了一下记录,分享给大家,有什么不足之处,请尽管指出。废话不多说,看代码。
关键词:

hbase分页查询实现 Hbase本身是没有分页查询的,我在网上找了很多资料来实现一个分页功能,在这里做了一下记录,分享给大家,有什么不足之处,请尽管指出。废话不多说, 看代码。 import java.io.IOException; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MasterNotRunningException; import org.apache.hadoop.hbase.ZooKeeperConnectionException; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.HTablePool; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; import org.apache.hadoop.hbase.util.Bytes; public class HBaseUtils { private static Configuration config = null; private static HBaseAdmin admin = null; private static HTablePool tp = null; static { // 加载集群配置 config = HBaseConfiguration.create(); config.set("hbase.zookeeper.quorum", "xx.xx.xx"); config.set("hbase.zookeeper.property.clientPort", "2181"); try { admin = new HBaseAdmin(config); // 创建表池(可伟略提高查询性能,具体说明请百度或官方API) tp = new HTablePool(config, 10); } catch (MasterNotRunningException e) { } catch (ZooKeeperConnectionException e) { } } /* * 获取hbase的表 */ public static HTableInterface getTable(String tableName) { if (StringUtils.isEmpty(tableName)) return null; return tp.getTable(getBytes(tableName)); } /*转换byte数组*/ public static byte[] getBytes(String str) { if (value == null) value = ""; return Bytes.toBytes(value); } /** * 查询数据 * @param tableKey 表标识 * @param queryKey 查询标识 * @param startRow 开始行 * @param paramsMap 参数集合 * @return 结果集 */ public static TBData getDataMap(String tableName, String startRow, String stopRow, Integer currentPage, Integer pageSize) throws IOException { List> mapList = null; mapList = new LinkedList>(); ResultScanner scanner = null; // 为分页创建的封装类对象,下面有给出具体属性 TBData tbData = null; try { if (et != null) { // 获取最大返回结果数量 if (pageSize == null || pageSize == 0L) pageSize = 100; if (currentPage == null || currentPage == 0) currentPage = 1; // 计算起始页和结束页 Integer firstPage = (currentPage - 1) * pageSize ; Integer endPage = firstPage + pageSize; // 从表池中取出HBASE表对象 HTableInterface table = getTable(et.getName()); String start = startRow; String stop = stopRow; // 获取筛选对象 Scan scan = getScan(startRow, stopRow); // 给筛选对象放入过滤器(true标识分页,具体方法在下面) scan.setFilter(packageFilters(true)); // 缓存1000条数据 scan.setCaching(1000); scan.setCacheBlocks(false); scanner = table.getScanner(scan); int i = 0; List rowList = new LinkedList(); // 遍历扫描器对象, 并将需要查询出来的数据row key取出 for (Result result : scanner) { String row = toStr(result.getRow()); if (i >= firstPage && i < endPage) { rowList.add(getBytes(row)); } i++; } // 获取取出的row key的GET对象 List getList = getList(rowList, efiList); Result[] results = table.get(getList); // 遍历结果 for (Result result : results) { Map fmap= packFamilyMap(result); Map rmap = packRowMap(map1); mapList.add(rmap); } // 封装分页对象 tbData = new TBData(); tbData.setCurrentPage(currentPage); tbData.setPageSize(pageSize); tbData.setTotalCount(i); tbData.setTotalPage(getTotalPage(pageSize, i)); tbData.setResultList(mapList); } } catch (IOException e) { e.printStackTrace(); } finally { closeScanner(scanner); } return tbData; } // 获取扫描器对象 private static Scan getScan(String startRow, String stopRow) { Scan scan = new Scan(); scan.setStartRow(getBytes(startRow)); scan.setStopRow(getBytes(stopRow)); return scan; } /** * 封装查询条件 */ private static FilterList packageFilters(boolean isPage) { FilterList filterList = null; // MUST_PASS_ALL(条件 AND) MUST_PASS_ONE(条件OR) filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL); Filter filter1 = null; Filter filter2 = null; filter1 = newFilter(getBytes("family1"),getBytes("column1"), CompareOp.EQUAL, getBytes("condition1")); filter2 = newFilter(getBytes("family2"),getBytes("column1"), CompareOp.LESS, getBytes("condition2")); filterList.addFilter(filter1); filterList.addFilter(filter2); if (isPage) { filterList.addFilter(new FirstKeyOnlyFilter()); } return filterList; } private static Filter newFilter(byte[] f,byte[]c, CompareOp op,byte[]v) { return new SingleColumnValueFilter(f, c, op, v); } private static void closeScanner(ResultScanner scanner) { if (scanner != null) scanner.close(); } /** * 封装每行数据 * @param dataMap * @return */ private static Map packRowMap(Map dataMap) { Map map = new LinkedHashMap(); for (byte[] key : dataMap.keySet()) { byte[] value = dataMap.get(key); map.put(toStr(key), toStr(value)); } return map; } /*根据ROW KEY集合获取GET对象集合*/ private static List getList(List rowList) { List list = new LinkedList(); for (byte[] row : rowList) { Get get = new Get(row); get.addColumn(getBytes("family1"),getBytes("column1")); get.addColumn(getBytes("family1"),getBytes("column2")); get.addColumn(getBytes("family2"),getBytes("column1")); list.add(get); } return list; } /** * 封装配置的所有字段列族 */ private static Map packFamilyMap(Result result) { Map dataMap = null; dataMap = new LinkedHashMap(); dataMap.putAll(result.getFamilyMap(getBytes("family1"))); dataMap.putAll(result.getFamilyMap(getBytes("family2"))); return dataMap; } private static String toStr(byte[] bt) { return Bytes.toString(bt); } public static void main(String[] args) throws IOException { // 拿出row key的起始行和结束行 // #<0<9<: String startRow = "aaaa#"; String stopRow = "aaaa:"; int currentPage = 1; int pageSize = 20; // 执行hbase查询 getDataMap("table",startRow ,stopRow , stopRow ,pageSize ); } } import java.util.List; import java.util.Map; public class TBData { private Integer currentPage; private Integer pageSize; private Integer totalCount; private Integer totalPage; private List> resultList; /** * @return the currentPage */ public Integer getCurrentPage() { return currentPage; } /** * @param currentPage the currentPage to set */ public void setCurrentPage(Integer currentPage) { this.currentPage = currentPage; } /** * @return the pageSize */ public Integer getPageSize() { return pageSize; } /** * @param pageSize the pageSize to set */ public void setPageSize(Integer pageSize) { this.pageSize = pageSize; } /** * @return the totalCount */ public Integer getTotalCount() { return totalCount; } /** * @param totalCount the totalCount to set */ public void setTotalCount(Integer totalCount) { this.totalCount = totalCount; } /** * @return the totalPage */ public Integer getTotalPage() { return totalPage; } /** * @param totalPage the totalPage to set */ public void setTotalPage(Integer totalPage) { this.totalPage = totalPage; } /** * @return the resultList */ public List> getResultList() { return resultList; } /** * @param resultList the resultList to set */ public void setResultList(List> resultList) { this.resultList = resultList; } }

下载文档到电脑,查找使用更方便

文档的实际排版效果,会与网站的显示效果略有不同!!

需要 5 金币 [ 分享文档获得金币 ] 5 人已下载

下载文档