Hive UDAF开发之同时计算最大值与最小值
卷首语
前一篇文章hive UDAF开发入门和运行过程详解（转）里面讲过UDAF的开发过程，其中说到如果要深入理解UDAF的执行，可以看看求平均值的UDF的源码
本人在看完源码后，也还是没能十分理解里面的内容，于是动手再自己开发一个新的函数，试图多实践中理解它
函数功能介绍
函数的功能比较蛋疼，我们都知道Hive中有几个常用的聚合函数：sum,max,min,avg
现在要用一个函数来同时实现俩个不同的功能，对于同一个key，要求返回指定value集合中的最大值与最小值
这里面涉及到一个难点，函数接收到的数据只有一个，但是要同时产生出俩个新的数据出来，且具备一定的逻辑关系
语言描述这东西我不大懂，想了好久，还是直接上代码得了。。。。。。。。。。。。。
源码
package org.juefan.udaf; import java.util.ArrayList; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.udf.generic.AbstractGenericUDAFResolver; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.util.StringUtils; /** * GenericUDAFMaxMin. */ @Description(name = "maxmin", value = "_FUNC_(x) - Returns the max and min value of a set of numbers") public class GenericUDAFMaxMin extends AbstractGenericUDAFResolver { static final Log LOG = LogFactory.getLog(GenericUDAFMaxMin.class.getName()); @Override public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticException { if (parameters.length != 1) { throw new UDFArgumentTypeException(parameters.length - 1, "Exactly one argument is expected."); } if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " + parameters[0].getTypeName() + " is passed."); } switch (((PrimitiveTypeInfo) parameters[0]).getPrimitiveCategory()) { case BYTE: case SHORT: case INT: case LONG: case FLOAT: case DOUBLE: case STRING: case TIMESTAMP: return new GenericUDAFMaxMinEvaluator(); case BOOLEAN: default: throw new UDFArgumentTypeException(0, "Only numeric or string type arguments are accepted but " + parameters[0].getTypeName() + " is passed."); } } /** * GenericUDAFMaxMinEvaluator. * */ public static class GenericUDAFMaxMinEvaluator extends GenericUDAFEvaluator { // For PARTIAL1 and COMPLETE PrimitiveObjectInspector inputOI; // For PARTIAL2 and FINAL StructObjectInspector soi; // 封装好的序列化数据接口，存储计算过程中的最大值与最小值 StructField maxField; StructField minField; // 存储数据，利用get()可直接返回double类型值 DoubleObjectInspector maxFieldOI; DoubleObjectInspector minFieldOI; // For PARTIAL1 and PARTIAL2 // 存储中间的结果 Object[] partialResult; // For FINAL and COMPLETE // 最终输出的数据 Text result; @Override public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException { assert (parameters.length == 1); super.init(m, parameters); // 初始化数据输入过程 if (m == Mode.PARTIAL1 || m == Mode.COMPLETE) { inputOI = (PrimitiveObjectInspector) parameters[0]; } else { // 如果接收到的数据是中间数据，则转换成相应的结构体 soi = (StructObjectInspector) parameters[0]; // 获取指定字段的序列化数据 maxField = soi.getStructFieldRef("max"); minField = soi.getStructFieldRef("min"); // 获取指定字段的实际数据 maxFieldOI = (DoubleObjectInspector) maxField.getFieldObjectInspector(); minFieldOI = (DoubleObjectInspector) minField.getFieldObjectInspector(); } // 初始化数据输出过程 if (m == Mode.PARTIAL1 || m == Mode.PARTIAL2) { // 输出的数据是一个结构体，其中包含了max和min的值 // 存储结构化数据类型 ArrayList<ObjectInspector> foi = new ArrayList<ObjectInspector>(); foi.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector); foi.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector); // 存储结构化数据的字段名称 ArrayList<String> fname = new ArrayList<String>(); fname.add("max"); fname.add("min"); partialResult = new Object[2]; partialResult[0] = new DoubleWritable(0); partialResult[1] = new DoubleWritable(0); return ObjectInspectorFactory.getStandardStructObjectInspector(fname, foi); } else { // 如果执行到了最后一步，则指定相应的输出数据类型 result = new Text(""); return PrimitiveObjectInspectorFactory.writableStringObjectInspector; } } static class AverageAgg implements AggregationBuffer { double max; double min; }; @Override public AggregationBuffer getNewAggregationBuffer() throws HiveException { AverageAgg result = new AverageAgg(); reset(result); return result; } @Override public void reset(AggregationBuffer agg) throws HiveException { AverageAgg myagg = (AverageAgg) agg; myagg.max = Double.MIN_VALUE; myagg.min = Double.MAX_VALUE; } boolean warned = false; @Override public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveException { assert (parameters.length == 1); Object p = parameters[0]; if (p != null) { AverageAgg myagg = (AverageAgg) agg; try { // 获取输入数据，并进行相应的大小判断 double v = PrimitiveObjectInspectorUtils.getDouble(p, inputOI); if(myagg.max < v){ myagg.max = v; } if(myagg.min > v){ myagg.min = v; } } catch (NumberFormatException e) { if (!warned) { warned = true; LOG.warn(getClass().getSimpleName() + " " + StringUtils.stringifyException(e)); LOG.warn(getClass().getSimpleName() + " ignoring similar exceptions."); } } } } @Override public Object terminatePartial(AggregationBuffer agg) throws HiveException { // 将中间计算出的结果封装好返回给下一步操作 AverageAgg myagg = (AverageAgg) agg; ((DoubleWritable) partialResult[0]).set(myagg.max); ((DoubleWritable) partialResult[1]).set(myagg.min); return partialResult; } @Override public void merge(AggregationBuffer agg, Object partial) throws HiveException { if (partial != null) { //此处partial接收到的是terminatePartial的输出数据 AverageAgg myagg = (AverageAgg) agg; Object partialmax = soi.getStructFieldData(partial, maxField); Object partialmin = soi.getStructFieldData(partial, minField); if(myagg.max < maxFieldOI.get(partialmax)){ myagg.max = maxFieldOI.get(partialmax); } if(myagg.min > minFieldOI.get(partialmin)){ myagg.min = minFieldOI.get(partialmin); } } } @Override public Object terminate(AggregationBuffer agg) throws HiveException { // 将最终的结果合并成字符串后输出 AverageAgg myagg = (AverageAgg) agg; if (myagg.max == 0) { return null; } else { result.set(myagg.max + "\t" + myagg.min); return result; } } } }
写完后还是觉得没有怎么理解透整个过程，所以上面的注释也就将就着看了，不保证一定正确的！
下午加上一些输出跟踪一下执行过程才行，不过代码的逻辑是没有问题的了，本人运行过！
注意！
本站转载的文章为个人学习借鉴使用，本站对版权不负任何法律责任。如果侵犯了您的隐私权益，请联系我们删除。