examples plus tests

.gitignore

@@ -0,0 +1 @@
+target/*

pom.xml

@@ -0,0 +1,81 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+<build>
+    <pluginManagement>
+      <plugins>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-surefire-plugin</artifactId>
+          <version>2.8</version>
+        </plugin>
+        <plugin>
+            <artifactId>maven-assembly-plugin</artifactId>
+            <configuration>
+                <archive>
+                    <manifest>
+                        <mainClass>com.matthewrathbone.example.RawMapreduce</mainClass>
+                    </manifest>
+                </archive>
+                <descriptorRefs>
+                    <descriptorRef>jar-with-dependencies</descriptorRef>
+                </descriptorRefs>
+            </configuration>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+  </build>
+
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>com.matthewrathbone.example</groupId>
+  <artifactId>hive-extensions</artifactId>
+  <packaging>jar</packaging>
+  <version>1.0-SNAPSHOT</version>
+  <name>hive-extensions</name>
+  <url>http://maven.apache.org</url>
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client</artifactId>
+      <version>2.0.0-mr1-cdh4.3.1</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-exec</artifactId>
+      <version>0.10.0-cdh4.3.1</version>
+      <scope>provided</scope>
+    </dependency>
+    <!-- TEST DEPENDENCIES -->
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-io</artifactId>
+      <version>1.3.2</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>commons-httpclient</groupId>
+      <artifactId>commons-httpclient</artifactId>
+      <version>3.1</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-test</artifactId>
+      <version>2.0.0-mr1-cdh4.1.2</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.8.2</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+    <repositories>
+    <repository>
+      <id>cloudera</id>
+      <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
+    </repository>
+  </repositories>
+</project>

src/main/java/com/matthewrathbone/example/ComplexUDFExample.java

@@ -0,0 +1,66 @@
+package com.matthewrathbone.example;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+
+class ComplexUDFExample extends GenericUDF {
+
+  ListObjectInspector listOI;
+  StringObjectInspector elementOI;
+
+  @Override
+  public String getDisplayString(String[] arg0) {
+    return "arrayContainsExample()"; // this should probably be better
+  }
+
+  @Override
+  public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
+    if (arguments.length != 2) {
+      throw new UDFArgumentLengthException("arrayContainsExample only takes 2 arguments: List<T>, T");
+    }
+    // 1. Check we received the right object types.
+    ObjectInspector a = arguments[0];
+    ObjectInspector b = arguments[1];
+    if (!(a instanceof ListObjectInspector) || !(b instanceof StringObjectInspector)) {
+      throw new UDFArgumentException("first argument must be a list / array, second argument must be a string");
+    }
+    this.listOI = (ListObjectInspector) a;
+    this.elementOI = (StringObjectInspector) b;
+
+    // 2. Check that the list contains strings
+    if(!(listOI.getListElementObjectInspector() instanceof StringObjectInspector)) {
+      throw new UDFArgumentException("first argument must be a list of strings");
+    }
+
+    // the return type of our function is a boolean, so we provide the correct object inspector
+    return PrimitiveObjectInspectorFactory.javaBooleanObjectInspector;
+  }
+
+  @Override
+  public Object evaluate(DeferredObject[] arguments) throws HiveException {
+
+    // get the list and string from the deferred objects using the object inspectors
+    List<String> list = (List<String>) this.listOI.getList(arguments[0].get());
+    String arg = elementOI.getPrimitiveJavaObject(arguments[1].get());
+
+    // check for nulls
+    if (list == null || arg == null) {
+      return null;
+    }
+
+    // see if our list contains the value we need
+    for(String s: list) {
+      if (arg.equals(s)) return new Boolean(true);
+    }
+    return new Boolean(false);
+  }
+
+}

src/main/java/com/matthewrathbone/example/SimpleUDFExample.java

@@ -0,0 +1,19 @@
+package com.matthewrathbone.example;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.io.Text;
+
+
+@Description(
+  name="SimpleUDFExample",
+  value="returns 'hello x', where x is whatever you give it (STRING)",
+  extended="SELECT simpleudfexample('world') from foo limit 1;"
+  )
+class SimpleUDFExample extends UDF {
+
+  public Text evaluate(Text input) {
+    if(input == null) return null;
+    return new Text("Hello " + input.toString());
+  }
+}

src/test/java/com/matthewrathbone/example/ComplexUDFExampleTest.java

@@ -0,0 +1,49 @@
+package com.matthewrathbone.example;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaBooleanObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.junit.Test;
+
+public class ComplexUDFExampleTest {
+
+
+  @Test
+  public void testComplexUDFReturnsCorrectValues() throws HiveException {
+
+    // set up the models we need
+    ComplexUDFExample example = new ComplexUDFExample();
+    ObjectInspector stringOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+    ObjectInspector listOI = ObjectInspectorFactory.getStandardListObjectInspector(stringOI);
+    JavaBooleanObjectInspector resultInspector = (JavaBooleanObjectInspector) example.initialize(new ObjectInspector[]{listOI, stringOI});
+
+    // create the actual UDF arguments
+    List<String> list = new ArrayList<String>();
+    list.add("a");
+    list.add("b");
+    list.add("c");
+
+    // test our results
+
+    // the value exists
+    Object result = example.evaluate(new DeferredObject[]{new DeferredJavaObject(list), new DeferredJavaObject("a")});
+    Assert.assertEquals(true, resultInspector.get(result));
+
+    // the value doesn't exist
+    Object result2 = example.evaluate(new DeferredObject[]{new DeferredJavaObject(list), new DeferredJavaObject("d")});
+    Assert.assertEquals(false, resultInspector.get(result2));
+
+    // arguments are null
+    Object result3 = example.evaluate(new DeferredObject[]{new DeferredJavaObject(null), new DeferredJavaObject(null)});
+    Assert.assertNull(result3);
+  }
+}

src/test/java/com/matthewrathbone/example/SimpleUDFExampleTest.java

@@ -0,0 +1,21 @@
+package com.matthewrathbone.example;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class SimpleUDFExampleTest {
+
+  @Test
+  public void testUDF() {
+    SimpleUDFExample example = new SimpleUDFExample();
+    Assert.assertEquals("Hello world", example.evaluate(new Text("world")).toString());
+  }
+
+  @Test
+  public void testUDFNullCheck() {
+    SimpleUDFExample example = new SimpleUDFExample();
+    Assert.assertNull(example.evaluate(null));
+  }
+}