diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java index 1258568941..4bfb4a49f1 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java @@ -133,8 +133,7 @@ public ParquetRewriter(RewriteOptions options) throws IOException { .collect(Collectors.toList()), out); - this.outSchema = getSchema(); - this.outSchema = pruneColumnsInSchema(outSchema, options.getPruneColumns()); + this.outSchema = pruneColumnsInSchema(getSchema(), options.getPruneColumns()); this.extraMetaData = getExtraMetadata(options); if (options.getMaskColumns() != null) { @@ -208,7 +207,6 @@ private MessageType getSchema() { .getFields() .forEach(x -> { if (!fieldNames.containsKey(x.getName())) { - LOG.info("Column {} is added to the output from inputFilesToJoin side", x.getName()); fieldNames.put(x.getName(), x); } else if (overwriteInputWithJoinColumns) { LOG.info("Column {} in inputFiles is overwritten by inputFilesToJoin side", x.getName()); diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/rewrite/ParquetRewriterTest.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/rewrite/ParquetRewriterTest.java index 8b799afc8e..ce612baec9 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/rewrite/ParquetRewriterTest.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/rewrite/ParquetRewriterTest.java @@ -18,7 +18,10 @@ */ package org.apache.parquet.hadoop.rewrite; -import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.*; +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY; +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE; +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FLOAT; +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64; import static org.apache.parquet.schema.Type.Repetition.OPTIONAL; import static org.apache.parquet.schema.Type.Repetition.REPEATED; import static org.apache.parquet.schema.Type.Repetition.REQUIRED; @@ -36,7 +39,14 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.function.BiFunction; import java.util.function.Function; import java.util.stream.Collectors;