ORC-1740: Avoid the dump tool repeatedly parsing ColumnStatistics

cxzl25 · williamhyun · commit c157afaac0a2 · 2024-07-10T15:51:30.000-07:00
### What changes were proposed in this pull request? This PR aims to avoid the dump tool repeatedly parsing ColumnStatistics. ### Why are the changes needed? `org.apache.orc.StripeStatistics#getColumnStatistics` always generates statistical information for all columns. When there are many columns, the parsing performance decreases. https://github.com/apache/orc/blob/c38e20d862ce19395558e092dd42033a000fe22d/java/core/src/java/org/apache/orc/StripeStatistics.java#L57-L66 ### How was this patch tested? local test and exist UT ### Was this patch authored or co-authored using generative AI tooling? No Closes #1972 from cxzl25/ORC-1740. Authored-by: sychen <sychen@ctrip.com> Signed-off-by: William Hyun <william@apache.org>
diff --git a/java/tools/src/java/org/apache/orc/tools/FileDump.java b/java/tools/src/java/org/apache/orc/tools/FileDump.java
@@ -357,9 +357,10 @@ private static void printMetaDataImpl(final String filename,
     for (int n = 0; n < stripeStats.size(); n++) {
       System.out.println("  Stripe " + (n + 1) + ":");
       StripeStatistics ss = stripeStats.get(n);
-      for (int i = 0; i < ss.getColumnStatistics().length; ++i) {
+      ColumnStatistics[] columnStatistics = ss.getColumnStatistics();
+      for (int i = 0; i < columnStatistics.length; ++i) {
         System.out.println("    Column " + i + ": " +
-            ss.getColumnStatistics()[i].toString());
+            columnStatistics[i].toString());
       }
     }
     ColumnStatistics[] stats = reader.getStatistics();
diff --git a/java/tools/src/java/org/apache/orc/tools/JsonFileDump.java b/java/tools/src/java/org/apache/orc/tools/JsonFileDump.java
@@ -112,10 +112,11 @@ public static void printJsonMetaData(List<String> files,
           writer.name("stripeNumber").value(n + 1);
           StripeStatistics ss = stripeStatistics.get(n);
           writer.name("columnStatistics").beginArray();
-          for (int i = 0; i < ss.getColumnStatistics().length; i++) {
+          ColumnStatistics[] columnStatistics = ss.getColumnStatistics();
+          for (int i = 0; i < columnStatistics.length; i++) {
             writer.beginObject();
             writer.name("columnId").value(i);
-            writeColumnStatistics(writer, ss.getColumnStatistics()[i]);
+            writeColumnStatistics(writer, columnStatistics[i]);
             writer.endObject();
           }
           writer.endArray();

Original file line number	Diff line number	Diff line change
`@@ -357,9 +357,10 @@ private static void printMetaDataImpl(final String filename,`
`357`	`357`	`for (int n = 0; n < stripeStats.size(); n++) {`
`358`	`358`	`System.out.println(" Stripe " + (n + 1) + ":");`
`359`	`359`	`StripeStatistics ss = stripeStats.get(n);`
`360`		`- for (int i = 0; i < ss.getColumnStatistics().length; ++i) {`
	`360`	`+ ColumnStatistics[] columnStatistics = ss.getColumnStatistics();`
	`361`	`+ for (int i = 0; i < columnStatistics.length; ++i) {`
`361`	`362`	`System.out.println(" Column " + i + ": " +`
`362`		`- ss.getColumnStatistics()[i].toString());`
	`363`	`+ columnStatistics[i].toString());`
`363`	`364`	`}`
`364`	`365`	`}`
`365`	`366`	`ColumnStatistics[] stats = reader.getStatistics();`