Class ParquetMetadataConverter
- java.lang.Object
-
- org.apache.parquet.format.converter.ParquetMetadataConverter
-
public class ParquetMetadataConverter extends Object
-
-
Nested Class Summary
Nested Classes Modifier and Type Class Description static class
ParquetMetadataConverter.MetadataFilter
-
Field Summary
Fields Modifier and Type Field Description static long
MAX_STATS_SIZE
static ParquetMetadataConverter.MetadataFilter
NO_FILTER
static ParquetMetadataConverter.MetadataFilter
SKIP_ROW_GROUPS
-
Constructor Summary
Constructors Constructor Description ParquetMetadataConverter()
ParquetMetadataConverter(int statisticsTruncateLength)
ParquetMetadataConverter(org.apache.hadoop.conf.Configuration conf)
Deprecated.will be removed in 2.0.0; useParquetMetadataConverter(ParquetReadOptions)
ParquetMetadataConverter(ParquetReadOptions options)
-
Method Summary
All Methods Static Methods Instance Methods Concrete Methods Deprecated Methods Modifier and Type Method Description ColumnChunkMetaData
buildColumnChunkMetaData(org.apache.parquet.format.ColumnMetaData metaData, org.apache.parquet.hadoop.metadata.ColumnPath columnPath, org.apache.parquet.schema.PrimitiveType type, String createdBy)
org.apache.parquet.column.EncodingStats
convertEncodingStats(List<org.apache.parquet.format.PageEncodingStats> stats)
List<org.apache.parquet.format.PageEncodingStats>
convertEncodingStats(org.apache.parquet.column.EncodingStats stats)
static org.apache.parquet.internal.column.columnindex.ColumnIndex
fromParquetColumnIndex(org.apache.parquet.schema.PrimitiveType type, org.apache.parquet.format.ColumnIndex parquetColumnIndex)
ParquetMetadata
fromParquetMetadata(org.apache.parquet.format.FileMetaData parquetMetadata)
ParquetMetadata
fromParquetMetadata(org.apache.parquet.format.FileMetaData parquetMetadata, InternalFileDecryptor fileDecryptor, boolean encryptedFooter)
ParquetMetadata
fromParquetMetadata(org.apache.parquet.format.FileMetaData parquetMetadata, InternalFileDecryptor fileDecryptor, boolean encryptedFooter, Map<org.apache.parquet.format.RowGroup,Long> rowGroupToRowIndexOffsetMap)
static org.apache.parquet.internal.column.columnindex.OffsetIndex
fromParquetOffsetIndex(org.apache.parquet.format.OffsetIndex parquetOffsetIndex)
org.apache.parquet.column.statistics.Statistics
fromParquetStatistics(String createdBy, org.apache.parquet.format.Statistics statistics, org.apache.parquet.schema.PrimitiveType type)
static org.apache.parquet.column.statistics.Statistics
fromParquetStatistics(String createdBy, org.apache.parquet.format.Statistics statistics, org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName type)
Deprecated.will be removed in 2.0.0.static org.apache.parquet.column.statistics.Statistics
fromParquetStatistics(org.apache.parquet.format.Statistics statistics, org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName type)
Deprecated.will be removed in 2.0.0.org.apache.parquet.format.Encoding
getEncoding(org.apache.parquet.column.Encoding encoding)
org.apache.parquet.column.Encoding
getEncoding(org.apache.parquet.format.Encoding encoding)
org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName
getPrimitive(org.apache.parquet.format.Type type)
static ParquetMetadataConverter.MetadataFilter
offsets(long... offsets)
static ParquetMetadataConverter.MetadataFilter
range(long startOffset, long endOffset)
[ startOffset, endOffset )ParquetMetadata
readParquetMetadata(InputStream from)
Deprecated.ParquetMetadata
readParquetMetadata(InputStream from, ParquetMetadataConverter.MetadataFilter filter)
ParquetMetadata
readParquetMetadata(InputStream from, ParquetMetadataConverter.MetadataFilter filter, InternalFileDecryptor fileDecryptor, boolean encryptedFooter, int combinedFooterLength)
static org.apache.parquet.format.BloomFilterHeader
toBloomFilterHeader(org.apache.parquet.column.values.bloomfilter.BloomFilter bloomFilter)
static org.apache.parquet.format.ColumnIndex
toParquetColumnIndex(org.apache.parquet.schema.PrimitiveType type, org.apache.parquet.internal.column.columnindex.ColumnIndex columnIndex)
org.apache.parquet.format.FileMetaData
toParquetMetadata(int currentVersion, ParquetMetadata parquetMetadata)
org.apache.parquet.format.FileMetaData
toParquetMetadata(int currentVersion, ParquetMetadata parquetMetadata, InternalFileEncryptor fileEncryptor)
static org.apache.parquet.format.OffsetIndex
toParquetOffsetIndex(org.apache.parquet.internal.column.columnindex.OffsetIndex offsetIndex)
static org.apache.parquet.format.Statistics
toParquetStatistics(org.apache.parquet.column.statistics.Statistics stats)
static org.apache.parquet.format.Statistics
toParquetStatistics(org.apache.parquet.column.statistics.Statistics stats, int truncateLength)
void
writeDataPageHeader(int uncompressedSize, int compressedSize, int valueCount, org.apache.parquet.column.Encoding rlEncoding, org.apache.parquet.column.Encoding dlEncoding, org.apache.parquet.column.Encoding valuesEncoding, OutputStream to)
Deprecated.void
writeDataPageHeader(int uncompressedSize, int compressedSize, int valueCount, org.apache.parquet.column.statistics.Statistics statistics, org.apache.parquet.column.Encoding rlEncoding, org.apache.parquet.column.Encoding dlEncoding, org.apache.parquet.column.Encoding valuesEncoding, OutputStream to)
Deprecated.void
writeDataPageV1Header(int uncompressedSize, int compressedSize, int valueCount, org.apache.parquet.column.Encoding rlEncoding, org.apache.parquet.column.Encoding dlEncoding, org.apache.parquet.column.Encoding valuesEncoding, int crc, OutputStream to)
void
writeDataPageV1Header(int uncompressedSize, int compressedSize, int valueCount, org.apache.parquet.column.Encoding rlEncoding, org.apache.parquet.column.Encoding dlEncoding, org.apache.parquet.column.Encoding valuesEncoding, int crc, OutputStream to, org.apache.parquet.format.BlockCipher.Encryptor blockEncryptor, byte[] pageHeaderAAD)
void
writeDataPageV1Header(int uncompressedSize, int compressedSize, int valueCount, org.apache.parquet.column.Encoding rlEncoding, org.apache.parquet.column.Encoding dlEncoding, org.apache.parquet.column.Encoding valuesEncoding, OutputStream to)
void
writeDataPageV1Header(int uncompressedSize, int compressedSize, int valueCount, org.apache.parquet.column.Encoding rlEncoding, org.apache.parquet.column.Encoding dlEncoding, org.apache.parquet.column.Encoding valuesEncoding, OutputStream to, org.apache.parquet.format.BlockCipher.Encryptor blockEncryptor, byte[] pageHeaderAAD)
void
writeDataPageV2Header(int uncompressedSize, int compressedSize, int valueCount, int nullCount, int rowCount, org.apache.parquet.column.Encoding dataEncoding, int rlByteLength, int dlByteLength, OutputStream to)
void
writeDataPageV2Header(int uncompressedSize, int compressedSize, int valueCount, int nullCount, int rowCount, org.apache.parquet.column.Encoding dataEncoding, int rlByteLength, int dlByteLength, OutputStream to, org.apache.parquet.format.BlockCipher.Encryptor blockEncryptor, byte[] pageHeaderAAD)
void
writeDataPageV2Header(int uncompressedSize, int compressedSize, int valueCount, int nullCount, int rowCount, org.apache.parquet.column.statistics.Statistics statistics, org.apache.parquet.column.Encoding dataEncoding, int rlByteLength, int dlByteLength, OutputStream to)
Deprecated.void
writeDictionaryPageHeader(int uncompressedSize, int compressedSize, int valueCount, org.apache.parquet.column.Encoding valuesEncoding, int crc, OutputStream to)
void
writeDictionaryPageHeader(int uncompressedSize, int compressedSize, int valueCount, org.apache.parquet.column.Encoding valuesEncoding, int crc, OutputStream to, org.apache.parquet.format.BlockCipher.Encryptor blockEncryptor, byte[] pageHeaderAAD)
void
writeDictionaryPageHeader(int uncompressedSize, int compressedSize, int valueCount, org.apache.parquet.column.Encoding valuesEncoding, OutputStream to)
void
writeDictionaryPageHeader(int uncompressedSize, int compressedSize, int valueCount, org.apache.parquet.column.Encoding valuesEncoding, OutputStream to, org.apache.parquet.format.BlockCipher.Encryptor blockEncryptor, byte[] pageHeaderAAD)
-
-
-
Field Detail
-
NO_FILTER
public static final ParquetMetadataConverter.MetadataFilter NO_FILTER
-
SKIP_ROW_GROUPS
public static final ParquetMetadataConverter.MetadataFilter SKIP_ROW_GROUPS
-
MAX_STATS_SIZE
public static final long MAX_STATS_SIZE
- See Also:
- Constant Field Values
-
-
Constructor Detail
-
ParquetMetadataConverter
public ParquetMetadataConverter()
-
ParquetMetadataConverter
public ParquetMetadataConverter(int statisticsTruncateLength)
-
ParquetMetadataConverter
@Deprecated public ParquetMetadataConverter(org.apache.hadoop.conf.Configuration conf)
Deprecated.will be removed in 2.0.0; useParquetMetadataConverter(ParquetReadOptions)
- Parameters:
conf
- a configuration
-
ParquetMetadataConverter
public ParquetMetadataConverter(ParquetReadOptions options)
-
-
Method Detail
-
toParquetMetadata
public org.apache.parquet.format.FileMetaData toParquetMetadata(int currentVersion, ParquetMetadata parquetMetadata)
-
toParquetMetadata
public org.apache.parquet.format.FileMetaData toParquetMetadata(int currentVersion, ParquetMetadata parquetMetadata, InternalFileEncryptor fileEncryptor)
-
getEncoding
public org.apache.parquet.column.Encoding getEncoding(org.apache.parquet.format.Encoding encoding)
-
getEncoding
public org.apache.parquet.format.Encoding getEncoding(org.apache.parquet.column.Encoding encoding)
-
convertEncodingStats
public org.apache.parquet.column.EncodingStats convertEncodingStats(List<org.apache.parquet.format.PageEncodingStats> stats)
-
convertEncodingStats
public List<org.apache.parquet.format.PageEncodingStats> convertEncodingStats(org.apache.parquet.column.EncodingStats stats)
-
toParquetStatistics
public static org.apache.parquet.format.Statistics toParquetStatistics(org.apache.parquet.column.statistics.Statistics stats)
-
toParquetStatistics
public static org.apache.parquet.format.Statistics toParquetStatistics(org.apache.parquet.column.statistics.Statistics stats, int truncateLength)
-
fromParquetStatistics
@Deprecated public static org.apache.parquet.column.statistics.Statistics fromParquetStatistics(org.apache.parquet.format.Statistics statistics, org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName type)
Deprecated.will be removed in 2.0.0.- Parameters:
statistics
- parquet format statisticstype
- a primitive type name- Returns:
- the statistics
-
fromParquetStatistics
@Deprecated public static org.apache.parquet.column.statistics.Statistics fromParquetStatistics(String createdBy, org.apache.parquet.format.Statistics statistics, org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName type)
Deprecated.will be removed in 2.0.0.- Parameters:
createdBy
- the created-by string from the filestatistics
- parquet format statisticstype
- a primitive type name- Returns:
- the statistics
-
fromParquetStatistics
public org.apache.parquet.column.statistics.Statistics fromParquetStatistics(String createdBy, org.apache.parquet.format.Statistics statistics, org.apache.parquet.schema.PrimitiveType type)
-
getPrimitive
public org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName getPrimitive(org.apache.parquet.format.Type type)
-
range
public static ParquetMetadataConverter.MetadataFilter range(long startOffset, long endOffset)
[ startOffset, endOffset )- Parameters:
startOffset
- a start offset (inclusive)endOffset
- an end offset (exclusive)- Returns:
- a range filter from the offsets
-
offsets
public static ParquetMetadataConverter.MetadataFilter offsets(long... offsets)
-
readParquetMetadata
@Deprecated public ParquetMetadata readParquetMetadata(InputStream from) throws IOException
Deprecated.- Throws:
IOException
-
readParquetMetadata
public ParquetMetadata readParquetMetadata(InputStream from, ParquetMetadataConverter.MetadataFilter filter) throws IOException
- Throws:
IOException
-
readParquetMetadata
public ParquetMetadata readParquetMetadata(InputStream from, ParquetMetadataConverter.MetadataFilter filter, InternalFileDecryptor fileDecryptor, boolean encryptedFooter, int combinedFooterLength) throws IOException
- Throws:
IOException
-
buildColumnChunkMetaData
public ColumnChunkMetaData buildColumnChunkMetaData(org.apache.parquet.format.ColumnMetaData metaData, org.apache.parquet.hadoop.metadata.ColumnPath columnPath, org.apache.parquet.schema.PrimitiveType type, String createdBy)
-
fromParquetMetadata
public ParquetMetadata fromParquetMetadata(org.apache.parquet.format.FileMetaData parquetMetadata) throws IOException
- Throws:
IOException
-
fromParquetMetadata
public ParquetMetadata fromParquetMetadata(org.apache.parquet.format.FileMetaData parquetMetadata, InternalFileDecryptor fileDecryptor, boolean encryptedFooter) throws IOException
- Throws:
IOException
-
fromParquetMetadata
public ParquetMetadata fromParquetMetadata(org.apache.parquet.format.FileMetaData parquetMetadata, InternalFileDecryptor fileDecryptor, boolean encryptedFooter, Map<org.apache.parquet.format.RowGroup,Long> rowGroupToRowIndexOffsetMap) throws IOException
- Throws:
IOException
-
writeDataPageHeader
@Deprecated public void writeDataPageHeader(int uncompressedSize, int compressedSize, int valueCount, org.apache.parquet.column.Encoding rlEncoding, org.apache.parquet.column.Encoding dlEncoding, org.apache.parquet.column.Encoding valuesEncoding, OutputStream to) throws IOException
Deprecated.- Throws:
IOException
-
writeDataPageHeader
@Deprecated public void writeDataPageHeader(int uncompressedSize, int compressedSize, int valueCount, org.apache.parquet.column.statistics.Statistics statistics, org.apache.parquet.column.Encoding rlEncoding, org.apache.parquet.column.Encoding dlEncoding, org.apache.parquet.column.Encoding valuesEncoding, OutputStream to) throws IOException
Deprecated.- Throws:
IOException
-
writeDataPageV2Header
@Deprecated public void writeDataPageV2Header(int uncompressedSize, int compressedSize, int valueCount, int nullCount, int rowCount, org.apache.parquet.column.statistics.Statistics statistics, org.apache.parquet.column.Encoding dataEncoding, int rlByteLength, int dlByteLength, OutputStream to) throws IOException
Deprecated.- Throws:
IOException
-
writeDataPageV1Header
public void writeDataPageV1Header(int uncompressedSize, int compressedSize, int valueCount, org.apache.parquet.column.Encoding rlEncoding, org.apache.parquet.column.Encoding dlEncoding, org.apache.parquet.column.Encoding valuesEncoding, OutputStream to) throws IOException
- Throws:
IOException
-
writeDataPageV1Header
public void writeDataPageV1Header(int uncompressedSize, int compressedSize, int valueCount, org.apache.parquet.column.Encoding rlEncoding, org.apache.parquet.column.Encoding dlEncoding, org.apache.parquet.column.Encoding valuesEncoding, OutputStream to, org.apache.parquet.format.BlockCipher.Encryptor blockEncryptor, byte[] pageHeaderAAD) throws IOException
- Throws:
IOException
-
writeDataPageV1Header
public void writeDataPageV1Header(int uncompressedSize, int compressedSize, int valueCount, org.apache.parquet.column.Encoding rlEncoding, org.apache.parquet.column.Encoding dlEncoding, org.apache.parquet.column.Encoding valuesEncoding, int crc, OutputStream to) throws IOException
- Throws:
IOException
-
writeDataPageV1Header
public void writeDataPageV1Header(int uncompressedSize, int compressedSize, int valueCount, org.apache.parquet.column.Encoding rlEncoding, org.apache.parquet.column.Encoding dlEncoding, org.apache.parquet.column.Encoding valuesEncoding, int crc, OutputStream to, org.apache.parquet.format.BlockCipher.Encryptor blockEncryptor, byte[] pageHeaderAAD) throws IOException
- Throws:
IOException
-
writeDataPageV2Header
public void writeDataPageV2Header(int uncompressedSize, int compressedSize, int valueCount, int nullCount, int rowCount, org.apache.parquet.column.Encoding dataEncoding, int rlByteLength, int dlByteLength, OutputStream to) throws IOException
- Throws:
IOException
-
writeDataPageV2Header
public void writeDataPageV2Header(int uncompressedSize, int compressedSize, int valueCount, int nullCount, int rowCount, org.apache.parquet.column.Encoding dataEncoding, int rlByteLength, int dlByteLength, OutputStream to, org.apache.parquet.format.BlockCipher.Encryptor blockEncryptor, byte[] pageHeaderAAD) throws IOException
- Throws:
IOException
-
writeDictionaryPageHeader
public void writeDictionaryPageHeader(int uncompressedSize, int compressedSize, int valueCount, org.apache.parquet.column.Encoding valuesEncoding, OutputStream to) throws IOException
- Throws:
IOException
-
writeDictionaryPageHeader
public void writeDictionaryPageHeader(int uncompressedSize, int compressedSize, int valueCount, org.apache.parquet.column.Encoding valuesEncoding, OutputStream to, org.apache.parquet.format.BlockCipher.Encryptor blockEncryptor, byte[] pageHeaderAAD) throws IOException
- Throws:
IOException
-
writeDictionaryPageHeader
public void writeDictionaryPageHeader(int uncompressedSize, int compressedSize, int valueCount, org.apache.parquet.column.Encoding valuesEncoding, int crc, OutputStream to) throws IOException
- Throws:
IOException
-
writeDictionaryPageHeader
public void writeDictionaryPageHeader(int uncompressedSize, int compressedSize, int valueCount, org.apache.parquet.column.Encoding valuesEncoding, int crc, OutputStream to, org.apache.parquet.format.BlockCipher.Encryptor blockEncryptor, byte[] pageHeaderAAD) throws IOException
- Throws:
IOException
-
toParquetColumnIndex
public static org.apache.parquet.format.ColumnIndex toParquetColumnIndex(org.apache.parquet.schema.PrimitiveType type, org.apache.parquet.internal.column.columnindex.ColumnIndex columnIndex)
-
fromParquetColumnIndex
public static org.apache.parquet.internal.column.columnindex.ColumnIndex fromParquetColumnIndex(org.apache.parquet.schema.PrimitiveType type, org.apache.parquet.format.ColumnIndex parquetColumnIndex)
-
toParquetOffsetIndex
public static org.apache.parquet.format.OffsetIndex toParquetOffsetIndex(org.apache.parquet.internal.column.columnindex.OffsetIndex offsetIndex)
-
fromParquetOffsetIndex
public static org.apache.parquet.internal.column.columnindex.OffsetIndex fromParquetOffsetIndex(org.apache.parquet.format.OffsetIndex parquetOffsetIndex)
-
toBloomFilterHeader
public static org.apache.parquet.format.BloomFilterHeader toBloomFilterHeader(org.apache.parquet.column.values.bloomfilter.BloomFilter bloomFilter)
-
-