public abstract class AbstractCassandraStorage
extends org.apache.pig.LoadFunc
implements org.apache.pig.StoreFuncInterface, org.apache.pig.LoadMetadata
| Modifier and Type | Class and Description |
|---|---|
protected class |
AbstractCassandraStorage.CfInfo |
protected static class |
AbstractCassandraStorage.MarshallerType |
| Modifier and Type | Field and Description |
|---|---|
protected java.lang.String |
column_family |
protected org.apache.hadoop.conf.Configuration |
conf |
protected java.lang.String |
DEFAULT_INPUT_FORMAT |
protected java.lang.String |
DEFAULT_OUTPUT_FORMAT |
protected java.lang.String |
initHostAddress |
protected java.lang.String |
inputFormatClass |
protected java.lang.String |
keyspace |
protected java.lang.String |
loadSignature |
protected static org.slf4j.Logger |
logger |
protected java.lang.String |
outputFormatClass |
static java.lang.String |
PARTITION_FILTER_SIGNATURE |
protected java.lang.String |
partitionerClass |
protected java.lang.String |
password |
static java.lang.String |
PIG_INITIAL_ADDRESS |
static java.lang.String |
PIG_INPUT_FORMAT |
static java.lang.String |
PIG_INPUT_INITIAL_ADDRESS |
static java.lang.String |
PIG_INPUT_PARTITIONER |
static java.lang.String |
PIG_INPUT_RPC_PORT |
static java.lang.String |
PIG_INPUT_SPLIT_SIZE |
static java.lang.String |
PIG_OUTPUT_FORMAT |
static java.lang.String |
PIG_OUTPUT_INITIAL_ADDRESS |
static java.lang.String |
PIG_OUTPUT_PARTITIONER |
static java.lang.String |
PIG_OUTPUT_RPC_PORT |
static java.lang.String |
PIG_PARTITIONER |
static java.lang.String |
PIG_RPC_PORT |
protected java.lang.String |
rpcPort |
protected int |
splitSize |
protected java.lang.String |
storeSignature |
protected boolean |
usePartitionFilter |
protected java.lang.String |
username |
| Constructor and Description |
|---|
AbstractCassandraStorage() |
| Modifier and Type | Method and Description |
|---|---|
protected java.lang.Object |
cassandraToObj(AbstractType validator,
java.nio.ByteBuffer value) |
protected static CfDef |
cfdefFromString(java.lang.String st)
convert string back to CfDef
|
protected static java.lang.String |
cfdefToString(CfDef cfDef)
convert CfDef to string
|
void |
checkSchema(org.apache.pig.ResourceSchema schema) |
void |
cleanupOnFailure(java.lang.String failure,
org.apache.hadoop.mapreduce.Job job) |
protected org.apache.pig.data.Tuple |
columnToTuple(Column col,
AbstractCassandraStorage.CfInfo cfInfo,
AbstractType comparator)
convert a column to a tuple
|
protected org.apache.pig.data.Tuple |
composeComposite(AbstractCompositeType comparator,
java.nio.ByteBuffer name)
Deconstructs a composite type to a Tuple.
|
protected CFDefinition |
getCfDefinition(java.lang.String ks,
java.lang.String cf,
Cassandra.Client client)
get CFDefinition of a column family
|
protected AbstractCassandraStorage.CfInfo |
getCfInfo(Cassandra.Client client)
return the CfInfo for the column family
|
protected AbstractCassandraStorage.CfInfo |
getCfInfo(java.lang.String signature)
get the columnfamily definition for the signature
|
protected java.util.List<ColumnDef> |
getColumnMeta(Cassandra.Client client,
boolean cassandraStorage,
boolean includeCompactValueColumn)
get column meta data
|
protected abstract java.util.List<ColumnDef> |
getColumnMetadata(Cassandra.Client client)
get a list of columns
|
protected java.util.Map<AbstractCassandraStorage.MarshallerType,AbstractType> |
getDefaultMarshallers(CfDef cfDef)
construct a map to store the mashaller type to cassandra data type mapping
|
protected java.lang.String |
getFullyQualifiedClassName(java.lang.String classname)
get the full class name
|
protected java.util.List<ColumnDef> |
getIndexes()
get a list of columns with defined index
|
protected IndexType |
getIndexType(java.lang.String type)
get index type from string
|
org.apache.hadoop.mapreduce.InputFormat |
getInputFormat() |
org.apache.hadoop.mapreduce.OutputFormat |
getOutputFormat()
output format
|
java.lang.String[] |
getPartitionKeys(java.lang.String location,
org.apache.hadoop.mapreduce.Job job)
return partition keys
|
protected byte |
getPigType(AbstractType type)
get pig type for the cassandra data type
|
static java.util.Map<java.lang.String,java.lang.String> |
getQueryMap(java.lang.String query)
decompose the query to store the parameters in a map
|
org.apache.pig.ResourceStatistics |
getStatistics(java.lang.String location,
org.apache.hadoop.mapreduce.Job job) |
protected java.util.Map<java.nio.ByteBuffer,AbstractType> |
getValidatorMap(CfDef cfDef)
get the validators
|
protected void |
initSchema(java.lang.String signature)
Methods to get the column family schema from Cassandra
|
protected abstract java.nio.ByteBuffer |
nullToBB() |
protected java.nio.ByteBuffer |
objToBB(java.lang.Object o)
convert object to ByteBuffer
|
protected AbstractType |
parseType(java.lang.String type)
parse the string to a cassandra data type
|
java.lang.String |
relativeToAbsolutePath(java.lang.String location,
org.apache.hadoop.fs.Path curDir) |
java.lang.String |
relToAbsPathForStoreLocation(java.lang.String location,
org.apache.hadoop.fs.Path curDir) |
protected void |
setConnectionInformation()
set hadoop cassandra connection settings
|
void |
setStoreFuncUDFContextSignature(java.lang.String signature)
StoreFunc methods
|
protected void |
setTupleValue(org.apache.pig.data.Tuple pair,
int position,
java.lang.Object value)
set the value to the position of the tuple
|
void |
setUDFContextSignature(java.lang.String signature) |
getAbsolutePath, getLoadCaster, getNext, getPathStrings, join, prepareToRead, setLocation, warnclone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, waitpublic static final java.lang.String PIG_INPUT_RPC_PORT
public static final java.lang.String PIG_INPUT_INITIAL_ADDRESS
public static final java.lang.String PIG_INPUT_PARTITIONER
public static final java.lang.String PIG_OUTPUT_RPC_PORT
public static final java.lang.String PIG_OUTPUT_INITIAL_ADDRESS
public static final java.lang.String PIG_OUTPUT_PARTITIONER
public static final java.lang.String PIG_RPC_PORT
public static final java.lang.String PIG_INITIAL_ADDRESS
public static final java.lang.String PIG_PARTITIONER
public static final java.lang.String PIG_INPUT_FORMAT
public static final java.lang.String PIG_OUTPUT_FORMAT
public static final java.lang.String PIG_INPUT_SPLIT_SIZE
protected java.lang.String DEFAULT_INPUT_FORMAT
protected java.lang.String DEFAULT_OUTPUT_FORMAT
public static final java.lang.String PARTITION_FILTER_SIGNATURE
protected static final org.slf4j.Logger logger
protected java.lang.String username
protected java.lang.String password
protected java.lang.String keyspace
protected java.lang.String column_family
protected java.lang.String loadSignature
protected java.lang.String storeSignature
protected org.apache.hadoop.conf.Configuration conf
protected java.lang.String inputFormatClass
protected java.lang.String outputFormatClass
protected int splitSize
protected java.lang.String partitionerClass
protected boolean usePartitionFilter
protected java.lang.String initHostAddress
protected java.lang.String rpcPort
protected org.apache.pig.data.Tuple composeComposite(AbstractCompositeType comparator, java.nio.ByteBuffer name) throws java.io.IOException
java.io.IOExceptionprotected org.apache.pig.data.Tuple columnToTuple(Column col, AbstractCassandraStorage.CfInfo cfInfo, AbstractType comparator) throws java.io.IOException
java.io.IOExceptionprotected void setTupleValue(org.apache.pig.data.Tuple pair,
int position,
java.lang.Object value)
throws org.apache.pig.backend.executionengine.ExecException
org.apache.pig.backend.executionengine.ExecExceptionprotected AbstractCassandraStorage.CfInfo getCfInfo(java.lang.String signature) throws java.io.IOException
java.io.IOExceptionprotected java.util.Map<AbstractCassandraStorage.MarshallerType,AbstractType> getDefaultMarshallers(CfDef cfDef) throws java.io.IOException
java.io.IOExceptionprotected java.util.Map<java.nio.ByteBuffer,AbstractType> getValidatorMap(CfDef cfDef) throws java.io.IOException
java.io.IOExceptionprotected AbstractType parseType(java.lang.String type) throws java.io.IOException
java.io.IOExceptionpublic org.apache.hadoop.mapreduce.InputFormat getInputFormat()
throws java.io.IOException
getInputFormat in class org.apache.pig.LoadFuncjava.io.IOExceptionpublic static java.util.Map<java.lang.String,java.lang.String> getQueryMap(java.lang.String query)
throws java.io.UnsupportedEncodingException
java.io.UnsupportedEncodingExceptionprotected void setConnectionInformation()
throws java.io.IOException
java.io.IOExceptionprotected java.lang.String getFullyQualifiedClassName(java.lang.String classname)
protected byte getPigType(AbstractType type)
public org.apache.pig.ResourceStatistics getStatistics(java.lang.String location,
org.apache.hadoop.mapreduce.Job job)
getStatistics in interface org.apache.pig.LoadMetadatapublic java.lang.String relativeToAbsolutePath(java.lang.String location,
org.apache.hadoop.fs.Path curDir)
throws java.io.IOException
relativeToAbsolutePath in class org.apache.pig.LoadFuncjava.io.IOExceptionpublic void setUDFContextSignature(java.lang.String signature)
setUDFContextSignature in class org.apache.pig.LoadFuncpublic void setStoreFuncUDFContextSignature(java.lang.String signature)
setStoreFuncUDFContextSignature in interface org.apache.pig.StoreFuncInterfacepublic java.lang.String relToAbsPathForStoreLocation(java.lang.String location,
org.apache.hadoop.fs.Path curDir)
throws java.io.IOException
relToAbsPathForStoreLocation in interface org.apache.pig.StoreFuncInterfacejava.io.IOExceptionpublic org.apache.hadoop.mapreduce.OutputFormat getOutputFormat()
throws java.io.IOException
getOutputFormat in interface org.apache.pig.StoreFuncInterfacejava.io.IOExceptionpublic void checkSchema(org.apache.pig.ResourceSchema schema)
throws java.io.IOException
checkSchema in interface org.apache.pig.StoreFuncInterfacejava.io.IOExceptionprotected abstract java.nio.ByteBuffer nullToBB()
protected java.nio.ByteBuffer objToBB(java.lang.Object o)
public void cleanupOnFailure(java.lang.String failure,
org.apache.hadoop.mapreduce.Job job)
cleanupOnFailure in interface org.apache.pig.StoreFuncInterfaceprotected void initSchema(java.lang.String signature)
throws java.io.IOException
java.io.IOExceptionprotected static java.lang.String cfdefToString(CfDef cfDef) throws java.io.IOException
java.io.IOExceptionprotected static CfDef cfdefFromString(java.lang.String st) throws java.io.IOException
java.io.IOExceptionprotected AbstractCassandraStorage.CfInfo getCfInfo(Cassandra.Client client) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, org.apache.thrift.TException, NotFoundException, InvalidRequestException, ConfigurationException, java.io.IOException
InvalidRequestExceptionUnavailableExceptionTimedOutExceptionSchemaDisagreementExceptionorg.apache.thrift.TExceptionNotFoundExceptionConfigurationExceptionjava.io.IOExceptionprotected abstract java.util.List<ColumnDef> getColumnMetadata(Cassandra.Client client) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, org.apache.thrift.TException, java.nio.charset.CharacterCodingException, InvalidRequestException, ConfigurationException, NotFoundException
InvalidRequestExceptionUnavailableExceptionTimedOutExceptionSchemaDisagreementExceptionorg.apache.thrift.TExceptionjava.nio.charset.CharacterCodingExceptionConfigurationExceptionNotFoundExceptionprotected java.util.List<ColumnDef> getColumnMeta(Cassandra.Client client, boolean cassandraStorage, boolean includeCompactValueColumn) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, org.apache.thrift.TException, java.nio.charset.CharacterCodingException, InvalidRequestException, ConfigurationException, NotFoundException
InvalidRequestExceptionUnavailableExceptionTimedOutExceptionSchemaDisagreementExceptionorg.apache.thrift.TExceptionjava.nio.charset.CharacterCodingExceptionConfigurationExceptionNotFoundExceptionprotected IndexType getIndexType(java.lang.String type)
public java.lang.String[] getPartitionKeys(java.lang.String location,
org.apache.hadoop.mapreduce.Job job)
throws java.io.IOException
getPartitionKeys in interface org.apache.pig.LoadMetadatajava.io.IOExceptionprotected java.util.List<ColumnDef> getIndexes() throws java.io.IOException
java.io.IOExceptionprotected CFDefinition getCfDefinition(java.lang.String ks, java.lang.String cf, Cassandra.Client client) throws NotFoundException, InvalidRequestException, org.apache.thrift.TException, InvalidRequestException, ConfigurationException
NotFoundExceptionInvalidRequestExceptionorg.apache.thrift.TExceptionConfigurationExceptionprotected java.lang.Object cassandraToObj(AbstractType validator, java.nio.ByteBuffer value)
Copyright © 2015 The Apache Software Foundation