ColumnHelper

Instance Constructors

new ColumnHelper(column: Column)

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def containsAll[T](candidates: T*)(implicit tt: ClassTag[T]): Column

A boolean, which is true if ALL of the given parameters are contained in the Array column
A boolean, which is true if ALL of the given parameters are contained in the Array column
```
df.smvSelectPlus($"arrayCol".containsAll(seqVals: _*) as "isFound")
```
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getName: String

Get the name of the column.
Get the name of the column.
returns
column alias for aliased columns, or expression string representation for unaliased columns
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def smvBinMode(): Column

Compute the mode given a double bin histogram
Compute the mode given a double bin histogram
```
df_with_double_histogram_bin.select('bin_histogram.smvBinMode())
```
def smvBinPercentile(percentile: Double): Column

Compute a given percentile given a double bin histogram
Compute a given percentile given a double bin histogram
```
df_with_double_histogram_bin.select('bin_histogram.smvBinPercentile(50.0))
```
def smvCoarseGrain(bin: Double): Column

Map double to the lower bound of bins with bin-size specified
Map double to the lower bound of bins with bin-size specified
```
$"amt".smvCoarseGrain(100)  // 122.34 => 100.0, 2230.21 => 2200.0
```
TODO: need to rename this function as this is just a special case of rounding!!!
def smvDay70: Column

Convert a timestamp to the number of months from 1970-01-01.
Convert a timestamp to the number of months from 1970-01-01.
```
lit("2012-02-29").smvStrToTimestamp("yyyy-MM-dd").smvDay70 // 15399
```
returns
number of days from 1970-01-01 (start from 0)
def smvDayOfMonth: Column

Extract day of month component from a timestamp.
Extract day of month component from a timestamp.
```
lit("2014-04-25").smvStrToTimestamp("yyyy-MM-dd").smvDayOfMonth // 25
```
returns
The day of month component as an integer (range 1-31) or null if input column is null
def smvDayOfWeek: Column

Extract day of the week component from a timestamp.
Extract day of the week component from a timestamp.
```
lit("2015-09-16").smvStrToTimestamp("yyyy-MM-dd").smvDayOfWeek // 3 (Wed)
```
returns
The day of the week component as an integer (range 1-7, 1 being Monday) or null if input column is null
def smvHour: Column

Extract hour component from a timestamp.
Extract hour component from a timestamp.
```
lit("2014-04-25 13:45").smvStrToTimestamp("yyyy-MM-dd HH:mm").smvHour // 13
```
returns
The hour component as an integer or null if input column is null
def smvIsAllIn[T](candidates: T*)(implicit tt: ClassTag[T]): Column

A boolean, which is true if ALL of the Array column's elements are in the given paraneter sequence
A boolean, which is true if ALL of the Array column's elements are in the given paraneter sequence
```
df.smvSelectPlus($"arrayCol".smvIsAllIn(seqVals: _*) as "isFound")
```
def smvIsAnyIn[T](candidates: T*)(implicit tt: ClassTag[T]): Column

A boolean, which is ture if ANY one of the Array column element is in the given parameter sequence
A boolean, which is ture if ANY one of the Array column element is in the given parameter sequence
```
df.smvSelectPlus($"arrayCol".smvIsAnyIn(seqVals: _*) as "isFound")
```
def smvMetaphone: Column

Map a string to it's Metaphone
def smvMonth: Column

Extract month component from a timestamp.
Extract month component from a timestamp.
```
lit("2014-04-25").smvStrToTimestamp("yyyy-MM-dd").smvMonth //4
```
returns
The month component as an integer or null if input column is null
def smvMonth70: Column

Convert a timestamp to the number of months from 1970-01.
Convert a timestamp to the number of months from 1970-01.
```
lit("2012-02-29").smvStrToTimestamp("yyyy-MM-dd").smvMonth70 // 505
```
returns
number of months from 1970-01 (start from 0)
def smvPlusDays(days: Column): Column

Add N days to Timestamp column.
Add N days to Timestamp column.
```
lit("2014-04-25").smvStrToTimestamp("yyyy-MM-dd").smvPlusDays($"ColumnName")
```
returns
The incremented Timestamp or null if input was null
def smvPlusDays(n: Int): Column

Add N days to Timestamp column.
Add N days to Timestamp column.
```
lit("2014-04-25").smvStrToTimestamp("yyyy-MM-dd").smvPlusDays(3)
```
returns
The incremented Timestamp or null if input was null
def smvPlusMonths(months: Column): Column

Add N months to Timestamp column.
Add N months to Timestamp column.
The calculation will do its best to only change the month field retaining the same day of month. However, in certain circumstances, it may be necessary to alter smaller fields. For example, 2007-03-31 plus one month cannot result in 2007-04-31, so the day of month is adjusted to 2007-04-30.
```
lit("2014-04-25").smvStrToTimestamp("yyyy-MM-dd").smvPlusMonths($"ColumnName")
```
returns
The incremented Timestamp or null if input was null
def smvPlusMonths(n: Int): Column

Add N months to Timestamp column.
Add N months to Timestamp column.
The calculation will do its best to only change the month field retaining the same day of month. However, in certain circumstances, it may be necessary to alter smaller fields. For example, 2007-03-31 plus one month cannot result in 2007-04-31, so the day of month is adjusted to 2007-04-30.
```
lit("2014-04-25").smvStrToTimestamp("yyyy-MM-dd").smvPlusMonths(1)
```
returns
The incremented Timestamp or null if input was null
def smvPlusWeeks(weeks: Column): Column

Add N weeks to Timestamp column.
Add N weeks to Timestamp column.
```
lit("2014-04-25").smvStrToTimestamp("yyyy-MM-dd").smvPlusWeeks($"ColumnName")
```
returns
The incremented Timestamp or null if input was null
def smvPlusWeeks(n: Int): Column

Add N weeks to Timestamp column.
Add N weeks to Timestamp column.
```
lit("2014-04-25").smvStrToTimestamp("yyyy-MM-dd").smvPlusWeeks(2)
```
returns
The incremented Timestamp or null if input was null
def smvPlusYears(years: Column): Column

Add N years to Timestamp column.
Add N years to Timestamp column.
```
lit("2014-04-25").smvStrToTimestamp("yyyy-MM-dd").smvPlusYears($"ColumnName")
```
returns
The incremented Timestamp or null if input was null
def smvPlusYears(n: Int): Column

Add N years to Timestamp column.
Add N years to Timestamp column.
```
lit("2014-04-25").smvStrToTimestamp("yyyy-MM-dd").smvPlusYears(2)
```
returns
The incremented Timestamp or null if input was null
def smvQuarter: Column

Extract quarter component from a timestamp.
Extract quarter component from a timestamp.
```
lit("2014-04-25").smvStrToTimestamp("yyyy-MM-dd").smvQuarter // 2
```
returns
The quarter component as an integer (1 based) or null if input column is null
def smvSafeDiv(other: Column, defaultv: Double): Column

Safely divide one column value by a Double constant.
def smvSafeDiv(denom: Column, defaultValue: Column): Column

Safely divide one column value by another.
Safely divide one column value by another. Note: only applies to columns of type Double
```
lit(1.0).smvSafeDiv(lit(0.0), 1000.0) => 1000.0
lit(1.0).smvSafeDiv(lit(null), 1000.0) => null
null.smvSafeDiv(?,?) => null
```
denom
The denomenator to divide by.
defaultValue
Default value to use if denom is 0.0

Annotations
@Experimental()
def smvStrToTimestamp(fmt: String): Column

Build a timestamp from a string.
Build a timestamp from a string. The format is the same as the Java Date format.
```
lit("2014-04-25").smvStrToTimestamp("yyyy-MM-dd")
```
returns
The timestamp or null if input string is null
def smvTimeToIndex: Column

smvTime helper to convert smvTime column to time index integer Example smvTime values (as String): "Q201301", "M201512", "D20141201" Example output 172, 551, 16405 (# of quarters, months, and days from 19700101)
def smvTimeToLabel: Column

smvTime helper to convert smvTime column to time index integer Example smvTime values (as String): "Q201301", "M201512", "D20141201" Example output 172, 551, 16405 (# of quarters, months, and days from 19700101)
def smvTimeToTimestamp: Column

smvTime helper to convert smvTime column to a timestamp at the beginning of the given time pireod.
smvTime helper to convert smvTime column to a timestamp at the beginning of the given time pireod. Example smvTime values (as String): "Q201301", "M201512", "D20141201" Example output "2013-01-01 00:00:00.0", "2015-12-01 00:00:00.0", "2014-12-01 00:00:00.0"
def smvTimeToType: Column

smvTime helper to convert smvTime column to time type string Example smvTime values (as String): "Q201301", "M201512", "D20141201" Example output type "quarter", "month", "day"
def smvYear: Column

Extract year component from a timestamp.
Extract year component from a timestamp.
```
lit("2014-04-25").smvStrToTimestamp("yyyy-MM-dd").smvYear // 2014
```
returns
The year component as an integer or null if input column is null
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toExpr: Expression

Convert Column to catalyst Expression.
Convert Column to catalyst Expression. This is needed here as the internal Expression in Column is marked spark private but we sometimes need access to the expression.
```
($"v" * 5).toExpr
```
Annotations
@DeveloperApi()
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package smv

class ColumnHelper extends AnyRef

Instance Constructors

new ColumnHelper(column: Column)

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def clone(): AnyRef

def containsAll[T](candidates: T*)(implicit tt: ClassTag[T]): Column

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def getName: String

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def smvBinMode(): Column

def smvBinPercentile(percentile: Double): Column

def smvCoarseGrain(bin: Double): Column

def smvDay70: Column

def smvDayOfMonth: Column

def smvDayOfWeek: Column

def smvHour: Column

def smvIsAllIn[T](candidates: T*)(implicit tt: ClassTag[T]): Column

def smvIsAnyIn[T](candidates: T*)(implicit tt: ClassTag[T]): Column

def smvMetaphone: Column

def smvMonth: Column

def smvMonth70: Column

def smvPlusDays(days: Column): Column

def smvPlusDays(n: Int): Column

def smvPlusMonths(months: Column): Column

def smvPlusMonths(n: Int): Column

def smvPlusWeeks(weeks: Column): Column

def smvPlusWeeks(n: Int): Column

def smvPlusYears(years: Column): Column

def smvPlusYears(n: Int): Column

def smvQuarter: Column

def smvSafeDiv(other: Column, defaultv: Double): Column

def smvSafeDiv(denom: Column, defaultValue: Column): Column

def smvStrToTimestamp(fmt: String): Column

def smvTimeToIndex: Column

def smvTimeToLabel: Column

def smvTimeToTimestamp: Column

def smvTimeToType: Column

def smvYear: Column

final def synchronized[T0](arg0: ⇒ T0): T0

def toExpr: Expression

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from AnyRef

Inherited from Any

Ungrouped