LoghubUtils

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

4. final def asInstanceOf[T0]: T0

   

   Definition Classes

   Any

5. def clone(): AnyRef

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( ... )

6. def createDirectStream(jssc: JavaStreamingContext, project: String, logStore: String, mConsumerGroup: String, accessKeyId: String, accessKeySecret: String, endpoint: String, zkParams: HashMap[String, String], mode: LogHubCursorPosition, cursorStartTime: Long): JavaInputDStream[String]

   

   Set cursorStartTime a valid value when using LogHubCursorPosition.SPECIAL_TIMER_CURSOR mode at the first time with current mConsumerGroup

   Set cursorStartTime a valid value when using LogHubCursorPosition.SPECIAL_TIMER_CURSOR mode at the first time with current mConsumerGroup

   Create direct loghub DStream.

   jssc

   StreamingContext.

   project

   The name of LogService project.

   logStore

   The name of logStore.

   mConsumerGroup

   The group name of loghub consumer. All consumer process which has the same group name will consumer specific logStore together.

   accessKeyId

   The Aliyun Access Key Id.

   accessKeySecret

   The Aliyun Access Key Secret.

   endpoint

   The endpoint of loghub.

   zkParams

   Zookeeper parameters.

   mode

   Set user defined cursor type.

   cursorStartTime

   Set user defined cursor position (Unix Timestamp).

   Annotations

   @Experimental()

7. def createDirectStream(jssc: JavaStreamingContext, project: String, logStore: String, mConsumerGroup: String, accessKeyId: String, accessKeySecret: String, endpoint: String, zkParams: HashMap[String, String], mode: LogHubCursorPosition): JavaInputDStream[String]

   

   Create direct loghub DStream.

   Create direct loghub DStream.

   jssc

   JavaStreamingContext.

   project

   The name of LogService project.

   logStore

   The name of logStore.

   mConsumerGroup

   The group name of loghub consumer. All consumer process which has the same group name will consumer specific logStore together.

   accessKeyId

   The Aliyun Access Key Id.

   accessKeySecret

   The Aliyun Access Key Secret.

   endpoint

   The endpoint of loghub.

   zkParams

   Zookeeper parameters.

   mode

   Set user defined cursor type.

   Annotations

   @Experimental()

8. def createDirectStream(ssc: StreamingContext, project: String, logStore: String, mConsumerGroup: String, accessKeyId: String, accessKeySecret: String, endpoint: String, zkParams: Map[String, String], mode: LogHubCursorPosition, cursorStartTime: Long): DStream[String]

   

   Set cursorStartTime a valid value when using LogHubCursorPosition.SPECIAL_TIMER_CURSOR mode at the first time with current mConsumerGroup

   Set cursorStartTime a valid value when using LogHubCursorPosition.SPECIAL_TIMER_CURSOR mode at the first time with current mConsumerGroup

   Create direct loghub DStream.

   ssc

   StreamingContext.

   project

   The name of LogService project.

   logStore

   The name of logStore.

   mConsumerGroup

   The group name of loghub consumer. All consumer process which has the same group name will consumer specific logStore together.

   accessKeyId

   The Aliyun Access Key Id.

   accessKeySecret

   The Aliyun Access Key Secret.

   endpoint

   The endpoint of loghub.

   zkParams

   Zookeeper parameters.

   mode

   Set user defined cursor type.

   cursorStartTime

   Set user defined cursor position (Unix Timestamp).

   Annotations

   @Experimental()

9. def createDirectStream(ssc: StreamingContext, project: String, logStore: String, mConsumerGroup: String, accessKeyId: String, accessKeySecret: String, endpoint: String, zkParams: Map[String, String], mode: LogHubCursorPosition): DStream[String]

   

   Create direct loghub DStream.

   Create direct loghub DStream.

   ssc

   StreamingContext.

   project

   The name of LogService project.

   logStore

   The name of logStore.

   mConsumerGroup

   The group name of loghub consumer. All consumer process which has the same group name will consumer specific logStore together.

   accessKeyId

   The Aliyun Access Key Id.

   accessKeySecret

   The Aliyun Access Key Secret.

   endpoint

   The endpoint of loghub.

   zkParams

   Zookeeper parameters.

   mode

   Set user defined cursor type.

   Annotations

   @Experimental()

10. def createRDD(sc: SparkContext, project: String, logStore: String, accessId: String, accessKey: String, endpoint: String, startTime: Long, endTime: Long = 1, parallelismInShard: Int = 1): RDD[String]

    

    for batch rdd

11. def createStream(jssc: JavaStreamingContext, logServiceProject: String, logStoreName: String, loghubConsumerGroupName: String, numReceivers: Int, storageLevel: StorageLevel, cursorPosition: LogHubCursorPosition, mLoghubCursorStartTime: Int, forceSpecial: Boolean): JavaDStream[Array[Byte]]

    

    Create loghub DStream.

    Create loghub DStream.

    jssc

    JavaStreamingContext.

    logServiceProject

    The name of LogService project.

    logStoreName

    The name of logStore.

    loghubConsumerGroupName

    The group name of loghub consumer. All consumer process which has the same group name will consumer specific logStore together.

    numReceivers

    The number of receivers.

    storageLevel

    Storage level to use for storing the received objects.

    cursorPosition

    Set user defined cursor type.

    mLoghubCursorStartTime

    Set user defined cursor position (Unix Timestamp).

    forceSpecial

    Whether to force to set consume position as the mLoghubCursorStartTime.

12. def createStream(jssc: JavaStreamingContext, logServiceProject: String, logStoreName: String, loghubConsumerGroupName: String, loghubEndpoint: String, numReceivers: Int, accessKeyId: String, accessKeySecret: String, storageLevel: StorageLevel, cursorPosition: LogHubCursorPosition, mLoghubCursorStartTime: Int, forceSpecial: Boolean): JavaDStream[Array[Byte]]

    

    Create loghub DStream.

    Create loghub DStream.

    jssc

    JavaStreamingContext.

    logServiceProject

    The name of LogService project.

    logStoreName

    The name of logStore.

    loghubConsumerGroupName

    The group name of loghub consumer. All consumer process which has the same group name will consumer specific logStore together.

    loghubEndpoint

    The endpoint of loghub.

    numReceivers

    The number of receivers.

    accessKeyId

    The Aliyun Access Key Id.

    accessKeySecret

    The Aliyun Access Key Secret.

    storageLevel

    Storage level to use for storing the received objects.

    cursorPosition

    Set user defined cursor type.

    mLoghubCursorStartTime

    Set user defined cursor position (Unix Timestamp).

    forceSpecial

    Whether to force to set consume position as the mLoghubCursorStartTime.

13. def createStream(jssc: JavaStreamingContext, logServiceProject: String, logStoreName: String, loghubConsumerGroupName: String, storageLevel: StorageLevel, cursorPosition: LogHubCursorPosition, mLoghubCursorStartTime: Int, forceSpecial: Boolean): JavaDStream[Array[Byte]]

    

    Create loghub DStream.

    Create loghub DStream.

    jssc

    JavaStreamingContext.

    logServiceProject

    The name of LogService project.

    logStoreName

    The name of logStore.

    loghubConsumerGroupName

    The group name of loghub consumer. All consumer process which has the same group name will consumer specific logStore together.

    storageLevel

    Storage level to use for storing the received objects.

    cursorPosition

    Set user defined cursor type.

    mLoghubCursorStartTime

    Set user defined cursor position (Unix Timestamp).

    forceSpecial

    Whether to force to set consume position as the mLoghubCursorStartTime.

14. def createStream(jssc: JavaStreamingContext, logServiceProject: String, logStoreName: String, loghubConsumerGroupName: String, loghubEndpoint: String, accessKeyId: String, accessKeySecret: String, storageLevel: StorageLevel, cursorPosition: LogHubCursorPosition, mLoghubCursorStartTime: Int, forceSpecial: Boolean): JavaDStream[Array[Byte]]

    

    Create loghub DStream.

    Create loghub DStream.

    jssc

    JavaStreamingContext.

    logServiceProject

    The name of LogService project.

    logStoreName

    The name of logStore.

    loghubConsumerGroupName

    The group name of loghub consumer. All consumer process which has the same group name will consumer specific logStore together.

    loghubEndpoint

    The endpoint of loghub.

    accessKeyId

    The Aliyun Access Key Id.

    accessKeySecret

    The Aliyun Access Key Secret.

    storageLevel

    Storage level to use for storing the received objects.

    cursorPosition

    Set user defined cursor type.

    mLoghubCursorStartTime

    Set user defined cursor position (Unix Timestamp).

    forceSpecial

    Whether to force to set consume position as the mLoghubCursorStartTime.

15. def createStream(jssc: JavaStreamingContext, logServiceProject: String, logStoreName: String, loghubConsumerGroupName: String, numReceivers: Int, storageLevel: StorageLevel): JavaDStream[Array[Byte]]

    

    Create loghub DStream.

    Create loghub DStream.

    jssc

    JavaStreamingContext.

    logServiceProject

    The name of LogService project.

    logStoreName

    The name of logStore.

    loghubConsumerGroupName

    The group name of loghub consumer. All consumer process which has the same group name will consumer specific logStore together.

    numReceivers

    The number of receivers.

    storageLevel

    Storage level to use for storing the received objects.

16. def createStream(jssc: JavaStreamingContext, logServiceProject: String, logStoreName: String, loghubConsumerGroupName: String, loghubEndpoint: String, numReceivers: Int, accessKeyId: String, accessKeySecret: String, storageLevel: StorageLevel): JavaDStream[Array[Byte]]

    

    Create loghub DStream.

    Create loghub DStream.

    jssc

    JavaStreamingContext.

    logServiceProject

    The name of LogService project.

    logStoreName

    The name of logStore.

    loghubConsumerGroupName

    The group name of loghub consumer. All consumer process which has the same group name will consumer specific logStore together.

    loghubEndpoint

    The endpoint of loghub.

    numReceivers

    The number of receivers.

    accessKeyId

    The Aliyun Access Key Id.

    accessKeySecret

    The Aliyun Access Key Secret.

    storageLevel

    Storage level to use for storing the received objects.

17. def createStream(jssc: JavaStreamingContext, logServiceProject: String, logStoreName: String, loghubConsumerGroupName: String, storageLevel: StorageLevel): JavaReceiverInputDStream[Array[Byte]]

    

    Create loghub DStream.

    Create loghub DStream.

    jssc

    JavaStreamingContext.

    logServiceProject

    The name of LogService project.

    logStoreName

    The name of logStore.

    loghubConsumerGroupName

    The group name of loghub consumer. All consumer process which has the same group name will consumer specific logStore together.

    storageLevel

    Storage level to use for storing the received objects.

18. def createStream(jssc: JavaStreamingContext, logServiceProject: String, logStoreName: String, loghubConsumerGroupName: String, loghubEndpoint: String, accessKeyId: String, accessKeySecret: String, storageLevel: StorageLevel): JavaReceiverInputDStream[Array[Byte]]

    

    Create loghub DStream.

    Create loghub DStream.

    jssc

    JavaStreamingContext.

    logServiceProject

    The name of LogService project.

    logStoreName

    The name of logStore.

    loghubConsumerGroupName

    The group name of loghub consumer. All consumer process which has the same group name will consumer specific logStore together.

    loghubEndpoint

    The endpoint of loghub.

    accessKeyId

    The Aliyun Access Key Id.

    accessKeySecret

    The Aliyun Access Key Secret.

    storageLevel

    Storage level to use for storing the received objects.

19. def createStream(ssc: StreamingContext, logServiceProject: String, logStoreName: String, loghubConsumerGroupName: String, numReceivers: Int, storageLevel: StorageLevel, cursorPosition: LogHubCursorPosition, mLoghubCursorStartTime: Int, forceSpecial: Boolean): DStream[Array[Byte]]

    

    Create loghub DStream.

    Create loghub DStream.

    ssc

    StreamingContext.

    logServiceProject

    The name of LogService project.

    logStoreName

    The name of logStore.

    loghubConsumerGroupName

    The group name of loghub consumer. All consumer process which has the same group name will consumer specific logStore together.

    numReceivers

    The number of receivers.

    storageLevel

    Storage level to use for storing the received objects.

    cursorPosition

    Set user defined cursor type.

    mLoghubCursorStartTime

    Set user defined cursor position (Unix Timestamp).

    forceSpecial

    Whether to force to set consume position as the mLoghubCursorStartTime.

20. def createStream(ssc: StreamingContext, logServiceProject: String, logStoreName: String, loghubConsumerGroupName: String, loghubEndpoint: String, numReceivers: Int, accessKeyId: String, accessKeySecret: String, storageLevel: StorageLevel, cursorPosition: LogHubCursorPosition, mLoghubCursorStartTime: Int, forceSpecial: Boolean): DStream[Array[Byte]]

    

    Create loghub DStream.

    Create loghub DStream.

    ssc

    StreamingContext.

    logServiceProject

    The name of LogService project.

    logStoreName

    The name of logStore.

    loghubConsumerGroupName

    The group name of loghub consumer. All consumer process which has the same group name will consumer specific logStore together.

    loghubEndpoint

    The endpoint of loghub.

    numReceivers

    The number of receivers.

    accessKeyId

    The Aliyun Access Key Id.

    accessKeySecret

    The Aliyun Access Key Secret.

    storageLevel

    Storage level to use for storing the received objects.

    cursorPosition

    Set user defined cursor type.

    mLoghubCursorStartTime

    Set user defined cursor position (Unix Timestamp).

    forceSpecial

    Whether to force to set consume position as the mLoghubCursorStartTime.

21. def createStream(ssc: StreamingContext, logServiceProject: String, logStoreName: String, loghubConsumerGroupName: String, storageLevel: StorageLevel, cursorPosition: LogHubCursorPosition, mLoghubCursorStartTime: Int, forceSpecial: Boolean): ReceiverInputDStream[Array[Byte]]

    

    Create loghub DStream.

    Create loghub DStream.

    ssc

    StreamingContext.

    logServiceProject

    The name of LogService project.

    logStoreName

    The name of logStore.

    loghubConsumerGroupName

    The group name of loghub consumer. All consumer process which has the same group name will consumer specific logStore together.

    storageLevel

    Storage level to use for storing the received objects.

    cursorPosition

    Set user defined cursor type.

    mLoghubCursorStartTime

    Set user defined cursor position (Unix Timestamp).

    forceSpecial

    Whether to force to set consume position as the mLoghubCursorStartTime.

22. def createStream(ssc: StreamingContext, logServiceProject: String, logStoreName: String, loghubConsumerGroupName: String, loghubEndpoint: String, accessKeyId: String, accessKeySecret: String, storageLevel: StorageLevel, cursorPosition: LogHubCursorPosition, mLoghubCursorStartTime: Int, forceSpecial: Boolean): ReceiverInputDStream[Array[Byte]]

    

    Create loghub DStream.

    Create loghub DStream.

    ssc

    StreamingContext.

    logServiceProject

    The name of LogService project.

    logStoreName

    The name of logStore.

    loghubConsumerGroupName

    The group name of loghub consumer. All consumer process which has the same group name will consumer specific logStore together.

    loghubEndpoint

    The endpoint of loghub.

    accessKeyId

    The Aliyun Access Key Id.

    accessKeySecret

    The Aliyun Access Key Secret.

    storageLevel

    Storage level to use for storing the received objects.

    cursorPosition

    Set user defined cursor type.

    mLoghubCursorStartTime

    Set user defined cursor position (Unix Timestamp).

    forceSpecial

    Whether to force to set consume position as the mLoghubCursorStartTime.

23. def createStream(ssc: StreamingContext, logServiceProject: String, logStoreName: String, loghubConsumerGroupName: String, numReceivers: Int, storageLevel: StorageLevel): DStream[Array[Byte]]

    

    Create loghub DStream.

    Create loghub DStream.

    val loghubProject = "sample-project"
    val logStream = "sample-logstore"
    val loghubGroupName = "sample-group"
    val numReceivers = 2
    val batchInterval = Milliseconds(5 * 1000)
    
    val conf = new SparkConf().setAppName("Test Loghub")
    val ssc = new StreamingContext(conf, batchInterval)
    val loghubStream = LoghubUtils.createStream(
      ssc,
      loghubProject,
      logStream,
      loghubGroupName,
      numReceivers,
      StorageLevel.MEMORY_AND_DISK)

    ssc

    StreamingContext.

    logServiceProject

    The name of LogService project.

    logStoreName

    The name of logStore.

    loghubConsumerGroupName

    The group name of loghub consumer. All consumer process which has the same group name will consumer specific logStore together.

    numReceivers

    The number of receivers.

    storageLevel

    Storage level to use for storing the received objects.

24. def createStream(ssc: StreamingContext, logServiceProject: String, logStoreName: String, loghubConsumerGroupName: String, loghubEndpoint: String, numReceivers: Int, accessKeyId: String, accessKeySecret: String, storageLevel: StorageLevel): DStream[Array[Byte]]

    

    Create loghub DStream.

    Create loghub DStream.

    val loghubProject = "sample-project"
    val logStream = "sample-logstore"
    val loghubGroupName = "sample-group"
    val endpoint = "cn-hangzhou-intranet.sls.aliyuncs.com"
    val accessKeyId = "kj7aY*******UYx6"
    val accessKeySecret = "AiNMAlxz*************1PxaPaL8t"
    val numReceivers = 2
    val batchInterval = Milliseconds(5 * 1000)
    
    val conf = new SparkConf().setAppName("Test Loghub")
    val ssc = new StreamingContext(conf, batchInterval)
    val loghubStream = LoghubUtils.createStream(
      ssc,
      loghubProject,
      logStream,
      loghubGroupName,
      endpoint,
      numReceivers,
      accessKeyId,
      accessKeySecret,
      StorageLevel.MEMORY_AND_DISK)

    ssc

    StreamingContext.

    logServiceProject

    The name of LogService project.

    logStoreName

    The name of logStore.

    loghubConsumerGroupName

    The group name of loghub consumer. All consumer process which has the same group name will consumer specific logStore together.

    loghubEndpoint

    The endpoint of loghub.

    numReceivers

    The number of receivers.

    accessKeyId

    The Aliyun Access Key Id.

    accessKeySecret

    The Aliyun Access Key Secret.

    storageLevel

    Storage level to use for storing the received objects.

25. def createStream(ssc: StreamingContext, logServiceProject: String, logStoreName: String, loghubConsumerGroupName: String, storageLevel: StorageLevel): ReceiverInputDStream[Array[Byte]]

    

    val loghubProject = "sample-project"
    val logStream = "sample-logstore"
    val loghubGroupName = "sample-group"
    val batchInterval = Milliseconds(5 * 1000)
    
    val conf = new SparkConf().setAppName("Test Loghub")
    val ssc = new StreamingContext(conf, batchInterval)
    val loghubStream = LoghubUtils.createStream(
      ssc,
      loghubProject,
      logStream,
      loghubGroupName,
      StorageLevel.MEMORY_AND_DISK)

    ssc

    StreamingContext.

    logServiceProject

    The name of LogService project.

    logStoreName

    The name of logStore.

    loghubConsumerGroupName

    The group name of loghub consumer. All consumer process which has the same group name will consumer specific logStore together.

    storageLevel

    Storage level to use for storing the received objects.

26. def createStream(ssc: StreamingContext, logServiceProject: String, logStoreName: String, loghubConsumerGroupName: String, loghubEndpoint: String, accessKeyId: String, accessKeySecret: String, storageLevel: StorageLevel): ReceiverInputDStream[Array[Byte]]

    

    val loghubProject = "sample-project"
    val logStream = "sample-logstore"
    val loghubGroupName = "sample-group"
    val endpoint = "cn-hangzhou-intranet.sls.aliyuncs.com"
    val accessKeyId = "kj7aY*******UYx6"
    val accessKeySecret = "AiNMAlxz*************1PxaPaL8t"
    val batchInterval = Milliseconds(5 * 1000)
    
    val conf = new SparkConf().setAppName("Test Loghub")
    val ssc = new StreamingContext(conf, batchInterval)
    val loghubStream = LoghubUtils.createStream(
      ssc,
      loghubProject,
      logStream,
      loghubGroupName,
      endpoint,
      accessKeyId,
      accessKeySecret,
      StorageLevel.MEMORY_AND_DISK)

    ssc

    StreamingContext.

    logServiceProject

    The name of LogService project.

    logStoreName

    The name of logStore.

    loghubConsumerGroupName

    The group name of loghub consumer. All consumer process which has the same group name will consumer specific logStore together.

    loghubEndpoint

    The endpoint of loghub.

    accessKeyId

    The Aliyun Access Key Id.

    accessKeySecret

    The Aliyun Access Key Secret.

    storageLevel

    Storage level to use for storing the received objects.

27. final def eq(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

28. def equals(arg0: Any): Boolean

    

    Definition Classes

    AnyRef → Any

29. def finalize(): Unit

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( classOf[java.lang.Throwable] )

30. final def getClass(): Class[_]

    

    Definition Classes

    AnyRef → Any

31. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

32. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

33. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

34. final def notify(): Unit

    

    Definition Classes

    AnyRef

35. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

36. final def synchronized[T0](arg0: ⇒ T0): T0

    

    Definition Classes

    AnyRef

37. def toString(): String

    

    Definition Classes

    AnyRef → Any

38. final def wait(): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

39. final def wait(arg0: Long, arg1: Int): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

40. final def wait(arg0: Long): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )