DataWorks:HBase veri kaynağı

{
    "type":"job",
    "version":"2.0",// The version number.
    "steps":[
        {
            "stepType":"hbase",// The plugin name.
            "parameter":{
                "mode":"normal",// Specifies the mode for reading data from HBase. Valid values: `normal` and `multiVersionFixedColumn`.
                "scanCacheSize":"256",// Specifies the number of rows to fetch from the server per RPC.
                "scanBatchSize":"100",// Specifies the number of columns to fetch from the server per RPC. 
                "hbaseVersion":"094x/11x",// The HBase version.
                "column":[// The fields to read.
                    {
                        "name":"rowkey",// The field name.
                        "type":"string"// The data type.
                    },
                    {
                        "name":"basic:age",
                        "type":"string"
                    },
                    {
                        "name":"basic:name",
                        "type":"string"
                    },
                    {
                        "name":"score:english",
                        "type":"string"
                    },
                    {
                        "name":"score:math",
                        "type":"string"
                    }
                ],
                "range":{// Specifies the rowkey range to read.
                    "endRowkey":"",// The end rowkey.
                    "isBinaryRowkey":true,// Specifies whether to use binary conversion for startRowkey and endRowkey. The default value is false.
                    "startRowkey":""// The start rowkey.
                },
                "maxVersion":"",// Specifies the number of versions to read in multi-version mode.
                "encoding":"UTF-8",// The encoding format.
                "table":"student",// The table name.
                "hbaseConfig":{// The connection configuration for the HBase cluster, in JSON format.
                    "hbase.zookeeper.quorum":"hostname",
                    "hbase.rootdir":"hdfs://ip:port/database",
                    "hbase.cluster.distributed":"true"
                }
            },
            "name":"Reader",
            "category":"reader"
        },
        {
            "stepType":"odps",// The plugin name for the destination. This example uses MaxCompute. You can replace it with another Writer plugin.
            "parameter":{
                "partition":"",// Partition information for the destination table. Not required for non-partitioned tables.
                "truncate":true,// Specifies whether to clear the destination table or partition before writing data.
                "datasource":"odps_datasource",// The MaxCompute data source name.
                "column":[// The destination fields.
                    "rowkey",
                    "basic_age",
                    "basic_name",
                    "score_english",
                    "score_math"
                ],
                "table":"student_target"// The name of the destination MaxCompute table.
            },
            "name":"Writer",
            "category":"writer"
        }
    ],
    "setting":{
        "errorLimit":{
            "record":"0"// The maximum number of error records to allow before the job fails.
        },
        "speed":{
            "throttle":true,// Specifies whether to enable rate limiting. If set to false, the mbps parameter is ignored.
            "concurrent":1,// The number of concurrent jobs.
            "mbps":"12"// The rate limit in megabytes per second (MB/s).
        }
    },
    "order":{
        "hops":[
            {
                "from":"Reader",
                "to":"Writer"
            }
        ]
    }
}

{
    "type":"job",
    "version":"2.0",
    "steps":[
        {
            "stepType":"hbase",// Plugin name.
            "parameter":{
                "mode":"multiVersionFixedColumn",// The mode for reading data from HBase. This example uses multiVersionFixedColumn mode.
                "scanCacheSize":"256",// The number of rows that the HBase client reads from the server in each remote procedure call (RPC).
                "scanBatchSize":"100",// The number of columns that the HBase client reads from the server in each RPC.
                "hbaseVersion":"20x",// HBase version.
                "datasource":"hbase_datasource",// HBase data source name.
                "column":[// The columns to read. The first column must be rowkey. The names of other columns must be in the "column family:qualifier" format.
                    {
                        "name":"rowkey",// The rowkey column.
                        "type":"string"
                    },
                    {
                        "name":"basic:age",// The age column in the basic column family.
                        "type":"string"
                    },
                    {
                        "name":"basic:name",// The name column in the basic column family.
                        "type":"string"
                    },
                    {
                        "name":"score:english",// The english column in the score column family.
                        "type":"string"
                    },
                    {
                        "name":"score:math",// The math column in the score column family.
                        "type":"string"
                    }
                ],
                "range":{
                    "isBinaryRowkey":false
                },
                "maxVersion":"-1",// The maximum number of versions to read. This parameter is required in multiVersionFixedColumn mode. A value of -1 specifies that all versions are read.
                "encoding":"UTF-8",// Encoding format.
                "table":"student"// HBase table name.
            },
            "name":"Reader",
            "category":"reader"
        },
        {
            "stepType":"odps",// The name of the destination plugin. This example uses MaxCompute.
            "parameter":{
                "partition":"",// The partition of the destination table. This parameter is not required for non-partitioned tables.
                "truncate":true,// If set to true, this clears the destination table or partition before writing data.
                "datasource":"odps_datasource",// MaxCompute data source name.
                "column":[// The destination has four fixed columns that correspond to the rowkey, family:qualifier, timestamp, and value from the source, respectively.
                    "row_key",
                    "cf",
                    "timestamp_col",
                    "value"
                ],
                "table":"hbase_multiversion_target"// The name of the destination MaxCompute table.
            },
            "name":"Writer",
            "category":"writer"
        }
    ],
    "setting":{
        "errorLimit":{
            "record":"0"// The maximum number of error records allowed.
        },
        "speed":{
            "throttle":false,// No rate limiting.
            "concurrent":2// Job concurrency.
        }
    },
    "order":{
        "hops":[
            {
                "from":"Reader",
                "to":"Writer"
            }
        ]
    }
}

{
    "type":"job",
    "version":"2.0",// The version number.
    "steps":[
        {
            "stepType":"stream",
            "parameter":{},
            "name":"Reader",
            "category":"reader"
        },
        {
            "stepType":"hbase",// The plugin name.
            "parameter":{
                "mode":"normal",// The write mode for HBase.
                "walFlag":"false",// Set to `false` to disable write-ahead logging (WAL).
                "hbaseVersion":"094x",// The HBase version.
                "rowkeyColumn":[// The columns that form the HBase rowkey.
                    {
                        "index":"0",// The index of the source data column.
                        "type":"string"// The data type for this part of the rowkey.
                    },
                    {
                        "index":"-1",
                        "type":"string",
                        "value":"_"
                    }
                ],
                "nullMode":"skip",// Specifies how to handle null values from the source.
                "column":[// The destination columns in the HBase table.
                    {
                        "name":"columnFamilyName1:columnName1",// The column name, in `family:qualifier` format.
                        "index":"0",// The index of the source data column.
                        "type":"string"// The data type of the column value.
                    },
                    {
                        "name":"columnFamilyName2:columnName2",
                        "index":"1",
                        "type":"string"
                    },
                    {
                        "name":"columnFamilyName3:columnName3",
                        "index":"2",
                        "type":"string"
                    }
                ],
                "encoding":"utf-8",// The character encoding.
                "table":"",// The name of the destination HBase table.
                "hbaseConfig":{// Configuration for the HBase cluster connection, in JSON format.
                    "hbase.zookeeper.quorum":"hostname",
                    "hbase.rootdir":"hdfs: //ip:port/database",
                    "hbase.cluster.distributed":"true"
                }
            },
            "name":"Writer",
            "category":"writer"
        }
    ],
    "setting":{
        "errorLimit":{
            "record":"0"// The maximum number of allowed error records.
        },
        "speed":{
            "throttle":true,// Enables (`true`) or disables (`false`) rate limiting. If `true`, the rate is defined by the `mbps` parameter.
            "concurrent":1, // The number of concurrent write tasks.
            "mbps":"12"// The maximum transfer rate in megabytes per second (MB/s).
        }
    },
    "order":{
        "hops":[
            {
                "from":"Reader",
                "to":"Writer"
            }
        ]
    }
}

{
    "type":"job",
    "version":"2.0",// Version number.
    "steps":[
        {
            "stepType":"hbase20xsql",// Plugin name.
            "parameter":{
                "queryServerAddress": "http://127.0.0.1:8765",  // Phoenix QueryServer endpoint.
                "serialization": "PROTOBUF",  // QueryServer serialization format.
                "table": "TEST",    // Table to read.
                "column": ["ID", "NAME"],   // Columns to read.
                "splitKey": "ID"    // Sharding key, which must be the primary key of the table.
            },
            "name":"Reader",
            "category":"reader"
        },
        {
            "stepType":"stream",
            "parameter":{},
            "name":"Writer",
            "category":"writer"
        }
    ],
    "setting":{
        "errorLimit":{
            "record":"0"// Maximum allowed error records.
        },
        "speed":{
            "throttle":true,// Toggles rate limiting. If true, the rate is limited by the mbps parameter.
            "concurrent":1,// Number of concurrent jobs.
            "mbps":"12"// Rate limit in MB/s.
        }
    },
    "order":{
        "hops":[
            {
                "from":"Reader",
                "to":"Writer"
            }
        ]
    }
}

{
  "type": "job",
  "version": "1.0",
  "configuration": {
    "setting": {
      "errorLimit": {
        "record": "0"
      },
      "speed": {
            "throttle":true,// Enables rate limiting. If set to false, the mbps parameter is ignored.
            "concurrent":1, // The number of concurrent jobs.
            "mbps":"1"// Rate limit in MB/s.
      }
    },
    "reader": {
      "plugin": "odps",
      "parameter": {
        "datasource": "",
        "table": "",
        "column": [],
        "partition": ""
      }
    },
    "plugin": "hbase11xsql",
    "parameter": {
      "table": "The name of the destination HBase table. The name is case-sensitive.",
      "hbaseConfig": {
        "hbase.zookeeper.quorum": "The ZooKeeper endpoint of the destination HBase cluster.",
        "zookeeper.znode.parent": "The znode of the destination HBase cluster."
      },
      "column": [
        "columnName"
      ],
      "batchSize": 256,
      "nullMode": "skip"
    }
  }
}