database-plugins/cloudsql-mysql-plugin/samples/BigQuery-to-CloudSQLMySQL.json at develop · data-integrations/database-plugins · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
{
  "artifact": {
    "name": "cdap-data-pipeline",
    "version": "6.1.2",
    "scope": "SYSTEM"
  },
  "description": "",
  "name": "bigquery-cloudsqlmysql",
  "config": {
    "resources": {
      "memoryMB": 1024,
      "virtualCores": 1
    },
    "driverResources": {
      "memoryMB": 1024,
      "virtualCores": 1
    },
    "connections": [
      {
        "from": "BigQuery",
        "to": "CloudSQL MySQL"
      },
      {
        "from": "CloudSQL MySQL",
        "to": "CloudSQL MySQL Execute"
      }
    ],
    "comments": [],
    "postActions": [],
    "properties": {},
    "processTimingEnabled": true,
    "stageLoggingEnabled": false,
    "stages": [
      {
        "name": "BigQuery",
        "plugin": {
          "name": "BigQueryTable",
          "type": "batchsource",
          "label": "BigQuery",
          "artifact": {
            "name": "google-cloud",
            "version": "0.14.2",
            "scope": "SYSTEM"
          },
          "properties": {
            "project": "auto-detect",
            "serviceFilePath": "auto-detect",
            "referenceName": "source",
            "dataset": "postgresOrders",
            "table": "orders",
            "schema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"ORDERKEY\",\"type\":[\"int\",\"null\"]},{\"name\":\"CUSTKEY\",\"type\":[\"int\",\"null\"]},{\"name\":\"ORDERSTATUS\",\"type\":[\"string\",\"null\"]},{\"name\":\"ORDERDATE\",\"type\":[{\"type\":\"int\",\"logicalType\":\"date\"},\"null\"]},{\"name\":\"ORDERPRIORITY\",\"type\":[\"string\",\"null\"]},{\"name\":\"CLERK\",\"type\":[\"string\",\"null\"]},{\"name\":\"SHIPPRIORITY\",\"type\":[\"int\",\"null\"]},{\"name\":\"O_COMMENT\",\"type\":[\"string\",\"null\"]}]}"
          }
        },
        "outputSchema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"ORDERKEY\",\"type\":[\"int\",\"null\"]},{\"name\":\"CUSTKEY\",\"type\":[\"int\",\"null\"]},{\"name\":\"ORDERSTATUS\",\"type\":[\"string\",\"null\"]},{\"name\":\"ORDERDATE\",\"type\":[{\"type\":\"int\",\"logicalType\":\"date\"},\"null\"]},{\"name\":\"ORDERPRIORITY\",\"type\":[\"string\",\"null\"]},{\"name\":\"CLERK\",\"type\":[\"string\",\"null\"]},{\"name\":\"SHIPPRIORITY\",\"type\":[\"int\",\"null\"]},{\"name\":\"O_COMMENT\",\"type\":[\"string\",\"null\"]}]}"
      },
      {
        "name": "CloudSQL MySQL",
        "plugin": {
          "name": "CloudSQLMySQL",
          "type": "batchsink",
          "label": "CloudSQL MySQL",
          "artifact": {
            "name": "cloudsql-mysql-plugin",
            "version": "1.4.0-SNAPSHOT",
            "scope": "USER"
          },
          "properties": {
            "jdbcPluginName": "cloudsql-mysql",
            "instanceType": "public",
            "transactionIsolationLevel": "TRANSACTION_REPEATABLE_READ",
            "connectionTimeout": "10",
            "referenceName": "sink",
            "database": "testdb",
            "user": "root",
            "password": "",
            "connectionName": "",
            "tableName": "orders"
          }
        },
        "outputSchema": [
          {
            "name": "etlSchemaBody",
            "schema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"ORDERKEY\",\"type\":[\"int\",\"null\"]},{\"name\":\"CUSTKEY\",\"type\":[\"int\",\"null\"]},{\"name\":\"ORDERSTATUS\",\"type\":[\"string\",\"null\"]},{\"name\":\"ORDERDATE\",\"type\":[{\"type\":\"int\",\"logicalType\":\"date\"},\"null\"]},{\"name\":\"ORDERPRIORITY\",\"type\":[\"string\",\"null\"]},{\"name\":\"CLERK\",\"type\":[\"string\",\"null\"]},{\"name\":\"SHIPPRIORITY\",\"type\":[\"int\",\"null\"]},{\"name\":\"O_COMMENT\",\"type\":[\"string\",\"null\"]}]}"
          }
        ],
        "inputSchema": [
          {
            "name": "BigQuery",
            "schema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"ORDERKEY\",\"type\":[\"int\",\"null\"]},{\"name\":\"CUSTKEY\",\"type\":[\"int\",\"null\"]},{\"name\":\"ORDERSTATUS\",\"type\":[\"string\",\"null\"]},{\"name\":\"ORDERDATE\",\"type\":[{\"type\":\"int\",\"logicalType\":\"date\"},\"null\"]},{\"name\":\"ORDERPRIORITY\",\"type\":[\"string\",\"null\"]},{\"name\":\"CLERK\",\"type\":[\"string\",\"null\"]},{\"name\":\"SHIPPRIORITY\",\"type\":[\"int\",\"null\"]},{\"name\":\"O_COMMENT\",\"type\":[\"string\",\"null\"]}]}"
          }
        ]
      },
      {
        "name": "CloudSQL MySQL Execute",
        "plugin": {
          "name": "CloudSQLMySQL",
          "type": "action",
          "label": "CloudSQL MySQL Execute",
          "artifact": {
            "name": "cloudsql-mysql-plugin",
            "version": "1.4.0-SNAPSHOT",
            "scope": "USER"
          },
          "properties": {
            "jdbcPluginName": "cloudsql-mysql",
            "instanceType": "public",
            "connectionTimeout": "10",
            "database": "testdb",
            "user": "root",
            "password": "",
            "connectionName": "",
            "query": "UPDATE orders SET CLERK=\"Clerk#000049252\" WHERE ORDERKEY=182250663;"
          }
        },
        "outputSchema": [
          {
            "name": "etlSchemaBody",
            "schema": ""
          }
        ]
      }
    ],
    "schedule": "0 * * * *",
    "engine": "spark",
    "numOfRecordsPreview": 100,
    "maxConcurrentRuns": 1
  }
}