database-plugins/cloudsql-postgresql-plugin/samples/CloudSQLPostgreSQL-to-BigQuery.json at develop · data-integrations/database-plugins · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
{
  "name": "cloudsqlpostgresql-bigquery",
  "description": "Data Pipeline Application",
  "artifact": {
    "name": "cdap-data-pipeline",
    "version": "6.1.2",
    "scope": "SYSTEM"
  },
  "config": {
    "stages": [
      {
        "name": "CloudSQL PostgreSQL",
        "plugin": {
          "name": "CloudSQLPostgreSQL",
          "type": "batchsource",
          "label": "CloudSQL PostgreSQL",
          "artifact": {
            "name": "cloudsql-postgresql-plugin",
            "version": "1.4.0-SNAPSHOT",
            "scope": "USER"
          },
          "properties": {
            "jdbcPluginName": "cloudsql-postgresql",
            "instanceType": "public",
            "numSplits": "500",
            "referenceName": "source",
            "database": "postgres",
            "user": "postgres",
            "password": "",
            "connectionName": "",
            "importQuery": "select * from orders where $CONDITIONS;",
            "boundingQuery": "select min(\"ORDERKEY\"), max(\"ORDERKEY\") from qin1tb_orders;",
            "splitBy": "\"ORDERKEY\"",
            "schema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"ORDERKEY\",\"type\":[\"int\",\"null\"]},{\"name\":\"CUSTKEY\",\"type\":[\"int\",\"null\"]},{\"name\":\"ORDERSTATUS\",\"type\":[\"string\",\"null\"]},{\"name\":\"ORDERDATE\",\"type\":[{\"type\":\"int\",\"logicalType\":\"date\"},\"null\"]},{\"name\":\"ORDERPRIORITY\",\"type\":[\"string\",\"null\"]},{\"name\":\"CLERK\",\"type\":[\"string\",\"null\"]},{\"name\":\"SHIPPRIORITY\",\"type\":[\"int\",\"null\"]},{\"name\":\"O_COMMENT\",\"type\":[\"string\",\"null\"]}]}"
          }
        },
        "outputSchema": [
          {
            "name": "etlSchemaBody",
            "schema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"ORDERKEY\",\"type\":[\"int\",\"null\"]},{\"name\":\"CUSTKEY\",\"type\":[\"int\",\"null\"]},{\"name\":\"ORDERSTATUS\",\"type\":[\"string\",\"null\"]},{\"name\":\"ORDERDATE\",\"type\":[{\"type\":\"int\",\"logicalType\":\"date\"},\"null\"]},{\"name\":\"ORDERPRIORITY\",\"type\":[\"string\",\"null\"]},{\"name\":\"CLERK\",\"type\":[\"string\",\"null\"]},{\"name\":\"SHIPPRIORITY\",\"type\":[\"int\",\"null\"]},{\"name\":\"O_COMMENT\",\"type\":[\"string\",\"null\"]}]}"
          }
        ],
        "type": "batchsource",
        "label": "CloudSQL PostgreSQL",
        "icon": "fa-plug"
      },
      {
        "name": "BigQuery",
        "plugin": {
          "name": "BigQueryTable",
          "type": "batchsink",
          "label": "BigQuery",
          "artifact": {
            "name": "google-cloud",
            "version": "0.14.2",
            "scope": "SYSTEM"
          },
          "properties": {
            "project": "auto-detect",
            "serviceFilePath": "auto-detect",
            "operation": "insert",
            "truncateTable": "false",
            "allowSchemaRelaxation": "false",
            "location": "US",
            "createPartitionedTable": "false",
            "partitionFilterRequired": "false",
            "schema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"ORDERKEY\",\"type\":[\"int\",\"null\"]},{\"name\":\"CUSTKEY\",\"type\":[\"int\",\"null\"]},{\"name\":\"ORDERSTATUS\",\"type\":[\"string\",\"null\"]},{\"name\":\"ORDERDATE\",\"type\":[{\"type\":\"int\",\"logicalType\":\"date\"},\"null\"]},{\"name\":\"ORDERPRIORITY\",\"type\":[\"string\",\"null\"]},{\"name\":\"CLERK\",\"type\":[\"string\",\"null\"]},{\"name\":\"SHIPPRIORITY\",\"type\":[\"int\",\"null\"]},{\"name\":\"O_COMMENT\",\"type\":[\"string\",\"null\"]}]}",
            "referenceName": "sink",
            "dataset": "postgresOrders",
            "table": "orders"
          }
        },
        "outputSchema": [
          {
            "name": "etlSchemaBody",
            "schema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"ORDERKEY\",\"type\":[\"int\",\"null\"]},{\"name\":\"CUSTKEY\",\"type\":[\"int\",\"null\"]},{\"name\":\"ORDERSTATUS\",\"type\":[\"string\",\"null\"]},{\"name\":\"ORDERDATE\",\"type\":[{\"type\":\"int\",\"logicalType\":\"date\"},\"null\"]},{\"name\":\"ORDERPRIORITY\",\"type\":[\"string\",\"null\"]},{\"name\":\"CLERK\",\"type\":[\"string\",\"null\"]},{\"name\":\"SHIPPRIORITY\",\"type\":[\"int\",\"null\"]},{\"name\":\"O_COMMENT\",\"type\":[\"string\",\"null\"]}]}"
          }
        ],
        "inputSchema": [
          {
            "name": "CloudSQL PostgreSQL",
            "schema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"ORDERKEY\",\"type\":[\"int\",\"null\"]},{\"name\":\"CUSTKEY\",\"type\":[\"int\",\"null\"]},{\"name\":\"ORDERSTATUS\",\"type\":[\"string\",\"null\"]},{\"name\":\"ORDERDATE\",\"type\":[{\"type\":\"int\",\"logicalType\":\"date\"},\"null\"]},{\"name\":\"ORDERPRIORITY\",\"type\":[\"string\",\"null\"]},{\"name\":\"CLERK\",\"type\":[\"string\",\"null\"]},{\"name\":\"SHIPPRIORITY\",\"type\":[\"int\",\"null\"]},{\"name\":\"O_COMMENT\",\"type\":[\"string\",\"null\"]}]}"
          }
        ],
        "type": "batchsink",
        "label": "BigQuery",
        "icon": "fa-plug"
      }
    ],
    "connections": [
      {
        "from": "CloudSQL PostgreSQL",
        "to": "BigQuery"
      }
    ],
    "comments": [],
    "resources": {
      "memoryMB": 1024,
      "virtualCores": 1
    },
    "driverResources": {
      "memoryMB": 1024,
      "virtualCores": 1
    },
    "postActions": [],
    "properties": {},
    "processTimingEnabled": true,
    "stageLoggingEnabled": false,
    "engine": "spark",
    "schedule": "0 * * * *",
    "maxConcurrentRuns": 1
  }
}