Discover ClickPipe source schema

This endpoint is in beta. API contract is stable, and no breaking changes are expected in the future.

Infers the schema (field names and ClickHouse data types) of a streaming ClickPipe source without creating a pipe. Supported for Kafka and Kinesis sources.

POST /v1/organizations/{organizationId}/services/{serviceId}/clickpipes/schemaDiscovery

Discover ClickPipe source schema

cURL

curl --request POST \
  --url https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes/schemaDiscovery \
  --header 'Authorization: Basic <encoded-value>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "source": {
    "kafka": {
      "brokers": "<string>",
      "topics": "<string>",
      "consumerGroup": "my-clickpipe-consumer-group",
      "iamRole": "arn:aws:iam::123456789012:role/MyRole",
      "offset": {
        "timestamp": "2021-01-01T00:00"
      },
      "schemaRegistry": {
        "url": "https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004",
        "authentication": "PLAIN",
        "caCertificate": "<string>",
        "credentials": {
          "username": "<string>",
          "password": "<string>"
        }
      },
      "caCertificate": "<string>",
      "reversePrivateEndpointIds": [
        "<string>"
      ],
      "exactlyOnce": true,
      "credentials": {
        "username": "postgres_user",
        "password": "your_secure_password"
      }
    },
    "kinesis": {
      "streamName": "my-stream",
      "region": "us-east-1",
      "useEnhancedFanOut": true,
      "timestamp": 1615766400,
      "iamRole": "arn:aws:iam::123456789012:role/MyRole",
      "accessKey": {
        "accessKeyId": "<string>",
        "secretKey": "<string>"
      }
    }
  }
}
'

import requests

url = "https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes/schemaDiscovery"

payload = { "source": {
        "kafka": {
            "brokers": "<string>",
            "topics": "<string>",
            "consumerGroup": "my-clickpipe-consumer-group",
            "iamRole": "arn:aws:iam::123456789012:role/MyRole",
            "offset": { "timestamp": "2021-01-01T00:00" },
            "schemaRegistry": {
                "url": "https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004",
                "authentication": "PLAIN",
                "caCertificate": "<string>",
                "credentials": {
                    "username": "<string>",
                    "password": "<string>"
                }
            },
            "caCertificate": "<string>",
            "reversePrivateEndpointIds": ["<string>"],
            "exactlyOnce": True,
            "credentials": {
                "username": "postgres_user",
                "password": "your_secure_password"
            }
        },
        "kinesis": {
            "streamName": "my-stream",
            "region": "us-east-1",
            "useEnhancedFanOut": True,
            "timestamp": 1615766400,
            "iamRole": "arn:aws:iam::123456789012:role/MyRole",
            "accessKey": {
                "accessKeyId": "<string>",
                "secretKey": "<string>"
            }
        }
    } }
headers = {
    "Authorization": "Basic <encoded-value>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Basic <encoded-value>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    source: {
      kafka: {
        brokers: '<string>',
        topics: '<string>',
        consumerGroup: 'my-clickpipe-consumer-group',
        iamRole: 'arn:aws:iam::123456789012:role/MyRole',
        offset: {timestamp: '2021-01-01T00:00'},
        schemaRegistry: {
          url: 'https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004',
          authentication: 'PLAIN',
          caCertificate: '<string>',
          credentials: {username: '<string>', password: '<string>'}
        },
        caCertificate: '<string>',
        reversePrivateEndpointIds: ['<string>'],
        exactlyOnce: true,
        credentials: {username: 'postgres_user', password: 'your_secure_password'}
      },
      kinesis: {
        streamName: 'my-stream',
        region: 'us-east-1',
        useEnhancedFanOut: true,
        timestamp: 1615766400,
        iamRole: 'arn:aws:iam::123456789012:role/MyRole',
        accessKey: {accessKeyId: '<string>', secretKey: '<string>'}
      }
    }
  })
};

fetch('https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes/schemaDiscovery', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes/schemaDiscovery",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'source' => [
        'kafka' => [
                'brokers' => '<string>',
                'topics' => '<string>',
                'consumerGroup' => 'my-clickpipe-consumer-group',
                'iamRole' => 'arn:aws:iam::123456789012:role/MyRole',
                'offset' => [
                                'timestamp' => '2021-01-01T00:00'
                ],
                'schemaRegistry' => [
                                'url' => 'https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004',
                                'authentication' => 'PLAIN',
                                'caCertificate' => '<string>',
                                'credentials' => [
                                                                'username' => '<string>',
                                                                'password' => '<string>'
                                ]
                ],
                'caCertificate' => '<string>',
                'reversePrivateEndpointIds' => [
                                '<string>'
                ],
                'exactlyOnce' => true,
                'credentials' => [
                                'username' => 'postgres_user',
                                'password' => 'your_secure_password'
                ]
        ],
        'kinesis' => [
                'streamName' => 'my-stream',
                'region' => 'us-east-1',
                'useEnhancedFanOut' => true,
                'timestamp' => 1615766400,
                'iamRole' => 'arn:aws:iam::123456789012:role/MyRole',
                'accessKey' => [
                                'accessKeyId' => '<string>',
                                'secretKey' => '<string>'
                ]
        ]
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Basic <encoded-value>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes/schemaDiscovery"

	payload := strings.NewReader("{\n  \"source\": {\n    \"kafka\": {\n      \"brokers\": \"<string>\",\n      \"topics\": \"<string>\",\n      \"consumerGroup\": \"my-clickpipe-consumer-group\",\n      \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n      \"offset\": {\n        \"timestamp\": \"2021-01-01T00:00\"\n      },\n      \"schemaRegistry\": {\n        \"url\": \"https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004\",\n        \"authentication\": \"PLAIN\",\n        \"caCertificate\": \"<string>\",\n        \"credentials\": {\n          \"username\": \"<string>\",\n          \"password\": \"<string>\"\n        }\n      },\n      \"caCertificate\": \"<string>\",\n      \"reversePrivateEndpointIds\": [\n        \"<string>\"\n      ],\n      \"exactlyOnce\": true,\n      \"credentials\": {\n        \"username\": \"postgres_user\",\n        \"password\": \"your_secure_password\"\n      }\n    },\n    \"kinesis\": {\n      \"streamName\": \"my-stream\",\n      \"region\": \"us-east-1\",\n      \"useEnhancedFanOut\": true,\n      \"timestamp\": 1615766400,\n      \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n      \"accessKey\": {\n        \"accessKeyId\": \"<string>\",\n        \"secretKey\": \"<string>\"\n      }\n    }\n  }\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Basic <encoded-value>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes/schemaDiscovery")
  .header("Authorization", "Basic <encoded-value>")
  .header("Content-Type", "application/json")
  .body("{\n  \"source\": {\n    \"kafka\": {\n      \"brokers\": \"<string>\",\n      \"topics\": \"<string>\",\n      \"consumerGroup\": \"my-clickpipe-consumer-group\",\n      \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n      \"offset\": {\n        \"timestamp\": \"2021-01-01T00:00\"\n      },\n      \"schemaRegistry\": {\n        \"url\": \"https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004\",\n        \"authentication\": \"PLAIN\",\n        \"caCertificate\": \"<string>\",\n        \"credentials\": {\n          \"username\": \"<string>\",\n          \"password\": \"<string>\"\n        }\n      },\n      \"caCertificate\": \"<string>\",\n      \"reversePrivateEndpointIds\": [\n        \"<string>\"\n      ],\n      \"exactlyOnce\": true,\n      \"credentials\": {\n        \"username\": \"postgres_user\",\n        \"password\": \"your_secure_password\"\n      }\n    },\n    \"kinesis\": {\n      \"streamName\": \"my-stream\",\n      \"region\": \"us-east-1\",\n      \"useEnhancedFanOut\": true,\n      \"timestamp\": 1615766400,\n      \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n      \"accessKey\": {\n        \"accessKeyId\": \"<string>\",\n        \"secretKey\": \"<string>\"\n      }\n    }\n  }\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes/schemaDiscovery")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Basic <encoded-value>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"source\": {\n    \"kafka\": {\n      \"brokers\": \"<string>\",\n      \"topics\": \"<string>\",\n      \"consumerGroup\": \"my-clickpipe-consumer-group\",\n      \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n      \"offset\": {\n        \"timestamp\": \"2021-01-01T00:00\"\n      },\n      \"schemaRegistry\": {\n        \"url\": \"https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004\",\n        \"authentication\": \"PLAIN\",\n        \"caCertificate\": \"<string>\",\n        \"credentials\": {\n          \"username\": \"<string>\",\n          \"password\": \"<string>\"\n        }\n      },\n      \"caCertificate\": \"<string>\",\n      \"reversePrivateEndpointIds\": [\n        \"<string>\"\n      ],\n      \"exactlyOnce\": true,\n      \"credentials\": {\n        \"username\": \"postgres_user\",\n        \"password\": \"your_secure_password\"\n      }\n    },\n    \"kinesis\": {\n      \"streamName\": \"my-stream\",\n      \"region\": \"us-east-1\",\n      \"useEnhancedFanOut\": true,\n      \"timestamp\": 1615766400,\n      \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n      \"accessKey\": {\n        \"accessKeyId\": \"<string>\",\n        \"secretKey\": \"<string>\"\n      }\n    }\n  }\n}"

response = http.request(request)
puts response.read_body

200

{
  "status": 200,
  "requestId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "result": {
    "fields": [
      {
        "name": "user_id",
        "type": "Int64",
        "optional": true
      }
    ]
  }
}

{
  "status": 400,
  "error": "<string>",
  "requestId": "3c90c3cc-0d44-4b50-8888-8dd25736052a"
}

{
  "status": 500,
  "error": "<string>",
  "requestId": "3c90c3cc-0d44-4b50-8888-8dd25736052a"
}

Beta

This endpoint is in beta. API contract is stable, and no breaking changes are expected in the future.

Infers the schema (field names and ClickHouse data types) of a streaming ClickPipe source without creating a pipe. Supported for Kafka and Kinesis sources.

Authorizations

Authorization · string · header · required

Use key ID and key secret obtained in ClickHouse Cloud console: https://clickhouse.com/docs/cloud/manage/openapi

Path Parameters

organizationId · string<uuid> · required

ID of the organization that owns the service.

serviceId · string<uuid> · required

ID of the service to run schema discovery against.

Body

application/json

source · object

Show child attributes

Response

200

application/json

Successful response

status · number

HTTP status code.

Example:

200

requestId · string<uuid>

Unique id assigned to every request. UUIDv4

result · object

Show child attributes